-diff --git a/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/default-navbar.tsx b/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/default-navbar.tsx
-index 2480b7f061f..8e5429dbde6 100644
---- a/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/default-navbar.tsx
-+++ b/services/web/frontend/js/features/ui/components/bootstrap-5/navbar/default-navbar.tsx
-@@ -1,4 +1,4 @@
--import { useState } from 'react'
-+import React, { useState } from 'react'
- import { sendMB } from '@/infrastructure/event-tracking'
- import { useTranslation } from 'react-i18next'
- import { Button, Container, Nav, Navbar } from 'react-bootstrap'
-@@ -13,9 +13,15 @@ import MaterialIcon from '@/shared/components/material-icon'
- import { useContactUsModal } from '@/shared/hooks/use-contact-us-modal'
- import { UserProvider } from '@/shared/context/user-context'
- import { X } from '@phosphor-icons/react'
-+import overleafWhiteLogo from '@/shared/svgs/overleaf-white.svg'
-+import overleafBlackLogo from '@/shared/svgs/overleaf-black.svg'
-+import type { CSSPropertiesWithVariables } from '../../../../../../../types/css-properties-with-variables'
-
--function DefaultNavbar(props: DefaultNavbarMetadata) {
-+function DefaultNavbar(
-+ props: DefaultNavbarMetadata & { overleafLogo?: string }
-+) {
- const {
-+ overleafLogo,
- customLogo,
- title,
- canDisplayAdminMenu,
-@@ -49,10 +55,20 @@ function DefaultNavbar(props: DefaultNavbarMetadata) {
- className="navbar-default navbar-main"
- expand="lg"
- onToggle={expanded => setExpanded(expanded)}
-+ style={
-+ {
-+ '--navbar-brand-image-default-url': `url("${overleafWhiteLogo}")`,
-+ '--navbar-brand-image-redesign-url': `url("${overleafBlackLogo}")`,
-+ } as CSSPropertiesWithVariables
-+ }
- >
-
-
--
-+
- {enableUpgradeButton ? (
-
) {
-+}: Pick & {
-+ overleafLogo?: string
-+}) {
- const { appName } = getMeta('ol-ExposedSettings')
--
- if (customLogo) {
- return (
- // eslint-disable-next-line jsx-a11y/anchor-has-content
-@@ -24,9 +26,16 @@ export default function HeaderLogoOrTitle({
-
- )
- } else {
-+ const style = overleafLogo
-+ ? {
-+ style: {
-+ backgroundImage: `url("${overleafLogo}")`,
-+ },
-+ }
-+ : null
- return (
- // eslint-disable-next-line jsx-a11y/anchor-has-content
--
-+
-+
-+
-+
-+
-diff --git a/services/web/frontend/js/shared/svgs/overleaf-white.svg b/services/web/frontend/js/shared/svgs/overleaf-white.svg
-new file mode 100644
-index 00000000000..2ced81aa46d
---- /dev/null
-+++ b/services/web/frontend/js/shared/svgs/overleaf-white.svg
-@@ -0,0 +1 @@
-+ {
- cy.url().then(url => {
- if (!url.endsWith('/project')) {
- cy.visit('/project')
- }
- })
- const interceptId = uuid()
- let projectId = ''
- if (!open) {
- cy.then(() => {
- // Register intercept just before creating the project, otherwise we might
- // intercept a request from a prior createProject invocation.
- cy.intercept(
- { method: 'GET', url: /\/project\/[a-fA-F0-9]{24}$/, times: 1 },
- req => {
- projectId = req.url.split('/').pop()!
- // Redirect back to the project dashboard, effectively reload the page.
- req.redirect('/project')
- }
- ).as(interceptId)
- })
- }
cy.findAllByRole('button').contains(newProjectButtonMatcher).click()
// FIXME: This should only look in the left menu
- // The upgrading tests create projects in older versions of Server Pro which used different casing of the project type. Use case-insensitive match.
- cy.findAllByText(type, { exact: false }).first().click()
+ cy.findAllByText(type).first().click()
cy.findByRole('dialog').within(() => {
cy.get('input').type(name)
cy.findByText('Create').click()
})
- if (open) {
- cy.url().should('match', /\/project\/[a-fA-F0-9]{24}/)
- waitForMainDocToLoad()
- return cy
- .url()
- .should('match', /\/project\/[a-fA-F0-9]{24}/)
- .then(url => url.split('/').pop())
- } else {
- const alias = `@${interceptId}` // IDEs do not like computed values in cy.wait().
- cy.wait(alias)
- return cy.then(() => projectId)
- }
-}
-
-export function openProjectByName(projectName: string) {
- cy.visit('/project')
- cy.findByText(projectName).click()
- waitForMainDocToLoad()
-}
-
-export function openProjectById(projectId: string) {
- cy.visit(`/project/${projectId}`)
- waitForMainDocToLoad()
-}
-
-export function openProjectViaLinkSharingAsAnon(url: string) {
- cy.visit(url)
- waitForMainDocToLoad()
-}
-
-export function openProjectViaLinkSharingAsUser(
- url: string,
- projectName: string,
- email: string
-) {
- cy.visit(url)
- cy.findByText(projectName) // wait for lazy loading
- cy.contains(`as ${email}`)
- cy.findByText('OK, join project').click()
- waitForMainDocToLoad()
-}
-
-export function openProjectViaInviteNotification(projectName: string) {
- cy.visit('/project')
- cy.findByText(projectName)
- .parent()
- .parent()
- .within(() => {
- cy.findByText('Join Project').click()
- })
- cy.findByText('Open Project').click()
- cy.url().should('match', /\/project\/[a-fA-F0-9]{24}/)
- waitForMainDocToLoad()
+ return cy
+ .url()
+ .should('match', /\/project\/[a-fA-F0-9]{24}/)
+ .then(url => url.split('/').pop())
}
function shareProjectByEmail(
projectName: string,
email: string,
- level: 'Viewer' | 'Editor'
+ level: 'Read only' | 'Can edit'
) {
- openProjectByName(projectName)
+ cy.visit('/project')
+ cy.findByText(projectName).click()
cy.findByText('Share').click()
cy.findByRole('dialog').within(() => {
- cy.findByLabelText('Add people', { selector: 'input' }).type(`${email},`)
- cy.findByLabelText('Add people', { selector: 'input' })
+ cy.get('input').type(`${email},`)
+ cy.get('input')
.parents('form')
- .within(() => {
- cy.findByTestId('add-collaborator-select')
- .click()
- .then(() => {
- cy.findByText(level).click()
- })
- })
- cy.findByText('Invite').click({ force: true })
- cy.findByText('Invite not yet accepted.')
+ .within(() => cy.findByText('Can edit').parent().select(level))
+ cy.findByText('Share').click({ force: true })
})
}
export function shareProjectByEmailAndAcceptInviteViaDash(
projectName: string,
email: string,
- level: 'Viewer' | 'Editor'
+ level: 'Read only' | 'Can edit'
) {
shareProjectByEmail(projectName, email, level)
login(email)
- openProjectViaInviteNotification(projectName)
+ cy.visit('/project')
+ cy.findByText(new RegExp(projectName))
+ .parent()
+ .parent()
+ .within(() => {
+ cy.findByText('Join Project').click()
+ })
}
export function shareProjectByEmailAndAcceptInviteViaEmail(
projectName: string,
email: string,
- level: 'Viewer' | 'Editor'
+ level: 'Read only' | 'Can edit'
) {
shareProjectByEmail(projectName, email, level)
@@ -153,27 +79,23 @@ export function shareProjectByEmailAndAcceptInviteViaEmail(
cy.findByText(/user would like you to join/)
cy.contains(new RegExp(`You are accepting this invite as ${email}`))
cy.findByText('Join Project').click()
- waitForMainDocToLoad()
}
export function enableLinkSharing() {
let linkSharingReadOnly: string
let linkSharingReadAndWrite: string
- const origin = new URL(Cypress.config().baseUrl!).origin
-
- waitForMainDocToLoad()
cy.findByText('Share').click()
cy.findByText('Turn on link sharing').click()
cy.findByText('Anyone with this link can view this project')
.next()
- .should('contain.text', origin + '/read')
+ .should('contain.text', 'http://sharelatex/')
.then(el => {
linkSharingReadOnly = el.text()
})
cy.findByText('Anyone with this link can edit this project')
.next()
- .should('contain.text', origin + '/')
+ .should('contain.text', 'http://sharelatex/')
.then(el => {
linkSharingReadAndWrite = el.text()
})
@@ -182,77 +104,3 @@ export function enableLinkSharing() {
return { linkSharingReadOnly, linkSharingReadAndWrite }
})
}
-
-export function waitForMainDocToLoad() {
- cy.log('Wait for main doc to load; it will steal the focus after loading')
- cy.get('.cm-content').should('contain.text', 'Introduction')
-}
-
-export function openFile(fileName: string, waitFor: string) {
- // force: The file-tree pane is too narrow to display the full name.
- cy.findByTestId('file-tree').findByText(fileName).click({ force: true })
-
- // wait until we've switched to the selected file
- cy.findByText('Loading…').should('not.exist')
- cy.findByText(waitFor)
-}
-
-export function createNewFile() {
- const fileName = `${uuid()}.tex`
-
- cy.log('create new project file')
- cy.get('button').contains('New file').click({ force: true })
- cy.findByRole('dialog').within(() => {
- cy.get('input').clear()
- cy.get('input').type(fileName)
- cy.findByText('Create').click()
- })
- // force: The file-tree pane is too narrow to display the full name.
- cy.findByTestId('file-tree').findByText(fileName).click({ force: true })
-
- // wait until we've switched to the newly created empty file
- cy.findByText('Loading…').should('not.exist')
- cy.get('.cm-line').should('have.length', 1)
-
- return fileName
-}
-
-export function prepareFileUploadTest(binary = false) {
- const name = `${uuid()}.txt`
- const content = `Test File Content ${name}${binary ? ' \x00' : ''}`
- cy.get('button').contains('Upload').click({ force: true })
- cy.get('input[type=file]')
- .first()
- .selectFile(
- {
- contents: Cypress.Buffer.from(content),
- fileName: name,
- lastModified: Date.now(),
- },
- { force: true }
- )
-
- // wait for the upload to finish
- cy.findByRole('treeitem', { name })
-
- return function check() {
- cy.findByRole('treeitem', { name }).click()
- if (binary) {
- cy.findByText(content).should('not.have.class', 'cm-line')
- } else {
- cy.findByText(content).should('have.class', 'cm-line')
- }
- }
-}
-
-export function testNewFileUpload() {
- it('can upload text file', () => {
- const check = prepareFileUploadTest(false)
- check()
- })
-
- it('can upload binary file', () => {
- const check = prepareFileUploadTest(true)
- check()
- })
-}
diff --git a/server-ce/test/history.spec.ts b/server-ce/test/history.spec.ts
index f0d7e74fb3..cc1950f240 100644
--- a/server-ce/test/history.spec.ts
+++ b/server-ce/test/history.spec.ts
@@ -40,6 +40,7 @@ describe('History', function () {
const CLASS_DELETION = 'ol-cm-deletion-marker'
it('should support labels, comparison and download', () => {
+ cy.visit('/project')
createProject('labels')
const recompile = throttledRecompile()
diff --git a/server-ce/test/host-admin.js b/server-ce/test/host-admin.js
index b3dcd72b1f..9e4cd5d360 100644
--- a/server-ce/test/host-admin.js
+++ b/server-ce/test/host-admin.js
@@ -29,17 +29,6 @@ const IMAGES = {
PRO: process.env.IMAGE_TAG_PRO.replace(/:.+/, ''),
}
-function defaultDockerComposeOverride() {
- return {
- services: {
- sharelatex: {
- environment: {},
- },
- 'git-bridge': {},
- },
- }
-}
-
let previousConfig = ''
function readDockerComposeOverride() {
@@ -49,7 +38,14 @@ function readDockerComposeOverride() {
if (error.code !== 'ENOENT') {
throw error
}
- return defaultDockerComposeOverride
+ return {
+ services: {
+ sharelatex: {
+ environment: {},
+ },
+ 'git-bridge': {},
+ },
+ }
}
}
@@ -81,21 +77,12 @@ app.use(bodyParser.json())
app.use((req, res, next) => {
// Basic access logs
console.log(req.method, req.url, req.body)
- const json = res.json
- res.json = body => {
- console.log(req.method, req.url, req.body, '->', body)
- json.call(res, body)
- }
- next()
-})
-app.use((req, res, next) => {
// Add CORS headers
const accessControlAllowOrigin =
process.env.ACCESS_CONTROL_ALLOW_ORIGIN || 'http://sharelatex'
res.setHeader('Access-Control-Allow-Origin', accessControlAllowOrigin)
res.setHeader('Access-Control-Allow-Headers', 'Content-Type')
res.setHeader('Access-Control-Max-Age', '3600')
- res.setHeader('Access-Control-Allow-Methods', 'DELETE, GET, HEAD, POST, PUT')
next()
})
@@ -144,9 +131,10 @@ const allowedVars = Joi.object(
'GIT_BRIDGE_HOST',
'GIT_BRIDGE_PORT',
'V1_HISTORY_URL',
+ 'DOCKER_RUNNER',
'SANDBOXED_COMPILES',
+ 'SANDBOXED_COMPILES_SIBLING_CONTAINERS',
'ALL_TEX_LIVE_DOCKER_IMAGE_NAMES',
- 'OVERLEAF_FILESTORE_MIGRATION_LEVEL',
'OVERLEAF_TEMPLATES_USER_ID',
'OVERLEAF_NEW_PROJECT_TEMPLATE_LINKS',
'OVERLEAF_ALLOW_PUBLIC_ACCESS',
@@ -208,7 +196,10 @@ function setVarsDockerCompose({ pro, vars, version, withDataDir }) {
)
}
- if (cfg.services.sharelatex.environment.SANDBOXED_COMPILES === 'true') {
+ if (
+ cfg.services.sharelatex.environment
+ .SANDBOXED_COMPILES_SIBLING_CONTAINERS === 'true'
+ ) {
cfg.services.sharelatex.environment.SANDBOXED_COMPILES_HOST_DIR =
PATHS.SANDBOXED_COMPILES_HOST_DIR
cfg.services.sharelatex.environment.TEX_LIVE_DOCKER_IMAGE =
@@ -333,19 +324,8 @@ app.get('/redis/keys', (req, res) => {
)
})
-app.delete('/data/user_files', (req, res) => {
- runDockerCompose(
- 'exec',
- ['sharelatex', 'rm', '-rf', '/var/lib/overleaf/data/user_files'],
- (error, stdout, stderr) => {
- res.json({ error, stdout, stderr })
- }
- )
-})
-
app.use(handleValidationErrors())
purgeDataDir()
-writeDockerComposeOverride(defaultDockerComposeOverride())
app.listen(80)
diff --git a/server-ce/test/package-lock.json b/server-ce/test/package-lock.json
index 05870284e8..6e84524ae0 100644
--- a/server-ce/test/package-lock.json
+++ b/server-ce/test/package-lock.json
@@ -12,10 +12,10 @@
"@types/pdf-parse": "^1.1.4",
"@types/uuid": "^9.0.8",
"adm-zip": "^0.5.12",
- "body-parser": "^1.20.3",
+ "body-parser": "^1.20.2",
"celebrate": "^15.0.3",
"cypress": "13.13.2",
- "express": "^4.21.2",
+ "express": "^4.19.2",
"isomorphic-git": "^1.25.10",
"js-yaml": "^4.1.0",
"pdf-parse": "^1.1.1",
@@ -609,9 +609,9 @@
"integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg=="
},
"node_modules/body-parser": {
- "version": "1.20.3",
- "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
- "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==",
+ "version": "1.20.2",
+ "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz",
+ "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==",
"dependencies": {
"bytes": "3.1.2",
"content-type": "~1.0.5",
@@ -621,7 +621,7 @@
"http-errors": "2.0.0",
"iconv-lite": "0.4.24",
"on-finished": "2.4.1",
- "qs": "6.13.0",
+ "qs": "6.11.0",
"raw-body": "2.5.2",
"type-is": "~1.6.18",
"unpipe": "1.0.0"
@@ -645,11 +645,11 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
},
"node_modules/body-parser/node_modules/qs": {
- "version": "6.13.0",
- "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
- "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
+ "version": "6.11.0",
+ "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
+ "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
"dependencies": {
- "side-channel": "^1.0.6"
+ "side-channel": "^1.0.4"
},
"engines": {
"node": ">=0.6"
@@ -718,33 +718,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
- "node_modules/call-bind-apply-helpers": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz",
- "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==",
- "dependencies": {
- "es-errors": "^1.3.0",
- "function-bind": "^1.1.2"
- },
- "engines": {
- "node": ">= 0.4"
- }
- },
- "node_modules/call-bound": {
- "version": "1.0.3",
- "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.3.tgz",
- "integrity": "sha512-YTd+6wGlNlPxSuri7Y6X8tY2dmm12UMH66RpKMhiX6rsk5wXXnYgbUcOt8kiS31/AjfoTOvCsE+w8nZQLQnzHA==",
- "dependencies": {
- "call-bind-apply-helpers": "^1.0.1",
- "get-intrinsic": "^1.2.6"
- },
- "engines": {
- "node": ">= 0.4"
- },
- "funding": {
- "url": "https://github.com/sponsors/ljharb"
- }
- },
"node_modules/caseless": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
@@ -918,9 +891,9 @@
}
},
"node_modules/cookie": {
- "version": "0.7.1",
- "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz",
- "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==",
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
+ "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==",
"engines": {
"node": ">= 0.6"
}
@@ -947,9 +920,9 @@
}
},
"node_modules/cross-spawn": {
- "version": "7.0.6",
- "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz",
- "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==",
+ "version": "7.0.3",
+ "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+ "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"dependencies": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
@@ -1167,19 +1140,6 @@
"resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz",
"integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg=="
},
- "node_modules/dunder-proto": {
- "version": "1.0.1",
- "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
- "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==",
- "dependencies": {
- "call-bind-apply-helpers": "^1.0.1",
- "es-errors": "^1.3.0",
- "gopd": "^1.2.0"
- },
- "engines": {
- "node": ">= 0.4"
- }
- },
"node_modules/ecc-jsbn": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
@@ -1200,9 +1160,9 @@
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
},
"node_modules/encodeurl": {
- "version": "2.0.0",
- "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
- "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
+ "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
"engines": {
"node": ">= 0.8"
}
@@ -1227,22 +1187,6 @@
"node": ">=8.6"
}
},
- "node_modules/es-define-property": {
- "version": "1.0.1",
- "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
- "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==",
- "engines": {
- "node": ">= 0.4"
- }
- },
- "node_modules/es-errors": {
- "version": "1.3.0",
- "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
- "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
- "engines": {
- "node": ">= 0.4"
- }
- },
"node_modules/es-get-iterator": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz",
@@ -1262,17 +1206,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
- "node_modules/es-object-atoms": {
- "version": "1.1.1",
- "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz",
- "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==",
- "dependencies": {
- "es-errors": "^1.3.0"
- },
- "engines": {
- "node": ">= 0.4"
- }
- },
"node_modules/escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
@@ -1333,36 +1266,36 @@
}
},
"node_modules/express": {
- "version": "4.21.2",
- "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
- "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
+ "version": "4.19.2",
+ "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz",
+ "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==",
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
- "body-parser": "1.20.3",
+ "body-parser": "1.20.2",
"content-disposition": "0.5.4",
"content-type": "~1.0.4",
- "cookie": "0.7.1",
+ "cookie": "0.6.0",
"cookie-signature": "1.0.6",
"debug": "2.6.9",
"depd": "2.0.0",
- "encodeurl": "~2.0.0",
+ "encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"etag": "~1.8.1",
- "finalhandler": "1.3.1",
+ "finalhandler": "1.2.0",
"fresh": "0.5.2",
"http-errors": "2.0.0",
- "merge-descriptors": "1.0.3",
+ "merge-descriptors": "1.0.1",
"methods": "~1.1.2",
"on-finished": "2.4.1",
"parseurl": "~1.3.3",
- "path-to-regexp": "0.1.12",
+ "path-to-regexp": "0.1.7",
"proxy-addr": "~2.0.7",
- "qs": "6.13.0",
+ "qs": "6.11.0",
"range-parser": "~1.2.1",
"safe-buffer": "5.2.1",
- "send": "0.19.0",
- "serve-static": "1.16.2",
+ "send": "0.18.0",
+ "serve-static": "1.15.0",
"setprototypeof": "1.2.0",
"statuses": "2.0.1",
"type-is": "~1.6.18",
@@ -1371,10 +1304,6 @@
},
"engines": {
"node": ">= 0.10.0"
- },
- "funding": {
- "type": "opencollective",
- "url": "https://opencollective.com/express"
}
},
"node_modules/express/node_modules/debug": {
@@ -1391,11 +1320,11 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
},
"node_modules/express/node_modules/qs": {
- "version": "6.13.0",
- "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
- "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
+ "version": "6.11.0",
+ "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz",
+ "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==",
"dependencies": {
- "side-channel": "^1.0.6"
+ "side-channel": "^1.0.4"
},
"engines": {
"node": ">=0.6"
@@ -1464,12 +1393,12 @@
}
},
"node_modules/finalhandler": {
- "version": "1.3.1",
- "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz",
- "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==",
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz",
+ "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==",
"dependencies": {
"debug": "2.6.9",
- "encodeurl": "~2.0.0",
+ "encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"on-finished": "2.4.1",
"parseurl": "~1.3.3",
@@ -1569,40 +1498,19 @@
}
},
"node_modules/get-intrinsic": {
- "version": "1.3.0",
- "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz",
- "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==",
+ "version": "1.2.2",
+ "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz",
+ "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==",
"dependencies": {
- "call-bind-apply-helpers": "^1.0.2",
- "es-define-property": "^1.0.1",
- "es-errors": "^1.3.0",
- "es-object-atoms": "^1.1.1",
"function-bind": "^1.1.2",
- "get-proto": "^1.0.1",
- "gopd": "^1.2.0",
- "has-symbols": "^1.1.0",
- "hasown": "^2.0.2",
- "math-intrinsics": "^1.1.0"
- },
- "engines": {
- "node": ">= 0.4"
+ "has-proto": "^1.0.1",
+ "has-symbols": "^1.0.3",
+ "hasown": "^2.0.0"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
- "node_modules/get-proto": {
- "version": "1.0.1",
- "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz",
- "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==",
- "dependencies": {
- "dunder-proto": "^1.0.1",
- "es-object-atoms": "^1.0.0"
- },
- "engines": {
- "node": ">= 0.4"
- }
- },
"node_modules/get-stream": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
@@ -1648,11 +1556,11 @@
}
},
"node_modules/gopd": {
- "version": "1.2.0",
- "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
- "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==",
- "engines": {
- "node": ">= 0.4"
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
+ "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
+ "dependencies": {
+ "get-intrinsic": "^1.1.3"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
@@ -1690,10 +1598,21 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/has-proto": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz",
+ "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==",
+ "engines": {
+ "node": ">= 0.4"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
"node_modules/has-symbols": {
- "version": "1.1.0",
- "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
- "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==",
+ "version": "1.0.3",
+ "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
+ "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
"engines": {
"node": ">= 0.4"
},
@@ -1716,9 +1635,9 @@
}
},
"node_modules/hasown": {
- "version": "2.0.2",
- "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
- "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz",
+ "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==",
"dependencies": {
"function-bind": "^1.1.2"
},
@@ -2365,14 +2284,6 @@
"lz-string": "bin/bin.js"
}
},
- "node_modules/math-intrinsics": {
- "version": "1.1.0",
- "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
- "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==",
- "engines": {
- "node": ">= 0.4"
- }
- },
"node_modules/media-typer": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
@@ -2382,12 +2293,9 @@
}
},
"node_modules/merge-descriptors": {
- "version": "1.0.3",
- "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz",
- "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==",
- "funding": {
- "url": "https://github.com/sponsors/sindresorhus"
- }
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
+ "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w=="
},
"node_modules/merge-stream": {
"version": "2.0.0",
@@ -2497,12 +2405,9 @@
}
},
"node_modules/object-inspect": {
- "version": "1.13.4",
- "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
- "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==",
- "engines": {
- "node": ">= 0.4"
- },
+ "version": "1.13.1",
+ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz",
+ "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==",
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
@@ -2621,9 +2526,9 @@
}
},
"node_modules/path-to-regexp": {
- "version": "0.1.12",
- "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz",
- "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ=="
+ "version": "0.1.7",
+ "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
+ "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ=="
},
"node_modules/pdf-parse": {
"version": "1.1.1",
@@ -2902,9 +2807,9 @@
}
},
"node_modules/send": {
- "version": "0.19.0",
- "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz",
- "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==",
+ "version": "0.18.0",
+ "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz",
+ "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==",
"dependencies": {
"debug": "2.6.9",
"depd": "2.0.0",
@@ -2937,28 +2842,20 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
},
- "node_modules/send/node_modules/encodeurl": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
- "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
- "engines": {
- "node": ">= 0.8"
- }
- },
"node_modules/send/node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/serve-static": {
- "version": "1.16.2",
- "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz",
- "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==",
+ "version": "1.15.0",
+ "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz",
+ "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==",
"dependencies": {
- "encodeurl": "~2.0.0",
+ "encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"parseurl": "~1.3.3",
- "send": "0.19.0"
+ "send": "0.18.0"
},
"engines": {
"node": ">= 0.8.0"
@@ -3028,68 +2925,13 @@
}
},
"node_modules/side-channel": {
- "version": "1.1.0",
- "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz",
- "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==",
+ "version": "1.0.4",
+ "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
+ "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
"dependencies": {
- "es-errors": "^1.3.0",
- "object-inspect": "^1.13.3",
- "side-channel-list": "^1.0.0",
- "side-channel-map": "^1.0.1",
- "side-channel-weakmap": "^1.0.2"
- },
- "engines": {
- "node": ">= 0.4"
- },
- "funding": {
- "url": "https://github.com/sponsors/ljharb"
- }
- },
- "node_modules/side-channel-list": {
- "version": "1.0.0",
- "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz",
- "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==",
- "dependencies": {
- "es-errors": "^1.3.0",
- "object-inspect": "^1.13.3"
- },
- "engines": {
- "node": ">= 0.4"
- },
- "funding": {
- "url": "https://github.com/sponsors/ljharb"
- }
- },
- "node_modules/side-channel-map": {
- "version": "1.0.1",
- "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz",
- "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==",
- "dependencies": {
- "call-bound": "^1.0.2",
- "es-errors": "^1.3.0",
- "get-intrinsic": "^1.2.5",
- "object-inspect": "^1.13.3"
- },
- "engines": {
- "node": ">= 0.4"
- },
- "funding": {
- "url": "https://github.com/sponsors/ljharb"
- }
- },
- "node_modules/side-channel-weakmap": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz",
- "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==",
- "dependencies": {
- "call-bound": "^1.0.2",
- "es-errors": "^1.3.0",
- "get-intrinsic": "^1.2.5",
- "object-inspect": "^1.13.3",
- "side-channel-map": "^1.0.1"
- },
- "engines": {
- "node": ">= 0.4"
+ "call-bind": "^1.0.0",
+ "get-intrinsic": "^1.0.2",
+ "object-inspect": "^1.9.0"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
diff --git a/server-ce/test/package.json b/server-ce/test/package.json
index 36ba3df2dd..674154dd39 100644
--- a/server-ce/test/package.json
+++ b/server-ce/test/package.json
@@ -15,10 +15,10 @@
"@types/pdf-parse": "^1.1.4",
"@types/uuid": "^9.0.8",
"adm-zip": "^0.5.12",
- "body-parser": "^1.20.3",
+ "body-parser": "^1.20.2",
"celebrate": "^15.0.3",
"cypress": "13.13.2",
- "express": "^4.21.2",
+ "express": "^4.19.2",
"isomorphic-git": "^1.25.10",
"js-yaml": "^4.1.0",
"pdf-parse": "^1.1.1",
diff --git a/server-ce/test/project-list.spec.ts b/server-ce/test/project-list.spec.ts
index 998fcf9ffb..9ee9ac9ca0 100644
--- a/server-ce/test/project-list.spec.ts
+++ b/server-ce/test/project-list.spec.ts
@@ -18,11 +18,11 @@ describe('Project List', () => {
describe('user with no projects', () => {
ensureUserExists({ email: WITHOUT_PROJECTS_USER })
- it("'Import from GitHub' is not displayed in the welcome page", () => {
+ it("'Import from Github' is not displayed in the welcome page", () => {
login(WITHOUT_PROJECTS_USER)
cy.visit('/project')
cy.findByText('Create a new project').click()
- cy.findByText(/Import from GitHub/i).should('not.exist')
+ cy.findByText(/Import from Github/i).should('not.exist')
})
})
@@ -31,27 +31,29 @@ describe('Project List', () => {
ensureUserExists({ email: REGULAR_USER })
before(() => {
- login(REGULAR_USER)
- createProject(projectName, { type: 'Example project', open: false })
- })
- beforeEach(function () {
login(REGULAR_USER)
cy.visit('/project')
+ createProject(projectName, { type: 'Example Project' })
})
it('Can download project sources', () => {
+ login(REGULAR_USER)
+ cy.visit('/project')
+
findProjectRow(projectName).within(() =>
cy.findByRole('button', { name: 'Download .zip file' }).click()
)
- const zipName = projectName.replaceAll('-', '_')
cy.task('readFileInZip', {
- pathToZip: `cypress/downloads/${zipName}.zip`,
+ pathToZip: `cypress/downloads/${projectName}.zip`,
fileToRead: 'main.tex',
}).should('contain', 'Your introduction goes here')
})
it('Can download project PDF', () => {
+ login(REGULAR_USER)
+ cy.visit('/project')
+
findProjectRow(projectName).within(() =>
cy.findByRole('button', { name: 'Download PDF' }).click()
)
@@ -65,6 +67,9 @@ describe('Project List', () => {
it('can assign and remove tags to projects', () => {
const tagName = uuid().slice(0, 7) // long tag names are truncated in the UI, which affects selectors
+ login(REGULAR_USER)
+ cy.visit('/project')
+
cy.log('select project')
cy.get(`[aria-label="Select ${projectName}"]`).click()
@@ -85,7 +90,9 @@ describe('Project List', () => {
cy.log('create a separate project to filter')
const nonTaggedProjectName = `project-${uuid()}`
login(REGULAR_USER)
- createProject(nonTaggedProjectName, { open: false })
+ cy.visit('/project')
+ createProject(nonTaggedProjectName)
+ cy.visit('/project')
cy.log('select project')
cy.get(`[aria-label="Select ${projectName}"]`).click()
diff --git a/server-ce/test/project-sharing.spec.ts b/server-ce/test/project-sharing.spec.ts
index 4da2209332..e14f36d778 100644
--- a/server-ce/test/project-sharing.spec.ts
+++ b/server-ce/test/project-sharing.spec.ts
@@ -4,9 +4,6 @@ import { ensureUserExists, login } from './helpers/login'
import {
createProject,
enableLinkSharing,
- openProjectByName,
- openProjectViaLinkSharingAsAnon,
- openProjectViaLinkSharingAsUser,
shareProjectByEmailAndAcceptInviteViaDash,
shareProjectByEmailAndAcceptInviteViaEmail,
} from './helpers/project'
@@ -34,6 +31,7 @@ describe('Project Sharing', function () {
function setupTestProject() {
login('user@example.com')
+ cy.visit('/project')
createProject(projectName)
// Add chat message
@@ -55,15 +53,8 @@ describe('Project Sharing', function () {
function expectContentReadOnlyAccess() {
cy.url().should('match', /\/project\/[a-fA-F0-9]{24}/)
- cy.findByRole('textbox', { name: /Source Editor editing/i }).should(
- 'contain.text',
- '\\maketitle'
- )
- cy.findByRole('textbox', { name: /Source Editor editing/i }).should(
- 'have.attr',
- 'contenteditable',
- 'false'
- )
+ cy.get('.cm-content').should('contain.text', '\\maketitle')
+ cy.get('.cm-content').should('have.attr', 'contenteditable', 'false')
}
function expectContentWriteAccess() {
@@ -71,23 +62,13 @@ describe('Project Sharing', function () {
cy.url().should('match', /\/project\/[a-fA-F0-9]{24}/)
const recompile = throttledRecompile()
// wait for the editor to finish loading
- cy.findByRole('textbox', { name: /Source Editor editing/i }).should(
- 'contain.text',
- '\\maketitle'
- )
+ cy.get('.cm-content').should('contain.text', '\\maketitle')
// the editor should be writable
- cy.findByRole('textbox', { name: /Source Editor editing/i }).should(
- 'have.attr',
- 'contenteditable',
- 'true'
- )
+ cy.get('.cm-content').should('have.attr', 'contenteditable', 'true')
cy.findByText('\\maketitle').parent().click()
cy.findByText('\\maketitle').parent().type(`\n\\section{{}${section}}`)
// should have written
- cy.findByRole('textbox', { name: /Source Editor editing/i }).should(
- 'contain.text',
- `\\section{${section}}`
- )
+ cy.get('.cm-content').should('contain.text', `\\section{${section}}`)
// check PDF
recompile()
cy.get('.pdf-viewer').should('contain.text', projectName)
@@ -171,10 +152,16 @@ describe('Project Sharing', function () {
beforeEach(function () {
login('user@example.com')
- shareProjectByEmailAndAcceptInviteViaEmail(projectName, email, 'Viewer')
+ shareProjectByEmailAndAcceptInviteViaEmail(
+ projectName,
+ email,
+ 'Read only'
+ )
})
it('should grant the collaborator read access', () => {
+ cy.visit('/project')
+ cy.findByText(projectName).click()
expectFullReadOnlyAccess()
expectProjectDashboardEntry()
})
@@ -186,12 +173,13 @@ describe('Project Sharing', function () {
beforeWithReRunOnTestRetry(function () {
login('user@example.com')
- shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Viewer')
+ shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Read only')
})
it('should grant the collaborator read access', () => {
login(email)
- openProjectByName(projectName)
+ cy.visit('/project')
+ cy.findByText(projectName).click()
expectFullReadOnlyAccess()
expectProjectDashboardEntry()
})
@@ -203,12 +191,13 @@ describe('Project Sharing', function () {
beforeWithReRunOnTestRetry(function () {
login('user@example.com')
- shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Editor')
+ shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Can edit')
})
it('should grant the collaborator write access', () => {
login(email)
- openProjectByName(projectName)
+ cy.visit('/project')
+ cy.findByText(projectName).click()
expectReadAndWriteAccess()
expectEditAuthoredAs('You')
expectProjectDashboardEntry()
@@ -223,11 +212,9 @@ describe('Project Sharing', function () {
it('should grant restricted read access', () => {
login(email)
- openProjectViaLinkSharingAsUser(
- linkSharingReadOnly,
- projectName,
- email
- )
+ cy.visit(linkSharingReadOnly)
+ cy.findByText(projectName) // wait for lazy loading
+ cy.findByText('Join Project').click()
expectRestrictedReadOnlyAccess()
expectProjectDashboardEntry()
})
@@ -239,11 +226,9 @@ describe('Project Sharing', function () {
it('should grant full write access', () => {
login(email)
- openProjectViaLinkSharingAsUser(
- linkSharingReadAndWrite,
- projectName,
- email
- )
+ cy.visit(linkSharingReadAndWrite)
+ cy.findByText(projectName) // wait for lazy loading
+ cy.findByText('Join Project').click()
expectReadAndWriteAccess()
expectEditAuthoredAs('You')
expectProjectDashboardEntry()
@@ -287,7 +272,7 @@ describe('Project Sharing', function () {
withDataDir: true,
})
it('should grant read access with read link', () => {
- openProjectViaLinkSharingAsAnon(linkSharingReadOnly)
+ cy.visit(linkSharingReadOnly)
expectRestrictedReadOnlyAccess()
})
@@ -307,12 +292,12 @@ describe('Project Sharing', function () {
})
it('should grant read access with read link', () => {
- openProjectViaLinkSharingAsAnon(linkSharingReadOnly)
+ cy.visit(linkSharingReadOnly)
expectRestrictedReadOnlyAccess()
})
it('should grant write access with write link', () => {
- openProjectViaLinkSharingAsAnon(linkSharingReadAndWrite)
+ cy.visit(linkSharingReadAndWrite)
expectReadAndWriteAccess()
expectEditAuthoredAs('Anonymous')
})
diff --git a/server-ce/test/sandboxed-compiles.spec.ts b/server-ce/test/sandboxed-compiles.spec.ts
index 71f5b43392..e50aa36283 100644
--- a/server-ce/test/sandboxed-compiles.spec.ts
+++ b/server-ce/test/sandboxed-compiles.spec.ts
@@ -1,7 +1,7 @@
import { ensureUserExists, login } from './helpers/login'
import { createProject } from './helpers/project'
import { isExcludedBySharding, startWith } from './helpers/config'
-import { throttledRecompile, stopCompile } from './helpers/compile'
+import { throttledRecompile } from './helpers/compile'
import { v4 as uuid } from 'uuid'
import { waitUntilScrollingFinished } from './helpers/waitUntilScrollingFinished'
import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry'
@@ -10,7 +10,9 @@ const LABEL_TEX_LIVE_VERSION = 'TeX Live version'
describe('SandboxedCompiles', function () {
const enabledVars = {
+ DOCKER_RUNNER: 'true',
SANDBOXED_COMPILES: 'true',
+ SANDBOXED_COMPILES_SIBLING_CONTAINERS: 'true',
ALL_TEX_LIVE_DOCKER_IMAGE_NAMES: '2023,2022',
}
@@ -27,6 +29,7 @@ describe('SandboxedCompiles', function () {
})
it('should offer TexLive images and switch the compiler', function () {
+ cy.visit('/project')
createProject('sandboxed')
const recompile = throttledRecompile()
cy.log('wait for compile')
@@ -43,7 +46,7 @@ describe('SandboxedCompiles', function () {
.findByText('2023')
.parent()
.select('2022')
- cy.get('.left-menu-modal-backdrop').click()
+ cy.get('#left-menu-modal').click()
cy.log('Trigger compile with other TeX Live version')
recompile()
@@ -56,47 +59,14 @@ describe('SandboxedCompiles', function () {
checkSyncTeX()
checkXeTeX()
checkRecompilesAfterErrors()
- checkStopCompile()
})
- function checkStopCompile() {
- it('users can stop a running compile', function () {
- login('user@example.com')
- createProject('test-project')
- // create an infinite loop in the main document
- // this will cause the compile to run indefinitely
- cy.findByText('\\maketitle').parent().click()
- cy.findByText('\\maketitle')
- .parent()
- .type('\n\\def\\x{{}Hello!\\par\\x}\\x')
- cy.log('Start compile')
- // We need to start the compile manually because we do not want to wait for it to finish
- cy.findByText('Recompile').click()
- // Now stop the compile and kill the latex process
- stopCompile({ delay: 1000 })
- cy.get('.logs-pane')
- .invoke('text')
- .should('match', /PDF Rendering Error|Compilation cancelled/)
- // Check that the previous compile is not running in the background by
- // disabling the infinite loop and recompiling
- cy.findByText('\\def').parent().click()
- cy.findByText('\\def').parent().type('{home}disabled loop% ')
- cy.findByText('Recompile').click()
- cy.get('.pdf-viewer').should('contain.text', 'disabled loop')
- cy.get('.logs-pane').should(
- 'not.contain.text',
- 'A previous compile is still running'
- )
- })
- }
-
function checkSyncTeX() {
- // TODO(25342): re-enable
- // eslint-disable-next-line mocha/no-skipped-tests
- describe.skip('SyncTeX', function () {
+ describe('SyncTeX', function () {
let projectName: string
beforeEach(function () {
projectName = `Project ${uuid()}`
+ cy.visit('/project')
createProject(projectName)
const recompile = throttledRecompile()
cy.findByText('\\maketitle').parent().click()
@@ -161,9 +131,7 @@ describe('SandboxedCompiles', function () {
})
cy.log('navigate to Section A')
- cy.findByRole('textbox', { name: /Source Editor editing/i }).within(
- () => cy.findByText('Section A').click()
- )
+ cy.get('.cm-content').within(() => cy.findByText('Section A').click())
cy.get('[aria-label="Go to code location in PDF"]').click()
cy.get('@title').then((title: any) => {
waitUntilScrollingFinished('.pdfjs-viewer-inner', title)
@@ -172,9 +140,7 @@ describe('SandboxedCompiles', function () {
})
cy.log('navigate to Section B')
- cy.findByRole('textbox', { name: /Source Editor editing/i }).within(
- () => cy.findByText('Section B').click()
- )
+ cy.get('.cm-content').within(() => cy.findByText('Section B').click())
cy.get('[aria-label="Go to code location in PDF"]').click()
cy.get('@sectionA').then((title: any) => {
waitUntilScrollingFinished('.pdfjs-viewer-inner', title)
@@ -188,6 +154,7 @@ describe('SandboxedCompiles', function () {
function checkRecompilesAfterErrors() {
it('recompiles even if there are Latex errors', function () {
login('user@example.com')
+ cy.visit('/project')
createProject('test-project')
const recompile = throttledRecompile()
cy.findByText('\\maketitle').parent().click()
@@ -203,6 +170,7 @@ describe('SandboxedCompiles', function () {
function checkXeTeX() {
it('should be able to use XeLaTeX', function () {
+ cy.visit('/project')
createProject('XeLaTeX')
const recompile = throttledRecompile()
cy.log('wait for compile')
@@ -219,7 +187,7 @@ describe('SandboxedCompiles', function () {
.findByText('pdfLaTeX')
.parent()
.select('XeLaTeX')
- cy.get('.left-menu-modal-backdrop').click()
+ cy.get('#left-menu-modal').click()
cy.log('Trigger compile with other compiler')
recompile()
@@ -236,13 +204,14 @@ describe('SandboxedCompiles', function () {
})
it('should not offer TexLive images and use default compiler', function () {
+ cy.visit('/project')
createProject('sandboxed')
cy.log('wait for compile')
cy.get('.pdf-viewer').should('contain.text', 'sandboxed')
- cy.log('Check which compiler version was used, expect 2025')
+ cy.log('Check which compiler version was used, expect 2024')
cy.get('[aria-label="View logs"]').click()
- cy.findByText(/This is pdfTeX, Version .+ \(TeX Live 2025\) /)
+ cy.findByText(/This is pdfTeX, Version .+ \(TeX Live 2024\) /)
cy.log('Check that there is no TeX Live version toggle')
cy.get('header').findByText('Menu').click()
@@ -263,7 +232,6 @@ describe('SandboxedCompiles', function () {
checkSyncTeX()
checkXeTeX()
checkRecompilesAfterErrors()
- checkStopCompile()
})
describe.skip('unavailable in CE', function () {
@@ -278,6 +246,5 @@ describe('SandboxedCompiles', function () {
checkSyncTeX()
checkXeTeX()
checkRecompilesAfterErrors()
- checkStopCompile()
})
})
diff --git a/server-ce/test/templates.spec.ts b/server-ce/test/templates.spec.ts
index 4959e149fc..b9cc3f87eb 100644
--- a/server-ce/test/templates.spec.ts
+++ b/server-ce/test/templates.spec.ts
@@ -47,9 +47,7 @@ describe('Templates', () => {
cy.url().should('match', /\/templates$/)
})
- // TODO(25342): re-enable
- // eslint-disable-next-line mocha/no-skipped-tests
- it.skip('should have templates feature', () => {
+ it('should have templates feature', () => {
login(TEMPLATES_USER)
const name = `Template ${Date.now()}`
const description = `Template Description ${Date.now()}`
@@ -66,7 +64,7 @@ describe('Templates', () => {
.get('textarea')
.type(description)
cy.findByText('Publish').click()
- cy.findByText('Publishing…').parent().should('be.disabled')
+ cy.findByText('Publishing…').should('be.disabled')
cy.findByText('Publish').should('not.exist')
cy.findByText('Unpublish', { timeout: 10_000 })
cy.findByText('Republish')
@@ -98,12 +96,12 @@ describe('Templates', () => {
.parent()
.parent()
.within(() => cy.get('input[type="checkbox"]').first().check())
- cy.get('.project-list-sidebar-scroll').within(() => {
+ cy.get('.project-list-sidebar-react').within(() => {
cy.findAllByText('New Tag').first().click()
})
cy.focused().type(tagName)
cy.findByText('Create').click()
- cy.get('.project-list-sidebar-scroll').within(() => {
+ cy.get('.project-list-sidebar-react').within(() => {
cy.findByText(tagName)
.parent()
.within(() => cy.get('.name').should('have.text', `${tagName} (1)`))
@@ -205,7 +203,6 @@ describe('Templates', () => {
.click()
cy.findAllByText('All Templates')
.first()
- .parent()
.should('have.attr', 'href', '/templates/all')
})
})
diff --git a/server-ce/test/upgrading.spec.ts b/server-ce/test/upgrading.spec.ts
index 16e0320dcc..86a3ea0cad 100644
--- a/server-ce/test/upgrading.spec.ts
+++ b/server-ce/test/upgrading.spec.ts
@@ -1,7 +1,7 @@
import { ensureUserExists, login } from './helpers/login'
import { isExcludedBySharding, startWith } from './helpers/config'
import { dockerCompose, runScript } from './helpers/hostAdminClient'
-import { createProject, openProjectByName } from './helpers/project'
+import { createProject } from './helpers/project'
import { throttledRecompile } from './helpers/compile'
import { v4 as uuid } from 'uuid'
@@ -38,6 +38,8 @@ describe('Upgrading', function () {
before(() => {
cy.log('Populate old instance')
login(USER)
+
+ cy.visit('/project')
createProject(PROJECT_NAME, {
newProjectButtonMatcher: startOptions.newProjectButtonMatcher,
})
@@ -57,7 +59,7 @@ describe('Upgrading', function () {
recompile()
cy.get('header').findByText('Menu').click()
cy.findByText('Source').click()
- cy.get('.left-menu-modal-backdrop').click({ force: true })
+ cy.get('#left-menu-modal').click()
}
cy.log('Check compile and history')
@@ -113,7 +115,8 @@ describe('Upgrading', function () {
})
it('should open the old project', () => {
- openProjectByName(PROJECT_NAME)
+ cy.visit('/project')
+ cy.findByText(PROJECT_NAME).click()
cy.url().should('match', /\/project\/[a-fA-F0-9]{24}/)
cy.findByRole('navigation').within(() => {
diff --git a/services/chat/.gitignore b/services/chat/.gitignore
new file mode 100644
index 0000000000..f0cf94b147
--- /dev/null
+++ b/services/chat/.gitignore
@@ -0,0 +1,12 @@
+**.swp
+
+public/build/
+
+node_modules/
+
+plato/
+
+**/*.map
+
+# managed by dev-environment$ bin/update_build_scripts
+.npmrc
diff --git a/services/chat/.nvmrc b/services/chat/.nvmrc
index fc37597bcc..123b052798 100644
--- a/services/chat/.nvmrc
+++ b/services/chat/.nvmrc
@@ -1 +1 @@
-22.17.0
+18.20.2
diff --git a/services/chat/Dockerfile b/services/chat/Dockerfile
index 66a8bc3ded..242daa44c0 100644
--- a/services/chat/Dockerfile
+++ b/services/chat/Dockerfile
@@ -2,7 +2,7 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:22.17.0 AS base
+FROM node:18.20.2 AS base
WORKDIR /overleaf/services/chat
diff --git a/services/chat/Makefile b/services/chat/Makefile
index 792f5d2cd6..cbe59db223 100644
--- a/services/chat/Makefile
+++ b/services/chat/Makefile
@@ -32,30 +32,12 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
-
-SHELLCHECK_OPTS = \
- --shell=bash \
- --external-sources
-SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
-SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
-
-shellcheck:
- @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
- koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
-
-shellcheck_fix:
- @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
- diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
- if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
- elif [ -n "$$diff" ]; then echo "$$file"; \
- else echo "\033[2m$$file\033[0m"; fi \
- done
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
format:
$(RUN_LINTING) format
@@ -81,7 +63,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck shellcheck test_unit test_acceptance
+test: format lint typecheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -116,6 +98,13 @@ test_acceptance_clean:
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
test_acceptance_pre_run:
+ $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+ $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+ while ! mongosh --eval "db.version()" > /dev/null; do \
+ echo "Waiting for Mongo..."; \
+ sleep 1; \
+ done; \
+ mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
endif
@@ -148,7 +137,6 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
- shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/chat/app/js/Features/Messages/MessageHttpController.js b/services/chat/app/js/Features/Messages/MessageHttpController.js
index 45208e2c03..a20d005864 100644
--- a/services/chat/app/js/Features/Messages/MessageHttpController.js
+++ b/services/chat/app/js/Features/Messages/MessageHttpController.js
@@ -74,10 +74,6 @@ export async function deleteMessage(context) {
return await callMessageHttpController(context, _deleteMessage)
}
-export async function deleteUserMessage(context) {
- return await callMessageHttpController(context, _deleteUserMessage)
-}
-
export async function getResolvedThreadIds(context) {
return await callMessageHttpController(context, _getResolvedThreadIds)
}
@@ -194,13 +190,6 @@ const _deleteMessage = async (req, res) => {
res.status(204)
}
-const _deleteUserMessage = async (req, res) => {
- const { projectId, threadId, userId, messageId } = req.params
- const room = await ThreadManager.findOrCreateThread(projectId, threadId)
- await MessageManager.deleteUserMessage(userId, room._id, messageId)
- res.status(204)
-}
-
const _getResolvedThreadIds = async (req, res) => {
const { projectId } = req.params
const resolvedThreadIds = await ThreadManager.getResolvedThreadIds(projectId)
diff --git a/services/chat/app/js/Features/Messages/MessageManager.js b/services/chat/app/js/Features/Messages/MessageManager.js
index efff22a2a4..cb8818e3b6 100644
--- a/services/chat/app/js/Features/Messages/MessageManager.js
+++ b/services/chat/app/js/Features/Messages/MessageManager.js
@@ -77,14 +77,6 @@ export async function deleteMessage(roomId, messageId) {
await db.messages.deleteOne(query)
}
-export async function deleteUserMessage(userId, roomId, messageId) {
- await db.messages.deleteOne({
- _id: new ObjectId(messageId),
- user_id: new ObjectId(userId),
- room_id: new ObjectId(roomId),
- })
-}
-
function _ensureIdsAreObjectIds(query) {
if (query.user_id && !(query.user_id instanceof ObjectId)) {
query.user_id = new ObjectId(query.user_id)
diff --git a/services/chat/app/js/server.js b/services/chat/app/js/server.js
index 80970fc22e..aa4ea42482 100644
--- a/services/chat/app/js/server.js
+++ b/services/chat/app/js/server.js
@@ -1,10 +1,10 @@
-import http from 'node:http'
+import http from 'http'
import metrics from '@overleaf/metrics'
import logger from '@overleaf/logger'
import express from 'express'
import exegesisExpress from 'exegesis-express'
-import path from 'node:path'
-import { fileURLToPath } from 'node:url'
+import path from 'path'
+import { fileURLToPath } from 'url'
import * as messagesController from './Features/Messages/MessageHttpController.js'
const __dirname = fileURLToPath(new URL('.', import.meta.url))
diff --git a/services/chat/buildscript.txt b/services/chat/buildscript.txt
index 35114bd2a4..8470e45e93 100644
--- a/services/chat/buildscript.txt
+++ b/services/chat/buildscript.txt
@@ -4,6 +4,6 @@ chat
--env-add=
--env-pass-through=
--esmock-loader=False
---node-version=22.17.0
+--node-version=18.20.2
--public-repo=False
---script-version=4.7.0
+--script-version=4.5.0
diff --git a/services/chat/chat.yaml b/services/chat/chat.yaml
index 35ed3d378d..3ccdf9bc30 100644
--- a/services/chat/chat.yaml
+++ b/services/chat/chat.yaml
@@ -177,34 +177,6 @@ paths:
'204':
description: No Content
description: 'Delete message with Message ID provided, from the Thread with ThreadID and ProjectID provided'
- '/project/{projectId}/thread/{threadId}/user/{userId}/messages/{messageId}':
- parameters:
- - schema:
- type: string
- name: projectId
- in: path
- required: true
- - schema:
- type: string
- name: threadId
- in: path
- required: true
- - schema:
- type: string
- name: userId
- in: path
- required: true
- - schema:
- type: string
- name: messageId
- in: path
- required: true
- delete:
- summary: Delete message written by a given user
- operationId: deleteUserMessage
- responses:
- '204':
- description: No Content
'/project/{projectId}/thread/{threadId}/resolve':
parameters:
- schema:
diff --git a/services/chat/config/settings.defaults.cjs b/services/chat/config/settings.defaults.cjs
index 4b7dc293a9..48734ff11e 100644
--- a/services/chat/config/settings.defaults.cjs
+++ b/services/chat/config/settings.defaults.cjs
@@ -1,9 +1,3 @@
-const http = require('node:http')
-const https = require('node:https')
-
-http.globalAgent.keepAlive = false
-https.globalAgent.keepAlive = false
-
module.exports = {
internal: {
chat: {
diff --git a/services/chat/docker-compose.ci.yml b/services/chat/docker-compose.ci.yml
index ca3303a079..6f1a608534 100644
--- a/services/chat/docker-compose.ci.yml
+++ b/services/chat/docker-compose.ci.yml
@@ -24,13 +24,10 @@ services:
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
- volumes:
- - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
- condition: service_started
+ condition: service_healthy
user: node
- entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run test:acceptance
@@ -42,14 +39,9 @@ services:
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
mongo:
- image: mongo:8.0.11
+ image: mongo:6.0.13
command: --replSet overleaf
- volumes:
- - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
- environment:
- MONGO_INITDB_DATABASE: sharelatex
- extra_hosts:
- # Required when using the automatic database setup for initializing the
- # replica set. This override is not needed when running the setup after
- # starting up mongo.
- - mongo:127.0.0.1
+ healthcheck:
+ test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
+ interval: 1s
+ retries: 20
diff --git a/services/chat/docker-compose.yml b/services/chat/docker-compose.yml
index e7b8ce7385..5474cc3574 100644
--- a/services/chat/docker-compose.yml
+++ b/services/chat/docker-compose.yml
@@ -6,7 +6,7 @@ version: "2.3"
services:
test_unit:
- image: node:22.17.0
+ image: node:18.20.2
volumes:
- .:/overleaf/services/chat
- ../../node_modules:/overleaf/node_modules
@@ -14,45 +14,37 @@ services:
working_dir: /overleaf/services/chat
environment:
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
user: node
test_acceptance:
- image: node:22.17.0
+ image: node:18.20.2
volumes:
- .:/overleaf/services/chat
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
working_dir: /overleaf/services/chat
environment:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
+ LOG_LEVEL: ERROR
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
user: node
depends_on:
mongo:
- condition: service_started
- entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
+ condition: service_healthy
command: npm run --silent test:acceptance
mongo:
- image: mongo:8.0.11
+ image: mongo:6.0.13
command: --replSet overleaf
- volumes:
- - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
- environment:
- MONGO_INITDB_DATABASE: sharelatex
- extra_hosts:
- # Required when using the automatic database setup for initializing the
- # replica set. This override is not needed when running the setup after
- # starting up mongo.
- - mongo:127.0.0.1
+ healthcheck:
+ test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
+ interval: 1s
+ retries: 20
diff --git a/services/chat/package.json b/services/chat/package.json
index f3d37eb6d3..f6132b969c 100644
--- a/services/chat/package.json
+++ b/services/chat/package.json
@@ -12,8 +12,8 @@
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
"lint": "eslint --max-warnings 0 --format unix .",
- "format": "prettier --list-different $PWD/'**/*.*js'",
- "format:fix": "prettier --write $PWD/'**/*.*js'",
+ "format": "prettier --list-different $PWD/'**/*.js'",
+ "format:fix": "prettier --write $PWD/'**/*.js'",
"lint:fix": "eslint --fix .",
"types:check": "tsc --noEmit"
},
@@ -24,15 +24,15 @@
"async": "^3.2.5",
"body-parser": "^1.20.3",
"exegesis-express": "^4.0.0",
- "express": "^4.21.2",
- "mongodb": "6.12.0"
+ "express": "^4.21.0",
+ "mongodb": "^6.1.0"
},
"devDependencies": {
"acorn": "^7.1.1",
"ajv": "^6.12.0",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
- "mocha": "^11.1.0",
+ "mocha": "^10.2.0",
"request": "^2.88.2",
"sandboxed-module": "^2.0.4",
"sinon": "^9.2.4",
diff --git a/services/chat/test/acceptance/js/helpers/ChatApp.js b/services/chat/test/acceptance/js/helpers/ChatApp.js
index 3a0baf5ce0..7116ea36f6 100644
--- a/services/chat/test/acceptance/js/helpers/ChatApp.js
+++ b/services/chat/test/acceptance/js/helpers/ChatApp.js
@@ -1,5 +1,5 @@
import { createServer } from '../../../../app/js/server.js'
-import { promisify } from 'node:util'
+import { promisify } from 'util'
export { db } from '../../../../app/js/mongodb.js'
diff --git a/services/clsi/.gitignore b/services/clsi/.gitignore
index a85e6b757a..360466227e 100644
--- a/services/clsi/.gitignore
+++ b/services/clsi/.gitignore
@@ -1,3 +1,14 @@
+**.swp
+node_modules
+test/acceptance/fixtures/tmp
compiles
output
+.DS_Store
+*~
cache
+.vagrant
+config/*
+npm-debug.log
+
+# managed by dev-environment$ bin/update_build_scripts
+.npmrc
diff --git a/services/clsi/.nvmrc b/services/clsi/.nvmrc
index fc37597bcc..123b052798 100644
--- a/services/clsi/.nvmrc
+++ b/services/clsi/.nvmrc
@@ -1 +1 @@
-22.17.0
+18.20.2
diff --git a/services/clsi/Dockerfile b/services/clsi/Dockerfile
index 77c26fab23..dc776e4cea 100644
--- a/services/clsi/Dockerfile
+++ b/services/clsi/Dockerfile
@@ -2,7 +2,7 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:22.17.0 AS base
+FROM node:18.20.2 AS base
WORKDIR /overleaf/services/clsi
COPY services/clsi/install_deps.sh /overleaf/services/clsi/
diff --git a/services/clsi/Makefile b/services/clsi/Makefile
index e02697f4e9..069c3c9426 100644
--- a/services/clsi/Makefile
+++ b/services/clsi/Makefile
@@ -24,6 +24,7 @@ DOCKER_COMPOSE_TEST_UNIT = \
clean:
-docker rmi ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
+ -docker rmi gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-docker rmi us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER)
-$(DOCKER_COMPOSE_TEST_UNIT) down --rmi local
-$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down --rmi local
@@ -32,30 +33,12 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
-
-SHELLCHECK_OPTS = \
- --shell=bash \
- --external-sources
-SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
-SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
-
-shellcheck:
- @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
- koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
-
-shellcheck_fix:
- @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
- diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
- if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
- elif [ -n "$$diff" ]; then echo "$$file"; \
- else echo "\033[2m$$file\033[0m"; fi \
- done
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
format:
$(RUN_LINTING) format
@@ -81,7 +64,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck shellcheck test_unit test_acceptance
+test: format lint typecheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -128,10 +111,11 @@ build:
--pull \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--tag ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+ --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
+ --tag gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME) \
+ --cache-from gcr.io/overleaf-ops/$(PROJECT_NAME):$(BRANCH_NAME) \
+ --cache-from gcr.io/overleaf-ops/$(PROJECT_NAME):main \
--tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) \
- --tag us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \
- --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):$(BRANCH_NAME) \
- --cache-from us-east1-docker.pkg.dev/overleaf-ops/ol-docker/$(PROJECT_NAME):main \
--file Dockerfile \
../..
@@ -148,7 +132,6 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
- shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/clsi/README.md b/services/clsi/README.md
index f1cf927d3d..0626bb4cfe 100644
--- a/services/clsi/README.md
+++ b/services/clsi/README.md
@@ -19,18 +19,18 @@ The CLSI can be configured through the following environment variables:
* `ALLOWED_IMAGES` - Space separated list of allowed Docker TeX Live images
* `CATCH_ERRORS` - Set to `true` to log uncaught exceptions
* `COMPILE_GROUP_DOCKER_CONFIGS` - JSON string of Docker configs for compile groups
-* `SANDBOXED_COMPILES` - Set to true to use sibling containers
-* `SANDBOXED_COMPILES_HOST_DIR_COMPILES` - Working directory for LaTeX compiles
-* `SANDBOXED_COMPILES_HOST_DIR_OUTPUT` - Output directory for LaTeX compiles
+* `COMPILES_HOST_DIR` - Working directory for LaTeX compiles
* `COMPILE_SIZE_LIMIT` - Sets the body-parser [limit](https://github.com/expressjs/body-parser#limit)
+* `DOCKER_RUNNER` - Set to true to use sibling containers
* `DOCKER_RUNTIME` -
* `FILESTORE_DOMAIN_OVERRIDE` - The url for the filestore service e.g.`http://$FILESTORE_HOST:3009`
* `FILESTORE_PARALLEL_FILE_DOWNLOADS` - Number of parallel file downloads
* `LISTEN_ADDRESS` - The address for the RESTful service to listen on. Set to `0.0.0.0` to listen on all network interfaces
* `PROCESS_LIFE_SPAN_LIMIT_MS` - Process life span limit in milliseconds
+* `SENTRY_DSN` - Sentry [Data Source Name](https://docs.sentry.io/product/sentry-basics/dsn-explainer/)
* `SMOKE_TEST` - Whether to run smoke tests
-* `TEXLIVE_IMAGE` - The TeX Live Docker image to use for sibling containers, e.g. `us-east1-docker.pkg.dev/overleaf-ops/ol-docker/texlive-full:2017.1`
-* `TEX_LIVE_IMAGE_NAME_OVERRIDE` - The name of the registry for the Docker image e.g. `us-east1-docker.pkg.dev/overleaf-ops/ol-docker`
+* `TEXLIVE_IMAGE` - The TeX Live Docker image to use for sibling containers, e.g. `gcr.io/overleaf-ops/texlive-full:2017.1`
+* `TEX_LIVE_IMAGE_NAME_OVERRIDE` - The name of the registry for the Docker image e.g. `gcr.io/overleaf-ops`
* `TEXLIVE_IMAGE_USER` - When using sibling containers, the user to run as in the TeX Live image. Defaults to `tex`
* `TEXLIVE_OPENOUT_ANY` - Sets the `openout_any` environment variable for TeX Live (see the `\openout` primitive [documentation](http://tug.org/texinfohtml/web2c.html#tex-invocation))
@@ -63,10 +63,10 @@ Then start the Docker container:
docker run --rm \
-p 127.0.0.1:3013:3013 \
-e LISTEN_ADDRESS=0.0.0.0 \
- -e SANDBOXED_COMPILES=true \
+ -e DOCKER_RUNNER=true \
-e TEXLIVE_IMAGE=texlive/texlive \
-e TEXLIVE_IMAGE_USER=root \
- -e SANDBOXED_COMPILES_HOST_DIR_COMPILES="$PWD/compiles" \
+ -e COMPILES_HOST_DIR="$PWD/compiles" \
-v "$PWD/compiles:/overleaf/services/clsi/compiles" \
-v "$PWD/cache:/overleaf/services/clsi/cache" \
-v /var/run/docker.sock:/var/run/docker.sock \
diff --git a/services/clsi/app.js b/services/clsi/app.js
index 872f612d9c..bcf89aaaf9 100644
--- a/services/clsi/app.js
+++ b/services/clsi/app.js
@@ -6,6 +6,9 @@ const ContentController = require('./app/js/ContentController')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
logger.initialize('clsi')
+if (Settings.sentry.dsn != null) {
+ logger.initializeErrorReporting(Settings.sentry.dsn)
+}
const Metrics = require('@overleaf/metrics')
const smokeTest = require('./test/smoke/js/SmokeTests')
@@ -13,7 +16,7 @@ const ContentTypeMapper = require('./app/js/ContentTypeMapper')
const Errors = require('./app/js/Errors')
const { createOutputZip } = require('./app/js/OutputController')
-const Path = require('node:path')
+const Path = require('path')
Metrics.open_sockets.monitor(true)
Metrics.memory.monitor(logger)
@@ -128,6 +131,26 @@ const ForbidSymlinks = require('./app/js/StaticServerForbidSymlinks')
// create a static server which does not allow access to any symlinks
// avoids possible mismatch of root directory between middleware check
// and serving the files
+const staticCompileServer = ForbidSymlinks(
+ express.static,
+ Settings.path.compilesDir,
+ {
+ setHeaders(res, path, stat) {
+ if (Path.basename(path) === 'output.pdf') {
+ // Calculate an etag in the same way as nginx
+ // https://github.com/tj/send/issues/65
+ const etag = (path, stat) =>
+ `"${Math.ceil(+stat.mtime / 1000).toString(16)}` +
+ '-' +
+ Number(stat.size).toString(16) +
+ '"'
+ res.set('Etag', etag(path, stat))
+ }
+ res.set('Content-Type', ContentTypeMapper.map(path))
+ },
+ }
+)
+
const staticOutputServer = ForbidSymlinks(
express.static,
Settings.path.outputDir,
@@ -193,6 +216,32 @@ app.get(
}
)
+app.get(
+ '/project/:project_id/user/:user_id/output/*',
+ function (req, res, next) {
+ // for specific user get the path to the top level file
+ logger.warn(
+ { url: req.url },
+ 'direct request for file in compile directory'
+ )
+ req.url = `/${req.params.project_id}-${req.params.user_id}/${req.params[0]}`
+ staticCompileServer(req, res, next)
+ }
+)
+
+app.get('/project/:project_id/output/*', function (req, res, next) {
+ logger.warn({ url: req.url }, 'direct request for file in compile directory')
+ if (req.query?.build?.match(OutputCacheManager.BUILD_REGEX)) {
+ // for specific build get the path from the OutputCacheManager (e.g. .clsi/buildId)
+ req.url =
+ `/${req.params.project_id}/` +
+ OutputCacheManager.path(req.query.build, `/${req.params[0]}`)
+ } else {
+ req.url = `/${req.params.project_id}/${req.params[0]}`
+ }
+ staticCompileServer(req, res, next)
+})
+
app.get('/oops', function (req, res, next) {
logger.error({ err: 'hello' }, 'test error')
res.send('error\n')
@@ -249,9 +298,6 @@ app.get('/health_check', function (req, res) {
if (Settings.processTooOld) {
return res.status(500).json({ processTooOld: true })
}
- if (ProjectPersistenceManager.isAnyDiskCriticalLow()) {
- return res.status(500).json({ diskCritical: true })
- }
smokeTest.sendLastResult(res)
})
@@ -261,8 +307,6 @@ app.use(function (error, req, res, next) {
if (error instanceof Errors.NotFoundError) {
logger.debug({ err: error, url: req.url }, 'not found error')
res.sendStatus(404)
- } else if (error instanceof Errors.InvalidParameter) {
- res.status(400).send(error.message)
} else if (error.code === 'EPIPE') {
// inspect container returns EPIPE when shutting down
res.sendStatus(503) // send 503 Unavailable response
@@ -272,8 +316,8 @@ app.use(function (error, req, res, next) {
}
})
-const net = require('node:net')
-const os = require('node:os')
+const net = require('net')
+const os = require('os')
let STATE = 'up'
@@ -299,18 +343,10 @@ const loadTcpServer = net.createServer(function (socket) {
}
const freeLoad = availableWorkingCpus - currentLoad
- let freeLoadPercentage = Math.round((freeLoad / availableWorkingCpus) * 100)
- if (ProjectPersistenceManager.isAnyDiskCriticalLow()) {
- freeLoadPercentage = 0
- }
- if (ProjectPersistenceManager.isAnyDiskLow()) {
- freeLoadPercentage = freeLoadPercentage / 2
- }
-
- if (
- Settings.internal.load_balancer_agent.allow_maintenance &&
- freeLoadPercentage <= 0
- ) {
+ const freeLoadPercentage = Math.round(
+ (freeLoad / availableWorkingCpus) * 100
+ )
+ if (freeLoadPercentage <= 0) {
// When its 0 the server is set to drain implicitly.
// Drain will move new projects to different servers.
// Drain will keep existing projects assigned to the same server.
@@ -318,11 +354,7 @@ const loadTcpServer = net.createServer(function (socket) {
socket.write(`maint, 0%\n`, 'ASCII')
} else {
// Ready will cancel the maint state.
- socket.write(`up, ready, ${Math.max(freeLoadPercentage, 1)}%\n`, 'ASCII')
- if (freeLoadPercentage <= 0) {
- // This metric records how often we would have gone into maintenance mode.
- Metrics.inc('clsi-prevented-maint')
- }
+ socket.write(`up, ready, ${freeLoadPercentage}%\n`, 'ASCII')
}
socket.end()
} else {
diff --git a/services/clsi/app/js/CLSICacheHandler.js b/services/clsi/app/js/CLSICacheHandler.js
deleted file mode 100644
index 26acd221f9..0000000000
--- a/services/clsi/app/js/CLSICacheHandler.js
+++ /dev/null
@@ -1,304 +0,0 @@
-const crypto = require('node:crypto')
-const fs = require('node:fs')
-const Path = require('node:path')
-const { pipeline } = require('node:stream/promises')
-const { createGzip, createGunzip } = require('node:zlib')
-const tarFs = require('tar-fs')
-const _ = require('lodash')
-const {
- fetchNothing,
- fetchStream,
- RequestFailedError,
-} = require('@overleaf/fetch-utils')
-const logger = require('@overleaf/logger')
-const Metrics = require('@overleaf/metrics')
-const Settings = require('@overleaf/settings')
-const { MeteredStream } = require('@overleaf/stream-utils')
-const { CACHE_SUBDIR } = require('./OutputCacheManager')
-const { isExtraneousFile } = require('./ResourceWriter')
-
-const TIMING_BUCKETS = [
- 0, 10, 100, 1000, 2000, 5000, 10000, 15000, 20000, 30000,
-]
-const MAX_ENTRIES_IN_OUTPUT_TAR = 100
-const OBJECT_ID_REGEX = /^[0-9a-f]{24}$/
-
-/**
- * @param {string} projectId
- * @return {{shard: string, url: string}}
- */
-function getShard(projectId) {
- // [timestamp 4bytes][random per machine 5bytes][counter 3bytes]
- // [32bit 4bytes]
- const last4Bytes = Buffer.from(projectId, 'hex').subarray(8, 12)
- const idx = last4Bytes.readUInt32BE() % Settings.apis.clsiCache.shards.length
- return Settings.apis.clsiCache.shards[idx]
-}
-
-/**
- * @param {string} projectId
- * @param {string} userId
- * @param {string} buildId
- * @param {string} editorId
- * @param {[{path: string}]} outputFiles
- * @param {string} compileGroup
- * @param {Record} options
- * @return {string | undefined}
- */
-function notifyCLSICacheAboutBuild({
- projectId,
- userId,
- buildId,
- editorId,
- outputFiles,
- compileGroup,
- options,
-}) {
- if (!Settings.apis.clsiCache.enabled) return undefined
- if (!OBJECT_ID_REGEX.test(projectId)) return undefined
- const { url, shard } = getShard(projectId)
-
- /**
- * @param {[{path: string}]} files
- */
- const enqueue = files => {
- Metrics.count('clsi_cache_enqueue_files', files.length)
- fetchNothing(`${url}/enqueue`, {
- method: 'POST',
- json: {
- projectId,
- userId,
- buildId,
- editorId,
- files,
- downloadHost: Settings.apis.clsi.downloadHost,
- clsiServerId: Settings.apis.clsi.clsiServerId,
- compileGroup,
- options,
- },
- signal: AbortSignal.timeout(15_000),
- }).catch(err => {
- logger.warn(
- { err, projectId, userId, buildId },
- 'enqueue for clsi cache failed'
- )
- })
- }
-
- // PDF preview
- enqueue(
- outputFiles
- .filter(
- f =>
- f.path === 'output.pdf' ||
- f.path === 'output.log' ||
- f.path === 'output.synctex.gz' ||
- f.path.endsWith('.blg')
- )
- .map(f => {
- if (f.path === 'output.pdf') {
- return _.pick(f, 'path', 'size', 'contentId', 'ranges')
- }
- return _.pick(f, 'path')
- })
- )
-
- // Compile Cache
- buildTarball({ projectId, userId, buildId, outputFiles })
- .then(() => {
- enqueue([{ path: 'output.tar.gz' }])
- })
- .catch(err => {
- logger.warn(
- { err, projectId, userId, buildId },
- 'build output.tar.gz for clsi cache failed'
- )
- })
-
- return shard
-}
-
-/**
- * @param {string} projectId
- * @param {string} userId
- * @param {string} buildId
- * @param {[{path: string}]} outputFiles
- * @return {Promise}
- */
-async function buildTarball({ projectId, userId, buildId, outputFiles }) {
- const timer = new Metrics.Timer('clsi_cache_build', 1, {}, TIMING_BUCKETS)
- const outputDir = Path.join(
- Settings.path.outputDir,
- userId ? `${projectId}-${userId}` : projectId,
- CACHE_SUBDIR,
- buildId
- )
-
- const files = outputFiles.filter(f => !isExtraneousFile(f.path))
- if (files.length > MAX_ENTRIES_IN_OUTPUT_TAR) {
- Metrics.inc('clsi_cache_build_too_many_entries')
- throw new Error('too many output files for output.tar.gz')
- }
- Metrics.count('clsi_cache_build_files', files.length)
-
- const path = Path.join(outputDir, 'output.tar.gz')
- try {
- await pipeline(
- tarFs.pack(outputDir, { entries: files.map(f => f.path) }),
- createGzip(),
- fs.createWriteStream(path)
- )
- } catch (err) {
- try {
- await fs.promises.unlink(path)
- } catch (e) {}
- throw err
- } finally {
- timer.done()
- }
-}
-
-/**
- * @param {string} projectId
- * @param {string} userId
- * @param {string} editorId
- * @param {string} buildId
- * @param {string} outputDir
- * @return {Promise}
- */
-async function downloadOutputDotSynctexFromCompileCache(
- projectId,
- userId,
- editorId,
- buildId,
- outputDir
-) {
- if (!Settings.apis.clsiCache.enabled) return false
- if (!OBJECT_ID_REGEX.test(projectId)) return false
-
- const timer = new Metrics.Timer(
- 'clsi_cache_download',
- 1,
- { method: 'synctex' },
- TIMING_BUCKETS
- )
- let stream
- try {
- stream = await fetchStream(
- `${getShard(projectId).url}/project/${projectId}/${
- userId ? `user/${userId}/` : ''
- }build/${editorId}-${buildId}/search/output/output.synctex.gz`,
- {
- method: 'GET',
- signal: AbortSignal.timeout(10_000),
- }
- )
- } catch (err) {
- if (err instanceof RequestFailedError && err.response.status === 404) {
- timer.done({ status: 'not-found' })
- return false
- }
- timer.done({ status: 'error' })
- throw err
- }
- await fs.promises.mkdir(outputDir, { recursive: true })
- const dst = Path.join(outputDir, 'output.synctex.gz')
- const tmp = dst + crypto.randomUUID()
- try {
- await pipeline(
- stream,
- new MeteredStream(Metrics, 'clsi_cache_egress', {
- path: 'output.synctex.gz',
- }),
- fs.createWriteStream(tmp)
- )
- await fs.promises.rename(tmp, dst)
- } catch (err) {
- try {
- await fs.promises.unlink(tmp)
- } catch {}
- throw err
- }
- timer.done({ status: 'success' })
- return true
-}
-
-/**
- * @param {string} projectId
- * @param {string} userId
- * @param {string} compileDir
- * @return {Promise}
- */
-async function downloadLatestCompileCache(projectId, userId, compileDir) {
- if (!Settings.apis.clsiCache.enabled) return false
- if (!OBJECT_ID_REGEX.test(projectId)) return false
-
- const url = `${getShard(projectId).url}/project/${projectId}/${
- userId ? `user/${userId}/` : ''
- }latest/output/output.tar.gz`
- const timer = new Metrics.Timer(
- 'clsi_cache_download',
- 1,
- { method: 'tar' },
- TIMING_BUCKETS
- )
- let stream
- try {
- stream = await fetchStream(url, {
- method: 'GET',
- signal: AbortSignal.timeout(10_000),
- })
- } catch (err) {
- if (err instanceof RequestFailedError && err.response.status === 404) {
- timer.done({ status: 'not-found' })
- return false
- }
- timer.done({ status: 'error' })
- throw err
- }
- let n = 0
- let abort = false
- await pipeline(
- stream,
- new MeteredStream(Metrics, 'clsi_cache_egress', { path: 'output.tar.gz' }),
- createGunzip(),
- tarFs.extract(compileDir, {
- // use ignore hook for counting entries (files+folders) and validation.
- // Include folders as they incur mkdir calls.
- ignore(_, header) {
- if (abort) return true // log once
- n++
- if (n > MAX_ENTRIES_IN_OUTPUT_TAR) {
- abort = true
- logger.warn(
- {
- url,
- compileDir,
- },
- 'too many entries in tar-ball from clsi-cache'
- )
- } else if (header.type !== 'file' && header.type !== 'directory') {
- abort = true
- logger.warn(
- {
- url,
- compileDir,
- entryType: header.type,
- },
- 'unexpected entry in tar-ball from clsi-cache'
- )
- }
- return abort
- },
- })
- )
- Metrics.count('clsi_cache_download_entries', n)
- timer.done({ status: 'success' })
- return !abort
-}
-
-module.exports = {
- notifyCLSICacheAboutBuild,
- downloadLatestCompileCache,
- downloadOutputDotSynctexFromCompileCache,
-}
diff --git a/services/clsi/app/js/CompileController.js b/services/clsi/app/js/CompileController.js
index b3343ee233..3da884eed7 100644
--- a/services/clsi/app/js/CompileController.js
+++ b/services/clsi/app/js/CompileController.js
@@ -1,4 +1,3 @@
-const Path = require('node:path')
const RequestParser = require('./RequestParser')
const CompileManager = require('./CompileManager')
const Settings = require('@overleaf/settings')
@@ -6,7 +5,6 @@ const Metrics = require('./Metrics')
const ProjectPersistenceManager = require('./ProjectPersistenceManager')
const logger = require('@overleaf/logger')
const Errors = require('./Errors')
-const { notifyCLSICacheAboutBuild } = require('./CLSICacheHandler')
let lastSuccessfulCompileTimestamp = 0
@@ -31,135 +29,101 @@ function compile(req, res, next) {
if (error) {
return next(error)
}
- const stats = {}
- const timings = {}
- CompileManager.doCompileWithLock(
- request,
- stats,
- timings,
- (error, result) => {
- let { buildId, outputFiles } = result || {}
- let code, status
- if (outputFiles == null) {
- outputFiles = []
- }
- if (error instanceof Errors.AlreadyCompilingError) {
- code = 423 // Http 423 Locked
- status = 'compile-in-progress'
- } else if (error instanceof Errors.FilesOutOfSyncError) {
- code = 409 // Http 409 Conflict
- status = 'retry'
- logger.warn(
- {
- projectId: request.project_id,
- userId: request.user_id,
- },
- 'files out of sync, please retry'
- )
- } else if (
- error?.code === 'EPIPE' ||
- error instanceof Errors.TooManyCompileRequestsError
- ) {
- // docker returns EPIPE when shutting down
- code = 503 // send 503 Unavailable response
- status = 'unavailable'
- } else if (error?.terminated) {
- status = 'terminated'
- } else if (error?.validate) {
- status = `validation-${error.validate}`
- } else if (error?.timedout) {
- status = 'timedout'
- logger.debug(
- { err: error, projectId: request.project_id },
- 'timeout running compile'
- )
- } else if (error) {
- status = 'error'
- code = 500
- logger.error(
- { err: error, projectId: request.project_id },
- 'error running compile'
- )
- } else {
- if (
- outputFiles.some(
- file => file.path === 'output.pdf' && file.size > 0
- )
- ) {
- status = 'success'
- lastSuccessfulCompileTimestamp = Date.now()
- } else if (request.stopOnFirstError) {
- status = 'stopped-on-first-error'
- } else {
- status = 'failure'
- logger.warn(
- { projectId: request.project_id, outputFiles },
- 'project failed to compile successfully, no output.pdf generated'
- )
- }
-
- // log an error if any core files are found
- if (outputFiles.some(file => file.path === 'core')) {
- logger.error(
- { projectId: request.project_id, req, outputFiles },
- 'core file found in output'
- )
- }
- }
-
- if (error) {
- outputFiles = error.outputFiles || []
- buildId = error.buildId
- }
-
- let clsiCacheShard
- if (
- status === 'success' &&
- request.editorId &&
- request.populateClsiCache
- ) {
- clsiCacheShard = notifyCLSICacheAboutBuild({
+ CompileManager.doCompileWithLock(request, (error, result) => {
+ let { buildId, outputFiles, stats, timings } = result || {}
+ let code, status
+ if (outputFiles == null) {
+ outputFiles = []
+ }
+ if (error instanceof Errors.AlreadyCompilingError) {
+ code = 423 // Http 423 Locked
+ status = 'compile-in-progress'
+ } else if (error instanceof Errors.FilesOutOfSyncError) {
+ code = 409 // Http 409 Conflict
+ status = 'retry'
+ logger.warn(
+ {
projectId: request.project_id,
userId: request.user_id,
- buildId: outputFiles[0].build,
- editorId: request.editorId,
- outputFiles,
- compileGroup: request.compileGroup,
- options: {
- compiler: request.compiler,
- draft: request.draft,
- imageName: request.imageName
- ? request.imageName
- : undefined,
- rootResourcePath: request.rootResourcePath,
- stopOnFirstError: request.stopOnFirstError,
- },
- })
+ },
+ 'files out of sync, please retry'
+ )
+ } else if (
+ error?.code === 'EPIPE' ||
+ error instanceof Errors.TooManyCompileRequestsError
+ ) {
+ // docker returns EPIPE when shutting down
+ code = 503 // send 503 Unavailable response
+ status = 'unavailable'
+ } else if (error?.terminated) {
+ status = 'terminated'
+ } else if (error?.validate) {
+ status = `validation-${error.validate}`
+ } else if (error?.timedout) {
+ status = 'timedout'
+ logger.debug(
+ { err: error, projectId: request.project_id },
+ 'timeout running compile'
+ )
+ } else if (error) {
+ status = 'error'
+ code = 500
+ logger.error(
+ { err: error, projectId: request.project_id },
+ 'error running compile'
+ )
+ } else {
+ if (
+ outputFiles.some(
+ file => file.path === 'output.pdf' && file.size > 0
+ )
+ ) {
+ status = 'success'
+ lastSuccessfulCompileTimestamp = Date.now()
+ } else if (request.stopOnFirstError) {
+ status = 'stopped-on-first-error'
+ } else {
+ status = 'failure'
+ logger.warn(
+ { projectId: request.project_id, outputFiles },
+ 'project failed to compile successfully, no output.pdf generated'
+ )
}
- timer.done()
- res.status(code || 200).send({
- compile: {
- status,
- error: error?.message || error,
- stats,
- timings,
- buildId,
- clsiCacheShard,
- outputUrlPrefix: Settings.apis.clsi.outputUrlPrefix,
- outputFiles: outputFiles.map(file => ({
- url:
- `${Settings.apis.clsi.url}/project/${request.project_id}` +
- (request.user_id != null
- ? `/user/${request.user_id}`
- : '') +
- `/build/${file.build}/output/${file.path}`,
- ...file,
- })),
- },
- })
+ // log an error if any core files are found
+ if (outputFiles.some(file => file.path === 'core')) {
+ logger.error(
+ { projectId: request.project_id, req, outputFiles },
+ 'core file found in output'
+ )
+ }
}
- )
+
+ if (error) {
+ outputFiles = error.outputFiles || []
+ buildId = error.buildId
+ }
+
+ timer.done()
+ res.status(code || 200).send({
+ compile: {
+ status,
+ error: error?.message || error,
+ stats,
+ timings,
+ buildId,
+ outputUrlPrefix: Settings.apis.clsi.outputUrlPrefix,
+ outputFiles: outputFiles.map(file => ({
+ url:
+ `${Settings.apis.clsi.url}/project/${request.project_id}` +
+ (request.user_id != null ? `/user/${request.user_id}` : '') +
+ (file.build != null ? `/build/${file.build}` : '') +
+ `/output/${file.path}`,
+ ...file,
+ })),
+ },
+ })
+ })
}
)
})
@@ -190,27 +154,30 @@ function clearCache(req, res, next) {
}
function syncFromCode(req, res, next) {
- const { file, editorId, buildId } = req.query
- const compileFromClsiCache = req.query.compileFromClsiCache === 'true'
+ const { file } = req.query
const line = parseInt(req.query.line, 10)
const column = parseInt(req.query.column, 10)
const { imageName } = req.query
const projectId = req.params.project_id
const userId = req.params.user_id
+
+ if (imageName && !_isImageNameAllowed(imageName)) {
+ return res.status(400).send('invalid image')
+ }
+
CompileManager.syncFromCode(
projectId,
userId,
file,
line,
column,
- { imageName, editorId, buildId, compileFromClsiCache },
- function (error, pdfPositions, downloadedFromCache) {
+ imageName,
+ function (error, pdfPositions) {
if (error) {
return next(error)
}
res.json({
pdf: pdfPositions,
- downloadedFromCache,
})
}
)
@@ -220,24 +187,26 @@ function syncFromPdf(req, res, next) {
const page = parseInt(req.query.page, 10)
const h = parseFloat(req.query.h)
const v = parseFloat(req.query.v)
- const { imageName, editorId, buildId } = req.query
- const compileFromClsiCache = req.query.compileFromClsiCache === 'true'
+ const { imageName } = req.query
const projectId = req.params.project_id
const userId = req.params.user_id
+
+ if (imageName && !_isImageNameAllowed(imageName)) {
+ return res.status(400).send('invalid image')
+ }
CompileManager.syncFromPdf(
projectId,
userId,
page,
h,
v,
- { imageName, editorId, buildId, compileFromClsiCache },
- function (error, codePositions, downloadedFromCache) {
+ imageName,
+ function (error, codePositions) {
if (error) {
return next(error)
}
res.json({
code: codePositions,
- downloadedFromCache,
})
}
)
@@ -248,6 +217,9 @@ function wordcount(req, res, next) {
const projectId = req.params.project_id
const userId = req.params.user_id
const { image } = req.query
+ if (image && !_isImageNameAllowed(image)) {
+ return res.status(400).send('invalid image')
+ }
logger.debug({ image, file, projectId }, 'word count request')
CompileManager.wordcount(
@@ -270,6 +242,12 @@ function status(req, res, next) {
res.send('OK')
}
+function _isImageNameAllowed(imageName) {
+ const ALLOWED_IMAGES =
+ Settings.clsi && Settings.clsi.docker && Settings.clsi.docker.allowedImages
+ return !ALLOWED_IMAGES || ALLOWED_IMAGES.includes(imageName)
+}
+
module.exports = {
compile,
stopCompile,
diff --git a/services/clsi/app/js/CompileManager.js b/services/clsi/app/js/CompileManager.js
index 1b66927412..9d1981263d 100644
--- a/services/clsi/app/js/CompileManager.js
+++ b/services/clsi/app/js/CompileManager.js
@@ -1,7 +1,7 @@
-const fsPromises = require('node:fs/promises')
-const os = require('node:os')
-const Path = require('node:path')
-const { callbackify } = require('node:util')
+const fsPromises = require('fs/promises')
+const os = require('os')
+const Path = require('path')
+const { callbackify } = require('util')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
@@ -19,11 +19,6 @@ const Errors = require('./Errors')
const CommandRunner = require('./CommandRunner')
const { emitPdfStats } = require('./ContentCacheMetrics')
const SynctexOutputParser = require('./SynctexOutputParser')
-const {
- downloadLatestCompileCache,
- downloadOutputDotSynctexFromCompileCache,
-} = require('./CLSICacheHandler')
-const { callbackifyMultiResult } = require('@overleaf/promise-utils')
const COMPILE_TIME_BUCKETS = [
// NOTE: These buckets are locked in per metric name.
@@ -47,22 +42,22 @@ function getOutputDir(projectId, userId) {
return Path.join(Settings.path.outputDir, getCompileName(projectId, userId))
}
-async function doCompileWithLock(request, stats, timings) {
+async function doCompileWithLock(request) {
const compileDir = getCompileDir(request.project_id, request.user_id)
- request.isInitialCompile =
- (await fsPromises.mkdir(compileDir, { recursive: true })) === compileDir
+ await fsPromises.mkdir(compileDir, { recursive: true })
// prevent simultaneous compiles
const lock = LockManager.acquire(compileDir)
try {
- return await doCompile(request, stats, timings)
+ return await doCompile(request)
} finally {
lock.release()
}
}
-async function doCompile(request, stats, timings) {
- const { project_id: projectId, user_id: userId } = request
+async function doCompile(request) {
const compileDir = getCompileDir(request.project_id, request.user_id)
+ const stats = {}
+ const timings = {}
const timerE2E = new Metrics.Timer(
'compile-e2e-v2',
@@ -70,25 +65,6 @@ async function doCompile(request, stats, timings) {
request.metricsOpts,
COMPILE_TIME_BUCKETS
)
- if (request.isInitialCompile) {
- stats.isInitialCompile = 1
- request.metricsOpts.compile = 'initial'
- if (request.compileFromClsiCache) {
- try {
- if (await downloadLatestCompileCache(projectId, userId, compileDir)) {
- stats.restoredClsiCache = 1
- request.metricsOpts.compile = 'from-clsi-cache'
- }
- } catch (err) {
- logger.warn(
- { err, projectId, userId },
- 'failed to populate compile dir from cache'
- )
- }
- }
- } else {
- request.metricsOpts.compile = 'recompile'
- }
const writeToDiskTimer = new Metrics.Timer(
'write-to-disk',
1,
@@ -320,7 +296,7 @@ async function doCompile(request, stats, timings) {
emitPdfStats(stats, timings, request)
}
- return { outputFiles, buildId }
+ return { outputFiles, stats, timings, buildId }
}
async function _saveOutputFiles({
@@ -337,16 +313,24 @@ async function _saveOutputFiles({
)
const outputDir = getOutputDir(request.project_id, request.user_id)
- const { outputFiles: rawOutputFiles, allEntries } =
+ let { outputFiles, allEntries } =
await OutputFileFinder.promises.findOutputFiles(resourceList, compileDir)
- const { buildId, outputFiles } =
- await OutputCacheManager.promises.saveOutputFiles(
+ let buildId
+
+ try {
+ const saveResult = await OutputCacheManager.promises.saveOutputFiles(
{ request, stats, timings },
- rawOutputFiles,
+ outputFiles,
compileDir,
outputDir
)
+ buildId = saveResult.buildId
+ outputFiles = saveResult.outputFiles
+ } catch (err) {
+ const { project_id: projectId, user_id: userId } = request
+ logger.err({ projectId, userId, err }, 'failed to save output files')
+ }
timings.output = timer.done()
return { outputFiles, allEntries, buildId }
@@ -432,7 +416,14 @@ async function _checkDirectory(compileDir) {
return true
}
-async function syncFromCode(projectId, userId, filename, line, column, opts) {
+async function syncFromCode(
+ projectId,
+ userId,
+ filename,
+ line,
+ column,
+ imageName
+) {
// If LaTeX was run in a virtual environment, the file path that synctex expects
// might not match the file path on the host. The .synctex.gz file however, will be accessed
// wherever it is on the host.
@@ -448,23 +439,15 @@ async function syncFromCode(projectId, userId, filename, line, column, opts) {
'-o',
outputFilePath,
]
- const { stdout, downloadedFromCache } = await _runSynctex(
- projectId,
- userId,
- command,
- opts
- )
+ const stdout = await _runSynctex(projectId, userId, command, imageName)
logger.debug(
{ projectId, userId, filename, line, column, command, stdout },
'synctex code output'
)
- return {
- codePositions: SynctexOutputParser.parseViewOutput(stdout),
- downloadedFromCache,
- }
+ return SynctexOutputParser.parseViewOutput(stdout)
}
-async function syncFromPdf(projectId, userId, page, h, v, opts) {
+async function syncFromPdf(projectId, userId, page, h, v, imageName) {
const compileName = getCompileName(projectId, userId)
const baseDir = Settings.path.synctexBaseDir(compileName)
const outputFilePath = `${baseDir}/output.pdf`
@@ -474,17 +457,9 @@ async function syncFromPdf(projectId, userId, page, h, v, opts) {
'-o',
`${page}:${h}:${v}:${outputFilePath}`,
]
- const { stdout, downloadedFromCache } = await _runSynctex(
- projectId,
- userId,
- command,
- opts
- )
+ const stdout = await _runSynctex(projectId, userId, command, imageName)
logger.debug({ projectId, userId, page, h, v, stdout }, 'synctex pdf output')
- return {
- pdfPositions: SynctexOutputParser.parseEditOutput(stdout, baseDir),
- downloadedFromCache,
- }
+ return SynctexOutputParser.parseEditOutput(stdout, baseDir)
}
async function _checkFileExists(dir, filename) {
@@ -511,90 +486,32 @@ async function _checkFileExists(dir, filename) {
}
}
-async function _runSynctex(projectId, userId, command, opts) {
- const { imageName, editorId, buildId, compileFromClsiCache } = opts
-
- if (imageName && !_isImageNameAllowed(imageName)) {
- throw new Errors.InvalidParameter('invalid image')
- }
- if (editorId && !/^[a-f0-9-]+$/.test(editorId)) {
- throw new Errors.InvalidParameter('invalid editorId')
- }
- if (buildId && !OutputCacheManager.BUILD_REGEX.test(buildId)) {
- throw new Errors.InvalidParameter('invalid buildId')
- }
-
- const outputDir = getOutputDir(projectId, userId)
- const runInOutputDir = buildId && CommandRunner.canRunSyncTeXInOutputDir()
-
- const directory = runInOutputDir
- ? Path.join(outputDir, OutputCacheManager.CACHE_SUBDIR, buildId)
- : getCompileDir(projectId, userId)
+async function _runSynctex(projectId, userId, command, imageName) {
+ const directory = getCompileDir(projectId, userId)
const timeout = 60 * 1000 // increased to allow for large projects
const compileName = getCompileName(projectId, userId)
- const compileGroup = runInOutputDir ? 'synctex-output' : 'synctex'
+ const compileGroup = 'synctex'
const defaultImageName =
Settings.clsi && Settings.clsi.docker && Settings.clsi.docker.image
- // eslint-disable-next-line @typescript-eslint/return-await
- return await OutputCacheManager.promises.queueDirOperation(
- outputDir,
- /**
- * @return {Promise<{stdout: string, downloadedFromCache: boolean}>}
- */
- async () => {
- let downloadedFromCache = false
- try {
- await _checkFileExists(directory, 'output.synctex.gz')
- } catch (err) {
- if (
- err instanceof Errors.NotFoundError &&
- compileFromClsiCache &&
- editorId &&
- buildId
- ) {
- try {
- downloadedFromCache =
- await downloadOutputDotSynctexFromCompileCache(
- projectId,
- userId,
- editorId,
- buildId,
- directory
- )
- } catch (err) {
- logger.warn(
- { err, projectId, userId, editorId, buildId },
- 'failed to download output.synctex.gz from clsi-cache'
- )
- }
- await _checkFileExists(directory, 'output.synctex.gz')
- } else {
- throw err
- }
- }
- try {
- const { stdout } = await CommandRunner.promises.run(
- compileName,
- command,
- directory,
- imageName || defaultImageName,
- timeout,
- {},
- compileGroup
- )
- return {
- stdout,
- downloadedFromCache,
- }
- } catch (error) {
- throw OError.tag(error, 'error running synctex', {
- command,
- projectId,
- userId,
- })
- }
- }
- )
+ await _checkFileExists(directory, 'output.synctex.gz')
+ try {
+ const output = await CommandRunner.promises.run(
+ compileName,
+ command,
+ directory,
+ imageName || defaultImageName,
+ timeout,
+ {},
+ compileGroup
+ )
+ return output.stdout
+ } catch (error) {
+ throw OError.tag(error, 'error running synctex', {
+ command,
+ projectId,
+ userId,
+ })
+ }
}
async function wordcount(projectId, userId, filename, image) {
@@ -606,10 +523,6 @@ async function wordcount(projectId, userId, filename, image) {
const compileName = getCompileName(projectId, userId)
const compileGroup = 'wordcount'
- if (image && !_isImageNameAllowed(image)) {
- throw new Errors.InvalidParameter('invalid image')
- }
-
try {
await fsPromises.mkdir(compileDir, { recursive: true })
} catch (err) {
@@ -697,25 +610,13 @@ function _parseWordcountFromOutput(output) {
return results
}
-function _isImageNameAllowed(imageName) {
- const ALLOWED_IMAGES =
- Settings.clsi && Settings.clsi.docker && Settings.clsi.docker.allowedImages
- return !ALLOWED_IMAGES || ALLOWED_IMAGES.includes(imageName)
-}
-
module.exports = {
doCompileWithLock: callbackify(doCompileWithLock),
stopCompile: callbackify(stopCompile),
clearProject: callbackify(clearProject),
clearExpiredProjects: callbackify(clearExpiredProjects),
- syncFromCode: callbackifyMultiResult(syncFromCode, [
- 'codePositions',
- 'downloadedFromCache',
- ]),
- syncFromPdf: callbackifyMultiResult(syncFromPdf, [
- 'pdfPositions',
- 'downloadedFromCache',
- ]),
+ syncFromCode: callbackify(syncFromCode),
+ syncFromPdf: callbackify(syncFromPdf),
wordcount: callbackify(wordcount),
promises: {
doCompileWithLock,
diff --git a/services/clsi/app/js/ContentCacheManager.js b/services/clsi/app/js/ContentCacheManager.js
index 5457c0dce0..016b264b86 100644
--- a/services/clsi/app/js/ContentCacheManager.js
+++ b/services/clsi/app/js/ContentCacheManager.js
@@ -2,10 +2,10 @@
* ContentCacheManager - maintains a cache of stream hashes from a PDF file
*/
-const { callbackify } = require('node:util')
-const fs = require('node:fs')
-const crypto = require('node:crypto')
-const Path = require('node:path')
+const { callbackify } = require('util')
+const fs = require('fs')
+const crypto = require('crypto')
+const Path = require('path')
const Settings = require('@overleaf/settings')
const OError = require('@overleaf/o-error')
const pLimit = require('p-limit')
diff --git a/services/clsi/app/js/ContentCacheMetrics.js b/services/clsi/app/js/ContentCacheMetrics.js
index 1e2b598286..0337876feb 100644
--- a/services/clsi/app/js/ContentCacheMetrics.js
+++ b/services/clsi/app/js/ContentCacheMetrics.js
@@ -1,6 +1,6 @@
const logger = require('@overleaf/logger')
const Metrics = require('./Metrics')
-const os = require('node:os')
+const os = require('os')
let CACHED_LOAD = {
expires: -1,
diff --git a/services/clsi/app/js/ContentController.js b/services/clsi/app/js/ContentController.js
index 96eba613ed..b154bea175 100644
--- a/services/clsi/app/js/ContentController.js
+++ b/services/clsi/app/js/ContentController.js
@@ -1,4 +1,4 @@
-const Path = require('node:path')
+const Path = require('path')
const send = require('send')
const Settings = require('@overleaf/settings')
const OutputCacheManager = require('./OutputCacheManager')
diff --git a/services/clsi/app/js/ContentTypeMapper.js b/services/clsi/app/js/ContentTypeMapper.js
index 5bf0c31423..6301dce489 100644
--- a/services/clsi/app/js/ContentTypeMapper.js
+++ b/services/clsi/app/js/ContentTypeMapper.js
@@ -4,7 +4,7 @@
// TODO: This file was created by bulk-decaffeinate.
// Fix any style issues and re-enable lint.
let ContentTypeMapper
-const Path = require('node:path')
+const Path = require('path')
// here we coerce html, css and js to text/plain,
// otherwise choose correct mime type based on file extension,
diff --git a/services/clsi/app/js/DockerRunner.js b/services/clsi/app/js/DockerRunner.js
index 97053c1875..45438ce35b 100644
--- a/services/clsi/app/js/DockerRunner.js
+++ b/services/clsi/app/js/DockerRunner.js
@@ -1,17 +1,26 @@
-const { promisify } = require('node:util')
+const { promisify } = require('util')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
const Docker = require('dockerode')
const dockerode = new Docker()
-const crypto = require('node:crypto')
+const crypto = require('crypto')
const async = require('async')
const LockManager = require('./DockerLockManager')
-const Path = require('node:path')
+const fs = require('fs')
+const Path = require('path')
const _ = require('lodash')
const ONE_HOUR_IN_MS = 60 * 60 * 1000
logger.debug('using docker runner')
+function usingSiblingContainers() {
+ return (
+ Settings != null &&
+ Settings.path != null &&
+ Settings.path.sandboxedCompilesHostDir != null
+ )
+}
+
let containerMonitorTimeout
let containerMonitorInterval
@@ -26,6 +35,24 @@ const DockerRunner = {
compileGroup,
callback
) {
+ if (usingSiblingContainers()) {
+ const _newPath = Settings.path.sandboxedCompilesHostDir
+ logger.debug(
+ { path: _newPath },
+ 'altering bind path for sibling containers'
+ )
+ // Server Pro, example:
+ // '/var/lib/overleaf/data/compiles/'
+ // ... becomes ...
+ // '/opt/overleaf_data/data/compiles/'
+ directory = Path.join(
+ Settings.path.sandboxedCompilesHostDir,
+ Path.basename(directory)
+ )
+ }
+
+ const volumes = { [directory]: '/compile' }
+
command = command.map(arg =>
arg.toString().replace('$COMPILE_DIR', '/compile')
)
@@ -45,32 +72,7 @@ const DockerRunner = {
image = `${Settings.texliveImageNameOveride}/${img[2]}`
}
- if (compileGroup === 'synctex-output') {
- // In: directory = '/overleaf/services/clsi/output/projectId-userId/generated-files/buildId'
- // directory.split('/').slice(-3) === 'projectId-userId/generated-files/buildId'
- // sandboxedCompilesHostDirOutput = '/host/output'
- // Out: directory = '/host/output/projectId-userId/generated-files/buildId'
- directory = Path.join(
- Settings.path.sandboxedCompilesHostDirOutput,
- ...directory.split('/').slice(-3)
- )
- } else {
- // In: directory = '/overleaf/services/clsi/compiles/projectId-userId'
- // Path.basename(directory) === 'projectId-userId'
- // sandboxedCompilesHostDirCompiles = '/host/compiles'
- // Out: directory = '/host/compiles/projectId-userId'
- directory = Path.join(
- Settings.path.sandboxedCompilesHostDirCompiles,
- Path.basename(directory)
- )
- }
-
- const volumes = { [directory]: '/compile' }
- if (
- compileGroup === 'synctex' ||
- compileGroup === 'synctex-output' ||
- compileGroup === 'wordcount'
- ) {
+ if (compileGroup === 'synctex' || compileGroup === 'wordcount') {
volumes[directory] += ':ro'
}
@@ -307,17 +309,50 @@ const DockerRunner = {
LockManager.runWithLock(
options.name,
releaseLock =>
- DockerRunner._startContainer(
- options,
- volumes,
- attachStreamHandler,
- releaseLock
- ),
+ // Check that volumes exist before starting the container.
+ // When a container is started with volume pointing to a
+ // non-existent directory then docker creates the directory but
+ // with root ownership.
+ DockerRunner._checkVolumes(options, volumes, err => {
+ if (err != null) {
+ return releaseLock(err)
+ }
+ DockerRunner._startContainer(
+ options,
+ volumes,
+ attachStreamHandler,
+ releaseLock
+ )
+ }),
+
callback
)
},
// Check that volumes exist and are directories
+ _checkVolumes(options, volumes, callback) {
+ if (usingSiblingContainers()) {
+ // Server Pro, with sibling-containers active, skip checks
+ return callback(null)
+ }
+
+ const checkVolume = (path, cb) =>
+ fs.stat(path, (err, stats) => {
+ if (err != null) {
+ return cb(err)
+ }
+ if (!stats.isDirectory()) {
+ return cb(new Error('not a directory'))
+ }
+ cb()
+ })
+ const jobs = []
+ for (const vol in volumes) {
+ jobs.push(cb => checkVolume(vol, cb))
+ }
+ async.series(jobs, callback)
+ },
+
_startContainer(options, volumes, attachStreamHandler, callback) {
callback = _.once(callback)
const { name } = options
@@ -582,10 +617,6 @@ const DockerRunner = {
containerMonitorInterval = undefined
}
},
-
- canRunSyncTeXInOutputDir() {
- return Boolean(Settings.path.sandboxedCompilesHostDirOutput)
- },
}
DockerRunner.startContainerMonitor()
diff --git a/services/clsi/app/js/DraftModeManager.js b/services/clsi/app/js/DraftModeManager.js
index cf8ababc47..a5b26348e6 100644
--- a/services/clsi/app/js/DraftModeManager.js
+++ b/services/clsi/app/js/DraftModeManager.js
@@ -1,5 +1,5 @@
-const fsPromises = require('node:fs/promises')
-const { callbackify } = require('node:util')
+const fsPromises = require('fs/promises')
+const { callbackify } = require('util')
const logger = require('@overleaf/logger')
async function injectDraftMode(filename) {
diff --git a/services/clsi/app/js/Errors.js b/services/clsi/app/js/Errors.js
index 64c3c7b59a..5c5fd3745a 100644
--- a/services/clsi/app/js/Errors.js
+++ b/services/clsi/app/js/Errors.js
@@ -35,7 +35,6 @@ class QueueLimitReachedError extends OError {}
class TimedOutError extends OError {}
class NoXrefTableError extends OError {}
class TooManyCompileRequestsError extends OError {}
-class InvalidParameter extends OError {}
module.exports = Errors = {
QueueLimitReachedError,
@@ -45,5 +44,4 @@ module.exports = Errors = {
AlreadyCompilingError,
NoXrefTableError,
TooManyCompileRequestsError,
- InvalidParameter,
}
diff --git a/services/clsi/app/js/LatexRunner.js b/services/clsi/app/js/LatexRunner.js
index beefa002ab..d956ee4949 100644
--- a/services/clsi/app/js/LatexRunner.js
+++ b/services/clsi/app/js/LatexRunner.js
@@ -1,9 +1,9 @@
-const Path = require('node:path')
-const { promisify } = require('node:util')
+const Path = require('path')
+const { promisify } = require('util')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
const CommandRunner = require('./CommandRunner')
-const fs = require('node:fs')
+const fs = require('fs')
const ProcessTable = {} // table of currently running jobs (pids or docker container names)
diff --git a/services/clsi/app/js/LocalCommandRunner.js b/services/clsi/app/js/LocalCommandRunner.js
index aa62825443..d909ec601c 100644
--- a/services/clsi/app/js/LocalCommandRunner.js
+++ b/services/clsi/app/js/LocalCommandRunner.js
@@ -12,8 +12,8 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let CommandRunner
-const { spawn } = require('node:child_process')
-const { promisify } = require('node:util')
+const { spawn } = require('child_process')
+const { promisify } = require('util')
const _ = require('lodash')
const logger = require('@overleaf/logger')
@@ -54,7 +54,6 @@ module.exports = CommandRunner = {
cwd: directory,
env,
stdio: ['pipe', 'pipe', 'ignore'],
- detached: true,
})
let stdout = ''
@@ -100,10 +99,6 @@ module.exports = CommandRunner = {
}
return callback()
},
-
- canRunSyncTeXInOutputDir() {
- return true
- },
}
module.exports.promises = {
diff --git a/services/clsi/app/js/OutputCacheManager.js b/services/clsi/app/js/OutputCacheManager.js
index a1a0a89aa7..13aad1bc74 100644
--- a/services/clsi/app/js/OutputCacheManager.js
+++ b/services/clsi/app/js/OutputCacheManager.js
@@ -1,12 +1,12 @@
let OutputCacheManager
-const { callbackify, promisify } = require('node:util')
+const { callbackify, promisify } = require('util')
const async = require('async')
-const fs = require('node:fs')
-const Path = require('node:path')
+const fs = require('fs')
+const Path = require('path')
const logger = require('@overleaf/logger')
const _ = require('lodash')
const Settings = require('@overleaf/settings')
-const crypto = require('node:crypto')
+const crypto = require('crypto')
const Metrics = require('./Metrics')
const OutputFileOptimiser = require('./OutputFileOptimiser')
@@ -83,13 +83,6 @@ async function cleanupDirectory(dir, options) {
})
}
-/**
- * @template T
- *
- * @param {string} dir
- * @param {() => Promise} fn
- * @return {Promise}
- */
async function queueDirOperation(dir, fn) {
const pending = PENDING_PROJECT_ACTIONS.get(dir) || Promise.resolve()
const p = pending.then(fn, fn).finally(() => {
@@ -105,11 +98,12 @@ module.exports = OutputCacheManager = {
CONTENT_SUBDIR: 'content',
CACHE_SUBDIR: 'generated-files',
ARCHIVE_SUBDIR: 'archived-logs',
- // build id is HEXDATE-HEXRANDOM from Date.now() and RandomBytes
- BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/,
- CONTENT_REGEX: /^[0-9a-f]+-[0-9a-f]+$/,
+ // build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
+ // for backwards compatibility, make the randombytes part optional
+ BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
+ CONTENT_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
CACHE_LIMIT: 2, // maximum number of cache directories
- CACHE_AGE: 90 * 60 * 1000, // up to 90 minutes old
+ CACHE_AGE: 60 * 60 * 1000, // up to one hour old
init,
queueDirOperation: callbackify(queueDirOperation),
@@ -143,11 +137,7 @@ module.exports = OutputCacheManager = {
outputDir,
callback
) {
- const getBuildId = cb => {
- if (request.buildId) return cb(null, request.buildId)
- OutputCacheManager.generateBuildId(cb)
- }
- getBuildId(function (err, buildId) {
+ OutputCacheManager.generateBuildId(function (err, buildId) {
if (err) {
return callback(err)
}
@@ -255,7 +245,7 @@ module.exports = OutputCacheManager = {
{ err, directory: cacheDir },
'error creating cache directory'
)
- callback(err)
+ callback(err, outputFiles)
} else {
// copy all the output files into the new cache directory
const results = []
@@ -273,6 +263,7 @@ module.exports = OutputCacheManager = {
return cb()
}
// copy other files into cache directory if valid
+ const newFile = _.clone(file)
const src = Path.join(compileDir, file.path)
const dst = Path.join(cacheDir, file.path)
OutputCacheManager._checkIfShouldCopy(
@@ -288,8 +279,8 @@ module.exports = OutputCacheManager = {
if (err) {
return cb(err)
}
- file.build = buildId
- results.push(file)
+ newFile.build = buildId // attach a build id if we cached the file
+ results.push(newFile)
cb()
})
}
@@ -297,7 +288,8 @@ module.exports = OutputCacheManager = {
},
function (err) {
if (err) {
- callback(err)
+ // pass back the original files if we encountered *any* error
+ callback(err, outputFiles)
// clean up the directory we just created
fs.rm(cacheDir, { force: true, recursive: true }, function (err) {
if (err) {
@@ -309,7 +301,7 @@ module.exports = OutputCacheManager = {
})
} else {
// pass back the list of new files in the cache
- callback(null, results)
+ callback(err, results)
// let file expiry run in the background, expire all previous files if per-user
cleanupDirectory(outputDir, {
keep: buildId,
@@ -684,5 +676,4 @@ OutputCacheManager.promises = {
saveOutputFilesInBuildDir: promisify(
OutputCacheManager.saveOutputFilesInBuildDir
),
- queueDirOperation,
}
diff --git a/services/clsi/app/js/OutputFileArchiveManager.js b/services/clsi/app/js/OutputFileArchiveManager.js
index 64c5198392..3c5a6c8197 100644
--- a/services/clsi/app/js/OutputFileArchiveManager.js
+++ b/services/clsi/app/js/OutputFileArchiveManager.js
@@ -7,7 +7,7 @@ const { NotFoundError } = require('./Errors')
const logger = require('@overleaf/logger')
// NOTE: Updating this list requires a corresponding change in
-// * services/web/frontend/js/features/pdf-preview/util/file-list.ts
+// * services/web/frontend/js/features/pdf-preview/util/file-list.js
const ignoreFiles = ['output.fls', 'output.fdb_latexmk']
function getContentDir(projectId, userId) {
@@ -93,11 +93,8 @@ module.exports = {
)
return outputFiles.filter(
- // Ignore the pdf, clsi-cache tar-ball and also ignore the files ignored by the frontend.
- ({ path }) =>
- path !== 'output.pdf' &&
- path !== 'output.tar.gz' &&
- !ignoreFiles.includes(path)
+ // Ignore the pdf and also ignore the files ignored by the frontend.
+ ({ path }) => path !== 'output.pdf' && !ignoreFiles.includes(path)
)
} catch (error) {
if (
diff --git a/services/clsi/app/js/OutputFileFinder.js b/services/clsi/app/js/OutputFileFinder.js
index e62038c614..8ca13183dc 100644
--- a/services/clsi/app/js/OutputFileFinder.js
+++ b/services/clsi/app/js/OutputFileFinder.js
@@ -1,5 +1,5 @@
-const Path = require('node:path')
-const fs = require('node:fs')
+const Path = require('path')
+const fs = require('fs')
const { callbackifyMultiResult } = require('@overleaf/promise-utils')
async function walkFolder(compileDir, d, files, allEntries) {
diff --git a/services/clsi/app/js/OutputFileOptimiser.js b/services/clsi/app/js/OutputFileOptimiser.js
index 09ca98672d..c97ef7ffdc 100644
--- a/services/clsi/app/js/OutputFileOptimiser.js
+++ b/services/clsi/app/js/OutputFileOptimiser.js
@@ -13,9 +13,9 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let OutputFileOptimiser
-const fs = require('node:fs')
-const Path = require('node:path')
-const { spawn } = require('node:child_process')
+const fs = require('fs')
+const Path = require('path')
+const { spawn } = require('child_process')
const logger = require('@overleaf/logger')
const Metrics = require('./Metrics')
const _ = require('lodash')
@@ -74,7 +74,9 @@ module.exports = OutputFileOptimiser = {
logger.debug({ args }, 'running qpdf command')
const timer = new Metrics.Timer('qpdf')
- const proc = spawn('qpdf', args, { stdio: 'ignore' })
+ const proc = spawn('qpdf', args)
+ let stdout = ''
+ proc.stdout.setEncoding('utf8').on('data', chunk => (stdout += chunk))
callback = _.once(callback) // avoid double call back for error and close event
proc.on('error', function (err) {
logger.warn({ err, args }, 'qpdf failed')
diff --git a/services/clsi/app/js/ProjectPersistenceManager.js b/services/clsi/app/js/ProjectPersistenceManager.js
index 41cdd07f4d..4dbe8b636e 100644
--- a/services/clsi/app/js/ProjectPersistenceManager.js
+++ b/services/clsi/app/js/ProjectPersistenceManager.js
@@ -13,90 +13,47 @@ const CompileManager = require('./CompileManager')
const async = require('async')
const logger = require('@overleaf/logger')
const oneDay = 24 * 60 * 60 * 1000
-const Metrics = require('@overleaf/metrics')
const Settings = require('@overleaf/settings')
-const { callbackify } = require('node:util')
-const Path = require('node:path')
-const fs = require('node:fs')
+const diskusage = require('diskusage')
+const { callbackify } = require('util')
+const Path = require('path')
+const fs = require('fs')
// projectId -> timestamp mapping.
const LAST_ACCESS = new Map()
-let ANY_DISK_LOW = false
-let ANY_DISK_CRITICAL_LOW = false
-
-async function collectDiskStats() {
+async function refreshExpiryTimeout() {
const paths = [
Settings.path.compilesDir,
Settings.path.outputDir,
Settings.path.clsiCacheDir,
]
-
- const diskStats = {}
- let anyDiskLow = false
- let anyDiskCriticalLow = false
for (const path of paths) {
try {
- const { blocks, bavail, bsize } = await fs.promises.statfs(path)
- const stats = {
- // Warning: these values will be wrong by a factor in Docker-for-Mac.
- // See https://github.com/docker/for-mac/issues/2136
- total: blocks * bsize, // Total size of the file system in bytes
- available: bavail * bsize, // Free space available to unprivileged users.
- }
- const diskAvailablePercent = (stats.available / stats.total) * 100
- Metrics.gauge('disk_available_percent', diskAvailablePercent, 1, {
- path,
- })
- const lowDisk = diskAvailablePercent < 10
- diskStats[path] = { stats, lowDisk }
+ const stats = await diskusage.check(path)
+ const lowDisk = stats.available / stats.total < 0.1
- const criticalLowDisk = diskAvailablePercent < 3
- anyDiskLow = anyDiskLow || lowDisk
- anyDiskCriticalLow = anyDiskCriticalLow || criticalLowDisk
+ const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
+ if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
+ logger.warn(
+ {
+ stats,
+ newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2),
+ },
+ 'disk running low on space, modifying EXPIRY_TIMEOUT'
+ )
+ ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
+ break
+ }
} catch (err) {
logger.err({ err, path }, 'error getting disk usage')
}
}
- ANY_DISK_LOW = anyDiskLow
- ANY_DISK_CRITICAL_LOW = anyDiskCriticalLow
- return diskStats
-}
-
-async function refreshExpiryTimeout() {
- for (const [path, { stats, lowDisk }] of Object.entries(
- await collectDiskStats()
- )) {
- const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
- if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
- logger.warn(
- {
- path,
- stats,
- newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2),
- },
- 'disk running low on space, modifying EXPIRY_TIMEOUT'
- )
- ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
- break
- }
- }
- Metrics.gauge(
- 'project_persistence_expiry_timeout',
- ProjectPersistenceManager.EXPIRY_TIMEOUT
- )
}
module.exports = ProjectPersistenceManager = {
EXPIRY_TIMEOUT: Settings.project_cache_length_ms || oneDay * 2.5,
- isAnyDiskLow() {
- return ANY_DISK_LOW
- },
- isAnyDiskCriticalLow() {
- return ANY_DISK_CRITICAL_LOW
- },
-
promises: {
refreshExpiryTimeout,
},
@@ -146,13 +103,6 @@ module.exports = ProjectPersistenceManager = {
}
)
})
-
- // Collect disk stats frequently to have them ready the next time /metrics is scraped (60s +- jitter) or every 5th scrape of the load agent (3s +- jitter).
- setInterval(() => {
- collectDiskStats().catch(err => {
- logger.err({ err }, 'low level error collecting disk stats')
- })
- }, 15_000)
},
markProjectAsJustAccessed(projectId, callback) {
diff --git a/services/clsi/app/js/RequestParser.js b/services/clsi/app/js/RequestParser.js
index 4e9d722921..61d3b9d229 100644
--- a/services/clsi/app/js/RequestParser.js
+++ b/services/clsi/app/js/RequestParser.js
@@ -1,9 +1,7 @@
const settings = require('@overleaf/settings')
-const OutputCacheManager = require('./OutputCacheManager')
const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex']
const MAX_TIMEOUT = 600
-const EDITOR_ID_REGEX = /^[a-f0-9-]{36}$/ // UUID
function parse(body, callback) {
const response = {}
@@ -29,24 +27,12 @@ function parse(body, callback) {
default: '',
type: 'string',
}),
- // Will be populated later. Must always be populated for prom library.
- compile: 'initial',
}
response.compiler = _parseAttribute('compiler', compile.options.compiler, {
validValues: VALID_COMPILERS,
default: 'pdflatex',
type: 'string',
})
- response.compileFromClsiCache = _parseAttribute(
- 'compileFromClsiCache',
- compile.options.compileFromClsiCache,
- { default: false, type: 'boolean' }
- )
- response.populateClsiCache = _parseAttribute(
- 'populateClsiCache',
- compile.options.populateClsiCache,
- { default: false, type: 'boolean' }
- )
response.enablePdfCaching = _parseAttribute(
'enablePdfCaching',
compile.options.enablePdfCaching,
@@ -149,15 +135,6 @@ function parse(body, callback) {
}
)
response.rootResourcePath = _checkPath(rootResourcePath)
-
- response.editorId = _parseAttribute('editorId', compile.options.editorId, {
- type: 'string',
- regex: EDITOR_ID_REGEX,
- })
- response.buildId = _parseAttribute('buildId', compile.options.buildId, {
- type: 'string',
- regex: OutputCacheManager.BUILD_REGEX,
- })
} catch (error1) {
const error = error1
return callback(error)
@@ -192,15 +169,11 @@ function _parseResource(resource) {
if (resource.url != null && typeof resource.url !== 'string') {
throw new Error('url attribute should be a string')
}
- if (resource.fallbackURL && typeof resource.fallbackURL !== 'string') {
- throw new Error('fallbackURL attribute should be a string')
- }
return {
path: resource.path,
modified,
url: resource.url,
- fallbackURL: resource.fallbackURL,
content: resource.content,
}
}
@@ -222,13 +195,6 @@ function _parseAttribute(name, attribute, options) {
throw new Error(`${name} attribute should be a ${options.type}`)
}
}
- if (options.type === 'string' && options.regex instanceof RegExp) {
- if (!options.regex.test(attribute)) {
- throw new Error(
- `${name} attribute does not match regex ${options.regex}`
- )
- }
- }
} else {
if (options.default != null) {
return options.default
diff --git a/services/clsi/app/js/ResourceStateManager.js b/services/clsi/app/js/ResourceStateManager.js
index a5f747e1cd..dbfb3c9fc4 100644
--- a/services/clsi/app/js/ResourceStateManager.js
+++ b/services/clsi/app/js/ResourceStateManager.js
@@ -1,5 +1,5 @@
-const Path = require('node:path')
-const fs = require('node:fs')
+const Path = require('path')
+const fs = require('fs')
const logger = require('@overleaf/logger')
const Errors = require('./Errors')
const SafeReader = require('./SafeReader')
diff --git a/services/clsi/app/js/ResourceWriter.js b/services/clsi/app/js/ResourceWriter.js
index bf88538746..11fb4500f4 100644
--- a/services/clsi/app/js/ResourceWriter.js
+++ b/services/clsi/app/js/ResourceWriter.js
@@ -13,10 +13,10 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ResourceWriter
-const { promisify } = require('node:util')
+const { promisify } = require('util')
const UrlCache = require('./UrlCache')
-const Path = require('node:path')
-const fs = require('node:fs')
+const Path = require('path')
+const fs = require('fs')
const async = require('async')
const OutputFileFinder = require('./OutputFileFinder')
const ResourceStateManager = require('./ResourceStateManager')
@@ -200,22 +200,73 @@ module.exports = ResourceWriter = {
return OutputFileFinder.findOutputFiles(
resources,
basePath,
- (error, outputFiles, allFiles) => {
+ function (error, outputFiles, allFiles) {
if (error != null) {
return callback(error)
}
const jobs = []
- for (const { path } of outputFiles || []) {
- const shouldDelete = ResourceWriter.isExtraneousFile(path)
- if (shouldDelete) {
- jobs.push(callback =>
- ResourceWriter._deleteFileIfNotDirectory(
- Path.join(basePath, path),
- callback
+ for (const file of Array.from(outputFiles || [])) {
+ ;(function (file) {
+ const { path } = file
+ let shouldDelete = true
+ if (
+ path.match(/^output\./) ||
+ path.match(/\.aux$/) ||
+ path.match(/^cache\//)
+ ) {
+ // knitr cache
+ shouldDelete = false
+ }
+ if (path.match(/^output-.*/)) {
+ // Tikz cached figures (default case)
+ shouldDelete = false
+ }
+ if (path.match(/\.(pdf|dpth|md5)$/)) {
+ // Tikz cached figures (by extension)
+ shouldDelete = false
+ }
+ if (
+ path.match(/\.(pygtex|pygstyle)$/) ||
+ path.match(/(^|\/)_minted-[^\/]+\//)
+ ) {
+ // minted files/directory
+ shouldDelete = false
+ }
+ if (
+ path.match(/\.md\.tex$/) ||
+ path.match(/(^|\/)_markdown_[^\/]+\//)
+ ) {
+ // markdown files/directory
+ shouldDelete = false
+ }
+ if (path.match(/-eps-converted-to\.pdf$/)) {
+ // Epstopdf generated files
+ shouldDelete = false
+ }
+ if (
+ path === 'output.pdf' ||
+ path === 'output.dvi' ||
+ path === 'output.log' ||
+ path === 'output.xdv' ||
+ path === 'output.stdout' ||
+ path === 'output.stderr'
+ ) {
+ shouldDelete = true
+ }
+ if (path === 'output.tex') {
+ // created by TikzManager if present in output files
+ shouldDelete = true
+ }
+ if (shouldDelete) {
+ return jobs.push(callback =>
+ ResourceWriter._deleteFileIfNotDirectory(
+ Path.join(basePath, path),
+ callback
+ )
)
- )
- }
+ }
+ })(file)
}
return async.series(jobs, function (error) {
@@ -228,59 +279,6 @@ module.exports = ResourceWriter = {
)
},
- isExtraneousFile(path) {
- let shouldDelete = true
- if (
- path.match(/^output\./) ||
- path.match(/\.aux$/) ||
- path.match(/^cache\//)
- ) {
- // knitr cache
- shouldDelete = false
- }
- if (path.match(/^output-.*/)) {
- // Tikz cached figures (default case)
- shouldDelete = false
- }
- if (path.match(/\.(pdf|dpth|md5)$/)) {
- // Tikz cached figures (by extension)
- shouldDelete = false
- }
- if (
- path.match(/\.(pygtex|pygstyle)$/) ||
- path.match(/(^|\/)_minted-[^\/]+\//)
- ) {
- // minted files/directory
- shouldDelete = false
- }
- if (path.match(/\.md\.tex$/) || path.match(/(^|\/)_markdown_[^\/]+\//)) {
- // markdown files/directory
- shouldDelete = false
- }
- if (path.match(/-eps-converted-to\.pdf$/)) {
- // Epstopdf generated files
- shouldDelete = false
- }
- if (
- path === 'output.tar.gz' ||
- path === 'output.synctex.gz' ||
- path === 'output.pdfxref' ||
- path === 'output.pdf' ||
- path === 'output.dvi' ||
- path === 'output.log' ||
- path === 'output.xdv' ||
- path === 'output.stdout' ||
- path === 'output.stderr'
- ) {
- shouldDelete = true
- }
- if (path === 'output.tex') {
- // created by TikzManager if present in output files
- shouldDelete = true
- }
- return shouldDelete
- },
-
_deleteFileIfNotDirectory(path, callback) {
if (callback == null) {
callback = function () {}
@@ -335,7 +333,6 @@ module.exports = ResourceWriter = {
return UrlCache.downloadUrlToFile(
projectId,
resource.url,
- resource.fallbackURL,
path,
resource.modified,
function (err) {
diff --git a/services/clsi/app/js/SafeReader.js b/services/clsi/app/js/SafeReader.js
index 8b1b5abb54..756747af47 100644
--- a/services/clsi/app/js/SafeReader.js
+++ b/services/clsi/app/js/SafeReader.js
@@ -12,7 +12,7 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let SafeReader
-const fs = require('node:fs')
+const fs = require('fs')
const logger = require('@overleaf/logger')
module.exports = SafeReader = {
diff --git a/services/clsi/app/js/StaticServerForbidSymlinks.js b/services/clsi/app/js/StaticServerForbidSymlinks.js
index a5ec774396..219408eb11 100644
--- a/services/clsi/app/js/StaticServerForbidSymlinks.js
+++ b/services/clsi/app/js/StaticServerForbidSymlinks.js
@@ -13,8 +13,8 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let ForbidSymlinks
-const Path = require('node:path')
-const fs = require('node:fs')
+const Path = require('path')
+const fs = require('fs')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
diff --git a/services/clsi/app/js/SynctexOutputParser.js b/services/clsi/app/js/SynctexOutputParser.js
index 5b2d237825..e54add65ae 100644
--- a/services/clsi/app/js/SynctexOutputParser.js
+++ b/services/clsi/app/js/SynctexOutputParser.js
@@ -1,4 +1,4 @@
-const Path = require('node:path')
+const Path = require('path')
/**
* Parse output from the `synctex view` command
diff --git a/services/clsi/app/js/TikzManager.js b/services/clsi/app/js/TikzManager.js
index ca9db6b005..7d5f6c1b81 100644
--- a/services/clsi/app/js/TikzManager.js
+++ b/services/clsi/app/js/TikzManager.js
@@ -11,9 +11,9 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
let TikzManager
-const fs = require('node:fs')
-const Path = require('node:path')
-const { promisify } = require('node:util')
+const fs = require('fs')
+const Path = require('path')
+const { promisify } = require('util')
const ResourceWriter = require('./ResourceWriter')
const SafeReader = require('./SafeReader')
const logger = require('@overleaf/logger')
diff --git a/services/clsi/app/js/UrlCache.js b/services/clsi/app/js/UrlCache.js
index 36703e7091..ae8eae6b2a 100644
--- a/services/clsi/app/js/UrlCache.js
+++ b/services/clsi/app/js/UrlCache.js
@@ -12,9 +12,9 @@
*/
const UrlFetcher = require('./UrlFetcher')
const Settings = require('@overleaf/settings')
-const fs = require('node:fs')
-const Path = require('node:path')
-const { callbackify } = require('node:util')
+const fs = require('fs')
+const Path = require('path')
+const { callbackify } = require('util')
const Metrics = require('./Metrics')
const PENDING_DOWNLOADS = new Map()
@@ -47,29 +47,14 @@ async function createProjectDir(projectId) {
await fs.promises.mkdir(getProjectDir(projectId), { recursive: true })
}
-async function downloadUrlToFile(
- projectId,
- url,
- fallbackURL,
- destPath,
- lastModified
-) {
+async function downloadUrlToFile(projectId, url, destPath, lastModified) {
const cachePath = getCachePath(projectId, url, lastModified)
try {
const timer = new Metrics.Timer('url_cache', {
status: 'cache-hit',
path: 'copy',
})
- try {
- await fs.promises.copyFile(cachePath, destPath)
- } catch (err) {
- if (err.code === 'ENOENT' && fallbackURL) {
- const fallbackPath = getCachePath(projectId, fallbackURL, lastModified)
- await fs.promises.copyFile(fallbackPath, destPath)
- } else {
- throw err
- }
- }
+ await fs.promises.copyFile(cachePath, destPath)
// the metric is only updated if the file is present in the cache
timer.done()
return
@@ -85,7 +70,7 @@ async function downloadUrlToFile(
path: 'download',
})
try {
- await download(url, fallbackURL, cachePath)
+ await download(url, cachePath)
} finally {
timer.done()
}
@@ -101,17 +86,13 @@ async function downloadUrlToFile(
}
}
-async function download(url, fallbackURL, cachePath) {
+async function download(url, cachePath) {
let pending = PENDING_DOWNLOADS.get(cachePath)
if (pending) {
return pending
}
- pending = UrlFetcher.promises.pipeUrlToFileWithRetry(
- url,
- fallbackURL,
- cachePath
- )
+ pending = UrlFetcher.promises.pipeUrlToFileWithRetry(url, cachePath)
PENDING_DOWNLOADS.set(cachePath, pending)
try {
await pending
diff --git a/services/clsi/app/js/UrlFetcher.js b/services/clsi/app/js/UrlFetcher.js
index 2c44f3a6dd..8d604d267b 100644
--- a/services/clsi/app/js/UrlFetcher.js
+++ b/services/clsi/app/js/UrlFetcher.js
@@ -1,21 +1,20 @@
-const fs = require('node:fs')
+const fs = require('fs')
const logger = require('@overleaf/logger')
const Settings = require('@overleaf/settings')
const {
CustomHttpAgent,
CustomHttpsAgent,
fetchStream,
- RequestFailedError,
} = require('@overleaf/fetch-utils')
-const { URL } = require('node:url')
-const { pipeline } = require('node:stream/promises')
+const { URL } = require('url')
+const { pipeline } = require('stream/promises')
const Metrics = require('./Metrics')
const MAX_CONNECT_TIME = 1000
const httpAgent = new CustomHttpAgent({ connectTimeout: MAX_CONNECT_TIME })
const httpsAgent = new CustomHttpsAgent({ connectTimeout: MAX_CONNECT_TIME })
-async function pipeUrlToFileWithRetry(url, fallbackURL, filePath) {
+async function pipeUrlToFileWithRetry(url, filePath) {
let remainingAttempts = 3
let lastErr
while (remainingAttempts-- > 0) {
@@ -23,7 +22,7 @@ async function pipeUrlToFileWithRetry(url, fallbackURL, filePath) {
path: lastErr ? ' retry' : 'fetch',
})
try {
- await pipeUrlToFile(url, fallbackURL, filePath)
+ await pipeUrlToFile(url, filePath)
timer.done({ status: 'success' })
return
} catch (err) {
@@ -38,7 +37,7 @@ async function pipeUrlToFileWithRetry(url, fallbackURL, filePath) {
throw lastErr
}
-async function pipeUrlToFile(url, fallbackURL, filePath) {
+async function pipeUrlToFile(url, filePath) {
const u = new URL(url)
if (
Settings.filestoreDomainOveride &&
@@ -46,55 +45,21 @@ async function pipeUrlToFile(url, fallbackURL, filePath) {
) {
url = `${Settings.filestoreDomainOveride}${u.pathname}${u.search}`
}
- if (fallbackURL) {
- const u2 = new URL(fallbackURL)
- if (
- Settings.filestoreDomainOveride &&
- u2.host !== Settings.apis.clsiPerf.host
- ) {
- fallbackURL = `${Settings.filestoreDomainOveride}${u2.pathname}${u2.search}`
- }
- }
- let stream
- try {
- stream = await fetchStream(url, {
- signal: AbortSignal.timeout(60 * 1000),
- // provide a function to get the agent for each request
- // as there may be multiple requests with different protocols
- // due to redirects.
- agent: _url => (_url.protocol === 'https:' ? httpsAgent : httpAgent),
- })
- } catch (err) {
- if (
- fallbackURL &&
- err instanceof RequestFailedError &&
- err.response.status === 404
- ) {
- stream = await fetchStream(fallbackURL, {
- signal: AbortSignal.timeout(60 * 1000),
- // provide a function to get the agent for each request
- // as there may be multiple requests with different protocols
- // due to redirects.
- agent: _url => (_url.protocol === 'https:' ? httpsAgent : httpAgent),
- })
- url = fallbackURL
- } else {
- throw err
- }
- }
-
- const source = inferSource(url)
- Metrics.inc('url_source', 1, { path: source })
+ const stream = await fetchStream(url, {
+ signal: AbortSignal.timeout(60 * 1000),
+ // provide a function to get the agent for each request
+ // as there may be multiple requests with different protocols
+ // due to redirects.
+ agent: _url => (_url.protocol === 'https:' ? httpsAgent : httpAgent),
+ })
const atomicWrite = filePath + '~'
try {
const output = fs.createWriteStream(atomicWrite)
await pipeline(stream, output)
await fs.promises.rename(atomicWrite, filePath)
- Metrics.count('UrlFetcher.downloaded_bytes', output.bytesWritten, {
- path: source,
- })
+ Metrics.count('UrlFetcher.downloaded_bytes', output.bytesWritten)
} catch (err) {
try {
await fs.promises.unlink(atomicWrite)
@@ -103,20 +68,6 @@ async function pipeUrlToFile(url, fallbackURL, filePath) {
}
}
-const BUCKET_REGEX = /\/bucket\/([^/]+)\/key\//
-
-function inferSource(url) {
- if (url.includes(Settings.apis.clsiPerf.host)) {
- return 'clsi-perf'
- } else if (url.includes('/project/') && url.includes('/file/')) {
- return 'user-files'
- } else if (url.includes('/key/')) {
- const match = url.match(BUCKET_REGEX)
- if (match) return match[1]
- }
- return 'unknown'
-}
-
module.exports.promises = {
pipeUrlToFileWithRetry,
}
diff --git a/services/clsi/app/js/XrefParser.js b/services/clsi/app/js/XrefParser.js
index 5f2d154679..76636acfe9 100644
--- a/services/clsi/app/js/XrefParser.js
+++ b/services/clsi/app/js/XrefParser.js
@@ -1,5 +1,5 @@
const { NoXrefTableError } = require('./Errors')
-const fs = require('node:fs')
+const fs = require('fs')
const { O_RDONLY, O_NOFOLLOW } = fs.constants
const MAX_XREF_FILE_SIZE = 1024 * 1024
diff --git a/services/clsi/buildscript.txt b/services/clsi/buildscript.txt
index 09c21888df..52113b47e0 100644
--- a/services/clsi/buildscript.txt
+++ b/services/clsi/buildscript.txt
@@ -1,11 +1,11 @@
clsi
--data-dirs=cache,compiles,output
--dependencies=
---docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker
---env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2017.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=us-east1-docker.pkg.dev/overleaf-ops/ol-docker,TEXLIVE_IMAGE_USER="tex",SANDBOXED_COMPILES="true",SANDBOXED_COMPILES_HOST_DIR_COMPILES=$PWD/compiles,SANDBOXED_COMPILES_HOST_DIR_OUTPUT=$PWD/output
+--docker-repos=gcr.io/overleaf-ops,us-east1-docker.pkg.dev/overleaf-ops/ol-docker
+--env-add=ENABLE_PDF_CACHING="true",PDF_CACHING_ENABLE_WORKER_POOL="true",ALLOWED_IMAGES=quay.io/sharelatex/texlive-full:2017.1,TEXLIVE_IMAGE=quay.io/sharelatex/texlive-full:2017.1,TEX_LIVE_IMAGE_NAME_OVERRIDE=gcr.io/overleaf-ops,TEXLIVE_IMAGE_USER="tex",DOCKER_RUNNER="true",COMPILES_HOST_DIR=$PWD/compiles
--env-pass-through=
--esmock-loader=False
---node-version=22.17.0
+--node-version=18.20.2
--public-repo=True
---script-version=4.7.0
+--script-version=4.5.0
--use-large-ci-runner=True
diff --git a/services/clsi/config/settings.defaults.js b/services/clsi/config/settings.defaults.js
index bd5614eb98..934ef6d8c8 100644
--- a/services/clsi/config/settings.defaults.js
+++ b/services/clsi/config/settings.defaults.js
@@ -1,8 +1,7 @@
-const Path = require('node:path')
-const os = require('node:os')
+const Path = require('path')
+const os = require('os')
-const isPreEmptible = process.env.PREEMPTIBLE === 'TRUE'
-const CLSI_SERVER_ID = os.hostname().replace('-ctr', '')
+const isPreEmptible = os.hostname().includes('pre-emp')
module.exports = {
compileSizeLimit: process.env.COMPILE_SIZE_LIMIT || '7mb',
@@ -34,10 +33,6 @@ module.exports = {
report_load: process.env.LOAD_BALANCER_AGENT_REPORT_LOAD !== 'false',
load_port: 3048,
local_port: 3049,
- allow_maintenance:
- (
- process.env.LOAD_BALANCER_AGENT_ALLOW_MAINTENANCE ?? ''
- ).toLowerCase() !== 'false',
},
},
apis: {
@@ -46,19 +41,12 @@ module.exports = {
url: `http://${process.env.CLSI_HOST || '127.0.0.1'}:3013`,
// External url prefix for output files, e.g. for requests via load-balancers.
outputUrlPrefix: `${process.env.ZONE ? `/zone/${process.env.ZONE}` : ''}`,
- clsiServerId: process.env.CLSI_SERVER_ID || CLSI_SERVER_ID,
-
- downloadHost: process.env.DOWNLOAD_HOST || 'http://localhost:3013',
},
clsiPerf: {
host: `${process.env.CLSI_PERF_HOST || '127.0.0.1'}:${
process.env.CLSI_PERF_PORT || '3043'
}`,
},
- clsiCache: {
- enabled: !!process.env.CLSI_CACHE_SHARDS,
- shards: JSON.parse(process.env.CLSI_CACHE_SHARDS || '[]'),
- },
},
smokeTest: process.env.SMOKE_TEST || false,
@@ -68,6 +56,10 @@ module.exports = {
texliveImageNameOveride: process.env.TEX_LIVE_IMAGE_NAME_OVERRIDE,
texliveOpenoutAny: process.env.TEXLIVE_OPENOUT_ANY,
texliveMaxPrintLine: process.env.TEXLIVE_MAX_PRINT_LINE,
+ sentry: {
+ dsn: process.env.SENTRY_DSN,
+ },
+
enablePdfCaching: process.env.ENABLE_PDF_CACHING === 'true',
enablePdfCachingDark: process.env.ENABLE_PDF_CACHING_DARK === 'true',
pdfCachingMinChunkSize:
@@ -93,21 +85,20 @@ if (process.env.ALLOWED_COMPILE_GROUPS) {
}
}
-if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') {
+if (process.env.DOCKER_RUNNER) {
+ let seccompProfilePath
module.exports.clsi = {
- dockerRunner: true,
+ dockerRunner: process.env.DOCKER_RUNNER === 'true',
docker: {
runtime: process.env.DOCKER_RUNTIME,
image:
- process.env.TEXLIVE_IMAGE ||
- process.env.TEX_LIVE_DOCKER_IMAGE ||
- 'quay.io/sharelatex/texlive-full:2017.1',
+ process.env.TEXLIVE_IMAGE || 'quay.io/sharelatex/texlive-full:2017.1',
env: {
HOME: '/tmp',
CLSI: 1,
},
socketPath: '/var/run/docker.sock',
- user: process.env.TEXLIVE_IMAGE_USER || 'www-data',
+ user: process.env.TEXLIVE_IMAGE_USER || 'tex',
},
optimiseInDocker: true,
expireProjectAfterIdleMs: 24 * 60 * 60 * 1000,
@@ -127,7 +118,6 @@ if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') {
const defaultCompileGroupConfig = {
wordcount: { 'HostConfig.AutoRemove': true },
synctex: { 'HostConfig.AutoRemove': true },
- 'synctex-output': { 'HostConfig.AutoRemove': true },
}
module.exports.clsi.docker.compileGroupConfig = Object.assign(
defaultCompileGroupConfig,
@@ -138,14 +128,11 @@ if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') {
process.exit(1)
}
- let seccompProfilePath
try {
seccompProfilePath = Path.resolve(__dirname, '../seccomp/clsi-profile.json')
- module.exports.clsi.docker.seccomp_profile =
- process.env.SECCOMP_PROFILE ||
- JSON.stringify(
- JSON.parse(require('node:fs').readFileSync(seccompProfilePath))
- )
+ module.exports.clsi.docker.seccomp_profile = JSON.stringify(
+ JSON.parse(require('fs').readFileSync(seccompProfilePath))
+ )
} catch (error) {
console.error(
error,
@@ -175,23 +162,5 @@ if ((process.env.DOCKER_RUNNER || process.env.SANDBOXED_COMPILES) === 'true') {
module.exports.path.synctexBaseDir = () => '/compile'
- module.exports.path.sandboxedCompilesHostDirCompiles =
- process.env.SANDBOXED_COMPILES_HOST_DIR_COMPILES ||
- process.env.SANDBOXED_COMPILES_HOST_DIR ||
- process.env.COMPILES_HOST_DIR
- if (!module.exports.path.sandboxedCompilesHostDirCompiles) {
- throw new Error(
- 'SANDBOXED_COMPILES enabled, but SANDBOXED_COMPILES_HOST_DIR_COMPILES not set'
- )
- }
-
- module.exports.path.sandboxedCompilesHostDirOutput =
- process.env.SANDBOXED_COMPILES_HOST_DIR_OUTPUT ||
- process.env.OUTPUT_HOST_DIR
- if (!module.exports.path.sandboxedCompilesHostDirOutput) {
- // TODO(das7pad): Enforce in a future major version of Server Pro.
- // throw new Error(
- // 'SANDBOXED_COMPILES enabled, but SANDBOXED_COMPILES_HOST_DIR_OUTPUT not set'
- // )
- }
+ module.exports.path.sandboxedCompilesHostDir = process.env.COMPILES_HOST_DIR
}
diff --git a/services/clsi/docker-compose.ci.yml b/services/clsi/docker-compose.ci.yml
index 77a45615b7..00f54c6e72 100644
--- a/services/clsi/docker-compose.ci.yml
+++ b/services/clsi/docker-compose.ci.yml
@@ -27,11 +27,10 @@ services:
PDF_CACHING_ENABLE_WORKER_POOL: "true"
ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
- TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
+ TEX_LIVE_IMAGE_NAME_OVERRIDE: gcr.io/overleaf-ops
TEXLIVE_IMAGE_USER: "tex"
- SANDBOXED_COMPILES: "true"
- SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
- SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
+ DOCKER_RUNNER: "true"
+ COMPILES_HOST_DIR: $PWD/compiles
volumes:
- ./compiles:/overleaf/services/clsi/compiles
- /var/run/docker.sock:/var/run/docker.sock
diff --git a/services/clsi/docker-compose.yml b/services/clsi/docker-compose.yml
index b8112a8e17..c72fb8b2c4 100644
--- a/services/clsi/docker-compose.yml
+++ b/services/clsi/docker-compose.yml
@@ -17,7 +17,6 @@ services:
working_dir: /overleaf/services/clsi
environment:
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
@@ -38,17 +37,16 @@ services:
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
+ LOG_LEVEL: ERROR
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
ENABLE_PDF_CACHING: "true"
PDF_CACHING_ENABLE_WORKER_POOL: "true"
ALLOWED_IMAGES: quay.io/sharelatex/texlive-full:2017.1
TEXLIVE_IMAGE: quay.io/sharelatex/texlive-full:2017.1
- TEX_LIVE_IMAGE_NAME_OVERRIDE: us-east1-docker.pkg.dev/overleaf-ops/ol-docker
+ TEX_LIVE_IMAGE_NAME_OVERRIDE: gcr.io/overleaf-ops
TEXLIVE_IMAGE_USER: "tex"
- SANDBOXED_COMPILES: "true"
- SANDBOXED_COMPILES_HOST_DIR_COMPILES: $PWD/compiles
- SANDBOXED_COMPILES_HOST_DIR_OUTPUT: $PWD/output
+ DOCKER_RUNNER: "true"
+ COMPILES_HOST_DIR: $PWD/compiles
command: npm run --silent test:acceptance
diff --git a/services/clsi/entrypoint.sh b/services/clsi/entrypoint.sh
index b45899ab17..9446ab9e2d 100755
--- a/services/clsi/entrypoint.sh
+++ b/services/clsi/entrypoint.sh
@@ -2,12 +2,13 @@
# add the node user to the docker group on the host
DOCKER_GROUP=$(stat -c '%g' /var/run/docker.sock)
-groupadd --non-unique --gid "${DOCKER_GROUP}" dockeronhost
+groupadd --non-unique --gid ${DOCKER_GROUP} dockeronhost
usermod -aG dockeronhost node
# compatibility: initial volume setup
mkdir -p /overleaf/services/clsi/cache && chown node:node /overleaf/services/clsi/cache
mkdir -p /overleaf/services/clsi/compiles && chown node:node /overleaf/services/clsi/compiles
+mkdir -p /overleaf/services/clsi/db && chown node:node /overleaf/services/clsi/db
mkdir -p /overleaf/services/clsi/output && chown node:node /overleaf/services/clsi/output
exec runuser -u node -- "$@"
diff --git a/services/clsi/kube.yaml b/services/clsi/kube.yaml
new file mode 100644
index 0000000000..d3fb04291e
--- /dev/null
+++ b/services/clsi/kube.yaml
@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: clsi
+ namespace: default
+spec:
+ type: LoadBalancer
+ ports:
+ - port: 80
+ protocol: TCP
+ targetPort: 80
+ selector:
+ run: clsi
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: clsi
+ namespace: default
+spec:
+ replicas: 2
+ template:
+ metadata:
+ labels:
+ run: clsi
+ spec:
+ containers:
+ - name: clsi
+ image: gcr.io/henry-terraform-admin/clsi
+ imagePullPolicy: Always
+ readinessProbe:
+ httpGet:
+ path: status
+ port: 80
+ periodSeconds: 5
+ initialDelaySeconds: 0
+ failureThreshold: 3
+ successThreshold: 1
+
+
+
diff --git a/services/clsi/nginx.conf b/services/clsi/nginx.conf
index 604eb93fbf..2290aeb444 100644
--- a/services/clsi/nginx.conf
+++ b/services/clsi/nginx.conf
@@ -46,7 +46,7 @@ server {
}
# handle output files for specific users
- location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
+ location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
if ($request_method = 'OPTIONS') {
# handle OPTIONS method for CORS requests
add_header 'Allow' 'GET,HEAD';
@@ -64,7 +64,7 @@ server {
alias /output/$1-$2/generated-files/$3/$4.blg;
}
# handle output files for anonymous users
- location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
+ location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
if ($request_method = 'OPTIONS') {
# handle OPTIONS method for CORS requests
add_header 'Allow' 'GET,HEAD';
diff --git a/services/clsi/package.json b/services/clsi/package.json
index fe31c430bd..8df44c7150 100644
--- a/services/clsi/package.json
+++ b/services/clsi/package.json
@@ -11,8 +11,8 @@
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
- "format": "prettier --list-different $PWD/'**/*.*js'",
- "format:fix": "prettier --write $PWD/'**/*.*js'",
+ "format": "prettier --list-different $PWD/'**/*.js'",
+ "format:fix": "prettier --write $PWD/'**/*.js'",
"lint:fix": "eslint --fix .",
"types:check": "tsc --noEmit"
},
@@ -23,24 +23,24 @@
"@overleaf/o-error": "*",
"@overleaf/promise-utils": "*",
"@overleaf/settings": "*",
- "@overleaf/stream-utils": "*",
"archiver": "5.3.2",
"async": "^3.2.5",
"body-parser": "^1.20.3",
"bunyan": "^1.8.15",
- "dockerode": "^4.0.7",
- "express": "^4.21.2",
+ "diskusage": "^1.1.3",
+ "dockerode": "^3.1.0",
+ "express": "^4.21.0",
"lodash": "^4.17.21",
"p-limit": "^3.1.0",
"request": "^2.88.2",
"send": "^0.19.0",
- "tar-fs": "^3.0.9",
"workerpool": "^6.1.5"
},
"devDependencies": {
+ "@types/workerpool": "^6.1.0",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
- "mocha": "^11.1.0",
+ "mocha": "^10.2.0",
"mock-fs": "^5.1.2",
"node-fetch": "^2.7.0",
"sandboxed-module": "^2.0.4",
diff --git a/services/clsi/scripts/demo-pdfjs-Xref.js b/services/clsi/scripts/demo-pdfjs-Xref.js
index 1f55c571a7..149e55ee9e 100644
--- a/services/clsi/scripts/demo-pdfjs-Xref.js
+++ b/services/clsi/scripts/demo-pdfjs-Xref.js
@@ -1,4 +1,4 @@
-const fs = require('node:fs')
+const fs = require('fs')
const { parseXrefTable } = require('../app/lib/pdfjs/parseXrefTable')
const pdfPath = process.argv[2]
diff --git a/services/clsi/seccomp/clsi-profile.json b/services/clsi/seccomp/clsi-profile.json
index ad95130f76..084354b15c 100644
--- a/services/clsi/seccomp/clsi-profile.json
+++ b/services/clsi/seccomp/clsi-profile.json
@@ -829,19 +829,13 @@
"args": []
},
{
- "name": "gettimeofday",
- "action": "SCMP_ACT_ALLOW",
- "args": []
- },
- {
- "name": "epoll_pwait",
- "action": "SCMP_ACT_ALLOW",
- "args": []
- },
- {
- "name": "poll",
- "action": "SCMP_ACT_ALLOW",
- "args": []
+ "name": "gettimeofday",
+ "action": "SCMP_ACT_ALLOW",
+ "args": []
+ }, {
+ "name": "epoll_pwait",
+ "action": "SCMP_ACT_ALLOW",
+ "args": []
}
]
-}
+}
\ No newline at end of file
diff --git a/services/clsi/test/acceptance/js/AllowedImageNamesTests.js b/services/clsi/test/acceptance/js/AllowedImageNamesTests.js
index 9cd7a65930..897f5d9c85 100644
--- a/services/clsi/test/acceptance/js/AllowedImageNamesTests.js
+++ b/services/clsi/test/acceptance/js/AllowedImageNamesTests.js
@@ -109,7 +109,6 @@ Hello world
width: 343.71106,
},
],
- downloadedFromCache: false,
})
done()
}
@@ -147,7 +146,6 @@ Hello world
expect(error).to.not.exist
expect(result).to.deep.equal({
code: [{ file: 'main.tex', line: 3, column: -1 }],
- downloadedFromCache: false,
})
done()
}
diff --git a/services/clsi/test/acceptance/js/BrokenLatexFileTests.js b/services/clsi/test/acceptance/js/BrokenLatexFileTests.js
index 46d07da092..71e9956c0d 100644
--- a/services/clsi/test/acceptance/js/BrokenLatexFileTests.js
+++ b/services/clsi/test/acceptance/js/BrokenLatexFileTests.js
@@ -11,7 +11,6 @@
const Client = require('./helpers/Client')
const request = require('request')
const ClsiApp = require('./helpers/ClsiApp')
-const { expect } = require('chai')
describe('Broken LaTeX file', function () {
before(function (done) {
@@ -59,27 +58,9 @@ Hello world
)
})
- it('should return a failure status', function () {
+ return it('should return a failure status', function () {
return this.body.compile.status.should.equal('failure')
})
-
- it('should return isInitialCompile flag', function () {
- expect(this.body.compile.stats.isInitialCompile).to.equal(1)
- })
-
- it('should return output files', function () {
- // NOTE: No output.pdf file.
- this.body.compile.outputFiles
- .map(f => f.path)
- .should.deep.equal([
- 'output.aux',
- 'output.fdb_latexmk',
- 'output.fls',
- 'output.log',
- 'output.stderr',
- 'output.stdout',
- ])
- })
})
return describe('on second run', function () {
@@ -99,26 +80,8 @@ Hello world
})
})
- it('should return a failure status', function () {
+ return it('should return a failure status', function () {
return this.body.compile.status.should.equal('failure')
})
-
- it('should not return isInitialCompile flag', function () {
- expect(this.body.compile.stats.isInitialCompile).to.not.exist
- })
-
- it('should return output files', function () {
- // NOTE: No output.pdf file.
- this.body.compile.outputFiles
- .map(f => f.path)
- .should.deep.equal([
- 'output.aux',
- 'output.fdb_latexmk',
- 'output.fls',
- 'output.log',
- 'output.stderr',
- 'output.stdout',
- ])
- })
})
})
diff --git a/services/clsi/test/acceptance/js/ExampleDocumentTests.js b/services/clsi/test/acceptance/js/ExampleDocumentTests.js
index b463584501..404f8c4e90 100644
--- a/services/clsi/test/acceptance/js/ExampleDocumentTests.js
+++ b/services/clsi/test/acceptance/js/ExampleDocumentTests.js
@@ -14,19 +14,19 @@
*/
const Client = require('./helpers/Client')
const fetch = require('node-fetch')
-const { pipeline } = require('node:stream')
-const fs = require('node:fs')
-const ChildProcess = require('node:child_process')
+const { pipeline } = require('stream')
+const fs = require('fs')
+const ChildProcess = require('child_process')
const ClsiApp = require('./helpers/ClsiApp')
const logger = require('@overleaf/logger')
-const Path = require('node:path')
+const Path = require('path')
const fixturePath = path => {
if (path.slice(0, 3) === 'tmp') {
return '/tmp/clsi_acceptance_tests' + path.slice(3)
}
return Path.join(__dirname, '../fixtures/', path)
}
-const process = require('node:process')
+const process = require('process')
console.log(
process.pid,
process.ppid,
diff --git a/services/clsi/test/acceptance/js/StopCompile.js b/services/clsi/test/acceptance/js/StopCompile.js
deleted file mode 100644
index 103a70f37d..0000000000
--- a/services/clsi/test/acceptance/js/StopCompile.js
+++ /dev/null
@@ -1,47 +0,0 @@
-const Client = require('./helpers/Client')
-const ClsiApp = require('./helpers/ClsiApp')
-const { expect } = require('chai')
-
-describe('Stop compile', function () {
- before(function (done) {
- this.request = {
- options: {
- timeout: 100,
- }, // seconds
- resources: [
- {
- path: 'main.tex',
- content: `\
-\\documentclass{article}
-\\begin{document}
-\\def\\x{Hello!\\par\\x}
-\\x
-\\end{document}\
-`,
- },
- ],
- }
- this.project_id = Client.randomId()
- ClsiApp.ensureRunning(() => {
- // start the compile in the background
- Client.compile(this.project_id, this.request, (error, res, body) => {
- this.compileResult = { error, res, body }
- })
- // wait for 1 second before stopping the compile
- setTimeout(() => {
- Client.stopCompile(this.project_id, (error, res, body) => {
- this.stopResult = { error, res, body }
- setTimeout(done, 1000) // allow time for the compile request to terminate
- })
- }, 1000)
- })
- })
-
- it('should force a compile response with an error status', function () {
- expect(this.stopResult.error).to.be.null
- expect(this.stopResult.res.statusCode).to.equal(204)
- expect(this.compileResult.res.statusCode).to.equal(200)
- expect(this.compileResult.body.compile.status).to.equal('terminated')
- expect(this.compileResult.body.compile.error).to.equal('terminated')
- })
-})
diff --git a/services/clsi/test/acceptance/js/SynctexTests.js b/services/clsi/test/acceptance/js/SynctexTests.js
index 049f260259..899898b5b2 100644
--- a/services/clsi/test/acceptance/js/SynctexTests.js
+++ b/services/clsi/test/acceptance/js/SynctexTests.js
@@ -13,7 +13,7 @@ const Client = require('./helpers/Client')
const request = require('request')
const { expect } = require('chai')
const ClsiApp = require('./helpers/ClsiApp')
-const crypto = require('node:crypto')
+const crypto = require('crypto')
describe('Syncing', function () {
before(function (done) {
@@ -67,7 +67,6 @@ Hello world
width: 343.71106,
},
],
- downloadedFromCache: false,
})
return done()
}
@@ -88,7 +87,6 @@ Hello world
}
expect(codePositions).to.deep.equal({
code: [{ file: 'main.tex', line: 3, column: -1 }],
- downloadedFromCache: false,
})
return done()
}
diff --git a/services/clsi/test/acceptance/js/TimeoutTests.js b/services/clsi/test/acceptance/js/TimeoutTests.js
index e9175d223c..bca8ae71d2 100644
--- a/services/clsi/test/acceptance/js/TimeoutTests.js
+++ b/services/clsi/test/acceptance/js/TimeoutTests.js
@@ -11,7 +11,6 @@
const Client = require('./helpers/Client')
const request = require('request')
const ClsiApp = require('./helpers/ClsiApp')
-const { expect } = require('chai')
describe('Timed out compile', function () {
before(function (done) {
@@ -55,10 +54,6 @@ describe('Timed out compile', function () {
return this.body.compile.status.should.equal('timedout')
})
- it('should return isInitialCompile flag', function () {
- expect(this.body.compile.stats.isInitialCompile).to.equal(1)
- })
-
return it('should return the log output file name', function () {
const outputFilePaths = this.body.compile.outputFiles.map(x => x.path)
return outputFilePaths.should.include('output.log')
diff --git a/services/clsi/test/acceptance/js/UrlCachingTests.js b/services/clsi/test/acceptance/js/UrlCachingTests.js
index 9fc9608204..424f1ad1b7 100644
--- a/services/clsi/test/acceptance/js/UrlCachingTests.js
+++ b/services/clsi/test/acceptance/js/UrlCachingTests.js
@@ -10,12 +10,10 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const express = require('express')
-const Path = require('node:path')
+const Path = require('path')
const Client = require('./helpers/Client')
const sinon = require('sinon')
const ClsiApp = require('./helpers/ClsiApp')
-const request = require('request')
-const Settings = require('@overleaf/settings')
const Server = {
run() {
@@ -37,21 +35,6 @@ const Server = {
}
})
- app.get('/not-found', (req, res, next) => {
- this.getFile(req.url)
- res.status(404).end()
- })
-
- app.get('/project/:projectId/file/:fileId', (req, res, next) => {
- this.getFile(req.url)
- return res.send(`${req.params.projectId}:${req.params.fileId}`)
- })
-
- app.get('/bucket/:bucket/key/*', (req, res, next) => {
- this.getFile(req.url)
- return res.send(`${req.params.bucket}:${req.params[0]}`)
- })
-
app.get('/:random_id/*', (req, res, next) => {
this.getFile(req.url)
req.url = `/${req.params[0]}`
@@ -235,24 +218,9 @@ describe('Url Caching', function () {
return Server.getFile.restore()
})
- it('should not download the image again', function () {
+ return it('should not download the image again', function () {
return Server.getFile.called.should.equal(false)
})
-
- it('should gather metrics', function (done) {
- request.get(`${Settings.apis.clsi.url}/metrics`, (err, res, body) => {
- if (err) return done(err)
- body
- .split('\n')
- .some(line => {
- return (
- line.startsWith('url_source') && line.includes('path="unknown"')
- )
- })
- .should.equal(true)
- done()
- })
- })
})
describe('When an image is in the cache and the last modified date is advanced', function () {
@@ -423,7 +391,7 @@ describe('Url Caching', function () {
})
})
- describe('After clearing the cache', function () {
+ return describe('After clearing the cache', function () {
before(function (done) {
this.project_id = Client.randomId()
this.file = `${Server.randomId()}/lion.png`
@@ -478,140 +446,4 @@ describe('Url Caching', function () {
return Server.getFile.called.should.equal(true)
})
})
-
- describe('fallbackURL', function () {
- describe('when the primary resource is available', function () {
- before(function (done) {
- this.project_id = Client.randomId()
- this.file = `/project/${Server.randomId()}/file/${Server.randomId()}`
- this.fallback = `/bucket/project-blobs/key/ab/cd/${Server.randomId()}`
- this.request = {
- resources: [
- {
- path: 'main.tex',
- content: `\
-\\documentclass{article}
-\\usepackage{graphicx}
-\\begin{document}
-\\includegraphics{lion.png}
-\\end{document}\
-`,
- },
- {
- path: 'lion.png',
- url: `http://filestore${this.file}`,
- fallbackURL: `http://filestore${this.fallback}`,
- },
- ],
- }
-
- sinon.spy(Server, 'getFile')
- return ClsiApp.ensureRunning(() => {
- return Client.compile(
- this.project_id,
- this.request,
- (error, res, body) => {
- this.error = error
- this.res = res
- this.body = body
- return done()
- }
- )
- })
- })
-
- after(function () {
- return Server.getFile.restore()
- })
-
- it('should download from the primary', function () {
- Server.getFile.calledWith(this.file).should.equal(true)
- })
- it('should not download from the fallback', function () {
- Server.getFile.calledWith(this.fallback).should.equal(false)
- })
-
- it('should gather metrics', function (done) {
- request.get(`${Settings.apis.clsi.url}/metrics`, (err, res, body) => {
- if (err) return done(err)
- body
- .split('\n')
- .some(line => {
- return (
- line.startsWith('url_source') &&
- line.includes('path="user-files"')
- )
- })
- .should.equal(true)
- done()
- })
- })
- })
-
- describe('when the primary resource is not available', function () {
- before(function (done) {
- this.project_id = Client.randomId()
- this.file = `/project/${Server.randomId()}/file/${Server.randomId()}`
- this.fallback = `/bucket/project-blobs/key/ab/cd/${Server.randomId()}`
- this.request = {
- resources: [
- {
- path: 'main.tex',
- content: `\
-\\documentclass{article}
-\\usepackage{graphicx}
-\\begin{document}
-\\includegraphics{lion.png}
-\\end{document}\
-`,
- },
- {
- path: 'lion.png',
- url: `http://filestore/not-found`,
- fallbackURL: `http://filestore${this.fallback}`,
- },
- ],
- }
-
- sinon.spy(Server, 'getFile')
- return ClsiApp.ensureRunning(() => {
- return Client.compile(
- this.project_id,
- this.request,
- (error, res, body) => {
- this.error = error
- this.res = res
- this.body = body
- return done()
- }
- )
- })
- })
-
- after(function () {
- return Server.getFile.restore()
- })
-
- it('should download from the fallback', function () {
- Server.getFile.calledWith(`/not-found`).should.equal(true)
- Server.getFile.calledWith(this.fallback).should.equal(true)
- })
-
- it('should gather metrics', function (done) {
- request.get(`${Settings.apis.clsi.url}/metrics`, (err, res, body) => {
- if (err) return done(err)
- body
- .split('\n')
- .some(line => {
- return (
- line.startsWith('url_source') &&
- line.includes('path="project-blobs"')
- )
- })
- .should.equal(true)
- done()
- })
- })
- })
- })
})
diff --git a/services/clsi/test/acceptance/js/WordcountTests.js b/services/clsi/test/acceptance/js/WordcountTests.js
index 626b5d7034..d3fa7d2b94 100644
--- a/services/clsi/test/acceptance/js/WordcountTests.js
+++ b/services/clsi/test/acceptance/js/WordcountTests.js
@@ -12,8 +12,8 @@
const Client = require('./helpers/Client')
const request = require('request')
const { expect } = require('chai')
-const path = require('node:path')
-const fs = require('node:fs')
+const path = require('path')
+const fs = require('fs')
const ClsiApp = require('./helpers/ClsiApp')
describe('Syncing', function () {
diff --git a/services/clsi/test/acceptance/js/helpers/Client.js b/services/clsi/test/acceptance/js/helpers/Client.js
index 49bf7390c6..389a26201e 100644
--- a/services/clsi/test/acceptance/js/helpers/Client.js
+++ b/services/clsi/test/acceptance/js/helpers/Client.js
@@ -13,7 +13,7 @@
let Client
const express = require('express')
const request = require('request')
-const fs = require('node:fs')
+const fs = require('fs')
const Settings = require('@overleaf/settings')
module.exports = Client = {
@@ -42,16 +42,6 @@ module.exports = Client = {
)
},
- stopCompile(projectId, callback) {
- if (callback == null) {
- callback = function () {}
- }
- return request.post(
- { url: `${this.host}/project/${projectId}/compile/stop` },
- callback
- )
- },
-
clearCache(projectId, callback) {
if (callback == null) {
callback = function () {}
diff --git a/services/clsi/test/acceptance/js/helpers/ClsiApp.js b/services/clsi/test/acceptance/js/helpers/ClsiApp.js
index 38308e9129..4736315df8 100644
--- a/services/clsi/test/acceptance/js/helpers/ClsiApp.js
+++ b/services/clsi/test/acceptance/js/helpers/ClsiApp.js
@@ -10,6 +10,8 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const app = require('../../../../app')
+require('@overleaf/logger').logger.level('info')
+const logger = require('@overleaf/logger')
const Settings = require('@overleaf/settings')
module.exports = {
@@ -35,6 +37,7 @@ module.exports = {
throw error
}
this.running = true
+ logger.info('clsi running in dev mode')
return (() => {
const result = []
diff --git a/services/clsi/test/bench/hashbench.js b/services/clsi/test/bench/hashbench.js
index 1e19af6a35..652ad39c7d 100644
--- a/services/clsi/test/bench/hashbench.js
+++ b/services/clsi/test/bench/hashbench.js
@@ -1,8 +1,8 @@
const ContentCacheManager = require('../../app/js/ContentCacheManager')
-const fs = require('node:fs')
-const crypto = require('node:crypto')
-const path = require('node:path')
-const os = require('node:os')
+const fs = require('fs')
+const crypto = require('crypto')
+const path = require('path')
+const os = require('os')
const async = require('async')
const _createHash = crypto.createHash
diff --git a/services/clsi/test/load/js/loadTest.js b/services/clsi/test/load/js/loadTest.js
index 506b51bf6f..196d8579ca 100644
--- a/services/clsi/test/load/js/loadTest.js
+++ b/services/clsi/test/load/js/loadTest.js
@@ -9,7 +9,7 @@
const request = require('request')
const Settings = require('@overleaf/settings')
const async = require('async')
-const fs = require('node:fs')
+const fs = require('fs')
const _ = require('lodash')
const concurentCompiles = 5
const totalCompiles = 50
diff --git a/services/clsi/test/setup.js b/services/clsi/test/setup.js
index b17507bf92..653a8a69b2 100644
--- a/services/clsi/test/setup.js
+++ b/services/clsi/test/setup.js
@@ -20,10 +20,5 @@ SandboxedModule.configure({
err() {},
},
},
- globals: { Buffer, console, process, URL, Math },
- sourceTransformers: {
- removeNodePrefix: function (source) {
- return source.replace(/require\(['"]node:/g, "require('")
- },
- },
+ globals: { Buffer, console, process, URL },
})
diff --git a/services/clsi/test/unit/js/CompileControllerTests.js b/services/clsi/test/unit/js/CompileControllerTests.js
index 2ac8d9c2d7..ac807d212e 100644
--- a/services/clsi/test/unit/js/CompileControllerTests.js
+++ b/services/clsi/test/unit/js/CompileControllerTests.js
@@ -1,11 +1,54 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('node:path').join(
+const { expect } = require('chai')
+const modulePath = require('path').join(
__dirname,
'../../../app/js/CompileController'
)
const Errors = require('../../../app/js/Errors')
+function tryImageNameValidation(method, imageNameField) {
+ describe('when allowedImages is set', function () {
+ beforeEach(function () {
+ this.Settings.clsi = { docker: {} }
+ this.Settings.clsi.docker.allowedImages = [
+ 'repo/image:tag1',
+ 'repo/image:tag2',
+ ]
+ this.res.send = sinon.stub()
+ this.res.status = sinon.stub().returns({ send: this.res.send })
+
+ this.CompileManager[method].reset()
+ })
+
+ describe('with an invalid image', function () {
+ beforeEach(function () {
+ this.req.query[imageNameField] = 'something/evil:1337'
+ this.CompileController[method](this.req, this.res, this.next)
+ })
+ it('should return a 400', function () {
+ expect(this.res.status.calledWith(400)).to.equal(true)
+ })
+ it('should not run the query', function () {
+ expect(this.CompileManager[method].called).to.equal(false)
+ })
+ })
+
+ describe('with a valid image', function () {
+ beforeEach(function () {
+ this.req.query[imageNameField] = 'repo/image:tag1'
+ this.CompileController[method](this.req, this.res, this.next)
+ })
+ it('should not return a 400', function () {
+ expect(this.res.status.calledWith(400)).to.equal(false)
+ })
+ it('should run the query', function () {
+ expect(this.CompileManager[method].called).to.equal(true)
+ })
+ })
+ })
+}
+
describe('CompileController', function () {
beforeEach(function () {
this.buildId = 'build-id-123'
@@ -18,11 +61,6 @@ describe('CompileController', function () {
clsi: {
url: 'http://clsi.example.com',
outputUrlPrefix: '/zone/b',
- downloadHost: 'http://localhost:3013',
- },
- clsiCache: {
- enabled: false,
- url: 'http://localhost:3044',
},
},
}),
@@ -30,11 +68,6 @@ describe('CompileController', function () {
Timer: sinon.stub().returns({ done: sinon.stub() }),
},
'./ProjectPersistenceManager': (this.ProjectPersistenceManager = {}),
- './CLSICacheHandler': {
- notifyCLSICacheAboutBuild: sinon.stub(),
- downloadLatestCompileCache: sinon.stub().resolves(),
- downloadOutputDotSynctexFromCompileCache: sinon.stub().resolves(),
- },
'./Errors': (this.Erros = Errors),
},
})
@@ -80,21 +113,16 @@ describe('CompileController', function () {
this.timings = { bar: 2 }
this.res.status = sinon.stub().returnsThis()
this.res.send = sinon.stub()
-
- this.CompileManager.doCompileWithLock = sinon
- .stub()
- .callsFake((_req, stats, timings, cb) => {
- Object.assign(stats, this.stats)
- Object.assign(timings, this.timings)
- cb(null, {
- outputFiles: this.output_files,
- buildId: this.buildId,
- })
- })
})
describe('successfully', function () {
beforeEach(function () {
+ this.CompileManager.doCompileWithLock = sinon.stub().yields(null, {
+ outputFiles: this.output_files,
+ stats: this.stats,
+ timings: this.timings,
+ buildId: this.buildId,
+ })
this.CompileController.compile(this.req, this.res)
})
@@ -129,7 +157,6 @@ describe('CompileController', function () {
url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
...file,
})),
- clsiCacheShard: undefined,
},
})
.should.equal(true)
@@ -139,6 +166,12 @@ describe('CompileController', function () {
describe('without a outputUrlPrefix', function () {
beforeEach(function () {
this.Settings.apis.clsi.outputUrlPrefix = ''
+ this.CompileManager.doCompileWithLock = sinon.stub().yields(null, {
+ outputFiles: this.output_files,
+ stats: this.stats,
+ timings: this.timings,
+ buildId: this.buildId,
+ })
this.CompileController.compile(this.req, this.res)
})
@@ -157,7 +190,6 @@ describe('CompileController', function () {
url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
...file,
})),
- clsiCacheShard: undefined,
},
})
.should.equal(true)
@@ -178,36 +210,33 @@ describe('CompileController', function () {
build: 1234,
},
]
- this.CompileManager.doCompileWithLock = sinon
- .stub()
- .callsFake((_req, stats, timings, cb) => {
- Object.assign(stats, this.stats)
- Object.assign(timings, this.timings)
- cb(null, {
- outputFiles: this.output_files,
- buildId: this.buildId,
- })
- })
+ this.CompileManager.doCompileWithLock = sinon.stub().yields(null, {
+ outputFiles: this.output_files,
+ stats: this.stats,
+ timings: this.timings,
+ buildId: this.buildId,
+ })
this.CompileController.compile(this.req, this.res)
})
it('should return the JSON response with status failure', function () {
this.res.status.calledWith(200).should.equal(true)
- this.res.send.should.have.been.calledWith({
- compile: {
- status: 'failure',
- error: null,
- stats: this.stats,
- timings: this.timings,
- outputUrlPrefix: '/zone/b',
- buildId: this.buildId,
- outputFiles: this.output_files.map(file => ({
- url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
- ...file,
- })),
- clsiCacheShard: undefined,
- },
- })
+ this.res.send
+ .calledWith({
+ compile: {
+ status: 'failure',
+ error: null,
+ stats: this.stats,
+ timings: this.timings,
+ outputUrlPrefix: '/zone/b',
+ buildId: this.buildId,
+ outputFiles: this.output_files.map(file => ({
+ url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
+ ...file,
+ })),
+ },
+ })
+ .should.equal(true)
})
})
@@ -226,36 +255,33 @@ describe('CompileController', function () {
build: 1234,
},
]
- this.CompileManager.doCompileWithLock = sinon
- .stub()
- .callsFake((_req, stats, timings, cb) => {
- Object.assign(stats, this.stats)
- Object.assign(timings, this.timings)
- cb(null, {
- outputFiles: this.output_files,
- buildId: this.buildId,
- })
- })
+ this.CompileManager.doCompileWithLock = sinon.stub().yields(null, {
+ outputFiles: this.output_files,
+ stats: this.stats,
+ timings: this.timings,
+ buildId: this.buildId,
+ })
this.CompileController.compile(this.req, this.res)
})
it('should return the JSON response with status failure', function () {
this.res.status.calledWith(200).should.equal(true)
- this.res.send.should.have.been.calledWith({
- compile: {
- status: 'failure',
- error: null,
- stats: this.stats,
- buildId: this.buildId,
- timings: this.timings,
- outputUrlPrefix: '/zone/b',
- outputFiles: this.output_files.map(file => ({
- url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
- ...file,
- })),
- clsiCacheShard: undefined,
- },
- })
+ this.res.send
+ .calledWith({
+ compile: {
+ status: 'failure',
+ error: null,
+ stats: this.stats,
+ buildId: this.buildId,
+ timings: this.timings,
+ outputUrlPrefix: '/zone/b',
+ outputFiles: this.output_files.map(file => ({
+ url: `${this.Settings.apis.clsi.url}/project/${this.project_id}/build/${file.build}/output/${file.path}`,
+ ...file,
+ })),
+ },
+ })
+ .should.equal(true)
})
})
@@ -265,11 +291,7 @@ describe('CompileController', function () {
error.buildId = this.buildId
this.CompileManager.doCompileWithLock = sinon
.stub()
- .callsFake((_req, stats, timings, cb) => {
- Object.assign(stats, this.stats)
- Object.assign(timings, this.timings)
- cb(error)
- })
+ .callsArgWith(1, error, null)
this.CompileController.compile(this.req, this.res)
})
@@ -283,9 +305,9 @@ describe('CompileController', function () {
outputUrlPrefix: '/zone/b',
outputFiles: [],
buildId: this.buildId,
- stats: this.stats,
- timings: this.timings,
- clsiCacheShard: undefined,
+ // JSON.stringify will omit these
+ stats: undefined,
+ timings: undefined,
},
})
.should.equal(true)
@@ -299,11 +321,7 @@ describe('CompileController', function () {
)
this.CompileManager.doCompileWithLock = sinon
.stub()
- .callsFake((_req, stats, timings, cb) => {
- Object.assign(stats, this.stats)
- Object.assign(timings, this.timings)
- cb(error)
- })
+ .callsArgWith(1, error, null)
this.CompileController.compile(this.req, this.res)
})
@@ -316,11 +334,9 @@ describe('CompileController', function () {
error: 'too many concurrent compile requests',
outputUrlPrefix: '/zone/b',
outputFiles: [],
- stats: this.stats,
- timings: this.timings,
- // JSON.stringify will omit these undefined values
buildId: undefined,
- clsiCacheShard: undefined,
+ stats: undefined,
+ timings: undefined,
},
})
.should.equal(true)
@@ -333,11 +349,7 @@ describe('CompileController', function () {
this.error.timedout = true
this.CompileManager.doCompileWithLock = sinon
.stub()
- .callsFake((_req, stats, timings, cb) => {
- Object.assign(stats, this.stats)
- Object.assign(timings, this.timings)
- cb(this.error)
- })
+ .callsArgWith(1, this.error, null)
this.CompileController.compile(this.req, this.res)
})
@@ -350,11 +362,10 @@ describe('CompileController', function () {
error: this.message,
outputUrlPrefix: '/zone/b',
outputFiles: [],
- stats: this.stats,
- timings: this.timings,
- // JSON.stringify will omit these undefined values
+ // JSON.stringify will omit these
buildId: undefined,
- clsiCacheShard: undefined,
+ stats: undefined,
+ timings: undefined,
},
})
.should.equal(true)
@@ -365,11 +376,7 @@ describe('CompileController', function () {
beforeEach(function () {
this.CompileManager.doCompileWithLock = sinon
.stub()
- .callsFake((_req, stats, timings, cb) => {
- Object.assign(stats, this.stats)
- Object.assign(timings, this.timings)
- cb(null, {})
- })
+ .callsArgWith(1, null, [])
this.CompileController.compile(this.req, this.res)
})
@@ -382,11 +389,10 @@ describe('CompileController', function () {
status: 'failure',
outputUrlPrefix: '/zone/b',
outputFiles: [],
- stats: this.stats,
- timings: this.timings,
- // JSON.stringify will omit these undefined values
+ // JSON.stringify will omit these
buildId: undefined,
- clsiCacheShard: undefined,
+ stats: undefined,
+ timings: undefined,
},
})
.should.equal(true)
@@ -410,7 +416,7 @@ describe('CompileController', function () {
this.CompileManager.syncFromCode = sinon
.stub()
- .yields(null, (this.pdfPositions = ['mock-positions']), true)
+ .yields(null, (this.pdfPositions = ['mock-positions']))
this.CompileController.syncFromCode(this.req, this.res, this.next)
})
@@ -430,10 +436,11 @@ describe('CompileController', function () {
this.res.json
.calledWith({
pdf: this.pdfPositions,
- downloadedFromCache: true,
})
.should.equal(true)
})
+
+ tryImageNameValidation('syncFromCode', 'imageName')
})
describe('syncFromPdf', function () {
@@ -452,7 +459,7 @@ describe('CompileController', function () {
this.CompileManager.syncFromPdf = sinon
.stub()
- .yields(null, (this.codePositions = ['mock-positions']), true)
+ .yields(null, (this.codePositions = ['mock-positions']))
this.CompileController.syncFromPdf(this.req, this.res, this.next)
})
@@ -466,10 +473,11 @@ describe('CompileController', function () {
this.res.json
.calledWith({
code: this.codePositions,
- downloadedFromCache: true,
})
.should.equal(true)
})
+
+ tryImageNameValidation('syncFromPdf', 'imageName')
})
describe('wordcount', function () {
@@ -503,5 +511,7 @@ describe('CompileController', function () {
})
.should.equal(true)
})
+
+ tryImageNameValidation('wordcount', 'image')
})
})
diff --git a/services/clsi/test/unit/js/CompileManagerTests.js b/services/clsi/test/unit/js/CompileManagerTests.js
index 30ef538ac3..6f5b5baf64 100644
--- a/services/clsi/test/unit/js/CompileManagerTests.js
+++ b/services/clsi/test/unit/js/CompileManagerTests.js
@@ -1,9 +1,9 @@
-const Path = require('node:path')
+const Path = require('path')
const SandboxedModule = require('sandboxed-module')
const { expect } = require('chai')
const sinon = require('sinon')
-const MODULE_PATH = require('node:path').join(
+const MODULE_PATH = require('path').join(
__dirname,
'../../../app/js/CompileManager'
)
@@ -35,7 +35,7 @@ describe('CompileManager', function () {
build: 1234,
},
]
- this.buildId = '00000000000-0000000000000000'
+ this.buildId = 'build-id-123'
this.commandOutput = 'Dummy output'
this.compileBaseDir = '/compile/dir'
this.outputBaseDir = '/output/dir'
@@ -61,10 +61,7 @@ describe('CompileManager', function () {
},
}
this.OutputCacheManager = {
- BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/,
- CACHE_SUBDIR: 'generated-files',
promises: {
- queueDirOperation: sinon.stub().callsArg(1),
saveOutputFiles: sinon
.stub()
.resolves({ outputFiles: this.buildFiles, buildId: this.buildId }),
@@ -90,10 +87,9 @@ describe('CompileManager', function () {
execFile: sinon.stub().yields(),
}
this.CommandRunner = {
- canRunSyncTeXInOutputDir: sinon.stub().returns(false),
promises: {
run: sinon.stub().callsFake((_1, _2, _3, _4, _5, _6, compileGroup) => {
- if (compileGroup === 'synctex' || compileGroup === 'synctex-output') {
+ if (compileGroup === 'synctex') {
return Promise.resolve({ stdout: this.commandOutput })
} else {
return Promise.resolve({
@@ -144,12 +140,6 @@ describe('CompileManager', function () {
.withArgs(Path.join(this.compileDir, 'output.synctex.gz'))
.resolves(this.fileStats)
- this.CLSICacheHandler = {
- notifyCLSICacheAboutBuild: sinon.stub(),
- downloadLatestCompileCache: sinon.stub().resolves(),
- downloadOutputDotSynctexFromCompileCache: sinon.stub().resolves(),
- }
-
this.CompileManager = SandboxedModule.require(MODULE_PATH, {
requires: {
'./LatexRunner': this.LatexRunner,
@@ -170,7 +160,6 @@ describe('CompileManager', function () {
'./LockManager': this.LockManager,
'./SynctexOutputParser': this.SynctexOutputParser,
'fs/promises': this.fsPromises,
- './CLSICacheHandler': this.CLSICacheHandler,
},
})
})
@@ -188,11 +177,6 @@ describe('CompileManager', function () {
flags: (this.flags = ['-file-line-error']),
compileGroup: (this.compileGroup = 'compile-group'),
stopOnFirstError: false,
- metricsOpts: {
- path: 'clsi-perf',
- method: 'minimal',
- compile: 'initial',
- },
}
this.env = {
OVERLEAF_PROJECT_ID: this.projectId,
@@ -204,7 +188,7 @@ describe('CompileManager', function () {
const error = new Error('locked')
this.LockManager.acquire.throws(error)
await expect(
- this.CompileManager.promises.doCompileWithLock(this.request, {}, {})
+ this.CompileManager.promises.doCompileWithLock(this.request)
).to.be.rejectedWith(error)
})
@@ -222,9 +206,7 @@ describe('CompileManager', function () {
describe('normally', function () {
beforeEach(async function () {
this.result = await this.CompileManager.promises.doCompileWithLock(
- this.request,
- {},
- {}
+ this.request
)
})
@@ -278,11 +260,7 @@ describe('CompileManager', function () {
describe('with draft mode', function () {
beforeEach(async function () {
this.request.draft = true
- await this.CompileManager.promises.doCompileWithLock(
- this.request,
- {},
- {}
- )
+ await this.CompileManager.promises.doCompileWithLock(this.request)
})
it('should inject the draft mode header', function () {
@@ -295,11 +273,7 @@ describe('CompileManager', function () {
describe('with a check option', function () {
beforeEach(async function () {
this.request.check = 'error'
- await this.CompileManager.promises.doCompileWithLock(
- this.request,
- {},
- {}
- )
+ await this.CompileManager.promises.doCompileWithLock(this.request)
})
it('should run chktex', function () {
@@ -331,11 +305,7 @@ describe('CompileManager', function () {
beforeEach(async function () {
this.request.rootResourcePath = 'main.Rtex'
this.request.check = 'error'
- await this.CompileManager.promises.doCompileWithLock(
- this.request,
- {},
- {}
- )
+ await this.CompileManager.promises.doCompileWithLock(this.request)
})
it('should not run chktex', function () {
@@ -364,7 +334,7 @@ describe('CompileManager', function () {
error.timedout = true
this.LatexRunner.promises.runLatex.rejects(error)
await expect(
- this.CompileManager.promises.doCompileWithLock(this.request, {}, {})
+ this.CompileManager.promises.doCompileWithLock(this.request)
).to.be.rejected
})
@@ -387,7 +357,7 @@ describe('CompileManager', function () {
error.terminated = true
this.LatexRunner.promises.runLatex.rejects(error)
await expect(
- this.CompileManager.promises.doCompileWithLock(this.request, {}, {})
+ this.CompileManager.promises.doCompileWithLock(this.request)
).to.be.rejected
})
@@ -467,83 +437,12 @@ describe('CompileManager', function () {
this.compileDir,
this.Settings.clsi.docker.image,
60000,
- {},
- 'synctex'
+ {}
)
})
it('should return the parsed output', function () {
- expect(this.result).to.deep.equal({
- codePositions: this.records,
- downloadedFromCache: false,
- })
- })
- })
-
- describe('from cache in docker', function () {
- beforeEach(async function () {
- this.CommandRunner.canRunSyncTeXInOutputDir.returns(true)
- this.Settings.path.synctexBaseDir
- .withArgs(`${this.projectId}-${this.userId}`)
- .returns('/compile')
-
- const errNotFound = new Error()
- errNotFound.code = 'ENOENT'
- this.outputDir = `${this.outputBaseDir}/${this.projectId}-${this.userId}/${this.OutputCacheManager.CACHE_SUBDIR}/${this.buildId}`
- const filename = Path.join(this.outputDir, 'output.synctex.gz')
- this.fsPromises.stat
- .withArgs(this.outputDir)
- .onFirstCall()
- .rejects(errNotFound)
- this.fsPromises.stat
- .withArgs(this.outputDir)
- .onSecondCall()
- .resolves(this.dirStats)
- this.fsPromises.stat.withArgs(filename).resolves(this.fileStats)
- this.CLSICacheHandler.downloadOutputDotSynctexFromCompileCache.resolves(
- true
- )
- this.result = await this.CompileManager.promises.syncFromCode(
- this.projectId,
- this.userId,
- this.filename,
- this.line,
- this.column,
- {
- imageName: 'image',
- editorId: '00000000-0000-0000-0000-000000000000',
- buildId: this.buildId,
- compileFromClsiCache: true,
- }
- )
- })
-
- it('should run in output dir', function () {
- const outputFilePath = '/compile/output.pdf'
- const inputFilePath = `/compile/${this.filename}`
- expect(this.CommandRunner.promises.run).to.have.been.calledWith(
- `${this.projectId}-${this.userId}`,
- [
- 'synctex',
- 'view',
- '-i',
- `${this.line}:${this.column}:${inputFilePath}`,
- '-o',
- outputFilePath,
- ],
- this.outputDir,
- 'image',
- 60000,
- {},
- 'synctex-output'
- )
- })
-
- it('should return the parsed output', function () {
- expect(this.result).to.deep.equal({
- codePositions: this.records,
- downloadedFromCache: true,
- })
+ expect(this.result).to.deep.equal(this.records)
})
})
@@ -556,7 +455,7 @@ describe('CompileManager', function () {
this.filename,
this.line,
this.column,
- { imageName: customImageName }
+ customImageName
)
})
@@ -576,8 +475,7 @@ describe('CompileManager', function () {
this.compileDir,
customImageName,
60000,
- {},
- 'synctex'
+ {}
)
})
})
@@ -599,7 +497,7 @@ describe('CompileManager', function () {
this.page,
this.h,
this.v,
- { imageName: '' }
+ ''
)
})
@@ -621,10 +519,7 @@ describe('CompileManager', function () {
})
it('should return the parsed output', function () {
- expect(this.result).to.deep.equal({
- pdfPositions: this.records,
- downloadedFromCache: false,
- })
+ expect(this.result).to.deep.equal(this.records)
})
})
@@ -637,7 +532,7 @@ describe('CompileManager', function () {
this.page,
this.h,
this.v,
- { imageName: customImageName }
+ customImageName
)
})
diff --git a/services/clsi/test/unit/js/ContentCacheManagerTests.js b/services/clsi/test/unit/js/ContentCacheManagerTests.js
index df3bce212b..be2d17039d 100644
--- a/services/clsi/test/unit/js/ContentCacheManagerTests.js
+++ b/services/clsi/test/unit/js/ContentCacheManagerTests.js
@@ -1,5 +1,5 @@
-const fs = require('node:fs')
-const Path = require('node:path')
+const fs = require('fs')
+const Path = require('path')
const { expect } = require('chai')
const MODULE_PATH = '../../../app/js/ContentCacheManager'
diff --git a/services/clsi/test/unit/js/ContentTypeMapperTests.js b/services/clsi/test/unit/js/ContentTypeMapperTests.js
index a413337153..a80ef991ca 100644
--- a/services/clsi/test/unit/js/ContentTypeMapperTests.js
+++ b/services/clsi/test/unit/js/ContentTypeMapperTests.js
@@ -11,7 +11,7 @@
*/
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/ContentTypeMapper'
)
diff --git a/services/clsi/test/unit/js/DockerLockManagerTests.js b/services/clsi/test/unit/js/DockerLockManagerTests.js
index f69179443c..5708faf292 100644
--- a/services/clsi/test/unit/js/DockerLockManagerTests.js
+++ b/services/clsi/test/unit/js/DockerLockManagerTests.js
@@ -11,12 +11,12 @@
*/
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/DockerLockManager'
)
-describe('DockerLockManager', function () {
+describe('LockManager', function () {
beforeEach(function () {
return (this.LockManager = SandboxedModule.require(modulePath, {
requires: {
diff --git a/services/clsi/test/unit/js/DockerRunnerTests.js b/services/clsi/test/unit/js/DockerRunnerTests.js
index d70aab52c7..8ff9c6107b 100644
--- a/services/clsi/test/unit/js/DockerRunnerTests.js
+++ b/services/clsi/test/unit/js/DockerRunnerTests.js
@@ -15,11 +15,11 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/DockerRunner'
)
-const Path = require('node:path')
+const Path = require('path')
describe('DockerRunner', function () {
beforeEach(function () {
@@ -76,11 +76,8 @@ describe('DockerRunner', function () {
this.env = {}
this.callback = sinon.stub()
this.project_id = 'project-id-123'
- this.volumes = { '/some/host/dir/compiles/directory': '/compile' }
+ this.volumes = { '/local/compile/directory': '/compile' }
this.Settings.clsi.docker.image = this.defaultImage = 'default-image'
- this.Settings.path.sandboxedCompilesHostDirCompiles =
- '/some/host/dir/compiles'
- this.Settings.path.sandboxedCompilesHostDirOutput = '/some/host/dir/output'
this.compileGroup = 'compile-group'
return (this.Settings.clsi.docker.env = { PATH: 'mock-path' })
})
@@ -154,8 +151,9 @@ describe('DockerRunner', function () {
})
})
- describe('standard compile', function () {
+ describe('when path.sandboxedCompilesHostDir is set', function () {
beforeEach(function () {
+ this.Settings.path.sandboxedCompilesHostDir = '/some/host/dir/compiles'
this.directory = '/var/lib/overleaf/data/compiles/xyz'
this.DockerRunner._runAndWaitForContainer = sinon
.stub()
@@ -185,99 +183,6 @@ describe('DockerRunner', function () {
})
})
- describe('synctex-output', function () {
- beforeEach(function () {
- this.directory = '/var/lib/overleaf/data/output/xyz/generated-files/id'
- this.DockerRunner._runAndWaitForContainer = sinon
- .stub()
- .callsArgWith(3, null, (this.output = 'mock-output'))
- this.DockerRunner.run(
- this.project_id,
- this.command,
- this.directory,
- this.image,
- this.timeout,
- this.env,
- 'synctex-output',
- this.callback
- )
- })
-
- it('should re-write the bind directory and set ro flag', function () {
- const volumes =
- this.DockerRunner._runAndWaitForContainer.lastCall.args[1]
- expect(volumes).to.deep.equal({
- '/some/host/dir/output/xyz/generated-files/id': '/compile:ro',
- })
- })
-
- it('should call the callback', function () {
- this.callback.calledWith(null, this.output).should.equal(true)
- })
- })
-
- describe('synctex', function () {
- beforeEach(function () {
- this.directory = '/var/lib/overleaf/data/compile/xyz'
- this.DockerRunner._runAndWaitForContainer = sinon
- .stub()
- .callsArgWith(3, null, (this.output = 'mock-output'))
- this.DockerRunner.run(
- this.project_id,
- this.command,
- this.directory,
- this.image,
- this.timeout,
- this.env,
- 'synctex',
- this.callback
- )
- })
-
- it('should re-write the bind directory', function () {
- const volumes =
- this.DockerRunner._runAndWaitForContainer.lastCall.args[1]
- expect(volumes).to.deep.equal({
- '/some/host/dir/compiles/xyz': '/compile:ro',
- })
- })
-
- it('should call the callback', function () {
- this.callback.calledWith(null, this.output).should.equal(true)
- })
- })
-
- describe('wordcount', function () {
- beforeEach(function () {
- this.directory = '/var/lib/overleaf/data/compile/xyz'
- this.DockerRunner._runAndWaitForContainer = sinon
- .stub()
- .callsArgWith(3, null, (this.output = 'mock-output'))
- this.DockerRunner.run(
- this.project_id,
- this.command,
- this.directory,
- this.image,
- this.timeout,
- this.env,
- 'wordcount',
- this.callback
- )
- })
-
- it('should re-write the bind directory', function () {
- const volumes =
- this.DockerRunner._runAndWaitForContainer.lastCall.args[1]
- expect(volumes).to.deep.equal({
- '/some/host/dir/compiles/xyz': '/compile:ro',
- })
- })
-
- it('should call the callback', function () {
- this.callback.calledWith(null, this.output).should.equal(true)
- })
- })
-
describe('when the run throws an error', function () {
beforeEach(function () {
let firstTime = true
@@ -485,7 +390,7 @@ describe('DockerRunner', function () {
const options =
this.DockerRunner._runAndWaitForContainer.lastCall.args[0]
return expect(options.HostConfig).to.deep.include({
- Binds: ['/some/host/dir/compiles/directory:/compile:rw'],
+ Binds: ['/local/compile/directory:/compile:rw'],
LogConfig: { Type: 'none', Config: {} },
CapDrop: 'ALL',
SecurityOpt: ['no-new-privileges'],
@@ -657,6 +562,82 @@ describe('DockerRunner', function () {
})
})
+ describe('when a volume does not exist', function () {
+ beforeEach(function () {
+ this.fs.stat = sinon.stub().yields(new Error('no such path'))
+ return this.DockerRunner.startContainer(
+ this.options,
+ this.volumes,
+ this.attachStreamHandler,
+ this.callback
+ )
+ })
+
+ it('should not try to create the container', function () {
+ return this.createContainer.called.should.equal(false)
+ })
+
+ it('should call the callback with an error', function () {
+ this.callback.calledWith(sinon.match(Error)).should.equal(true)
+ })
+ })
+
+ describe('when a volume exists but is not a directory', function () {
+ beforeEach(function () {
+ this.fs.stat = sinon.stub().yields(null, {
+ isDirectory() {
+ return false
+ },
+ })
+ return this.DockerRunner.startContainer(
+ this.options,
+ this.volumes,
+ this.attachStreamHandler,
+ this.callback
+ )
+ })
+
+ it('should not try to create the container', function () {
+ return this.createContainer.called.should.equal(false)
+ })
+
+ it('should call the callback with an error', function () {
+ this.callback.calledWith(sinon.match(Error)).should.equal(true)
+ })
+ })
+
+ describe('when a volume does not exist, but sibling-containers are used', function () {
+ beforeEach(function () {
+ this.fs.stat = sinon.stub().yields(new Error('no such path'))
+ this.Settings.path.sandboxedCompilesHostDir = '/some/path'
+ this.container.start = sinon.stub().yields()
+ return this.DockerRunner.startContainer(
+ this.options,
+ this.volumes,
+ () => {},
+ this.callback
+ )
+ })
+
+ afterEach(function () {
+ return delete this.Settings.path.sandboxedCompilesHostDir
+ })
+
+ it('should start the container with the given name', function () {
+ this.getContainer.calledWith(this.options.name).should.equal(true)
+ return this.container.start.called.should.equal(true)
+ })
+
+ it('should not try to create the container', function () {
+ return this.createContainer.called.should.equal(false)
+ })
+
+ return it('should call the callback', function () {
+ this.callback.called.should.equal(true)
+ return this.callback.calledWith(new Error()).should.equal(false)
+ })
+ })
+
return describe('when the container tries to be created, but already has been (race condition)', function () {})
})
diff --git a/services/clsi/test/unit/js/DraftModeManagerTests.js b/services/clsi/test/unit/js/DraftModeManagerTests.js
index eda83380e7..acaae0f10a 100644
--- a/services/clsi/test/unit/js/DraftModeManagerTests.js
+++ b/services/clsi/test/unit/js/DraftModeManagerTests.js
@@ -1,5 +1,5 @@
-const Path = require('node:path')
-const fsPromises = require('node:fs/promises')
+const Path = require('path')
+const fsPromises = require('fs/promises')
const { expect } = require('chai')
const mockFs = require('mock-fs')
const SandboxedModule = require('sandboxed-module')
diff --git a/services/clsi/test/unit/js/LatexRunnerTests.js b/services/clsi/test/unit/js/LatexRunnerTests.js
index 0d250dd517..ca36d7308f 100644
--- a/services/clsi/test/unit/js/LatexRunnerTests.js
+++ b/services/clsi/test/unit/js/LatexRunnerTests.js
@@ -2,7 +2,7 @@ const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const MODULE_PATH = require('node:path').join(
+const MODULE_PATH = require('path').join(
__dirname,
'../../../app/js/LatexRunner'
)
diff --git a/services/clsi/test/unit/js/LockManagerTests.js b/services/clsi/test/unit/js/LockManagerTests.js
index 7005b3e5a3..cd0d34cd4a 100644
--- a/services/clsi/test/unit/js/LockManagerTests.js
+++ b/services/clsi/test/unit/js/LockManagerTests.js
@@ -1,7 +1,7 @@
const { expect } = require('chai')
const sinon = require('sinon')
const SandboxedModule = require('sandboxed-module')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/LockManager'
)
@@ -21,7 +21,6 @@ describe('LockManager', function () {
compileConcurrencyLimit: 5,
}),
'./Errors': (this.Erros = Errors),
- './RequestParser': { MAX_TIMEOUT: 600 },
},
})
})
diff --git a/services/clsi/test/unit/js/OutputControllerTests.js b/services/clsi/test/unit/js/OutputControllerTests.js
index ee5c9c2a7a..33367af26a 100644
--- a/services/clsi/test/unit/js/OutputControllerTests.js
+++ b/services/clsi/test/unit/js/OutputControllerTests.js
@@ -1,6 +1,6 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const MODULE_PATH = require('node:path').join(
+const MODULE_PATH = require('path').join(
__dirname,
'../../../app/js/OutputController'
)
@@ -19,7 +19,7 @@ describe('OutputController', function () {
'./OutputFileArchiveManager': {
archiveFilesForBuild: this.archiveFilesForBuild,
},
- 'stream/promises': {
+ 'node:stream/promises': {
pipeline: this.pipeline,
},
},
diff --git a/services/clsi/test/unit/js/OutputFileArchiveManagerTests.js b/services/clsi/test/unit/js/OutputFileArchiveManagerTests.js
index d6817f3559..bd62b00825 100644
--- a/services/clsi/test/unit/js/OutputFileArchiveManagerTests.js
+++ b/services/clsi/test/unit/js/OutputFileArchiveManagerTests.js
@@ -2,7 +2,7 @@ const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { assert, expect } = require('chai')
-const MODULE_PATH = require('node:path').join(
+const MODULE_PATH = require('path').join(
__dirname,
'../../../app/js/OutputFileArchiveManager'
)
@@ -50,7 +50,7 @@ describe('OutputFileArchiveManager', function () {
'./OutputFileFinder': this.OutputFileFinder,
'./OutputCacheManager': this.OutputCacheManger,
archiver: this.archiver,
- 'fs/promises': this.fs,
+ 'node:fs/promises': this.fs,
'@overleaf/settings': {
path: {
outputDir: this.outputDir,
diff --git a/services/clsi/test/unit/js/OutputFileFinderTests.js b/services/clsi/test/unit/js/OutputFileFinderTests.js
index c9e1b443be..cf596a6271 100644
--- a/services/clsi/test/unit/js/OutputFileFinderTests.js
+++ b/services/clsi/test/unit/js/OutputFileFinderTests.js
@@ -1,6 +1,6 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/OutputFileFinder'
)
diff --git a/services/clsi/test/unit/js/OutputFileOptimiserTests.js b/services/clsi/test/unit/js/OutputFileOptimiserTests.js
index 1dd1a751b9..d74c6a3534 100644
--- a/services/clsi/test/unit/js/OutputFileOptimiserTests.js
+++ b/services/clsi/test/unit/js/OutputFileOptimiserTests.js
@@ -12,13 +12,13 @@
*/
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/OutputFileOptimiser'
)
-const path = require('node:path')
+const path = require('path')
const { expect } = require('chai')
-const { EventEmitter } = require('node:events')
+const { EventEmitter } = require('events')
describe('OutputFileOptimiser', function () {
beforeEach(function () {
diff --git a/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js b/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js
index 4f42411fba..c5f30b1ea9 100644
--- a/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js
+++ b/services/clsi/test/unit/js/ProjectPersistenceManagerTests.js
@@ -13,7 +13,7 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const assert = require('chai').assert
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/ProjectPersistenceManager'
)
@@ -21,16 +21,11 @@ const tk = require('timekeeper')
describe('ProjectPersistenceManager', function () {
beforeEach(function () {
- this.fsPromises = {
- statfs: sinon.stub(),
- }
-
this.ProjectPersistenceManager = SandboxedModule.require(modulePath, {
requires: {
- '@overleaf/metrics': (this.Metrics = { gauge: sinon.stub() }),
'./UrlCache': (this.UrlCache = {}),
'./CompileManager': (this.CompileManager = {}),
- fs: { promises: this.fsPromises },
+ diskusage: (this.diskusage = { check: sinon.stub() }),
'@overleaf/settings': (this.settings = {
project_cache_length_ms: 1000,
path: {
@@ -48,17 +43,12 @@ describe('ProjectPersistenceManager', function () {
describe('refreshExpiryTimeout', function () {
it('should leave expiry alone if plenty of disk', function (done) {
- this.fsPromises.statfs.resolves({
- blocks: 100,
- bsize: 1,
- bavail: 40,
+ this.diskusage.check.resolves({
+ available: 40,
+ total: 100,
})
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
- this.Metrics.gauge.should.have.been.calledWith(
- 'disk_available_percent',
- 40
- )
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(
this.settings.project_cache_length_ms
)
@@ -67,41 +57,31 @@ describe('ProjectPersistenceManager', function () {
})
it('should drop EXPIRY_TIMEOUT 10% if low disk usage', function (done) {
- this.fsPromises.statfs.resolves({
- blocks: 100,
- bsize: 1,
- bavail: 5,
+ this.diskusage.check.resolves({
+ available: 5,
+ total: 100,
})
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
- this.Metrics.gauge.should.have.been.calledWith(
- 'disk_available_percent',
- 5
- )
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(900)
done()
})
})
it('should not drop EXPIRY_TIMEOUT to below 50% of project_cache_length_ms', function (done) {
- this.fsPromises.statfs.resolves({
- blocks: 100,
- bsize: 1,
- bavail: 5,
+ this.diskusage.check.resolves({
+ available: 5,
+ total: 100,
})
this.ProjectPersistenceManager.EXPIRY_TIMEOUT = 500
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
- this.Metrics.gauge.should.have.been.calledWith(
- 'disk_available_percent',
- 5
- )
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(500)
done()
})
})
it('should not modify EXPIRY_TIMEOUT if there is an error getting disk values', function (done) {
- this.fsPromises.statfs.rejects(new Error())
+ this.diskusage.check.throws(new Error())
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(1000)
done()
diff --git a/services/clsi/test/unit/js/RequestParserTests.js b/services/clsi/test/unit/js/RequestParserTests.js
index 437c3c4fbe..4cf31a2b37 100644
--- a/services/clsi/test/unit/js/RequestParserTests.js
+++ b/services/clsi/test/unit/js/RequestParserTests.js
@@ -1,7 +1,7 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/RequestParser'
)
@@ -30,7 +30,6 @@ describe('RequestParser', function () {
this.RequestParser = SandboxedModule.require(modulePath, {
requires: {
'@overleaf/settings': (this.settings = {}),
- './OutputCacheManager': { BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/ },
},
})
})
@@ -275,37 +274,6 @@ describe('RequestParser', function () {
})
})
- describe('with a valid buildId', function () {
- beforeEach(function (done) {
- this.validRequest.compile.options.buildId = '195a4869176-a4ad60bee7bf35e4'
- this.RequestParser.parse(this.validRequest, (error, data) => {
- if (error) return done(error)
- this.data = data
- done()
- })
- })
-
- it('should return an error', function () {
- this.data.buildId.should.equal('195a4869176-a4ad60bee7bf35e4')
- })
- })
-
- describe('with a bad buildId', function () {
- beforeEach(function () {
- this.validRequest.compile.options.buildId = 'foo/bar'
- this.RequestParser.parse(this.validRequest, this.callback)
- })
-
- it('should return an error', function () {
- this.callback
- .calledWithMatch({
- message:
- 'buildId attribute does not match regex /^[0-9a-f]+-[0-9a-f]+$/',
- })
- .should.equal(true)
- })
- })
-
describe('with a resource with a valid date', function () {
beforeEach(function () {
this.date = '12:00 01/02/03'
diff --git a/services/clsi/test/unit/js/ResourceStateManagerTests.js b/services/clsi/test/unit/js/ResourceStateManagerTests.js
index 823c81616f..0a97d7b705 100644
--- a/services/clsi/test/unit/js/ResourceStateManagerTests.js
+++ b/services/clsi/test/unit/js/ResourceStateManagerTests.js
@@ -12,11 +12,11 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/ResourceStateManager'
)
-const Path = require('node:path')
+const Path = require('path')
const Errors = require('../../../app/js/Errors')
describe('ResourceStateManager', function () {
diff --git a/services/clsi/test/unit/js/ResourceWriterTests.js b/services/clsi/test/unit/js/ResourceWriterTests.js
index c2e09ce9cf..3b18a3b195 100644
--- a/services/clsi/test/unit/js/ResourceWriterTests.js
+++ b/services/clsi/test/unit/js/ResourceWriterTests.js
@@ -13,11 +13,11 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/ResourceWriter'
)
-const path = require('node:path')
+const path = require('path')
describe('ResourceWriter', function () {
beforeEach(function () {
@@ -378,13 +378,12 @@ describe('ResourceWriter', function () {
this.fs.mkdir = sinon.stub().callsArg(2)
this.resource = {
path: 'main.tex',
- url: 'http://www.example.com/primary/main.tex',
- fallbackURL: 'http://fallback.example.com/fallback/main.tex',
+ url: 'http://www.example.com/main.tex',
modified: Date.now(),
}
this.UrlCache.downloadUrlToFile = sinon
.stub()
- .callsArgWith(5, 'fake error downloading file')
+ .callsArgWith(4, 'fake error downloading file')
return this.ResourceWriter._writeResourceToDisk(
this.project_id,
this.resource,
@@ -406,7 +405,6 @@ describe('ResourceWriter', function () {
.calledWith(
this.project_id,
this.resource.url,
- this.resource.fallbackURL,
path.join(this.basePath, this.resource.path),
this.resource.modified
)
diff --git a/services/clsi/test/unit/js/StaticServerForbidSymlinksTests.js b/services/clsi/test/unit/js/StaticServerForbidSymlinksTests.js
index 53507fe3f2..0a3806a1e6 100644
--- a/services/clsi/test/unit/js/StaticServerForbidSymlinksTests.js
+++ b/services/clsi/test/unit/js/StaticServerForbidSymlinksTests.js
@@ -10,8 +10,8 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const SandboxedModule = require('sandboxed-module')
-const assert = require('node:assert')
-const path = require('node:path')
+const assert = require('assert')
+const path = require('path')
const sinon = require('sinon')
const modulePath = path.join(
__dirname,
diff --git a/services/clsi/test/unit/js/SynctexOutputParserTests.js b/services/clsi/test/unit/js/SynctexOutputParserTests.js
index b999a6adb1..19bf2f6d37 100644
--- a/services/clsi/test/unit/js/SynctexOutputParserTests.js
+++ b/services/clsi/test/unit/js/SynctexOutputParserTests.js
@@ -1,4 +1,4 @@
-const Path = require('node:path')
+const Path = require('path')
const SandboxedModule = require('sandboxed-module')
const { expect } = require('chai')
diff --git a/services/clsi/test/unit/js/TikzManager.js b/services/clsi/test/unit/js/TikzManager.js
index ee651f6b1d..8e01194955 100644
--- a/services/clsi/test/unit/js/TikzManager.js
+++ b/services/clsi/test/unit/js/TikzManager.js
@@ -10,7 +10,7 @@
*/
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/TikzManager'
)
diff --git a/services/clsi/test/unit/js/UrlCacheTests.js b/services/clsi/test/unit/js/UrlCacheTests.js
index a3dc2fac3c..a5c1e60c0a 100644
--- a/services/clsi/test/unit/js/UrlCacheTests.js
+++ b/services/clsi/test/unit/js/UrlCacheTests.js
@@ -13,17 +13,13 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const modulePath = require('node:path').join(
- __dirname,
- '../../../app/js/UrlCache'
-)
+const modulePath = require('path').join(__dirname, '../../../app/js/UrlCache')
describe('UrlCache', function () {
beforeEach(function () {
this.callback = sinon.stub()
this.url =
'http://filestore/project/60b0dd39c418bc00598a0d22/file/60ae721ffb1d920027d3201f'
- this.fallbackURL = 'http://filestore/bucket/project-blobs/key/ab/cd/ef'
this.project_id = '60b0dd39c418bc00598a0d22'
return (this.UrlCache = SandboxedModule.require(modulePath, {
requires: {
@@ -55,29 +51,6 @@ describe('UrlCache', function () {
this.UrlCache.downloadUrlToFile(
this.project_id,
this.url,
- this.fallbackURL,
- this.destPath,
- this.lastModified,
- error => {
- expect(error).to.not.exist
- expect(
- this.UrlFetcher.promises.pipeUrlToFileWithRetry.called
- ).to.equal(false)
- done()
- }
- )
- })
-
- it('should not download on the semi-happy path', function (done) {
- const codedError = new Error()
- codedError.code = 'ENOENT'
- this.fs.promises.copyFile.onCall(0).rejects(codedError)
- this.fs.promises.copyFile.onCall(1).resolves()
-
- this.UrlCache.downloadUrlToFile(
- this.project_id,
- this.url,
- this.fallbackURL,
this.destPath,
this.lastModified,
error => {
@@ -94,13 +67,11 @@ describe('UrlCache', function () {
const codedError = new Error()
codedError.code = 'ENOENT'
this.fs.promises.copyFile.onCall(0).rejects(codedError)
- this.fs.promises.copyFile.onCall(1).rejects(codedError)
- this.fs.promises.copyFile.onCall(2).resolves()
+ this.fs.promises.copyFile.onCall(1).resolves()
this.UrlCache.downloadUrlToFile(
this.project_id,
this.url,
- this.fallbackURL,
this.destPath,
this.lastModified,
error => {
@@ -120,7 +91,6 @@ describe('UrlCache', function () {
this.UrlCache.downloadUrlToFile(
this.project_id,
this.url,
- this.fallbackURL,
this.destPath,
this.lastModified,
error => {
diff --git a/services/clsi/test/unit/js/pdfjsTests.js b/services/clsi/test/unit/js/pdfjsTests.js
index bc8b775b43..ef85ccc704 100644
--- a/services/clsi/test/unit/js/pdfjsTests.js
+++ b/services/clsi/test/unit/js/pdfjsTests.js
@@ -1,5 +1,5 @@
-const fs = require('node:fs')
-const Path = require('node:path')
+const fs = require('fs')
+const Path = require('path')
const { expect } = require('chai')
const { parseXrefTable } = require('../../../app/js/XrefParser')
const { NoXrefTableError } = require('../../../app/js/Errors')
diff --git a/services/references/.gitignore b/services/contacts/.gitignore
similarity index 100%
rename from services/references/.gitignore
rename to services/contacts/.gitignore
diff --git a/services/contacts/.nvmrc b/services/contacts/.nvmrc
index fc37597bcc..123b052798 100644
--- a/services/contacts/.nvmrc
+++ b/services/contacts/.nvmrc
@@ -1 +1 @@
-22.17.0
+18.20.2
diff --git a/services/contacts/Dockerfile b/services/contacts/Dockerfile
index b59cada7b3..2b2d8dae61 100644
--- a/services/contacts/Dockerfile
+++ b/services/contacts/Dockerfile
@@ -2,7 +2,7 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:22.17.0 AS base
+FROM node:18.20.2 AS base
WORKDIR /overleaf/services/contacts
diff --git a/services/contacts/Makefile b/services/contacts/Makefile
index 3309e298e8..1f2b7a10fb 100644
--- a/services/contacts/Makefile
+++ b/services/contacts/Makefile
@@ -32,30 +32,12 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
-
-SHELLCHECK_OPTS = \
- --shell=bash \
- --external-sources
-SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
-SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
-
-shellcheck:
- @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
- koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
-
-shellcheck_fix:
- @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
- diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
- if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
- elif [ -n "$$diff" ]; then echo "$$file"; \
- else echo "\033[2m$$file\033[0m"; fi \
- done
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
format:
$(RUN_LINTING) format
@@ -81,7 +63,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck shellcheck test_unit test_acceptance
+test: format lint typecheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -116,6 +98,13 @@ test_acceptance_clean:
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
test_acceptance_pre_run:
+ $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+ $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+ while ! mongosh --eval "db.version()" > /dev/null; do \
+ echo "Waiting for Mongo..."; \
+ sleep 1; \
+ done; \
+ mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
endif
@@ -148,7 +137,6 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
- shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/contacts/buildscript.txt b/services/contacts/buildscript.txt
index b20764246c..cfbc56040e 100644
--- a/services/contacts/buildscript.txt
+++ b/services/contacts/buildscript.txt
@@ -4,6 +4,6 @@ contacts
--env-add=
--env-pass-through=
--esmock-loader=True
---node-version=22.17.0
+--node-version=18.20.2
--public-repo=False
---script-version=4.7.0
+--script-version=4.5.0
diff --git a/services/contacts/config/settings.defaults.cjs b/services/contacts/config/settings.defaults.cjs
index 7ffdb83ce5..fb716d1812 100644
--- a/services/contacts/config/settings.defaults.cjs
+++ b/services/contacts/config/settings.defaults.cjs
@@ -1,9 +1,5 @@
-const http = require('node:http')
-const https = require('node:https')
-
+const http = require('http')
http.globalAgent.maxSockets = 300
-http.globalAgent.keepAlive = false
-https.globalAgent.keepAlive = false
module.exports = {
internal: {
diff --git a/services/contacts/docker-compose.ci.yml b/services/contacts/docker-compose.ci.yml
index ca3303a079..6f1a608534 100644
--- a/services/contacts/docker-compose.ci.yml
+++ b/services/contacts/docker-compose.ci.yml
@@ -24,13 +24,10 @@ services:
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
- volumes:
- - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
- condition: service_started
+ condition: service_healthy
user: node
- entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run test:acceptance
@@ -42,14 +39,9 @@ services:
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
mongo:
- image: mongo:8.0.11
+ image: mongo:6.0.13
command: --replSet overleaf
- volumes:
- - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
- environment:
- MONGO_INITDB_DATABASE: sharelatex
- extra_hosts:
- # Required when using the automatic database setup for initializing the
- # replica set. This override is not needed when running the setup after
- # starting up mongo.
- - mongo:127.0.0.1
+ healthcheck:
+ test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
+ interval: 1s
+ retries: 20
diff --git a/services/contacts/docker-compose.yml b/services/contacts/docker-compose.yml
index 474ea224f8..71a8aaa5f9 100644
--- a/services/contacts/docker-compose.yml
+++ b/services/contacts/docker-compose.yml
@@ -6,7 +6,7 @@ version: "2.3"
services:
test_unit:
- image: node:22.17.0
+ image: node:18.20.2
volumes:
- .:/overleaf/services/contacts
- ../../node_modules:/overleaf/node_modules
@@ -14,45 +14,37 @@ services:
working_dir: /overleaf/services/contacts
environment:
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
user: node
test_acceptance:
- image: node:22.17.0
+ image: node:18.20.2
volumes:
- .:/overleaf/services/contacts
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
working_dir: /overleaf/services/contacts
environment:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
+ LOG_LEVEL: ERROR
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
user: node
depends_on:
mongo:
- condition: service_started
- entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
+ condition: service_healthy
command: npm run --silent test:acceptance
mongo:
- image: mongo:8.0.11
+ image: mongo:6.0.13
command: --replSet overleaf
- volumes:
- - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
- environment:
- MONGO_INITDB_DATABASE: sharelatex
- extra_hosts:
- # Required when using the automatic database setup for initializing the
- # replica set. This override is not needed when running the setup after
- # starting up mongo.
- - mongo:127.0.0.1
+ healthcheck:
+ test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
+ interval: 1s
+ retries: 20
diff --git a/services/contacts/package.json b/services/contacts/package.json
index db707e55c9..7906016a77 100644
--- a/services/contacts/package.json
+++ b/services/contacts/package.json
@@ -6,14 +6,14 @@
"main": "app.js",
"scripts": {
"start": "node app.js",
- "test:acceptance:_run": "mocha --loader=esmock --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
+ "test:acceptance:_run": "LOG_LEVEL=fatal mocha --loader=esmock --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
- "test:unit:_run": "mocha --loader=esmock --recursive --reporter spec $@ test/unit/js",
+ "test:unit:_run": "LOG_LEVEL=fatal mocha --loader=esmock --recursive --reporter spec $@ test/unit/js",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
- "format": "prettier --list-different $PWD/'**/*.*js'",
- "format:fix": "prettier --write $PWD/'**/*.*js'",
+ "format": "prettier --list-different $PWD/'**/*.js'",
+ "format:fix": "prettier --write $PWD/'**/*.js'",
"lint:fix": "eslint --fix .",
"types:check": "tsc --noEmit"
},
@@ -24,8 +24,8 @@
"async": "^3.2.5",
"body-parser": "^1.20.3",
"bunyan": "^1.8.15",
- "express": "^4.21.2",
- "mongodb": "6.12.0",
+ "express": "^4.21.0",
+ "mongodb": "^6.1.0",
"request": "~2.88.2",
"underscore": "~1.13.1"
},
@@ -33,7 +33,7 @@
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"esmock": "^2.6.3",
- "mocha": "^11.1.0",
+ "mocha": "^10.2.0",
"sinon": "~9.0.1",
"sinon-chai": "^3.7.0",
"typescript": "^5.0.4"
diff --git a/services/docstore/.gitignore b/services/docstore/.gitignore
new file mode 100644
index 0000000000..84bf300f7f
--- /dev/null
+++ b/services/docstore/.gitignore
@@ -0,0 +1,8 @@
+node_modules
+forever
+
+# managed by dev-environment$ bin/update_build_scripts
+.npmrc
+
+# Jetbrains IDEs
+.idea
diff --git a/services/docstore/.nvmrc b/services/docstore/.nvmrc
index fc37597bcc..123b052798 100644
--- a/services/docstore/.nvmrc
+++ b/services/docstore/.nvmrc
@@ -1 +1 @@
-22.17.0
+18.20.2
diff --git a/services/docstore/Dockerfile b/services/docstore/Dockerfile
index f24f9ddaf7..d1a661fc7b 100644
--- a/services/docstore/Dockerfile
+++ b/services/docstore/Dockerfile
@@ -2,7 +2,7 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:22.17.0 AS base
+FROM node:18.20.2 AS base
WORKDIR /overleaf/services/docstore
diff --git a/services/docstore/Makefile b/services/docstore/Makefile
index 2b3596b0b4..25068f95e6 100644
--- a/services/docstore/Makefile
+++ b/services/docstore/Makefile
@@ -32,30 +32,12 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
-
-SHELLCHECK_OPTS = \
- --shell=bash \
- --external-sources
-SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
-SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
-
-shellcheck:
- @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
- koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
-
-shellcheck_fix:
- @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
- diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
- if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
- elif [ -n "$$diff" ]; then echo "$$file"; \
- else echo "\033[2m$$file\033[0m"; fi \
- done
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
format:
$(RUN_LINTING) format
@@ -81,7 +63,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck shellcheck test_unit test_acceptance
+test: format lint typecheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -116,6 +98,13 @@ test_acceptance_clean:
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
test_acceptance_pre_run:
+ $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+ $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+ while ! mongosh --eval "db.version()" > /dev/null; do \
+ echo "Waiting for Mongo..."; \
+ sleep 1; \
+ done; \
+ mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
endif
@@ -148,7 +137,6 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
- shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/docstore/app.js b/services/docstore/app.js
index ef755c4bb1..51ad785065 100644
--- a/services/docstore/app.js
+++ b/services/docstore/app.js
@@ -1,7 +1,7 @@
// Metrics must be initialized before importing anything else
require('@overleaf/metrics/initialize')
-const Events = require('node:events')
+const Events = require('events')
const Metrics = require('@overleaf/metrics')
const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
@@ -50,14 +50,6 @@ app.param('doc_id', function (req, res, next, docId) {
app.get('/project/:project_id/doc-deleted', HttpController.getAllDeletedDocs)
app.get('/project/:project_id/doc', HttpController.getAllDocs)
app.get('/project/:project_id/ranges', HttpController.getAllRanges)
-app.get(
- '/project/:project_id/comment-thread-ids',
- HttpController.getCommentThreadIds
-)
-app.get(
- '/project/:project_id/tracked-changes-user-ids',
- HttpController.getTrackedChangesUserIds
-)
app.get('/project/:project_id/has-ranges', HttpController.projectHasRanges)
app.get('/project/:project_id/doc/:doc_id', HttpController.getDoc)
app.get('/project/:project_id/doc/:doc_id/deleted', HttpController.isDocDeleted)
@@ -96,17 +88,14 @@ app.get('/status', (req, res) => res.send('docstore is alive'))
app.use(handleValidationErrors())
app.use(function (error, req, res, next) {
+ logger.error({ err: error, req }, 'request errored')
if (error instanceof Errors.NotFoundError) {
- logger.warn({ req }, 'not found')
res.sendStatus(404)
} else if (error instanceof Errors.DocModifiedError) {
- logger.warn({ req }, 'conflict: doc modified')
res.sendStatus(409)
} else if (error instanceof Errors.DocVersionDecrementedError) {
- logger.warn({ req }, 'conflict: doc version decremented')
res.sendStatus(409)
} else {
- logger.error({ err: error, req }, 'request errored')
res.status(500).send('Oops, something went wrong')
}
})
diff --git a/services/docstore/app/js/DocArchiveManager.js b/services/docstore/app/js/DocArchiveManager.js
index d03ee161a8..035448f4a1 100644
--- a/services/docstore/app/js/DocArchiveManager.js
+++ b/services/docstore/app/js/DocArchiveManager.js
@@ -1,18 +1,35 @@
-const MongoManager = require('./MongoManager')
+const { callbackify } = require('util')
+const MongoManager = require('./MongoManager').promises
const Errors = require('./Errors')
const logger = require('@overleaf/logger')
const Settings = require('@overleaf/settings')
-const crypto = require('node:crypto')
+const crypto = require('crypto')
const { ReadableString } = require('@overleaf/stream-utils')
const RangeManager = require('./RangeManager')
const PersistorManager = require('./PersistorManager')
const pMap = require('p-map')
-const { streamToBuffer } = require('./StreamToBuffer')
const { BSON } = require('mongodb-legacy')
const PARALLEL_JOBS = Settings.parallelArchiveJobs
const UN_ARCHIVE_BATCH_SIZE = Settings.unArchiveBatchSize
+module.exports = {
+ archiveAllDocs: callbackify(archiveAllDocs),
+ archiveDoc: callbackify(archiveDoc),
+ unArchiveAllDocs: callbackify(unArchiveAllDocs),
+ unarchiveDoc: callbackify(unarchiveDoc),
+ destroyProject: callbackify(destroyProject),
+ getDoc: callbackify(getDoc),
+ promises: {
+ archiveAllDocs,
+ archiveDoc,
+ unArchiveAllDocs,
+ unarchiveDoc,
+ destroyProject,
+ getDoc,
+ },
+}
+
async function archiveAllDocs(projectId) {
if (!_isArchivingEnabled()) {
return
@@ -44,8 +61,6 @@ async function archiveDoc(projectId, docId) {
throw new Error('doc has no lines')
}
- RangeManager.fixCommentIds(doc)
-
// warn about any oversized docs already in mongo
const linesSize = BSON.calculateObjectSize(doc.lines || {})
const rangesSize = BSON.calculateObjectSize(doc.ranges || {})
@@ -121,7 +136,7 @@ async function getDoc(projectId, docId) {
key
)
stream.resume()
- const buffer = await streamToBuffer(projectId, docId, stream)
+ const buffer = await _streamToBuffer(projectId, docId, stream)
const md5 = crypto.createHash('md5').update(buffer).digest('hex')
if (sourceMd5 !== md5) {
throw new Errors.Md5MismatchError('md5 mismatch when downloading doc', {
@@ -172,6 +187,34 @@ async function destroyProject(projectId) {
await Promise.all(tasks)
}
+async function _streamToBuffer(projectId, docId, stream) {
+ const chunks = []
+ let size = 0
+ let logged = false
+ const logIfTooLarge = finishedReading => {
+ if (size <= Settings.max_doc_length) return
+ // Log progress once and then again at the end.
+ if (logged && !finishedReading) return
+ logger.warn(
+ { projectId, docId, size, finishedReading },
+ 'potentially large doc pulled down from gcs'
+ )
+ logged = true
+ }
+ return await new Promise((resolve, reject) => {
+ stream.on('data', chunk => {
+ size += chunk.byteLength
+ logIfTooLarge(false)
+ chunks.push(chunk)
+ })
+ stream.on('error', reject)
+ stream.on('end', () => {
+ logIfTooLarge(true)
+ resolve(Buffer.concat(chunks))
+ })
+ })
+}
+
function _deserializeArchivedDoc(buffer) {
const doc = JSON.parse(buffer)
@@ -209,12 +252,3 @@ function _isArchivingEnabled() {
return true
}
-
-module.exports = {
- archiveAllDocs,
- archiveDoc,
- unArchiveAllDocs,
- unarchiveDoc,
- destroyProject,
- getDoc,
-}
diff --git a/services/docstore/app/js/DocManager.js b/services/docstore/app/js/DocManager.js
index c9e8dadc2c..80e8ae4527 100644
--- a/services/docstore/app/js/DocManager.js
+++ b/services/docstore/app/js/DocManager.js
@@ -5,7 +5,8 @@ const _ = require('lodash')
const DocArchive = require('./DocArchiveManager')
const RangeManager = require('./RangeManager')
const Settings = require('@overleaf/settings')
-const { setTimeout } = require('node:timers/promises')
+const { callbackifyAll } = require('@overleaf/promise-utils')
+const { setTimeout } = require('timers/promises')
/**
* @import { Document } from 'mongodb'
@@ -28,7 +29,7 @@ const DocManager = {
throw new Error('must include inS3 when getting doc')
}
- const doc = await MongoManager.findDoc(projectId, docId, filter)
+ const doc = await MongoManager.promises.findDoc(projectId, docId, filter)
if (doc == null) {
throw new Errors.NotFoundError(
@@ -37,19 +38,15 @@ const DocManager = {
}
if (doc.inS3) {
- await DocArchive.unarchiveDoc(projectId, docId)
+ await DocArchive.promises.unarchiveDoc(projectId, docId)
return await DocManager._getDoc(projectId, docId, filter)
}
- if (filter.ranges) {
- RangeManager.fixCommentIds(doc)
- }
-
return doc
},
async isDocDeleted(projectId, docId) {
- const doc = await MongoManager.findDoc(projectId, docId, {
+ const doc = await MongoManager.promises.findDoc(projectId, docId, {
deleted: true,
})
@@ -77,7 +74,7 @@ const DocManager = {
// returns the doc without any version information
async _peekRawDoc(projectId, docId) {
- const doc = await MongoManager.findDoc(projectId, docId, {
+ const doc = await MongoManager.promises.findDoc(projectId, docId, {
lines: true,
rev: true,
deleted: true,
@@ -94,7 +91,7 @@ const DocManager = {
if (doc.inS3) {
// skip the unarchiving to mongo when getting a doc
- const archivedDoc = await DocArchive.getDoc(projectId, docId)
+ const archivedDoc = await DocArchive.promises.getDoc(projectId, docId)
Object.assign(doc, archivedDoc)
}
@@ -105,7 +102,7 @@ const DocManager = {
// without unarchiving it (avoids unnecessary writes to mongo)
async peekDoc(projectId, docId) {
const doc = await DocManager._peekRawDoc(projectId, docId)
- await MongoManager.checkRevUnchanged(doc)
+ await MongoManager.promises.checkRevUnchanged(doc)
return doc
},
@@ -114,18 +111,16 @@ const DocManager = {
lines: true,
inS3: true,
})
- if (!doc) throw new Errors.NotFoundError()
- if (!Array.isArray(doc.lines)) throw new Errors.DocWithoutLinesError()
- return doc.lines.join('\n')
+ return doc
},
async getAllDeletedDocs(projectId, filter) {
- return await MongoManager.getProjectsDeletedDocs(projectId, filter)
+ return await MongoManager.promises.getProjectsDeletedDocs(projectId, filter)
},
async getAllNonDeletedDocs(projectId, filter) {
- await DocArchive.unArchiveAllDocs(projectId)
- const docs = await MongoManager.getProjectsDocs(
+ await DocArchive.promises.unArchiveAllDocs(projectId)
+ const docs = await MongoManager.promises.getProjectsDocs(
projectId,
{ include_deleted: false },
filter
@@ -133,46 +128,15 @@ const DocManager = {
if (docs == null) {
throw new Errors.NotFoundError(`No docs for project ${projectId}`)
}
- if (filter.ranges) {
- for (const doc of docs) {
- RangeManager.fixCommentIds(doc)
- }
- }
return docs
},
- async getCommentThreadIds(projectId) {
- const docs = await DocManager.getAllNonDeletedDocs(projectId, {
- _id: true,
- ranges: true,
- })
- const byDoc = new Map()
- for (const doc of docs) {
- const ids = new Set()
- for (const comment of doc.ranges?.comments || []) {
- ids.add(comment.op.t)
- }
- if (ids.size > 0) byDoc.set(doc._id.toString(), Array.from(ids))
- }
- return Object.fromEntries(byDoc.entries())
- },
-
- async getTrackedChangesUserIds(projectId) {
- const docs = await DocManager.getAllNonDeletedDocs(projectId, {
- ranges: true,
- })
- const userIds = new Set()
- for (const doc of docs) {
- for (const change of doc.ranges?.changes || []) {
- if (change.metadata.user_id === 'anonymous-user') continue
- userIds.add(change.metadata.user_id)
- }
- }
- return Array.from(userIds)
- },
-
async projectHasRanges(projectId) {
- const docs = await MongoManager.getProjectsDocs(projectId, {}, { _id: 1 })
+ const docs = await MongoManager.promises.getProjectsDocs(
+ projectId,
+ {},
+ { _id: 1 }
+ )
const docIds = docs.map(doc => doc._id)
for (const docId of docIds) {
const doc = await DocManager.peekDoc(projectId, docId)
@@ -283,7 +247,7 @@ const DocManager = {
}
modified = true
- await MongoManager.upsertIntoDocCollection(
+ await MongoManager.promises.upsertIntoDocCollection(
projectId,
docId,
doc?.rev,
@@ -298,7 +262,11 @@ const DocManager = {
async patchDoc(projectId, docId, meta) {
const projection = { _id: 1, deleted: true }
- const doc = await MongoManager.findDoc(projectId, docId, projection)
+ const doc = await MongoManager.promises.findDoc(
+ projectId,
+ docId,
+ projection
+ )
if (!doc) {
throw new Errors.NotFoundError(
`No such project/doc to delete: ${projectId}/${docId}`
@@ -307,7 +275,7 @@ const DocManager = {
if (meta.deleted && Settings.docstore.archiveOnSoftDelete) {
// The user will not read this doc anytime soon. Flush it out of mongo.
- DocArchive.archiveDoc(projectId, docId).catch(err => {
+ DocArchive.promises.archiveDoc(projectId, docId).catch(err => {
logger.warn(
{ projectId, docId, err },
'archiving a single doc in the background failed'
@@ -315,8 +283,15 @@ const DocManager = {
})
}
- await MongoManager.patchDoc(projectId, docId, meta)
+ await MongoManager.promises.patchDoc(projectId, docId, meta)
},
}
-module.exports = DocManager
+module.exports = {
+ ...callbackifyAll(DocManager, {
+ multiResult: {
+ updateDoc: ['modified', 'rev'],
+ },
+ }),
+ promises: DocManager,
+}
diff --git a/services/docstore/app/js/Errors.js b/services/docstore/app/js/Errors.js
index 7b150cc0db..bbdbe75c08 100644
--- a/services/docstore/app/js/Errors.js
+++ b/services/docstore/app/js/Errors.js
@@ -10,13 +10,10 @@ class DocRevValueError extends OError {}
class DocVersionDecrementedError extends OError {}
-class DocWithoutLinesError extends OError {}
-
module.exports = {
Md5MismatchError,
DocModifiedError,
DocRevValueError,
DocVersionDecrementedError,
- DocWithoutLinesError,
...Errors,
}
diff --git a/services/docstore/app/js/HealthChecker.js b/services/docstore/app/js/HealthChecker.js
index a5b7ad7e9a..65dc92cbba 100644
--- a/services/docstore/app/js/HealthChecker.js
+++ b/services/docstore/app/js/HealthChecker.js
@@ -1,35 +1,67 @@
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
const { db, ObjectId } = require('./mongodb')
+const request = require('request')
+const async = require('async')
const _ = require('lodash')
-const crypto = require('node:crypto')
+const crypto = require('crypto')
const settings = require('@overleaf/settings')
const { port } = settings.internal.docstore
const logger = require('@overleaf/logger')
-const { fetchNothing, fetchJson } = require('@overleaf/fetch-utils')
-async function check() {
- const docId = new ObjectId()
- const projectId = new ObjectId(settings.docstore.healthCheck.project_id)
- const url = `http://127.0.0.1:${port}/project/${projectId}/doc/${docId}`
- const lines = [
- 'smoke test - delete me',
- `${crypto.randomBytes(32).toString('hex')}`,
- ]
- logger.debug({ lines, url, docId, projectId }, 'running health check')
- let body
- try {
- await fetchNothing(url, {
- method: 'POST',
- json: { lines, version: 42, ranges: {} },
- signal: AbortSignal.timeout(3_000),
- })
- body = await fetchJson(url, { signal: AbortSignal.timeout(3_000) })
- } finally {
- await db.docs.deleteOne({ _id: docId, project_id: projectId })
- }
- if (!_.isEqual(body?.lines, lines)) {
- throw new Error(`health check lines not equal ${body.lines} != ${lines}`)
- }
-}
module.exports = {
- check,
+ check(callback) {
+ const docId = new ObjectId()
+ const projectId = new ObjectId(settings.docstore.healthCheck.project_id)
+ const url = `http://127.0.0.1:${port}/project/${projectId}/doc/${docId}`
+ const lines = [
+ 'smoke test - delete me',
+ `${crypto.randomBytes(32).toString('hex')}`,
+ ]
+ const getOpts = () => ({
+ url,
+ timeout: 3000,
+ })
+ logger.debug({ lines, url, docId, projectId }, 'running health check')
+ const jobs = [
+ function (cb) {
+ const opts = getOpts()
+ opts.json = { lines, version: 42, ranges: {} }
+ return request.post(opts, cb)
+ },
+ function (cb) {
+ const opts = getOpts()
+ opts.json = true
+ return request.get(opts, function (err, res, body) {
+ if (err != null) {
+ logger.err({ err }, 'docstore returned a error in health check get')
+ return cb(err)
+ } else if (res == null) {
+ return cb(new Error('no response from docstore with get check'))
+ } else if ((res != null ? res.statusCode : undefined) !== 200) {
+ return cb(new Error(`status code not 200, its ${res.statusCode}`))
+ } else if (
+ _.isEqual(body != null ? body.lines : undefined, lines) &&
+ (body != null ? body._id : undefined) === docId.toString()
+ ) {
+ return cb()
+ } else {
+ return cb(
+ new Error(
+ `health check lines not equal ${body.lines} != ${lines}`
+ )
+ )
+ }
+ })
+ },
+ cb => db.docs.deleteOne({ _id: docId, project_id: projectId }, cb),
+ ]
+ return async.series(jobs, callback)
+ },
}
diff --git a/services/docstore/app/js/HttpController.js b/services/docstore/app/js/HttpController.js
index 50c4302aeb..1c4e137033 100644
--- a/services/docstore/app/js/HttpController.js
+++ b/services/docstore/app/js/HttpController.js
@@ -4,104 +4,143 @@ const DocArchive = require('./DocArchiveManager')
const HealthChecker = require('./HealthChecker')
const Errors = require('./Errors')
const Settings = require('@overleaf/settings')
-const { expressify } = require('@overleaf/promise-utils')
-async function getDoc(req, res) {
+function getDoc(req, res, next) {
const { doc_id: docId, project_id: projectId } = req.params
const includeDeleted = req.query.include_deleted === 'true'
logger.debug({ projectId, docId }, 'getting doc')
- const doc = await DocManager.getFullDoc(projectId, docId)
- logger.debug({ docId, projectId }, 'got doc')
- if (doc.deleted && !includeDeleted) {
- res.sendStatus(404)
- } else {
- res.json(_buildDocView(doc))
- }
+ DocManager.getFullDoc(projectId, docId, function (error, doc) {
+ if (error) {
+ return next(error)
+ }
+ logger.debug({ docId, projectId }, 'got doc')
+ if (doc == null) {
+ res.sendStatus(404)
+ } else if (doc.deleted && !includeDeleted) {
+ res.sendStatus(404)
+ } else {
+ res.json(_buildDocView(doc))
+ }
+ })
}
-async function peekDoc(req, res) {
+function peekDoc(req, res, next) {
const { doc_id: docId, project_id: projectId } = req.params
logger.debug({ projectId, docId }, 'peeking doc')
- const doc = await DocManager.peekDoc(projectId, docId)
- res.setHeader('x-doc-status', doc.inS3 ? 'archived' : 'active')
- res.json(_buildDocView(doc))
+ DocManager.peekDoc(projectId, docId, function (error, doc) {
+ if (error) {
+ return next(error)
+ }
+ if (doc == null) {
+ res.sendStatus(404)
+ } else {
+ res.setHeader('x-doc-status', doc.inS3 ? 'archived' : 'active')
+ res.json(_buildDocView(doc))
+ }
+ })
}
-async function isDocDeleted(req, res) {
+function isDocDeleted(req, res, next) {
const { doc_id: docId, project_id: projectId } = req.params
- const deleted = await DocManager.isDocDeleted(projectId, docId)
- res.json({ deleted })
+ DocManager.isDocDeleted(projectId, docId, function (error, deleted) {
+ if (error) {
+ return next(error)
+ }
+ res.json({ deleted })
+ })
}
-async function getRawDoc(req, res) {
+function getRawDoc(req, res, next) {
const { doc_id: docId, project_id: projectId } = req.params
logger.debug({ projectId, docId }, 'getting raw doc')
- const content = await DocManager.getDocLines(projectId, docId)
- res.setHeader('content-type', 'text/plain')
- res.send(content)
+ DocManager.getDocLines(projectId, docId, function (error, doc) {
+ if (error) {
+ return next(error)
+ }
+ if (doc == null) {
+ res.sendStatus(404)
+ } else {
+ res.setHeader('content-type', 'text/plain')
+ res.send(_buildRawDocView(doc))
+ }
+ })
}
-async function getAllDocs(req, res) {
+function getAllDocs(req, res, next) {
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'getting all docs')
- const docs = await DocManager.getAllNonDeletedDocs(projectId, {
- lines: true,
- rev: true,
- })
- const docViews = _buildDocsArrayView(projectId, docs)
- for (const docView of docViews) {
- if (!docView.lines) {
- logger.warn({ projectId, docId: docView._id }, 'missing doc lines')
- docView.lines = []
+ DocManager.getAllNonDeletedDocs(
+ projectId,
+ { lines: true, rev: true },
+ function (error, docs) {
+ if (docs == null) {
+ docs = []
+ }
+ if (error) {
+ return next(error)
+ }
+ const docViews = _buildDocsArrayView(projectId, docs)
+ for (const docView of docViews) {
+ if (!docView.lines) {
+ logger.warn({ projectId, docId: docView._id }, 'missing doc lines')
+ docView.lines = []
+ }
+ }
+ res.json(docViews)
}
- }
- res.json(docViews)
-}
-
-async function getAllDeletedDocs(req, res) {
- const { project_id: projectId } = req.params
- logger.debug({ projectId }, 'getting all deleted docs')
- const docs = await DocManager.getAllDeletedDocs(projectId, {
- name: true,
- deletedAt: true,
- })
- res.json(
- docs.map(doc => ({
- _id: doc._id.toString(),
- name: doc.name,
- deletedAt: doc.deletedAt,
- }))
)
}
-async function getAllRanges(req, res) {
+function getAllDeletedDocs(req, res, next) {
+ const { project_id: projectId } = req.params
+ logger.debug({ projectId }, 'getting all deleted docs')
+ DocManager.getAllDeletedDocs(
+ projectId,
+ { name: true, deletedAt: true },
+ function (error, docs) {
+ if (error) {
+ return next(error)
+ }
+ res.json(
+ docs.map(doc => ({
+ _id: doc._id.toString(),
+ name: doc.name,
+ deletedAt: doc.deletedAt,
+ }))
+ )
+ }
+ )
+}
+
+function getAllRanges(req, res, next) {
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'getting all ranges')
- const docs = await DocManager.getAllNonDeletedDocs(projectId, {
- ranges: true,
+ DocManager.getAllNonDeletedDocs(
+ projectId,
+ { ranges: true },
+ function (error, docs) {
+ if (docs == null) {
+ docs = []
+ }
+ if (error) {
+ return next(error)
+ }
+ res.json(_buildDocsArrayView(projectId, docs))
+ }
+ )
+}
+
+function projectHasRanges(req, res, next) {
+ const { project_id: projectId } = req.params
+ DocManager.projectHasRanges(projectId, (err, projectHasRanges) => {
+ if (err) {
+ return next(err)
+ }
+ res.json({ projectHasRanges })
})
- res.json(_buildDocsArrayView(projectId, docs))
}
-async function getCommentThreadIds(req, res) {
- const { project_id: projectId } = req.params
- const threadIds = await DocManager.getCommentThreadIds(projectId)
- res.json(threadIds)
-}
-
-async function getTrackedChangesUserIds(req, res) {
- const { project_id: projectId } = req.params
- const userIds = await DocManager.getTrackedChangesUserIds(projectId)
- res.json(userIds)
-}
-
-async function projectHasRanges(req, res) {
- const { project_id: projectId } = req.params
- const projectHasRanges = await DocManager.projectHasRanges(projectId)
- res.json({ projectHasRanges })
-}
-
-async function updateDoc(req, res) {
+function updateDoc(req, res, next) {
const { doc_id: docId, project_id: projectId } = req.params
const lines = req.body?.lines
const version = req.body?.version
@@ -133,20 +172,25 @@ async function updateDoc(req, res) {
}
logger.debug({ projectId, docId }, 'got http request to update doc')
- const { modified, rev } = await DocManager.updateDoc(
+ DocManager.updateDoc(
projectId,
docId,
lines,
version,
- ranges
+ ranges,
+ function (error, modified, rev) {
+ if (error) {
+ return next(error)
+ }
+ res.json({
+ modified,
+ rev,
+ })
+ }
)
- res.json({
- modified,
- rev,
- })
}
-async function patchDoc(req, res) {
+function patchDoc(req, res, next) {
const { doc_id: docId, project_id: projectId } = req.params
logger.debug({ projectId, docId }, 'patching doc')
@@ -159,8 +203,12 @@ async function patchDoc(req, res) {
logger.fatal({ field }, 'joi validation for pathDoc is broken')
}
})
- await DocManager.patchDoc(projectId, docId, meta)
- res.sendStatus(204)
+ DocManager.patchDoc(projectId, docId, meta, function (error) {
+ if (error) {
+ return next(error)
+ }
+ res.sendStatus(204)
+ })
}
function _buildDocView(doc) {
@@ -173,6 +221,10 @@ function _buildDocView(doc) {
return docView
}
+function _buildRawDocView(doc) {
+ return (doc?.lines ?? []).join('\n')
+}
+
function _buildDocsArrayView(projectId, docs) {
const docViews = []
for (const doc of docs) {
@@ -189,69 +241,79 @@ function _buildDocsArrayView(projectId, docs) {
return docViews
}
-async function archiveAllDocs(req, res) {
+function archiveAllDocs(req, res, next) {
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'archiving all docs')
- await DocArchive.archiveAllDocs(projectId)
- res.sendStatus(204)
+ DocArchive.archiveAllDocs(projectId, function (error) {
+ if (error) {
+ return next(error)
+ }
+ res.sendStatus(204)
+ })
}
-async function archiveDoc(req, res) {
+function archiveDoc(req, res, next) {
const { doc_id: docId, project_id: projectId } = req.params
logger.debug({ projectId, docId }, 'archiving a doc')
- await DocArchive.archiveDoc(projectId, docId)
- res.sendStatus(204)
+ DocArchive.archiveDoc(projectId, docId, function (error) {
+ if (error) {
+ return next(error)
+ }
+ res.sendStatus(204)
+ })
}
-async function unArchiveAllDocs(req, res) {
+function unArchiveAllDocs(req, res, next) {
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'unarchiving all docs')
- try {
- await DocArchive.unArchiveAllDocs(projectId)
- } catch (err) {
- if (err instanceof Errors.DocRevValueError) {
- logger.warn({ err }, 'Failed to unarchive doc')
- return res.sendStatus(409)
+ DocArchive.unArchiveAllDocs(projectId, function (err) {
+ if (err) {
+ if (err instanceof Errors.DocRevValueError) {
+ logger.warn({ err }, 'Failed to unarchive doc')
+ return res.sendStatus(409)
+ }
+ return next(err)
}
- throw err
- }
- res.sendStatus(200)
+ res.sendStatus(200)
+ })
}
-async function destroyProject(req, res) {
+function destroyProject(req, res, next) {
const { project_id: projectId } = req.params
logger.debug({ projectId }, 'destroying all docs')
- await DocArchive.destroyProject(projectId)
- res.sendStatus(204)
+ DocArchive.destroyProject(projectId, function (error) {
+ if (error) {
+ return next(error)
+ }
+ res.sendStatus(204)
+ })
}
-async function healthCheck(req, res) {
- try {
- await HealthChecker.check()
- } catch (err) {
- logger.err({ err }, 'error performing health check')
- res.sendStatus(500)
- return
- }
- res.sendStatus(200)
+function healthCheck(req, res) {
+ HealthChecker.check(function (err) {
+ if (err) {
+ logger.err({ err }, 'error performing health check')
+ res.sendStatus(500)
+ } else {
+ res.sendStatus(200)
+ }
+ })
}
module.exports = {
- getDoc: expressify(getDoc),
- peekDoc: expressify(peekDoc),
- isDocDeleted: expressify(isDocDeleted),
- getRawDoc: expressify(getRawDoc),
- getAllDocs: expressify(getAllDocs),
- getAllDeletedDocs: expressify(getAllDeletedDocs),
- getAllRanges: expressify(getAllRanges),
- getTrackedChangesUserIds: expressify(getTrackedChangesUserIds),
- getCommentThreadIds: expressify(getCommentThreadIds),
- projectHasRanges: expressify(projectHasRanges),
- updateDoc: expressify(updateDoc),
- patchDoc: expressify(patchDoc),
- archiveAllDocs: expressify(archiveAllDocs),
- archiveDoc: expressify(archiveDoc),
- unArchiveAllDocs: expressify(unArchiveAllDocs),
- destroyProject: expressify(destroyProject),
- healthCheck: expressify(healthCheck),
+ getDoc,
+ peekDoc,
+ isDocDeleted,
+ getRawDoc,
+ getAllDocs,
+ getAllDeletedDocs,
+ getAllRanges,
+ projectHasRanges,
+ updateDoc,
+ patchDoc,
+ archiveAllDocs,
+ archiveDoc,
+ unArchiveAllDocs,
+ destroyProject,
+ healthCheck,
}
diff --git a/services/docstore/app/js/MongoManager.js b/services/docstore/app/js/MongoManager.js
index ef101f91c0..87d8af8a1b 100644
--- a/services/docstore/app/js/MongoManager.js
+++ b/services/docstore/app/js/MongoManager.js
@@ -1,6 +1,7 @@
const { db, ObjectId } = require('./mongodb')
const Settings = require('@overleaf/settings')
const Errors = require('./Errors')
+const { callbackify } = require('util')
const ARCHIVING_LOCK_DURATION_MS = Settings.archivingLockDurationMs
@@ -240,17 +241,34 @@ async function destroyProject(projectId) {
}
module.exports = {
- findDoc,
- getProjectsDeletedDocs,
- getProjectsDocs,
- getArchivedProjectDocs,
- getNonArchivedProjectDocIds,
- getNonDeletedArchivedProjectDocs,
- upsertIntoDocCollection,
- restoreArchivedDoc,
- patchDoc,
- getDocForArchiving,
- markDocAsArchived,
- checkRevUnchanged,
- destroyProject,
+ findDoc: callbackify(findDoc),
+ getProjectsDeletedDocs: callbackify(getProjectsDeletedDocs),
+ getProjectsDocs: callbackify(getProjectsDocs),
+ getArchivedProjectDocs: callbackify(getArchivedProjectDocs),
+ getNonArchivedProjectDocIds: callbackify(getNonArchivedProjectDocIds),
+ getNonDeletedArchivedProjectDocs: callbackify(
+ getNonDeletedArchivedProjectDocs
+ ),
+ upsertIntoDocCollection: callbackify(upsertIntoDocCollection),
+ restoreArchivedDoc: callbackify(restoreArchivedDoc),
+ patchDoc: callbackify(patchDoc),
+ getDocForArchiving: callbackify(getDocForArchiving),
+ markDocAsArchived: callbackify(markDocAsArchived),
+ checkRevUnchanged: callbackify(checkRevUnchanged),
+ destroyProject: callbackify(destroyProject),
+ promises: {
+ findDoc,
+ getProjectsDeletedDocs,
+ getProjectsDocs,
+ getArchivedProjectDocs,
+ getNonArchivedProjectDocIds,
+ getNonDeletedArchivedProjectDocs,
+ upsertIntoDocCollection,
+ restoreArchivedDoc,
+ patchDoc,
+ getDocForArchiving,
+ markDocAsArchived,
+ checkRevUnchanged,
+ destroyProject,
+ },
}
diff --git a/services/docstore/app/js/RangeManager.js b/services/docstore/app/js/RangeManager.js
index 2fbadf9468..f36f68fe35 100644
--- a/services/docstore/app/js/RangeManager.js
+++ b/services/docstore/app/js/RangeManager.js
@@ -49,25 +49,15 @@ module.exports = RangeManager = {
updateMetadata(change.metadata)
}
for (const comment of Array.from(ranges.comments || [])) {
- // Two bugs resulted in mismatched ids, prefer the thread id from the op: https://github.com/overleaf/internal/issues/23272
- comment.id = RangeManager._safeObjectId(comment.op?.t || comment.id)
- if (comment.op) comment.op.t = comment.id
-
- // resolved property is added to comments when they are obtained from history, but this state doesn't belong in mongo docs collection
- // more info: https://github.com/overleaf/internal/issues/24371#issuecomment-2913095174
- delete comment.op?.resolved
+ comment.id = RangeManager._safeObjectId(comment.id)
+ if ((comment.op != null ? comment.op.t : undefined) != null) {
+ comment.op.t = RangeManager._safeObjectId(comment.op.t)
+ }
updateMetadata(comment.metadata)
}
return ranges
},
- fixCommentIds(doc) {
- for (const comment of doc?.ranges?.comments || []) {
- // Two bugs resulted in mismatched ids, prefer the thread id from the op: https://github.com/overleaf/internal/issues/23272
- if (comment.op?.t) comment.id = comment.op.t
- }
- },
-
_safeObjectId(data) {
try {
return new ObjectId(data)
diff --git a/services/docstore/app/js/StreamToBuffer.js b/services/docstore/app/js/StreamToBuffer.js
deleted file mode 100644
index 09215a7367..0000000000
--- a/services/docstore/app/js/StreamToBuffer.js
+++ /dev/null
@@ -1,24 +0,0 @@
-const { LoggerStream, WritableBuffer } = require('@overleaf/stream-utils')
-const Settings = require('@overleaf/settings')
-const logger = require('@overleaf/logger/logging-manager')
-const { pipeline } = require('node:stream/promises')
-
-module.exports = {
- streamToBuffer,
-}
-
-async function streamToBuffer(projectId, docId, stream) {
- const loggerTransform = new LoggerStream(
- Settings.max_doc_length,
- (size, isFlush) => {
- logger.warn(
- { projectId, docId, size, finishedReading: isFlush },
- 'potentially large doc pulled down from gcs'
- )
- }
- )
-
- const buffer = new WritableBuffer()
- await pipeline(stream, loggerTransform, buffer)
- return buffer.contents()
-}
diff --git a/services/docstore/buildscript.txt b/services/docstore/buildscript.txt
index 4526aa959c..be2a937cd7 100644
--- a/services/docstore/buildscript.txt
+++ b/services/docstore/buildscript.txt
@@ -4,6 +4,6 @@ docstore
--env-add=
--env-pass-through=
--esmock-loader=False
---node-version=22.17.0
+--node-version=18.20.2
--public-repo=True
---script-version=4.7.0
+--script-version=4.5.0
diff --git a/services/docstore/config/settings.defaults.js b/services/docstore/config/settings.defaults.js
index 9ad506a9bd..b91cdc87ec 100644
--- a/services/docstore/config/settings.defaults.js
+++ b/services/docstore/config/settings.defaults.js
@@ -1,9 +1,5 @@
-const http = require('node:http')
-const https = require('node:https')
-
+const http = require('http')
http.globalAgent.maxSockets = 300
-http.globalAgent.keepAlive = false
-https.globalAgent.keepAlive = false
const Settings = {
internal: {
diff --git a/services/docstore/docker-compose.ci.yml b/services/docstore/docker-compose.ci.yml
index cdb4783c5a..a8847e8996 100644
--- a/services/docstore/docker-compose.ci.yml
+++ b/services/docstore/docker-compose.ci.yml
@@ -27,15 +27,12 @@ services:
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
- volumes:
- - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
- condition: service_started
+ condition: service_healthy
gcs:
condition: service_healthy
user: node
- entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run test:acceptance
@@ -47,17 +44,12 @@ services:
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
mongo:
- image: mongo:8.0.11
+ image: mongo:6.0.13
command: --replSet overleaf
- volumes:
- - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
- environment:
- MONGO_INITDB_DATABASE: sharelatex
- extra_hosts:
- # Required when using the automatic database setup for initializing the
- # replica set. This override is not needed when running the setup after
- # starting up mongo.
- - mongo:127.0.0.1
+ healthcheck:
+ test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
+ interval: 1s
+ retries: 20
gcs:
image: fsouza/fake-gcs-server:1.45.2
command: ["--port=9090", "--scheme=http"]
diff --git a/services/docstore/docker-compose.yml b/services/docstore/docker-compose.yml
index a9099c7e7b..4555d6ed54 100644
--- a/services/docstore/docker-compose.yml
+++ b/services/docstore/docker-compose.yml
@@ -6,7 +6,7 @@ version: "2.3"
services:
test_unit:
- image: node:22.17.0
+ image: node:18.20.2
volumes:
- .:/overleaf/services/docstore
- ../../node_modules:/overleaf/node_modules
@@ -14,19 +14,17 @@ services:
working_dir: /overleaf/services/docstore
environment:
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
user: node
test_acceptance:
- image: node:22.17.0
+ image: node:18.20.2
volumes:
- .:/overleaf/services/docstore
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
working_dir: /overleaf/services/docstore
environment:
ELASTIC_SEARCH_DSN: es:9200
@@ -36,30 +34,24 @@ services:
GCS_PROJECT_ID: fake
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
+ LOG_LEVEL: ERROR
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
user: node
depends_on:
mongo:
- condition: service_started
+ condition: service_healthy
gcs:
condition: service_healthy
- entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run --silent test:acceptance
mongo:
- image: mongo:8.0.11
+ image: mongo:6.0.13
command: --replSet overleaf
- volumes:
- - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
- environment:
- MONGO_INITDB_DATABASE: sharelatex
- extra_hosts:
- # Required when using the automatic database setup for initializing the
- # replica set. This override is not needed when running the setup after
- # starting up mongo.
- - mongo:127.0.0.1
+ healthcheck:
+ test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
+ interval: 1s
+ retries: 20
gcs:
image: fsouza/fake-gcs-server:1.45.2
diff --git a/services/docstore/package.json b/services/docstore/package.json
index bf5857fd49..5ffd892f09 100644
--- a/services/docstore/package.json
+++ b/services/docstore/package.json
@@ -11,13 +11,12 @@
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
- "format": "prettier --list-different $PWD/'**/*.*js'",
- "format:fix": "prettier --write $PWD/'**/*.*js'",
+ "format": "prettier --list-different $PWD/'**/*.js'",
+ "format:fix": "prettier --write $PWD/'**/*.js'",
"lint:fix": "eslint --fix .",
"types:check": "tsc --noEmit"
},
"dependencies": {
- "@overleaf/fetch-utils": "*",
"@overleaf/logger": "*",
"@overleaf/metrics": "*",
"@overleaf/o-error": "*",
@@ -29,9 +28,9 @@
"body-parser": "^1.20.3",
"bunyan": "^1.8.15",
"celebrate": "^15.0.3",
- "express": "^4.21.2",
+ "express": "^4.21.0",
"lodash": "^4.17.21",
- "mongodb-legacy": "6.1.3",
+ "mongodb-legacy": "^6.0.1",
"p-map": "^4.0.0",
"request": "^2.88.2"
},
@@ -39,7 +38,7 @@
"@google-cloud/storage": "^6.10.1",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
- "mocha": "^11.1.0",
+ "mocha": "^10.2.0",
"sandboxed-module": "~2.0.4",
"sinon": "~9.0.2",
"sinon-chai": "^3.7.0",
diff --git a/services/docstore/test/acceptance/deps/healthcheck.sh b/services/docstore/test/acceptance/deps/healthcheck.sh
index 675c205be6..cd19cea637 100644
--- a/services/docstore/test/acceptance/deps/healthcheck.sh
+++ b/services/docstore/test/acceptance/deps/healthcheck.sh
@@ -1,9 +1,9 @@
#!/bin/sh
# health check to allow 404 status code as valid
-STATUSCODE=$(curl --silent --output /dev/null --write-out "%{http_code}" "$1")
+STATUSCODE=$(curl --silent --output /dev/null --write-out "%{http_code}" $1)
# will be 000 on non-http error (e.g. connection failure)
-if test "$STATUSCODE" -ge 500 || test "$STATUSCODE" -lt 200; then
+if test $STATUSCODE -ge 500 || test $STATUSCODE -lt 200; then
exit 1
fi
exit 0
diff --git a/services/docstore/test/acceptance/js/ArchiveDocsTests.js b/services/docstore/test/acceptance/js/ArchiveDocsTests.js
index 7e254c7e84..f85b845a18 100644
--- a/services/docstore/test/acceptance/js/ArchiveDocsTests.js
+++ b/services/docstore/test/acceptance/js/ArchiveDocsTests.js
@@ -399,7 +399,7 @@ describe('Archiving', function () {
this.project_id = new ObjectId()
this.timeout(1000 * 30)
const quarterMegInBytes = 250000
- const bigLine = require('node:crypto')
+ const bigLine = require('crypto')
.randomBytes(quarterMegInBytes)
.toString('hex')
this.doc = {
@@ -1001,15 +1001,6 @@ describe('Archiving', function () {
},
version: 2,
}
- this.fixedRanges = {
- ...this.doc.ranges,
- comments: [
- {
- ...this.doc.ranges.comments[0],
- id: this.doc.ranges.comments[0].op.t,
- },
- ],
- }
return DocstoreClient.createDoc(
this.project_id,
this.doc._id,
@@ -1057,7 +1048,7 @@ describe('Archiving', function () {
throw error
}
s3Doc.lines.should.deep.equal(this.doc.lines)
- const ranges = JSON.parse(JSON.stringify(this.fixedRanges)) // ObjectId -> String
+ const ranges = JSON.parse(JSON.stringify(this.doc.ranges)) // ObjectId -> String
s3Doc.ranges.should.deep.equal(ranges)
return done()
}
@@ -1084,7 +1075,7 @@ describe('Archiving', function () {
throw error
}
doc.lines.should.deep.equal(this.doc.lines)
- doc.ranges.should.deep.equal(this.fixedRanges)
+ doc.ranges.should.deep.equal(this.doc.ranges)
expect(doc.inS3).not.to.exist
return done()
})
diff --git a/services/docstore/test/acceptance/js/GettingAllDocsTests.js b/services/docstore/test/acceptance/js/GettingAllDocsTests.js
index 57851b2c3b..8fe5e7d91b 100644
--- a/services/docstore/test/acceptance/js/GettingAllDocsTests.js
+++ b/services/docstore/test/acceptance/js/GettingAllDocsTests.js
@@ -20,73 +20,30 @@ const DocstoreClient = require('./helpers/DocstoreClient')
describe('Getting all docs', function () {
beforeEach(function (done) {
this.project_id = new ObjectId()
- this.threadId1 = new ObjectId().toString()
- this.threadId2 = new ObjectId().toString()
this.docs = [
{
_id: new ObjectId(),
lines: ['one', 'two', 'three'],
- ranges: {
- comments: [
- { id: new ObjectId().toString(), op: { t: this.threadId1 } },
- ],
- changes: [
- {
- id: new ObjectId().toString(),
- metadata: { user_id: 'user-id-1' },
- },
- ],
- },
+ ranges: { mock: 'one' },
rev: 2,
},
{
_id: new ObjectId(),
lines: ['aaa', 'bbb', 'ccc'],
- ranges: {
- changes: [
- {
- id: new ObjectId().toString(),
- metadata: { user_id: 'user-id-2' },
- },
- ],
- },
+ ranges: { mock: 'two' },
rev: 4,
},
{
_id: new ObjectId(),
lines: ['111', '222', '333'],
- ranges: {
- comments: [
- { id: new ObjectId().toString(), op: { t: this.threadId2 } },
- ],
- changes: [
- {
- id: new ObjectId().toString(),
- metadata: { user_id: 'anonymous-user' },
- },
- ],
- },
+ ranges: { mock: 'three' },
rev: 6,
},
]
- this.fixedRanges = this.docs.map(doc => {
- if (!doc.ranges?.comments?.length) return doc.ranges
- return {
- ...doc.ranges,
- comments: [
- { ...doc.ranges.comments[0], id: doc.ranges.comments[0].op.t },
- ],
- }
- })
this.deleted_doc = {
_id: new ObjectId(),
lines: ['deleted'],
- ranges: {
- comments: [{ id: new ObjectId().toString(), op: { t: 'thread-id-3' } }],
- changes: [
- { id: new ObjectId().toString(), metadata: { user_id: 'user-id-3' } },
- ],
- },
+ ranges: { mock: 'four' },
rev: 8,
}
const version = 42
@@ -139,7 +96,7 @@ describe('Getting all docs', function () {
})
})
- it('getAllRanges should return all the (non-deleted) doc ranges', function (done) {
+ return it('getAllRanges should return all the (non-deleted) doc ranges', function (done) {
return DocstoreClient.getAllRanges(this.project_id, (error, res, docs) => {
if (error != null) {
throw error
@@ -147,38 +104,9 @@ describe('Getting all docs', function () {
docs.length.should.equal(this.docs.length)
for (let i = 0; i < docs.length; i++) {
const doc = docs[i]
- doc.ranges.should.deep.equal(this.fixedRanges[i])
+ doc.ranges.should.deep.equal(this.docs[i].ranges)
}
return done()
})
})
-
- it('getTrackedChangesUserIds should return all the user ids from (non-deleted) ranges', function (done) {
- DocstoreClient.getTrackedChangesUserIds(
- this.project_id,
- (error, res, userIds) => {
- if (error != null) {
- throw error
- }
- userIds.should.deep.equal(['user-id-1', 'user-id-2'])
- done()
- }
- )
- })
-
- it('getCommentThreadIds should return all the thread ids from (non-deleted) ranges', function (done) {
- DocstoreClient.getCommentThreadIds(
- this.project_id,
- (error, res, threadIds) => {
- if (error != null) {
- throw error
- }
- threadIds.should.deep.equal({
- [this.docs[0]._id.toString()]: [this.threadId1],
- [this.docs[2]._id.toString()]: [this.threadId2],
- })
- done()
- }
- )
- })
})
diff --git a/services/docstore/test/acceptance/js/GettingDocsTests.js b/services/docstore/test/acceptance/js/GettingDocsTests.js
index 1cfc53c5c6..121b3c1e24 100644
--- a/services/docstore/test/acceptance/js/GettingDocsTests.js
+++ b/services/docstore/test/acceptance/js/GettingDocsTests.js
@@ -28,26 +28,10 @@ describe('Getting a doc', function () {
op: { i: 'foo', p: 3 },
meta: {
user_id: new ObjectId().toString(),
- ts: new Date().toJSON(),
+ ts: new Date().toString(),
},
},
],
- comments: [
- {
- id: new ObjectId().toString(),
- op: { c: 'comment', p: 1, t: new ObjectId().toString() },
- metadata: {
- user_id: new ObjectId().toString(),
- ts: new Date().toJSON(),
- },
- },
- ],
- }
- this.fixedRanges = {
- ...this.ranges,
- comments: [
- { ...this.ranges.comments[0], id: this.ranges.comments[0].op.t },
- ],
}
return DocstoreApp.ensureRunning(() => {
return DocstoreClient.createDoc(
@@ -76,7 +60,7 @@ describe('Getting a doc', function () {
if (error) return done(error)
doc.lines.should.deep.equal(this.lines)
doc.version.should.equal(this.version)
- doc.ranges.should.deep.equal(this.fixedRanges)
+ doc.ranges.should.deep.equal(this.ranges)
return done()
}
)
@@ -130,7 +114,7 @@ describe('Getting a doc', function () {
if (error) return done(error)
doc.lines.should.deep.equal(this.lines)
doc.version.should.equal(this.version)
- doc.ranges.should.deep.equal(this.fixedRanges)
+ doc.ranges.should.deep.equal(this.ranges)
doc.deleted.should.equal(true)
return done()
}
diff --git a/services/docstore/test/acceptance/js/HealthCheckerTest.js b/services/docstore/test/acceptance/js/HealthCheckerTest.js
deleted file mode 100644
index b25a45312b..0000000000
--- a/services/docstore/test/acceptance/js/HealthCheckerTest.js
+++ /dev/null
@@ -1,28 +0,0 @@
-const { db } = require('../../../app/js/mongodb')
-const DocstoreApp = require('./helpers/DocstoreApp')
-const DocstoreClient = require('./helpers/DocstoreClient')
-const { expect } = require('chai')
-
-describe('HealthChecker', function () {
- beforeEach('start', function (done) {
- DocstoreApp.ensureRunning(done)
- })
- beforeEach('clear docs collection', async function () {
- await db.docs.deleteMany({})
- })
- let res
- beforeEach('run health check', function (done) {
- DocstoreClient.healthCheck((err, _res) => {
- res = _res
- done(err)
- })
- })
-
- it('should return 200', function () {
- res.statusCode.should.equal(200)
- })
-
- it('should not leave any cruft behind', async function () {
- expect(await db.docs.find({}).toArray()).to.deep.equal([])
- })
-})
diff --git a/services/docstore/test/acceptance/js/helpers/DocstoreApp.js b/services/docstore/test/acceptance/js/helpers/DocstoreApp.js
index 5e837b1277..03db0ea322 100644
--- a/services/docstore/test/acceptance/js/helpers/DocstoreApp.js
+++ b/services/docstore/test/acceptance/js/helpers/DocstoreApp.js
@@ -1,4 +1,5 @@
const app = require('../../../../app')
+require('@overleaf/logger').logger.level('error')
const settings = require('@overleaf/settings')
module.exports = {
diff --git a/services/docstore/test/acceptance/js/helpers/DocstoreClient.js b/services/docstore/test/acceptance/js/helpers/DocstoreClient.js
index cb8bce2579..790ec8f237 100644
--- a/services/docstore/test/acceptance/js/helpers/DocstoreClient.js
+++ b/services/docstore/test/acceptance/js/helpers/DocstoreClient.js
@@ -100,26 +100,6 @@ module.exports = DocstoreClient = {
)
},
- getCommentThreadIds(projectId, callback) {
- request.get(
- {
- url: `http://127.0.0.1:${settings.internal.docstore.port}/project/${projectId}/comment-thread-ids`,
- json: true,
- },
- callback
- )
- },
-
- getTrackedChangesUserIds(projectId, callback) {
- request.get(
- {
- url: `http://127.0.0.1:${settings.internal.docstore.port}/project/${projectId}/tracked-changes-user-ids`,
- json: true,
- },
- callback
- )
- },
-
updateDoc(projectId, docId, lines, version, ranges, callback) {
return request.post(
{
@@ -201,13 +181,6 @@ module.exports = DocstoreClient = {
)
},
- healthCheck(callback) {
- request.get(
- `http://127.0.0.1:${settings.internal.docstore.port}/health_check`,
- callback
- )
- },
-
getS3Doc(projectId, docId, callback) {
getStringFromPersistor(
Persistor,
diff --git a/services/docstore/test/setup.js b/services/docstore/test/setup.js
index 92b86c9384..809624a2ee 100644
--- a/services/docstore/test/setup.js
+++ b/services/docstore/test/setup.js
@@ -3,7 +3,7 @@ const sinon = require('sinon')
const sinonChai = require('sinon-chai')
const chaiAsPromised = require('chai-as-promised')
const SandboxedModule = require('sandboxed-module')
-const timersPromises = require('node:timers/promises')
+const timersPromises = require('timers/promises')
// ensure every ObjectId has the id string as a property for correct comparisons
require('mongodb-legacy').ObjectId.cacheHexString = true
@@ -37,11 +37,6 @@ SandboxedModule.configure({
'mongodb-legacy': require('mongodb-legacy'),
},
globals: { Buffer, JSON, Math, console, process },
- sourceTransformers: {
- removeNodePrefix: function (source) {
- return source.replace(/require\(['"]node:/g, "require('")
- },
- },
})
exports.mochaHooks = {
diff --git a/services/docstore/test/unit/js/DocArchiveManagerTests.js b/services/docstore/test/unit/js/DocArchiveManagerTests.js
index 2ec1cb2016..13046d86fc 100644
--- a/services/docstore/test/unit/js/DocArchiveManagerTests.js
+++ b/services/docstore/test/unit/js/DocArchiveManagerTests.js
@@ -4,7 +4,6 @@ const modulePath = '../../../app/js/DocArchiveManager.js'
const SandboxedModule = require('sandboxed-module')
const { ObjectId } = require('mongodb-legacy')
const Errors = require('../../../app/js/Errors')
-const StreamToBuffer = require('../../../app/js/StreamToBuffer')
describe('DocArchiveManager', function () {
let DocArchiveManager,
@@ -23,15 +22,13 @@ describe('DocArchiveManager', function () {
md5Sum,
projectId,
readStream,
- stream,
- streamToBuffer
+ stream
beforeEach(function () {
md5Sum = 'decafbad'
RangeManager = {
jsonRangesToMongo: sinon.stub().returns({ mongo: 'ranges' }),
- fixCommentIds: sinon.stub(),
}
Settings = {
docstore: {
@@ -143,33 +140,17 @@ describe('DocArchiveManager', function () {
}
MongoManager = {
- markDocAsArchived: sinon.stub().resolves(),
- restoreArchivedDoc: sinon.stub().resolves(),
- upsertIntoDocCollection: sinon.stub().resolves(),
- getProjectsDocs: sinon.stub().resolves(mongoDocs),
- getNonDeletedArchivedProjectDocs: getArchivedProjectDocs,
- getNonArchivedProjectDocIds,
- getArchivedProjectDocs,
- findDoc: sinon.stub().callsFake(fakeGetDoc),
- getDocForArchiving: sinon.stub().callsFake(fakeGetDoc),
- destroyProject: sinon.stub().resolves(),
- }
-
- // Wrap streamToBuffer so that we can pass in something that it expects (in
- // this case, a Promise) rather than a stubbed stream object
- streamToBuffer = {
- streamToBuffer: async () => {
- const inputStream = new Promise(resolve => {
- stream.on('data', data => resolve(data))
- })
-
- const value = await StreamToBuffer.streamToBuffer(
- 'testProjectId',
- 'testDocId',
- inputStream
- )
-
- return value
+ promises: {
+ markDocAsArchived: sinon.stub().resolves(),
+ restoreArchivedDoc: sinon.stub().resolves(),
+ upsertIntoDocCollection: sinon.stub().resolves(),
+ getProjectsDocs: sinon.stub().resolves(mongoDocs),
+ getNonDeletedArchivedProjectDocs: getArchivedProjectDocs,
+ getNonArchivedProjectDocIds,
+ getArchivedProjectDocs,
+ findDoc: sinon.stub().callsFake(fakeGetDoc),
+ getDocForArchiving: sinon.stub().callsFake(fakeGetDoc),
+ destroyProject: sinon.stub().resolves(),
},
}
@@ -182,20 +163,15 @@ describe('DocArchiveManager', function () {
'./RangeManager': RangeManager,
'./PersistorManager': PersistorManager,
'./Errors': Errors,
- './StreamToBuffer': streamToBuffer,
},
})
})
describe('archiveDoc', function () {
it('should resolve when passed a valid document', async function () {
- await expect(DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)).to
- .eventually.be.fulfilled
- })
-
- it('should fix comment ids', async function () {
- await DocArchiveManager.archiveDoc(projectId, mongoDocs[1]._id)
- expect(RangeManager.fixCommentIds).to.have.been.called
+ await expect(
+ DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
+ ).to.eventually.be.fulfilled
})
it('should throw an error if the doc has no lines', async function () {
@@ -203,26 +179,26 @@ describe('DocArchiveManager', function () {
doc.lines = null
await expect(
- DocArchiveManager.archiveDoc(projectId, doc._id)
+ DocArchiveManager.promises.archiveDoc(projectId, doc._id)
).to.eventually.be.rejectedWith('doc has no lines')
})
it('should add the schema version', async function () {
- await DocArchiveManager.archiveDoc(projectId, mongoDocs[1]._id)
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[1]._id)
expect(StreamUtils.ReadableString).to.have.been.calledWith(
sinon.match(/"schema_v":1/)
)
})
it('should calculate the hex md5 sum of the content', async function () {
- await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
expect(Crypto.createHash).to.have.been.calledWith('md5')
expect(HashUpdate).to.have.been.calledWith(archivedDocJson)
expect(HashDigest).to.have.been.calledWith('hex')
})
it('should pass the md5 hash to the object persistor for verification', async function () {
- await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
expect(PersistorManager.sendStream).to.have.been.calledWith(
sinon.match.any,
@@ -233,7 +209,7 @@ describe('DocArchiveManager', function () {
})
it('should pass the correct bucket and key to the persistor', async function () {
- await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
expect(PersistorManager.sendStream).to.have.been.calledWith(
Settings.docstore.bucket,
@@ -242,7 +218,7 @@ describe('DocArchiveManager', function () {
})
it('should create a stream from the encoded json and send it', async function () {
- await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
expect(StreamUtils.ReadableString).to.have.been.calledWith(
archivedDocJson
)
@@ -254,8 +230,8 @@ describe('DocArchiveManager', function () {
})
it('should mark the doc as archived', async function () {
- await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
- expect(MongoManager.markDocAsArchived).to.have.been.calledWith(
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
+ expect(MongoManager.promises.markDocAsArchived).to.have.been.calledWith(
projectId,
mongoDocs[0]._id,
mongoDocs[0].rev
@@ -268,8 +244,8 @@ describe('DocArchiveManager', function () {
})
it('should bail out early', async function () {
- await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
- expect(MongoManager.getDocForArchiving).to.not.have.been.called
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
+ expect(MongoManager.promises.getDocForArchiving).to.not.have.been.called
})
})
@@ -286,7 +262,7 @@ describe('DocArchiveManager', function () {
it('should return an error', async function () {
await expect(
- DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
+ DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
).to.eventually.be.rejectedWith('null bytes detected')
})
})
@@ -297,19 +273,21 @@ describe('DocArchiveManager', function () {
describe('when the doc is in S3', function () {
beforeEach(function () {
- MongoManager.findDoc = sinon.stub().resolves({ inS3: true, rev })
+ MongoManager.promises.findDoc = sinon
+ .stub()
+ .resolves({ inS3: true, rev })
docId = mongoDocs[0]._id
lines = ['doc', 'lines']
rev = 123
})
it('should resolve when passed a valid document', async function () {
- await expect(DocArchiveManager.unarchiveDoc(projectId, docId)).to
- .eventually.be.fulfilled
+ await expect(DocArchiveManager.promises.unarchiveDoc(projectId, docId))
+ .to.eventually.be.fulfilled
})
it('should test md5 validity with the raw buffer', async function () {
- await DocArchiveManager.unarchiveDoc(projectId, docId)
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
expect(HashUpdate).to.have.been.calledWith(
sinon.match.instanceOf(Buffer)
)
@@ -318,17 +296,15 @@ describe('DocArchiveManager', function () {
it('should throw an error if the md5 does not match', async function () {
PersistorManager.getObjectMd5Hash.resolves('badf00d')
await expect(
- DocArchiveManager.unarchiveDoc(projectId, docId)
+ DocArchiveManager.promises.unarchiveDoc(projectId, docId)
).to.eventually.be.rejected.and.be.instanceof(Errors.Md5MismatchError)
})
it('should restore the doc in Mongo', async function () {
- await DocArchiveManager.unarchiveDoc(projectId, docId)
- expect(MongoManager.restoreArchivedDoc).to.have.been.calledWith(
- projectId,
- docId,
- archivedDoc
- )
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ expect(
+ MongoManager.promises.restoreArchivedDoc
+ ).to.have.been.calledWith(projectId, docId, archivedDoc)
})
describe('when archiving is not configured', function () {
@@ -338,15 +314,15 @@ describe('DocArchiveManager', function () {
it('should error out on archived doc', async function () {
await expect(
- DocArchiveManager.unarchiveDoc(projectId, docId)
+ DocArchiveManager.promises.unarchiveDoc(projectId, docId)
).to.eventually.be.rejected.and.match(
/found archived doc, but archiving backend is not configured/
)
})
it('should return early on non-archived doc', async function () {
- MongoManager.findDoc = sinon.stub().resolves({ rev })
- await DocArchiveManager.unarchiveDoc(projectId, docId)
+ MongoManager.promises.findDoc = sinon.stub().resolves({ rev })
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
expect(PersistorManager.getObjectMd5Hash).to.not.have.been.called
})
})
@@ -364,12 +340,10 @@ describe('DocArchiveManager', function () {
})
it('should return the docs lines', async function () {
- await DocArchiveManager.unarchiveDoc(projectId, docId)
- expect(MongoManager.restoreArchivedDoc).to.have.been.calledWith(
- projectId,
- docId,
- { lines, rev }
- )
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ expect(
+ MongoManager.promises.restoreArchivedDoc
+ ).to.have.been.calledWith(projectId, docId, { lines, rev })
})
})
@@ -388,16 +362,14 @@ describe('DocArchiveManager', function () {
})
it('should return the doc lines and ranges', async function () {
- await DocArchiveManager.unarchiveDoc(projectId, docId)
- expect(MongoManager.restoreArchivedDoc).to.have.been.calledWith(
- projectId,
- docId,
- {
- lines,
- ranges: { mongo: 'ranges' },
- rev: 456,
- }
- )
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ expect(
+ MongoManager.promises.restoreArchivedDoc
+ ).to.have.been.calledWith(projectId, docId, {
+ lines,
+ ranges: { mongo: 'ranges' },
+ rev: 456,
+ })
})
})
@@ -411,12 +383,10 @@ describe('DocArchiveManager', function () {
})
it('should return only the doc lines', async function () {
- await DocArchiveManager.unarchiveDoc(projectId, docId)
- expect(MongoManager.restoreArchivedDoc).to.have.been.calledWith(
- projectId,
- docId,
- { lines, rev: 456 }
- )
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ expect(
+ MongoManager.promises.restoreArchivedDoc
+ ).to.have.been.calledWith(projectId, docId, { lines, rev: 456 })
})
})
@@ -430,12 +400,10 @@ describe('DocArchiveManager', function () {
})
it('should use the rev obtained from Mongo', async function () {
- await DocArchiveManager.unarchiveDoc(projectId, docId)
- expect(MongoManager.restoreArchivedDoc).to.have.been.calledWith(
- projectId,
- docId,
- { lines, rev }
- )
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
+ expect(
+ MongoManager.promises.restoreArchivedDoc
+ ).to.have.been.calledWith(projectId, docId, { lines, rev })
})
})
@@ -450,7 +418,7 @@ describe('DocArchiveManager', function () {
it('should throw an error', async function () {
await expect(
- DocArchiveManager.unarchiveDoc(projectId, docId)
+ DocArchiveManager.promises.unarchiveDoc(projectId, docId)
).to.eventually.be.rejectedWith(
"I don't understand the doc format in s3"
)
@@ -460,8 +428,8 @@ describe('DocArchiveManager', function () {
})
it('should not do anything if the file is already unarchived', async function () {
- MongoManager.findDoc.resolves({ inS3: false })
- await DocArchiveManager.unarchiveDoc(projectId, docId)
+ MongoManager.promises.findDoc.resolves({ inS3: false })
+ await DocArchiveManager.promises.unarchiveDoc(projectId, docId)
expect(PersistorManager.getObjectStream).not.to.have.been.called
})
@@ -470,7 +438,7 @@ describe('DocArchiveManager', function () {
.stub()
.rejects(new Errors.NotFoundError())
await expect(
- DocArchiveManager.unarchiveDoc(projectId, docId)
+ DocArchiveManager.promises.unarchiveDoc(projectId, docId)
).to.eventually.be.rejected.and.be.instanceof(Errors.NotFoundError)
})
})
@@ -478,11 +446,13 @@ describe('DocArchiveManager', function () {
describe('destroyProject', function () {
describe('when archiving is enabled', function () {
beforeEach(async function () {
- await DocArchiveManager.destroyProject(projectId)
+ await DocArchiveManager.promises.destroyProject(projectId)
})
it('should delete the project in Mongo', function () {
- expect(MongoManager.destroyProject).to.have.been.calledWith(projectId)
+ expect(MongoManager.promises.destroyProject).to.have.been.calledWith(
+ projectId
+ )
})
it('should delete the project in the persistor', function () {
@@ -496,11 +466,13 @@ describe('DocArchiveManager', function () {
describe('when archiving is disabled', function () {
beforeEach(async function () {
Settings.docstore.backend = ''
- await DocArchiveManager.destroyProject(projectId)
+ await DocArchiveManager.promises.destroyProject(projectId)
})
it('should delete the project in Mongo', function () {
- expect(MongoManager.destroyProject).to.have.been.calledWith(projectId)
+ expect(MongoManager.promises.destroyProject).to.have.been.calledWith(
+ projectId
+ )
})
it('should not delete the project in the persistor', function () {
@@ -511,35 +483,33 @@ describe('DocArchiveManager', function () {
describe('archiveAllDocs', function () {
it('should resolve with valid arguments', async function () {
- await expect(DocArchiveManager.archiveAllDocs(projectId)).to.eventually.be
- .fulfilled
+ await expect(DocArchiveManager.promises.archiveAllDocs(projectId)).to
+ .eventually.be.fulfilled
})
it('should archive all project docs which are not in s3', async function () {
- await DocArchiveManager.archiveAllDocs(projectId)
+ await DocArchiveManager.promises.archiveAllDocs(projectId)
// not inS3
- expect(MongoManager.markDocAsArchived).to.have.been.calledWith(
+ expect(MongoManager.promises.markDocAsArchived).to.have.been.calledWith(
projectId,
mongoDocs[0]._id
)
- expect(MongoManager.markDocAsArchived).to.have.been.calledWith(
+ expect(MongoManager.promises.markDocAsArchived).to.have.been.calledWith(
projectId,
mongoDocs[1]._id
)
- expect(MongoManager.markDocAsArchived).to.have.been.calledWith(
+ expect(MongoManager.promises.markDocAsArchived).to.have.been.calledWith(
projectId,
mongoDocs[4]._id
)
// inS3
- expect(MongoManager.markDocAsArchived).not.to.have.been.calledWith(
- projectId,
- mongoDocs[2]._id
- )
- expect(MongoManager.markDocAsArchived).not.to.have.been.calledWith(
- projectId,
- mongoDocs[3]._id
- )
+ expect(
+ MongoManager.promises.markDocAsArchived
+ ).not.to.have.been.calledWith(projectId, mongoDocs[2]._id)
+ expect(
+ MongoManager.promises.markDocAsArchived
+ ).not.to.have.been.calledWith(projectId, mongoDocs[3]._id)
})
describe('when archiving is not configured', function () {
@@ -548,20 +518,21 @@ describe('DocArchiveManager', function () {
})
it('should bail out early', async function () {
- await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
- expect(MongoManager.getNonArchivedProjectDocIds).to.not.have.been.called
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
+ expect(MongoManager.promises.getNonArchivedProjectDocIds).to.not.have
+ .been.called
})
})
})
describe('unArchiveAllDocs', function () {
it('should resolve with valid arguments', async function () {
- await expect(DocArchiveManager.unArchiveAllDocs(projectId)).to.eventually
- .be.fulfilled
+ await expect(DocArchiveManager.promises.unArchiveAllDocs(projectId)).to
+ .eventually.be.fulfilled
})
it('should unarchive all inS3 docs', async function () {
- await DocArchiveManager.unArchiveAllDocs(projectId)
+ await DocArchiveManager.promises.unArchiveAllDocs(projectId)
for (const doc of archivedDocs) {
expect(PersistorManager.getObjectStream).to.have.been.calledWith(
@@ -577,9 +548,9 @@ describe('DocArchiveManager', function () {
})
it('should bail out early', async function () {
- await DocArchiveManager.archiveDoc(projectId, mongoDocs[0]._id)
- expect(MongoManager.getNonDeletedArchivedProjectDocs).to.not.have.been
- .called
+ await DocArchiveManager.promises.archiveDoc(projectId, mongoDocs[0]._id)
+ expect(MongoManager.promises.getNonDeletedArchivedProjectDocs).to.not
+ .have.been.called
})
})
})
diff --git a/services/docstore/test/unit/js/DocManagerTests.js b/services/docstore/test/unit/js/DocManagerTests.js
index 67a2f26547..be833f4b3f 100644
--- a/services/docstore/test/unit/js/DocManagerTests.js
+++ b/services/docstore/test/unit/js/DocManagerTests.js
@@ -1,10 +1,7 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { expect } = require('chai')
-const modulePath = require('node:path').join(
- __dirname,
- '../../../app/js/DocManager'
-)
+const modulePath = require('path').join(__dirname, '../../../app/js/DocManager')
const { ObjectId } = require('mongodb-legacy')
const Errors = require('../../../app/js/Errors')
@@ -17,22 +14,25 @@ describe('DocManager', function () {
this.version = 42
this.MongoManager = {
- findDoc: sinon.stub(),
- getProjectsDocs: sinon.stub(),
- patchDoc: sinon.stub().resolves(),
- upsertIntoDocCollection: sinon.stub().resolves(),
+ promises: {
+ findDoc: sinon.stub(),
+ getProjectsDocs: sinon.stub(),
+ patchDoc: sinon.stub().resolves(),
+ upsertIntoDocCollection: sinon.stub().resolves(),
+ },
}
this.DocArchiveManager = {
- unarchiveDoc: sinon.stub(),
- unArchiveAllDocs: sinon.stub(),
- archiveDoc: sinon.stub().resolves(),
+ promises: {
+ unarchiveDoc: sinon.stub(),
+ unArchiveAllDocs: sinon.stub(),
+ archiveDoc: sinon.stub().resolves(),
+ },
}
this.RangeManager = {
jsonRangesToMongo(r) {
return r
},
shouldUpdateRanges: sinon.stub().returns(false),
- fixCommentIds: sinon.stub(),
}
this.settings = { docstore: {} }
@@ -49,7 +49,7 @@ describe('DocManager', function () {
describe('getFullDoc', function () {
beforeEach(function () {
- this.DocManager._getDoc = sinon.stub()
+ this.DocManager.promises._getDoc = sinon.stub()
this.doc = {
_id: this.doc_id,
lines: ['2134'],
@@ -57,10 +57,13 @@ describe('DocManager', function () {
})
it('should call get doc with a quick filter', async function () {
- this.DocManager._getDoc.resolves(this.doc)
- const doc = await this.DocManager.getFullDoc(this.project_id, this.doc_id)
+ this.DocManager.promises._getDoc.resolves(this.doc)
+ const doc = await this.DocManager.promises.getFullDoc(
+ this.project_id,
+ this.doc_id
+ )
doc.should.equal(this.doc)
- this.DocManager._getDoc
+ this.DocManager.promises._getDoc
.calledWith(this.project_id, this.doc_id, {
lines: true,
rev: true,
@@ -73,27 +76,27 @@ describe('DocManager', function () {
})
it('should return error when get doc errors', async function () {
- this.DocManager._getDoc.rejects(this.stubbedError)
+ this.DocManager.promises._getDoc.rejects(this.stubbedError)
await expect(
- this.DocManager.getFullDoc(this.project_id, this.doc_id)
+ this.DocManager.promises.getFullDoc(this.project_id, this.doc_id)
).to.be.rejectedWith(this.stubbedError)
})
})
describe('getRawDoc', function () {
beforeEach(function () {
- this.DocManager._getDoc = sinon.stub()
+ this.DocManager.promises._getDoc = sinon.stub()
this.doc = { lines: ['2134'] }
})
it('should call get doc with a quick filter', async function () {
- this.DocManager._getDoc.resolves(this.doc)
- const content = await this.DocManager.getDocLines(
+ this.DocManager.promises._getDoc.resolves(this.doc)
+ const doc = await this.DocManager.promises.getDocLines(
this.project_id,
this.doc_id
)
- content.should.equal(this.doc.lines.join('\n'))
- this.DocManager._getDoc
+ doc.should.equal(this.doc)
+ this.DocManager.promises._getDoc
.calledWith(this.project_id, this.doc_id, {
lines: true,
inS3: true,
@@ -102,46 +105,11 @@ describe('DocManager', function () {
})
it('should return error when get doc errors', async function () {
- this.DocManager._getDoc.rejects(this.stubbedError)
+ this.DocManager.promises._getDoc.rejects(this.stubbedError)
await expect(
- this.DocManager.getDocLines(this.project_id, this.doc_id)
+ this.DocManager.promises.getDocLines(this.project_id, this.doc_id)
).to.be.rejectedWith(this.stubbedError)
})
-
- it('should return error when get doc does not exist', async function () {
- this.DocManager._getDoc.resolves(null)
- await expect(
- this.DocManager.getDocLines(this.project_id, this.doc_id)
- ).to.be.rejectedWith(Errors.NotFoundError)
- })
-
- it('should return error when get doc has no lines', async function () {
- this.DocManager._getDoc.resolves({})
- await expect(
- this.DocManager.getDocLines(this.project_id, this.doc_id)
- ).to.be.rejectedWith(Errors.DocWithoutLinesError)
- })
- })
-
- describe('_getDoc', function () {
- it('should return error when get doc does not exist', async function () {
- this.MongoManager.findDoc.resolves(null)
- await expect(
- this.DocManager._getDoc(this.project_id, this.doc_id, { inS3: true })
- ).to.be.rejectedWith(Errors.NotFoundError)
- })
-
- it('should fix comment ids', async function () {
- this.MongoManager.findDoc.resolves({
- _id: this.doc_id,
- ranges: {},
- })
- await this.DocManager._getDoc(this.project_id, this.doc_id, {
- inS3: true,
- ranges: true,
- })
- expect(this.RangeManager.fixCommentIds).to.have.been.called
- })
})
describe('getDoc', function () {
@@ -157,25 +125,26 @@ describe('DocManager', function () {
describe('when using a filter', function () {
beforeEach(function () {
- this.MongoManager.findDoc.resolves(this.doc)
+ this.MongoManager.promises.findDoc.resolves(this.doc)
})
it('should error if inS3 is not set to true', async function () {
await expect(
- this.DocManager._getDoc(this.project_id, this.doc_id, {
+ this.DocManager.promises._getDoc(this.project_id, this.doc_id, {
inS3: false,
})
).to.be.rejected
})
it('should always get inS3 even when no filter is passed', async function () {
- await expect(this.DocManager._getDoc(this.project_id, this.doc_id)).to
- .be.rejected
- this.MongoManager.findDoc.called.should.equal(false)
+ await expect(
+ this.DocManager.promises._getDoc(this.project_id, this.doc_id)
+ ).to.be.rejected
+ this.MongoManager.promises.findDoc.called.should.equal(false)
})
it('should not error if inS3 is set to true', async function () {
- await this.DocManager._getDoc(this.project_id, this.doc_id, {
+ await this.DocManager.promises._getDoc(this.project_id, this.doc_id, {
inS3: true,
})
})
@@ -183,8 +152,8 @@ describe('DocManager', function () {
describe('when the doc is in the doc collection', function () {
beforeEach(async function () {
- this.MongoManager.findDoc.resolves(this.doc)
- this.result = await this.DocManager._getDoc(
+ this.MongoManager.promises.findDoc.resolves(this.doc)
+ this.result = await this.DocManager.promises._getDoc(
this.project_id,
this.doc_id,
{ version: true, inS3: true }
@@ -192,7 +161,7 @@ describe('DocManager', function () {
})
it('should get the doc from the doc collection', function () {
- this.MongoManager.findDoc
+ this.MongoManager.promises.findDoc
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
@@ -205,9 +174,9 @@ describe('DocManager', function () {
describe('when MongoManager.findDoc errors', function () {
it('should return the error', async function () {
- this.MongoManager.findDoc.rejects(this.stubbedError)
+ this.MongoManager.promises.findDoc.rejects(this.stubbedError)
await expect(
- this.DocManager._getDoc(this.project_id, this.doc_id, {
+ this.DocManager.promises._getDoc(this.project_id, this.doc_id, {
version: true,
inS3: true,
})
@@ -230,15 +199,15 @@ describe('DocManager', function () {
version: 2,
inS3: false,
}
- this.MongoManager.findDoc.resolves(this.doc)
- this.DocArchiveManager.unarchiveDoc.callsFake(
+ this.MongoManager.promises.findDoc.resolves(this.doc)
+ this.DocArchiveManager.promises.unarchiveDoc.callsFake(
async (projectId, docId) => {
- this.MongoManager.findDoc.resolves({
+ this.MongoManager.promises.findDoc.resolves({
...this.unarchivedDoc,
})
}
)
- this.result = await this.DocManager._getDoc(
+ this.result = await this.DocManager.promises._getDoc(
this.project_id,
this.doc_id,
{
@@ -249,13 +218,13 @@ describe('DocManager', function () {
})
it('should call the DocArchive to unarchive the doc', function () {
- this.DocArchiveManager.unarchiveDoc
+ this.DocArchiveManager.promises.unarchiveDoc
.calledWith(this.project_id, this.doc_id)
.should.equal(true)
})
it('should look up the doc twice', function () {
- this.MongoManager.findDoc.calledTwice.should.equal(true)
+ this.MongoManager.promises.findDoc.calledTwice.should.equal(true)
})
it('should return the doc', function () {
@@ -267,9 +236,9 @@ describe('DocManager', function () {
describe('when the doc does not exist in the docs collection', function () {
it('should return a NotFoundError', async function () {
- this.MongoManager.findDoc.resolves(null)
+ this.MongoManager.promises.findDoc.resolves(null)
await expect(
- this.DocManager._getDoc(this.project_id, this.doc_id, {
+ this.DocManager.promises._getDoc(this.project_id, this.doc_id, {
version: true,
inS3: true,
})
@@ -290,27 +259,23 @@ describe('DocManager', function () {
lines: ['mock-lines'],
},
]
- this.MongoManager.getProjectsDocs.resolves(this.docs)
- this.DocArchiveManager.unArchiveAllDocs.resolves(this.docs)
- this.filter = { lines: true, ranges: true }
- this.result = await this.DocManager.getAllNonDeletedDocs(
+ this.MongoManager.promises.getProjectsDocs.resolves(this.docs)
+ this.DocArchiveManager.promises.unArchiveAllDocs.resolves(this.docs)
+ this.filter = { lines: true }
+ this.result = await this.DocManager.promises.getAllNonDeletedDocs(
this.project_id,
this.filter
)
})
it('should get the project from the database', function () {
- this.MongoManager.getProjectsDocs.should.have.been.calledWith(
+ this.MongoManager.promises.getProjectsDocs.should.have.been.calledWith(
this.project_id,
{ include_deleted: false },
this.filter
)
})
- it('should fix comment ids', async function () {
- expect(this.RangeManager.fixCommentIds).to.have.been.called
- })
-
it('should return the docs', function () {
expect(this.result).to.deep.equal(this.docs)
})
@@ -318,10 +283,13 @@ describe('DocManager', function () {
describe('when there are no docs for the project', function () {
it('should return a NotFoundError', async function () {
- this.MongoManager.getProjectsDocs.resolves(null)
- this.DocArchiveManager.unArchiveAllDocs.resolves(null)
+ this.MongoManager.promises.getProjectsDocs.resolves(null)
+ this.DocArchiveManager.promises.unArchiveAllDocs.resolves(null)
await expect(
- this.DocManager.getAllNonDeletedDocs(this.project_id, this.filter)
+ this.DocManager.promises.getAllNonDeletedDocs(
+ this.project_id,
+ this.filter
+ )
).to.be.rejectedWith(`No docs for project ${this.project_id}`)
})
})
@@ -332,7 +300,7 @@ describe('DocManager', function () {
beforeEach(function () {
this.lines = ['mock', 'doc', 'lines']
this.rev = 77
- this.MongoManager.findDoc.resolves({
+ this.MongoManager.promises.findDoc.resolves({
_id: new ObjectId(this.doc_id),
})
this.meta = {}
@@ -340,7 +308,7 @@ describe('DocManager', function () {
describe('standard path', function () {
beforeEach(async function () {
- await this.DocManager.patchDoc(
+ await this.DocManager.promises.patchDoc(
this.project_id,
this.doc_id,
this.meta
@@ -348,14 +316,14 @@ describe('DocManager', function () {
})
it('should get the doc', function () {
- expect(this.MongoManager.findDoc).to.have.been.calledWith(
+ expect(this.MongoManager.promises.findDoc).to.have.been.calledWith(
this.project_id,
this.doc_id
)
})
it('should persist the meta', function () {
- expect(this.MongoManager.patchDoc).to.have.been.calledWith(
+ expect(this.MongoManager.promises.patchDoc).to.have.been.calledWith(
this.project_id,
this.doc_id,
this.meta
@@ -368,7 +336,7 @@ describe('DocManager', function () {
this.settings.docstore.archiveOnSoftDelete = false
this.meta.deleted = true
- await this.DocManager.patchDoc(
+ await this.DocManager.promises.patchDoc(
this.project_id,
this.doc_id,
this.meta
@@ -376,7 +344,8 @@ describe('DocManager', function () {
})
it('should not flush the doc out of mongo', function () {
- expect(this.DocArchiveManager.archiveDoc).to.not.have.been.called
+ expect(this.DocArchiveManager.promises.archiveDoc).to.not.have.been
+ .called
})
})
@@ -384,7 +353,7 @@ describe('DocManager', function () {
beforeEach(async function () {
this.settings.docstore.archiveOnSoftDelete = false
this.meta.deleted = false
- await this.DocManager.patchDoc(
+ await this.DocManager.promises.patchDoc(
this.project_id,
this.doc_id,
this.meta
@@ -392,7 +361,8 @@ describe('DocManager', function () {
})
it('should not flush the doc out of mongo', function () {
- expect(this.DocArchiveManager.archiveDoc).to.not.have.been.called
+ expect(this.DocArchiveManager.promises.archiveDoc).to.not.have.been
+ .called
})
})
@@ -404,7 +374,7 @@ describe('DocManager', function () {
describe('when the background flush succeeds', function () {
beforeEach(async function () {
- await this.DocManager.patchDoc(
+ await this.DocManager.promises.patchDoc(
this.project_id,
this.doc_id,
this.meta
@@ -416,18 +386,17 @@ describe('DocManager', function () {
})
it('should flush the doc out of mongo', function () {
- expect(this.DocArchiveManager.archiveDoc).to.have.been.calledWith(
- this.project_id,
- this.doc_id
- )
+ expect(
+ this.DocArchiveManager.promises.archiveDoc
+ ).to.have.been.calledWith(this.project_id, this.doc_id)
})
})
describe('when the background flush fails', function () {
beforeEach(async function () {
this.err = new Error('foo')
- this.DocArchiveManager.archiveDoc.rejects(this.err)
- await this.DocManager.patchDoc(
+ this.DocArchiveManager.promises.archiveDoc.rejects(this.err)
+ await this.DocManager.promises.patchDoc(
this.project_id,
this.doc_id,
this.meta
@@ -450,9 +419,9 @@ describe('DocManager', function () {
describe('when the doc does not exist', function () {
it('should return a NotFoundError', async function () {
- this.MongoManager.findDoc.resolves(null)
+ this.MongoManager.promises.findDoc.resolves(null)
await expect(
- this.DocManager.patchDoc(this.project_id, this.doc_id, {})
+ this.DocManager.promises.patchDoc(this.project_id, this.doc_id, {})
).to.be.rejectedWith(
`No such project/doc to delete: ${this.project_id}/${this.doc_id}`
)
@@ -498,13 +467,13 @@ describe('DocManager', function () {
ranges: this.originalRanges,
}
- this.DocManager._getDoc = sinon.stub()
+ this.DocManager.promises._getDoc = sinon.stub()
})
describe('when only the doc lines have changed', function () {
beforeEach(async function () {
- this.DocManager._getDoc = sinon.stub().resolves(this.doc)
- this.result = await this.DocManager.updateDoc(
+ this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
+ this.result = await this.DocManager.promises.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -514,7 +483,7 @@ describe('DocManager', function () {
})
it('should get the existing doc', function () {
- this.DocManager._getDoc
+ this.DocManager.promises._getDoc
.calledWith(this.project_id, this.doc_id, {
version: true,
rev: true,
@@ -526,7 +495,7 @@ describe('DocManager', function () {
})
it('should upsert the document to the doc collection', function () {
- this.MongoManager.upsertIntoDocCollection
+ this.MongoManager.promises.upsertIntoDocCollection
.calledWith(this.project_id, this.doc_id, this.rev, {
lines: this.newDocLines,
})
@@ -540,9 +509,9 @@ describe('DocManager', function () {
describe('when the doc ranges have changed', function () {
beforeEach(async function () {
- this.DocManager._getDoc = sinon.stub().resolves(this.doc)
+ this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
this.RangeManager.shouldUpdateRanges.returns(true)
- this.result = await this.DocManager.updateDoc(
+ this.result = await this.DocManager.promises.updateDoc(
this.project_id,
this.doc_id,
this.oldDocLines,
@@ -552,7 +521,7 @@ describe('DocManager', function () {
})
it('should upsert the ranges', function () {
- this.MongoManager.upsertIntoDocCollection
+ this.MongoManager.promises.upsertIntoDocCollection
.calledWith(this.project_id, this.doc_id, this.rev, {
ranges: this.newRanges,
})
@@ -566,8 +535,8 @@ describe('DocManager', function () {
describe('when only the version has changed', function () {
beforeEach(async function () {
- this.DocManager._getDoc = sinon.stub().resolves(this.doc)
- this.result = await this.DocManager.updateDoc(
+ this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
+ this.result = await this.DocManager.promises.updateDoc(
this.project_id,
this.doc_id,
this.oldDocLines,
@@ -577,7 +546,7 @@ describe('DocManager', function () {
})
it('should update the version', function () {
- this.MongoManager.upsertIntoDocCollection.should.have.been.calledWith(
+ this.MongoManager.promises.upsertIntoDocCollection.should.have.been.calledWith(
this.project_id,
this.doc_id,
this.rev,
@@ -592,8 +561,8 @@ describe('DocManager', function () {
describe('when the doc has not changed at all', function () {
beforeEach(async function () {
- this.DocManager._getDoc = sinon.stub().resolves(this.doc)
- this.result = await this.DocManager.updateDoc(
+ this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
+ this.result = await this.DocManager.promises.updateDoc(
this.project_id,
this.doc_id,
this.oldDocLines,
@@ -603,7 +572,9 @@ describe('DocManager', function () {
})
it('should not update the ranges or lines or version', function () {
- this.MongoManager.upsertIntoDocCollection.called.should.equal(false)
+ this.MongoManager.promises.upsertIntoDocCollection.called.should.equal(
+ false
+ )
})
it('should return the old rev and modified == false', function () {
@@ -614,7 +585,7 @@ describe('DocManager', function () {
describe('when the version is null', function () {
it('should return an error', async function () {
await expect(
- this.DocManager.updateDoc(
+ this.DocManager.promises.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -628,7 +599,7 @@ describe('DocManager', function () {
describe('when the lines are null', function () {
it('should return an error', async function () {
await expect(
- this.DocManager.updateDoc(
+ this.DocManager.promises.updateDoc(
this.project_id,
this.doc_id,
null,
@@ -642,7 +613,7 @@ describe('DocManager', function () {
describe('when the ranges are null', function () {
it('should return an error', async function () {
await expect(
- this.DocManager.updateDoc(
+ this.DocManager.promises.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -656,9 +627,9 @@ describe('DocManager', function () {
describe('when there is a generic error getting the doc', function () {
beforeEach(async function () {
this.error = new Error('doc could not be found')
- this.DocManager._getDoc = sinon.stub().rejects(this.error)
+ this.DocManager.promises._getDoc = sinon.stub().rejects(this.error)
await expect(
- this.DocManager.updateDoc(
+ this.DocManager.promises.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -669,15 +640,16 @@ describe('DocManager', function () {
})
it('should not upsert the document to the doc collection', function () {
- this.MongoManager.upsertIntoDocCollection.should.not.have.been.called
+ this.MongoManager.promises.upsertIntoDocCollection.should.not.have.been
+ .called
})
})
describe('when the version was decremented', function () {
it('should return an error', async function () {
- this.DocManager._getDoc = sinon.stub().resolves(this.doc)
+ this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
await expect(
- this.DocManager.updateDoc(
+ this.DocManager.promises.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -690,8 +662,8 @@ describe('DocManager', function () {
describe('when the doc lines have not changed', function () {
beforeEach(async function () {
- this.DocManager._getDoc = sinon.stub().resolves(this.doc)
- this.result = await this.DocManager.updateDoc(
+ this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
+ this.result = await this.DocManager.promises.updateDoc(
this.project_id,
this.doc_id,
this.oldDocLines.slice(),
@@ -701,7 +673,9 @@ describe('DocManager', function () {
})
it('should not update the doc', function () {
- this.MongoManager.upsertIntoDocCollection.called.should.equal(false)
+ this.MongoManager.promises.upsertIntoDocCollection.called.should.equal(
+ false
+ )
})
it('should return the existing rev', function () {
@@ -711,8 +685,8 @@ describe('DocManager', function () {
describe('when the doc does not exist', function () {
beforeEach(async function () {
- this.DocManager._getDoc = sinon.stub().resolves(null)
- this.result = await this.DocManager.updateDoc(
+ this.DocManager.promises._getDoc = sinon.stub().resolves(null)
+ this.result = await this.DocManager.promises.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -722,7 +696,7 @@ describe('DocManager', function () {
})
it('should upsert the document to the doc collection', function () {
- this.MongoManager.upsertIntoDocCollection.should.have.been.calledWith(
+ this.MongoManager.promises.upsertIntoDocCollection.should.have.been.calledWith(
this.project_id,
this.doc_id,
undefined,
@@ -741,12 +715,12 @@ describe('DocManager', function () {
describe('when another update is racing', function () {
beforeEach(async function () {
- this.DocManager._getDoc = sinon.stub().resolves(this.doc)
- this.MongoManager.upsertIntoDocCollection
+ this.DocManager.promises._getDoc = sinon.stub().resolves(this.doc)
+ this.MongoManager.promises.upsertIntoDocCollection
.onFirstCall()
.rejects(new Errors.DocRevValueError())
this.RangeManager.shouldUpdateRanges.returns(true)
- this.result = await this.DocManager.updateDoc(
+ this.result = await this.DocManager.promises.updateDoc(
this.project_id,
this.doc_id,
this.newDocLines,
@@ -756,7 +730,7 @@ describe('DocManager', function () {
})
it('should upsert the doc twice', function () {
- this.MongoManager.upsertIntoDocCollection.should.have.been.calledWith(
+ this.MongoManager.promises.upsertIntoDocCollection.should.have.been.calledWith(
this.project_id,
this.doc_id,
this.rev,
@@ -766,7 +740,8 @@ describe('DocManager', function () {
version: this.version + 1,
}
)
- this.MongoManager.upsertIntoDocCollection.should.have.been.calledTwice
+ this.MongoManager.promises.upsertIntoDocCollection.should.have.been
+ .calledTwice
})
it('should return the new rev', function () {
diff --git a/services/docstore/test/unit/js/HttpControllerTests.js b/services/docstore/test/unit/js/HttpControllerTests.js
index ab491ec150..2ed46a9d70 100644
--- a/services/docstore/test/unit/js/HttpControllerTests.js
+++ b/services/docstore/test/unit/js/HttpControllerTests.js
@@ -1,7 +1,7 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { assert, expect } = require('chai')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/HttpController'
)
@@ -14,7 +14,7 @@ describe('HttpController', function () {
max_doc_length: 2 * 1024 * 1024,
}
this.DocArchiveManager = {
- unArchiveAllDocs: sinon.stub().returns(),
+ unArchiveAllDocs: sinon.stub().yields(),
}
this.DocManager = {}
this.HttpController = SandboxedModule.require(modulePath, {
@@ -54,13 +54,15 @@ describe('HttpController', function () {
describe('getDoc', function () {
describe('without deleted docs', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.params = {
project_id: this.projectId,
doc_id: this.docId,
}
- this.DocManager.getFullDoc = sinon.stub().resolves(this.doc)
- await this.HttpController.getDoc(this.req, this.res, this.next)
+ this.DocManager.getFullDoc = sinon
+ .stub()
+ .callsArgWith(2, null, this.doc)
+ this.HttpController.getDoc(this.req, this.res, this.next)
})
it('should get the document with the version (including deleted)', function () {
@@ -87,24 +89,26 @@ describe('HttpController', function () {
project_id: this.projectId,
doc_id: this.docId,
}
- this.DocManager.getFullDoc = sinon.stub().resolves(this.deletedDoc)
+ this.DocManager.getFullDoc = sinon
+ .stub()
+ .callsArgWith(2, null, this.deletedDoc)
})
- it('should get the doc from the doc manager', async function () {
- await this.HttpController.getDoc(this.req, this.res, this.next)
+ it('should get the doc from the doc manager', function () {
+ this.HttpController.getDoc(this.req, this.res, this.next)
this.DocManager.getFullDoc
.calledWith(this.projectId, this.docId)
.should.equal(true)
})
- it('should return 404 if the query string delete is not set ', async function () {
- await this.HttpController.getDoc(this.req, this.res, this.next)
+ it('should return 404 if the query string delete is not set ', function () {
+ this.HttpController.getDoc(this.req, this.res, this.next)
this.res.sendStatus.calledWith(404).should.equal(true)
})
- it('should return the doc as JSON if include_deleted is set to true', async function () {
+ it('should return the doc as JSON if include_deleted is set to true', function () {
this.req.query.include_deleted = 'true'
- await this.HttpController.getDoc(this.req, this.res, this.next)
+ this.HttpController.getDoc(this.req, this.res, this.next)
this.res.json
.calledWith({
_id: this.docId,
@@ -119,15 +123,13 @@ describe('HttpController', function () {
})
describe('getRawDoc', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.params = {
project_id: this.projectId,
doc_id: this.docId,
}
- this.DocManager.getDocLines = sinon
- .stub()
- .resolves(this.doc.lines.join('\n'))
- await this.HttpController.getRawDoc(this.req, this.res, this.next)
+ this.DocManager.getDocLines = sinon.stub().callsArgWith(2, null, this.doc)
+ this.HttpController.getRawDoc(this.req, this.res, this.next)
})
it('should get the document without the version', function () {
@@ -152,7 +154,7 @@ describe('HttpController', function () {
describe('getAllDocs', function () {
describe('normally', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.params = { project_id: this.projectId }
this.docs = [
{
@@ -166,8 +168,10 @@ describe('HttpController', function () {
rev: 4,
},
]
- this.DocManager.getAllNonDeletedDocs = sinon.stub().resolves(this.docs)
- await this.HttpController.getAllDocs(this.req, this.res, this.next)
+ this.DocManager.getAllNonDeletedDocs = sinon
+ .stub()
+ .callsArgWith(2, null, this.docs)
+ this.HttpController.getAllDocs(this.req, this.res, this.next)
})
it('should get all the (non-deleted) docs', function () {
@@ -195,7 +199,7 @@ describe('HttpController', function () {
})
describe('with null lines', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.params = { project_id: this.projectId }
this.docs = [
{
@@ -209,8 +213,10 @@ describe('HttpController', function () {
rev: 4,
},
]
- this.DocManager.getAllNonDeletedDocs = sinon.stub().resolves(this.docs)
- await this.HttpController.getAllDocs(this.req, this.res, this.next)
+ this.DocManager.getAllNonDeletedDocs = sinon
+ .stub()
+ .callsArgWith(2, null, this.docs)
+ this.HttpController.getAllDocs(this.req, this.res, this.next)
})
it('should return the doc with fallback lines', function () {
@@ -232,7 +238,7 @@ describe('HttpController', function () {
})
describe('with a null doc', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.params = { project_id: this.projectId }
this.docs = [
{
@@ -247,8 +253,10 @@ describe('HttpController', function () {
rev: 4,
},
]
- this.DocManager.getAllNonDeletedDocs = sinon.stub().resolves(this.docs)
- await this.HttpController.getAllDocs(this.req, this.res, this.next)
+ this.DocManager.getAllNonDeletedDocs = sinon
+ .stub()
+ .callsArgWith(2, null, this.docs)
+ this.HttpController.getAllDocs(this.req, this.res, this.next)
})
it('should return the non null docs as JSON', function () {
@@ -284,7 +292,7 @@ describe('HttpController', function () {
describe('getAllRanges', function () {
describe('normally', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.params = { project_id: this.projectId }
this.docs = [
{
@@ -296,8 +304,10 @@ describe('HttpController', function () {
ranges: { mock_ranges: 'two' },
},
]
- this.DocManager.getAllNonDeletedDocs = sinon.stub().resolves(this.docs)
- await this.HttpController.getAllRanges(this.req, this.res, this.next)
+ this.DocManager.getAllNonDeletedDocs = sinon
+ .stub()
+ .callsArgWith(2, null, this.docs)
+ this.HttpController.getAllRanges(this.req, this.res, this.next)
})
it('should get all the (non-deleted) doc ranges', function () {
@@ -332,17 +342,16 @@ describe('HttpController', function () {
})
describe('when the doc lines exist and were updated', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.body = {
lines: (this.lines = ['hello', 'world']),
version: (this.version = 42),
ranges: (this.ranges = { changes: 'mock' }),
}
- this.rev = 5
this.DocManager.updateDoc = sinon
.stub()
- .resolves({ modified: true, rev: this.rev })
- await this.HttpController.updateDoc(this.req, this.res, this.next)
+ .yields(null, true, (this.rev = 5))
+ this.HttpController.updateDoc(this.req, this.res, this.next)
})
it('should update the document', function () {
@@ -365,17 +374,16 @@ describe('HttpController', function () {
})
describe('when the doc lines exist and were not updated', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.body = {
lines: (this.lines = ['hello', 'world']),
version: (this.version = 42),
ranges: {},
}
- this.rev = 5
this.DocManager.updateDoc = sinon
.stub()
- .resolves({ modified: false, rev: this.rev })
- await this.HttpController.updateDoc(this.req, this.res, this.next)
+ .yields(null, false, (this.rev = 5))
+ this.HttpController.updateDoc(this.req, this.res, this.next)
})
it('should return a modified status', function () {
@@ -386,12 +394,10 @@ describe('HttpController', function () {
})
describe('when the doc lines are not provided', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.body = { version: 42, ranges: {} }
- this.DocManager.updateDoc = sinon
- .stub()
- .resolves({ modified: false, rev: 0 })
- await this.HttpController.updateDoc(this.req, this.res, this.next)
+ this.DocManager.updateDoc = sinon.stub().yields(null, false)
+ this.HttpController.updateDoc(this.req, this.res, this.next)
})
it('should not update the document', function () {
@@ -404,12 +410,10 @@ describe('HttpController', function () {
})
describe('when the doc version are not provided', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.body = { version: 42, lines: ['hello world'] }
- this.DocManager.updateDoc = sinon
- .stub()
- .resolves({ modified: false, rev: 0 })
- await this.HttpController.updateDoc(this.req, this.res, this.next)
+ this.DocManager.updateDoc = sinon.stub().yields(null, false)
+ this.HttpController.updateDoc(this.req, this.res, this.next)
})
it('should not update the document', function () {
@@ -422,12 +426,10 @@ describe('HttpController', function () {
})
describe('when the doc ranges is not provided', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.body = { lines: ['foo'], version: 42 }
- this.DocManager.updateDoc = sinon
- .stub()
- .resolves({ modified: false, rev: 0 })
- await this.HttpController.updateDoc(this.req, this.res, this.next)
+ this.DocManager.updateDoc = sinon.stub().yields(null, false)
+ this.HttpController.updateDoc(this.req, this.res, this.next)
})
it('should not update the document', function () {
@@ -440,20 +442,13 @@ describe('HttpController', function () {
})
describe('when the doc body is too large', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.body = {
lines: (this.lines = Array(2049).fill('a'.repeat(1024))),
version: (this.version = 42),
ranges: (this.ranges = { changes: 'mock' }),
}
- this.DocManager.updateDoc = sinon
- .stub()
- .resolves({ modified: false, rev: 0 })
- await this.HttpController.updateDoc(this.req, this.res, this.next)
- })
-
- it('should not update the document', function () {
- this.DocManager.updateDoc.called.should.equal(false)
+ this.HttpController.updateDoc(this.req, this.res, this.next)
})
it('should return a 413 (too large) response', function () {
@@ -467,14 +462,14 @@ describe('HttpController', function () {
})
describe('patchDoc', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.params = {
project_id: this.projectId,
doc_id: this.docId,
}
this.req.body = { name: 'foo.tex' }
- this.DocManager.patchDoc = sinon.stub().resolves()
- await this.HttpController.patchDoc(this.req, this.res, this.next)
+ this.DocManager.patchDoc = sinon.stub().yields(null)
+ this.HttpController.patchDoc(this.req, this.res, this.next)
})
it('should delete the document', function () {
@@ -489,11 +484,11 @@ describe('HttpController', function () {
})
describe('with an invalid payload', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.body = { cannot: 'happen' }
- this.DocManager.patchDoc = sinon.stub().resolves()
- await this.HttpController.patchDoc(this.req, this.res, this.next)
+ this.DocManager.patchDoc = sinon.stub().yields(null)
+ this.HttpController.patchDoc(this.req, this.res, this.next)
})
it('should log a message', function () {
@@ -514,10 +509,10 @@ describe('HttpController', function () {
})
describe('archiveAllDocs', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.params = { project_id: this.projectId }
- this.DocArchiveManager.archiveAllDocs = sinon.stub().resolves()
- await this.HttpController.archiveAllDocs(this.req, this.res, this.next)
+ this.DocArchiveManager.archiveAllDocs = sinon.stub().callsArg(1)
+ this.HttpController.archiveAllDocs(this.req, this.res, this.next)
})
it('should archive the project', function () {
@@ -537,12 +532,9 @@ describe('HttpController', function () {
})
describe('on success', function () {
- beforeEach(async function () {
- await this.HttpController.unArchiveAllDocs(
- this.req,
- this.res,
- this.next
- )
+ beforeEach(function (done) {
+ this.res.sendStatus.callsFake(() => done())
+ this.HttpController.unArchiveAllDocs(this.req, this.res, this.next)
})
it('returns a 200', function () {
@@ -551,15 +543,12 @@ describe('HttpController', function () {
})
describe("when the archived rev doesn't match", function () {
- beforeEach(async function () {
- this.DocArchiveManager.unArchiveAllDocs.rejects(
+ beforeEach(function (done) {
+ this.res.sendStatus.callsFake(() => done())
+ this.DocArchiveManager.unArchiveAllDocs.yields(
new Errors.DocRevValueError('bad rev')
)
- await this.HttpController.unArchiveAllDocs(
- this.req,
- this.res,
- this.next
- )
+ this.HttpController.unArchiveAllDocs(this.req, this.res, this.next)
})
it('returns a 409', function () {
@@ -569,10 +558,10 @@ describe('HttpController', function () {
})
describe('destroyProject', function () {
- beforeEach(async function () {
+ beforeEach(function () {
this.req.params = { project_id: this.projectId }
- this.DocArchiveManager.destroyProject = sinon.stub().resolves()
- await this.HttpController.destroyProject(this.req, this.res, this.next)
+ this.DocArchiveManager.destroyProject = sinon.stub().callsArg(1)
+ this.HttpController.destroyProject(this.req, this.res, this.next)
})
it('should destroy the docs', function () {
diff --git a/services/docstore/test/unit/js/MongoManagerTests.js b/services/docstore/test/unit/js/MongoManagerTests.js
index b96b661df4..da8f59ae07 100644
--- a/services/docstore/test/unit/js/MongoManagerTests.js
+++ b/services/docstore/test/unit/js/MongoManagerTests.js
@@ -1,6 +1,6 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/MongoManager'
)
@@ -41,7 +41,7 @@ describe('MongoManager', function () {
this.doc = { name: 'mock-doc' }
this.db.docs.findOne = sinon.stub().resolves(this.doc)
this.filter = { lines: true }
- this.result = await this.MongoManager.findDoc(
+ this.result = await this.MongoManager.promises.findDoc(
this.projectId,
this.docId,
this.filter
@@ -70,7 +70,11 @@ describe('MongoManager', function () {
describe('patchDoc', function () {
beforeEach(async function () {
this.meta = { name: 'foo.tex' }
- await this.MongoManager.patchDoc(this.projectId, this.docId, this.meta)
+ await this.MongoManager.promises.patchDoc(
+ this.projectId,
+ this.docId,
+ this.meta
+ )
})
it('should pass the parameter along', function () {
@@ -100,7 +104,7 @@ describe('MongoManager', function () {
describe('with included_deleted = false', function () {
beforeEach(async function () {
- this.result = await this.MongoManager.getProjectsDocs(
+ this.result = await this.MongoManager.promises.getProjectsDocs(
this.projectId,
{ include_deleted: false },
this.filter
@@ -128,7 +132,7 @@ describe('MongoManager', function () {
describe('with included_deleted = true', function () {
beforeEach(async function () {
- this.result = await this.MongoManager.getProjectsDocs(
+ this.result = await this.MongoManager.promises.getProjectsDocs(
this.projectId,
{ include_deleted: true },
this.filter
@@ -163,7 +167,7 @@ describe('MongoManager', function () {
this.db.docs.find = sinon.stub().returns({
toArray: sinon.stub().resolves([this.doc1, this.doc2, this.doc3]),
})
- this.result = await this.MongoManager.getProjectsDeletedDocs(
+ this.result = await this.MongoManager.promises.getProjectsDeletedDocs(
this.projectId,
this.filter
)
@@ -199,7 +203,7 @@ describe('MongoManager', function () {
})
it('should upsert the document', async function () {
- await this.MongoManager.upsertIntoDocCollection(
+ await this.MongoManager.promises.upsertIntoDocCollection(
this.projectId,
this.docId,
this.oldRev,
@@ -219,7 +223,7 @@ describe('MongoManager', function () {
it('should handle update error', async function () {
this.db.docs.updateOne.rejects(this.stubbedErr)
await expect(
- this.MongoManager.upsertIntoDocCollection(
+ this.MongoManager.promises.upsertIntoDocCollection(
this.projectId,
this.docId,
this.rev,
@@ -231,7 +235,7 @@ describe('MongoManager', function () {
})
it('should insert without a previous rev', async function () {
- await this.MongoManager.upsertIntoDocCollection(
+ await this.MongoManager.promises.upsertIntoDocCollection(
this.projectId,
this.docId,
null,
@@ -250,7 +254,7 @@ describe('MongoManager', function () {
it('should handle generic insert error', async function () {
this.db.docs.insertOne.rejects(this.stubbedErr)
await expect(
- this.MongoManager.upsertIntoDocCollection(
+ this.MongoManager.promises.upsertIntoDocCollection(
this.projectId,
this.docId,
null,
@@ -262,7 +266,7 @@ describe('MongoManager', function () {
it('should handle duplicate insert error', async function () {
this.db.docs.insertOne.rejects({ code: 11000 })
await expect(
- this.MongoManager.upsertIntoDocCollection(
+ this.MongoManager.promises.upsertIntoDocCollection(
this.projectId,
this.docId,
null,
@@ -276,7 +280,7 @@ describe('MongoManager', function () {
beforeEach(async function () {
this.projectId = new ObjectId()
this.db.docs.deleteMany = sinon.stub().resolves()
- await this.MongoManager.destroyProject(this.projectId)
+ await this.MongoManager.promises.destroyProject(this.projectId)
})
it('should destroy all docs', function () {
@@ -293,13 +297,13 @@ describe('MongoManager', function () {
it('should not error when the rev has not changed', async function () {
this.db.docs.findOne = sinon.stub().resolves({ rev: 1 })
- await this.MongoManager.checkRevUnchanged(this.doc)
+ await this.MongoManager.promises.checkRevUnchanged(this.doc)
})
it('should return an error when the rev has changed', async function () {
this.db.docs.findOne = sinon.stub().resolves({ rev: 2 })
await expect(
- this.MongoManager.checkRevUnchanged(this.doc)
+ this.MongoManager.promises.checkRevUnchanged(this.doc)
).to.be.rejectedWith(Errors.DocModifiedError)
})
@@ -307,14 +311,14 @@ describe('MongoManager', function () {
this.db.docs.findOne = sinon.stub().resolves({ rev: 2 })
this.doc = { _id: new ObjectId(), name: 'mock-doc', rev: NaN }
await expect(
- this.MongoManager.checkRevUnchanged(this.doc)
+ this.MongoManager.promises.checkRevUnchanged(this.doc)
).to.be.rejectedWith(Errors.DocRevValueError)
})
it('should return a value error if checked doc rev is NaN', async function () {
this.db.docs.findOne = sinon.stub().resolves({ rev: NaN })
await expect(
- this.MongoManager.checkRevUnchanged(this.doc)
+ this.MongoManager.promises.checkRevUnchanged(this.doc)
).to.be.rejectedWith(Errors.DocRevValueError)
})
})
@@ -330,7 +334,7 @@ describe('MongoManager', function () {
describe('complete doc', function () {
beforeEach(async function () {
- await this.MongoManager.restoreArchivedDoc(
+ await this.MongoManager.promises.restoreArchivedDoc(
this.projectId,
this.docId,
this.archivedDoc
@@ -360,7 +364,7 @@ describe('MongoManager', function () {
describe('without ranges', function () {
beforeEach(async function () {
delete this.archivedDoc.ranges
- await this.MongoManager.restoreArchivedDoc(
+ await this.MongoManager.promises.restoreArchivedDoc(
this.projectId,
this.docId,
this.archivedDoc
@@ -391,7 +395,7 @@ describe('MongoManager', function () {
it('throws a DocRevValueError', async function () {
this.db.docs.updateOne.resolves({ matchedCount: 0 })
await expect(
- this.MongoManager.restoreArchivedDoc(
+ this.MongoManager.promises.restoreArchivedDoc(
this.projectId,
this.docId,
this.archivedDoc
diff --git a/services/docstore/test/unit/js/RangeManagerTests.js b/services/docstore/test/unit/js/RangeManagerTests.js
index ba99280a7a..0d2a353558 100644
--- a/services/docstore/test/unit/js/RangeManagerTests.js
+++ b/services/docstore/test/unit/js/RangeManagerTests.js
@@ -12,7 +12,7 @@
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
const { assert, expect } = require('chai')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../app/js/RangeManager'
)
@@ -30,7 +30,7 @@ describe('RangeManager', function () {
})
describe('jsonRangesToMongo', function () {
- it('should convert ObjectIds and dates to proper objects and fix comment id', function () {
+ it('should convert ObjectIds and dates to proper objects', function () {
const changeId = new ObjectId().toString()
const commentId = new ObjectId().toString()
const userId = new ObjectId().toString()
@@ -66,7 +66,7 @@ describe('RangeManager', function () {
],
comments: [
{
- id: new ObjectId(threadId),
+ id: new ObjectId(commentId),
op: { c: 'foo', p: 3, t: new ObjectId(threadId) },
},
],
@@ -110,6 +110,7 @@ describe('RangeManager', function () {
return it('should be consistent when transformed through json -> mongo -> json', function () {
const changeId = new ObjectId().toString()
+ const commentId = new ObjectId().toString()
const userId = new ObjectId().toString()
const threadId = new ObjectId().toString()
const ts = new Date().toJSON()
@@ -126,7 +127,7 @@ describe('RangeManager', function () {
],
comments: [
{
- id: threadId,
+ id: commentId,
op: { c: 'foo', p: 3, t: threadId },
},
],
@@ -141,7 +142,6 @@ describe('RangeManager', function () {
return describe('shouldUpdateRanges', function () {
beforeEach(function () {
- const threadId = new ObjectId()
this.ranges = {
changes: [
{
@@ -155,8 +155,8 @@ describe('RangeManager', function () {
],
comments: [
{
- id: threadId,
- op: { c: 'foo', p: 3, t: threadId },
+ id: new ObjectId(),
+ op: { c: 'foo', p: 3, t: new ObjectId() },
},
],
}
diff --git a/services/document-updater/.gitignore b/services/document-updater/.gitignore
new file mode 100644
index 0000000000..624e78f096
--- /dev/null
+++ b/services/document-updater/.gitignore
@@ -0,0 +1,52 @@
+compileFolder
+
+Compiled source #
+###################
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.so
+
+# Packages #
+############
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# Logs and databases #
+######################
+*.log
+*.sql
+*.sqlite
+
+# OS generated files #
+######################
+.DS_Store?
+ehthumbs.db
+Icon?
+Thumbs.db
+
+/node_modules/*
+
+
+
+forever/
+
+**.swp
+
+# Redis cluster
+**/appendonly.aof
+**/dump.rdb
+**/nodes.conf
+
+# managed by dev-environment$ bin/update_build_scripts
+.npmrc
diff --git a/services/document-updater/.nvmrc b/services/document-updater/.nvmrc
index fc37597bcc..123b052798 100644
--- a/services/document-updater/.nvmrc
+++ b/services/document-updater/.nvmrc
@@ -1 +1 @@
-22.17.0
+18.20.2
diff --git a/services/document-updater/Dockerfile b/services/document-updater/Dockerfile
index 720b619c41..06c1610c66 100644
--- a/services/document-updater/Dockerfile
+++ b/services/document-updater/Dockerfile
@@ -2,7 +2,7 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:22.17.0 AS base
+FROM node:18.20.2 AS base
WORKDIR /overleaf/services/document-updater
diff --git a/services/document-updater/Makefile b/services/document-updater/Makefile
index 46dfced5c9..de0cf0eabe 100644
--- a/services/document-updater/Makefile
+++ b/services/document-updater/Makefile
@@ -32,30 +32,12 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
-
-SHELLCHECK_OPTS = \
- --shell=bash \
- --external-sources
-SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
-SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
-
-shellcheck:
- @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
- koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
-
-shellcheck_fix:
- @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
- diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
- if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
- elif [ -n "$$diff" ]; then echo "$$file"; \
- else echo "\033[2m$$file\033[0m"; fi \
- done
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
format:
$(RUN_LINTING) format
@@ -81,7 +63,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck shellcheck test_unit test_acceptance
+test: format lint typecheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -116,6 +98,13 @@ test_acceptance_clean:
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
test_acceptance_pre_run:
+ $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+ $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+ while ! mongosh --eval "db.version()" > /dev/null; do \
+ echo "Waiting for Mongo..."; \
+ sleep 1; \
+ done; \
+ mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
endif
@@ -148,7 +137,6 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
- shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/document-updater/app.js b/services/document-updater/app.js
index 65c9895377..fab33150c3 100644
--- a/services/document-updater/app.js
+++ b/services/document-updater/app.js
@@ -9,6 +9,10 @@ logger.initialize('document-updater')
logger.logger.addSerializers(require('./app/js/LoggerSerializers'))
+if (Settings.sentry != null && Settings.sentry.dsn != null) {
+ logger.initializeErrorReporting(Settings.sentry.dsn)
+}
+
const RedisManager = require('./app/js/RedisManager')
const DispatchManager = require('./app/js/DispatchManager')
const DeleteQueueManager = require('./app/js/DeleteQueueManager')
@@ -135,10 +139,6 @@ app.use((req, res, next) => {
})
app.get('/project/:project_id/doc/:doc_id', HttpController.getDoc)
-app.get(
- '/project/:project_id/doc/:doc_id/comment/:comment_id',
- HttpController.getComment
-)
app.get('/project/:project_id/doc/:doc_id/peek', HttpController.peekDoc)
// temporarily keep the GET method for backwards compatibility
app.get('/project/:project_id/doc', HttpController.getProjectDocsAndFlushIfOld)
@@ -147,13 +147,8 @@ app.post(
'/project/:project_id/get_and_flush_if_old',
HttpController.getProjectDocsAndFlushIfOld
)
-app.get(
- '/project/:project_id/last_updated_at',
- HttpController.getProjectLastUpdatedAt
-)
app.post('/project/:project_id/clearState', HttpController.clearProjectState)
app.post('/project/:project_id/doc/:doc_id', HttpController.setDoc)
-app.post('/project/:project_id/doc/:doc_id/append', HttpController.appendToDoc)
app.post(
'/project/:project_id/doc/:doc_id/flush',
HttpController.flushDocIfLoaded
diff --git a/services/document-updater/app/js/DiffCodec.js b/services/document-updater/app/js/DiffCodec.js
index 17da409386..245903ca13 100644
--- a/services/document-updater/app/js/DiffCodec.js
+++ b/services/document-updater/app/js/DiffCodec.js
@@ -1,6 +1,4 @@
-const OError = require('@overleaf/o-error')
const DMP = require('diff-match-patch')
-const { TextOperation } = require('overleaf-editor-core')
const dmp = new DMP()
// Do not attempt to produce a diff for more than 100ms
@@ -18,7 +16,8 @@ module.exports = {
const ops = []
let position = 0
for (const diff of diffs) {
- const [type, content] = diff
+ const type = diff[0]
+ const content = diff[1]
if (type === this.ADDED) {
ops.push({
i: content,
@@ -38,63 +37,4 @@ module.exports = {
}
return ops
},
-
- /**
- * @param {import("overleaf-editor-core").StringFileData} file
- * @param {string} after
- * @return {TextOperation}
- */
- diffAsHistoryOTEditOperation(file, after) {
- const beforeWithoutTrackedDeletes = file.getContent({
- filterTrackedDeletes: true,
- })
- const diffs = dmp.diff_main(beforeWithoutTrackedDeletes, after)
- dmp.diff_cleanupSemantic(diffs)
-
- const trackedChanges = file.trackedChanges.asSorted()
- let nextTc = trackedChanges.shift()
-
- const op = new TextOperation()
- for (const diff of diffs) {
- let [type, content] = diff
- if (type === this.ADDED) {
- op.insert(content)
- } else if (type === this.REMOVED || type === this.UNCHANGED) {
- while (op.baseLength + content.length > nextTc?.range.start) {
- if (nextTc.tracking.type === 'delete') {
- const untilRange = nextTc.range.start - op.baseLength
- if (type === this.REMOVED) {
- op.remove(untilRange)
- } else if (type === this.UNCHANGED) {
- op.retain(untilRange)
- }
- op.retain(nextTc.range.end - nextTc.range.start)
- content = content.slice(untilRange)
- }
- nextTc = trackedChanges.shift()
- }
- if (type === this.REMOVED) {
- op.remove(content.length)
- } else if (type === this.UNCHANGED) {
- op.retain(content.length)
- }
- } else {
- throw new Error('Unknown type')
- }
- }
- while (nextTc) {
- if (
- nextTc.tracking.type !== 'delete' ||
- nextTc.range.start !== op.baseLength
- ) {
- throw new OError(
- 'StringFileData.trackedChanges out of sync: unexpected range after end of diff',
- { nextTc, baseLength: op.baseLength }
- )
- }
- op.retain(nextTc.range.end - nextTc.range.start)
- nextTc = trackedChanges.shift()
- }
- return op
- },
}
diff --git a/services/document-updater/app/js/DocumentManager.js b/services/document-updater/app/js/DocumentManager.js
index 3fb3d10a6e..157a99ccf6 100644
--- a/services/document-updater/app/js/DocumentManager.js
+++ b/services/document-updater/app/js/DocumentManager.js
@@ -9,18 +9,10 @@ const HistoryManager = require('./HistoryManager')
const Errors = require('./Errors')
const RangesManager = require('./RangesManager')
const { extractOriginOrSource } = require('./Utils')
-const { getTotalSizeOfLines } = require('./Limits')
-const Settings = require('@overleaf/settings')
-const { StringFileData } = require('overleaf-editor-core')
const MAX_UNFLUSHED_AGE = 300 * 1000 // 5 mins, document should be flushed to mongo this time after a change
const DocumentManager = {
- /**
- * @param {string} projectId
- * @param {string} docId
- * @return {Promise<{lines: (string[] | StringFileRawData), version: number, ranges: Ranges, resolvedCommentIds: any[], pathname: string, projectHistoryId: string, unflushedTime: any, alreadyLoaded: boolean, historyRangesSupport: boolean, type: OTType}>}
- */
async getDoc(projectId, docId) {
const {
lines,
@@ -81,7 +73,6 @@ const DocumentManager = {
unflushedTime: null,
alreadyLoaded: false,
historyRangesSupport,
- type: Array.isArray(lines) ? 'sharejs-text-ot' : 'history-ot',
}
} else {
return {
@@ -94,25 +85,16 @@ const DocumentManager = {
unflushedTime,
alreadyLoaded: true,
historyRangesSupport,
- type: Array.isArray(lines) ? 'sharejs-text-ot' : 'history-ot',
}
}
},
async getDocAndRecentOps(projectId, docId, fromVersion) {
- const { lines, version, ranges, pathname, projectHistoryId, type } =
+ const { lines, version, ranges, pathname, projectHistoryId } =
await DocumentManager.getDoc(projectId, docId)
if (fromVersion === -1) {
- return {
- lines,
- version,
- ops: [],
- ranges,
- pathname,
- projectHistoryId,
- type,
- }
+ return { lines, version, ops: [], ranges, pathname, projectHistoryId }
} else {
const ops = await RedisManager.promises.getPreviousDocOps(
docId,
@@ -126,90 +108,39 @@ const DocumentManager = {
ranges,
pathname,
projectHistoryId,
- type,
}
}
},
- async appendToDoc(projectId, docId, linesToAppend, originOrSource, userId) {
- let { lines: currentLines, type } = await DocumentManager.getDoc(
- projectId,
- docId
- )
- if (type === 'history-ot') {
- const file = StringFileData.fromRaw(currentLines)
- // TODO(24596): tc support for history-ot
- currentLines = file.getLines()
- }
- const currentLineSize = getTotalSizeOfLines(currentLines)
- const addedSize = getTotalSizeOfLines(linesToAppend)
- const newlineSize = '\n'.length
-
- if (currentLineSize + newlineSize + addedSize > Settings.max_doc_length) {
- throw new Errors.FileTooLargeError(
- 'doc would become too large if appending this text'
- )
- }
-
- return await DocumentManager.setDoc(
- projectId,
- docId,
- currentLines.concat(linesToAppend),
- originOrSource,
- userId,
- false,
- false
- )
- },
-
- async setDoc(
- projectId,
- docId,
- newLines,
- originOrSource,
- userId,
- undoing,
- external
- ) {
+ async setDoc(projectId, docId, newLines, originOrSource, userId, undoing) {
if (newLines == null) {
throw new Error('No lines were provided to setDoc')
}
- // Circular dependencies. Import at runtime.
- const HistoryOTUpdateManager = require('./HistoryOTUpdateManager')
const UpdateManager = require('./UpdateManager')
-
const {
lines: oldLines,
version,
alreadyLoaded,
- type,
} = await DocumentManager.getDoc(projectId, docId)
+ if (oldLines != null && oldLines.length > 0 && oldLines[0].text != null) {
+ logger.debug(
+ { docId, projectId, oldLines, newLines },
+ 'document is JSON so not updating'
+ )
+ return
+ }
+
logger.debug(
{ docId, projectId, oldLines, newLines },
'setting a document via http'
)
-
- let op
- if (type === 'history-ot') {
- const file = StringFileData.fromRaw(oldLines)
- const operation = DiffCodec.diffAsHistoryOTEditOperation(
- file,
- newLines.join('\n')
- )
- if (operation.isNoop()) {
- op = []
- } else {
- op = [operation.toJSON()]
- }
- } else {
- op = DiffCodec.diffAsShareJsOp(oldLines, newLines)
- if (undoing) {
- for (const o of op || []) {
- o.u = true
- } // Turn on undo flag for each op for track changes
- }
+ const op = DiffCodec.diffAsShareJsOp(oldLines, newLines)
+ if (undoing) {
+ for (const o of op || []) {
+ o.u = true
+ } // Turn on undo flag for each op for track changes
}
const { origin, source } = extractOriginOrSource(originOrSource)
@@ -219,12 +150,10 @@ const DocumentManager = {
op,
v: version,
meta: {
+ type: 'external',
user_id: userId,
},
}
- if (external) {
- update.meta.type = 'external'
- }
if (origin) {
update.meta.origin = origin
} else if (source) {
@@ -244,11 +173,7 @@ const DocumentManager = {
// this update, otherwise the doc would never be
// removed from redis.
if (op.length > 0) {
- if (type === 'history-ot') {
- await HistoryOTUpdateManager.applyUpdate(projectId, docId, update)
- } else {
- await UpdateManager.promises.applyUpdate(projectId, docId, update)
- }
+ await UpdateManager.promises.applyUpdate(projectId, docId, update)
}
// If the document was loaded already, then someone has it open
@@ -269,7 +194,7 @@ const DocumentManager = {
},
async flushDocIfLoaded(projectId, docId) {
- let {
+ const {
lines,
version,
ranges,
@@ -290,11 +215,6 @@ const DocumentManager = {
logger.debug({ projectId, docId, version }, 'flushing doc')
Metrics.inc('flush-doc-if-loaded', 1, { status: 'modified' })
- if (!Array.isArray(lines)) {
- const file = StringFileData.fromRaw(lines)
- // TODO(24596): tc support for history-ot
- lines = file.getLines()
- }
const result = await PersistenceManager.promises.setDoc(
projectId,
docId,
@@ -344,14 +264,7 @@ const DocumentManager = {
throw new Errors.NotFoundError(`document not found: ${docId}`)
}
- // TODO(24596): tc support for history-ot
- const newRanges = RangesManager.acceptChanges(
- projectId,
- docId,
- changeIds,
- ranges,
- lines
- )
+ const newRanges = RangesManager.acceptChanges(changeIds, ranges)
await RedisManager.promises.updateDocument(
projectId,
@@ -410,22 +323,6 @@ const DocumentManager = {
}
},
- async getComment(projectId, docId, commentId) {
- // TODO(24596): tc support for history-ot
- const { ranges } = await DocumentManager.getDoc(projectId, docId)
-
- const comment = ranges?.comments?.find(comment => comment.id === commentId)
-
- if (!comment) {
- throw new Errors.NotFoundError({
- message: 'comment not found',
- info: { commentId },
- })
- }
-
- return { comment }
- },
-
async deleteComment(projectId, docId, commentId, userId) {
const { lines, version, ranges, pathname, historyRangesSupport } =
await DocumentManager.getDoc(projectId, docId)
@@ -433,7 +330,6 @@ const DocumentManager = {
throw new Errors.NotFoundError(`document not found: ${docId}`)
}
- // TODO(24596): tc support for history-ot
const newRanges = RangesManager.deleteComment(commentId, ranges)
await RedisManager.promises.updateDocument(
@@ -473,7 +369,7 @@ const DocumentManager = {
},
async getDocAndFlushIfOld(projectId, docId) {
- let { lines, version, unflushedTime, alreadyLoaded } =
+ const { lines, version, unflushedTime, alreadyLoaded } =
await DocumentManager.getDoc(projectId, docId)
// if doc was already loaded see if it needs to be flushed
@@ -485,12 +381,6 @@ const DocumentManager = {
await DocumentManager.flushDocIfLoaded(projectId, docId)
}
- if (!Array.isArray(lines)) {
- const file = StringFileData.fromRaw(lines)
- // TODO(24596): tc support for history-ot
- lines = file.getLines()
- }
-
return { lines, version }
},
@@ -566,16 +456,6 @@ const DocumentManager = {
)
},
- async getCommentWithLock(projectId, docId, commentId) {
- const UpdateManager = require('./UpdateManager')
- return await UpdateManager.promises.lockUpdatesAndDo(
- DocumentManager.getComment,
- projectId,
- docId,
- commentId
- )
- },
-
async getDocAndRecentOpsWithLock(projectId, docId, fromVersion) {
const UpdateManager = require('./UpdateManager')
return await UpdateManager.promises.lockUpdatesAndDo(
@@ -595,15 +475,7 @@ const DocumentManager = {
)
},
- async setDocWithLock(
- projectId,
- docId,
- lines,
- source,
- userId,
- undoing,
- external
- ) {
+ async setDocWithLock(projectId, docId, lines, source, userId, undoing) {
const UpdateManager = require('./UpdateManager')
return await UpdateManager.promises.lockUpdatesAndDo(
DocumentManager.setDoc,
@@ -612,20 +484,7 @@ const DocumentManager = {
lines,
source,
userId,
- undoing,
- external
- )
- },
-
- async appendToDocWithLock(projectId, docId, lines, source, userId) {
- const UpdateManager = require('./UpdateManager')
- return await UpdateManager.promises.lockUpdatesAndDo(
- DocumentManager.appendToDoc,
- projectId,
- docId,
- lines,
- source,
- userId
+ undoing
)
},
@@ -743,7 +602,6 @@ module.exports = {
'ranges',
'pathname',
'projectHistoryId',
- 'type',
],
getDocAndRecentOpsWithLock: [
'lines',
@@ -752,9 +610,7 @@ module.exports = {
'ranges',
'pathname',
'projectHistoryId',
- 'type',
],
- getCommentWithLock: ['comment'],
},
}),
promises: DocumentManager,
diff --git a/services/document-updater/app/js/Errors.js b/services/document-updater/app/js/Errors.js
index ac1f5875fa..a43f69ad35 100644
--- a/services/document-updater/app/js/Errors.js
+++ b/services/document-updater/app/js/Errors.js
@@ -5,15 +5,6 @@ class OpRangeNotAvailableError extends OError {}
class ProjectStateChangedError extends OError {}
class DeleteMismatchError extends OError {}
class FileTooLargeError extends OError {}
-class OTTypeMismatchError extends OError {
- /**
- * @param {OTType} got
- * @param {OTType} want
- */
- constructor(got, want) {
- super('ot type mismatch', { got, want })
- }
-}
module.exports = {
NotFoundError,
@@ -21,5 +12,4 @@ module.exports = {
ProjectStateChangedError,
DeleteMismatchError,
FileTooLargeError,
- OTTypeMismatchError,
}
diff --git a/services/document-updater/app/js/HistoryManager.js b/services/document-updater/app/js/HistoryManager.js
index d9a8459525..3a91b29cb8 100644
--- a/services/document-updater/app/js/HistoryManager.js
+++ b/services/document-updater/app/js/HistoryManager.js
@@ -62,7 +62,6 @@ const HistoryManager = {
// record updates for project history
if (
HistoryManager.shouldFlushHistoryOps(
- projectId,
projectOpsLength,
ops.length,
HistoryManager.FLUSH_PROJECT_EVERY_N_OPS
@@ -78,8 +77,7 @@ const HistoryManager = {
}
},
- shouldFlushHistoryOps(projectId, length, opsLength, threshold) {
- if (Settings.shortHistoryQueues.includes(projectId)) return true
+ shouldFlushHistoryOps(length, opsLength, threshold) {
if (!length) {
return false
} // don't flush unless we know the length
@@ -108,12 +106,10 @@ const HistoryManager = {
projectHistoryId,
docs,
files,
- opts,
function (error) {
if (error) {
return callback(error)
}
- if (opts.resyncProjectStructureOnly) return callback()
const DocumentManager = require('./DocumentManager')
const resyncDoc = (doc, cb) => {
DocumentManager.resyncDocContentsWithLock(
diff --git a/services/document-updater/app/js/HistoryOTUpdateManager.js b/services/document-updater/app/js/HistoryOTUpdateManager.js
deleted file mode 100644
index 5a8b92099e..0000000000
--- a/services/document-updater/app/js/HistoryOTUpdateManager.js
+++ /dev/null
@@ -1,158 +0,0 @@
-// @ts-check
-
-const Profiler = require('./Profiler')
-const DocumentManager = require('./DocumentManager')
-const Errors = require('./Errors')
-const RedisManager = require('./RedisManager')
-const {
- EditOperationBuilder,
- StringFileData,
- EditOperationTransformer,
-} = require('overleaf-editor-core')
-const Metrics = require('./Metrics')
-const ProjectHistoryRedisManager = require('./ProjectHistoryRedisManager')
-const HistoryManager = require('./HistoryManager')
-const RealTimeRedisManager = require('./RealTimeRedisManager')
-
-/**
- * @typedef {import("./types").Update} Update
- * @typedef {import("./types").HistoryOTEditOperationUpdate} HistoryOTEditOperationUpdate
- */
-
-/**
- * @param {Update} update
- * @return {update is HistoryOTEditOperationUpdate}
- */
-function isHistoryOTEditOperationUpdate(update) {
- return (
- update &&
- 'doc' in update &&
- 'op' in update &&
- 'v' in update &&
- Array.isArray(update.op) &&
- EditOperationBuilder.isValid(update.op[0])
- )
-}
-
-/**
- * Try to apply an update to the given document
- *
- * @param {string} projectId
- * @param {string} docId
- * @param {HistoryOTEditOperationUpdate} update
- * @param {Profiler} profiler
- */
-async function tryApplyUpdate(projectId, docId, update, profiler) {
- let { lines, version, pathname, type } =
- await DocumentManager.promises.getDoc(projectId, docId)
- profiler.log('getDoc')
-
- if (lines == null || version == null) {
- throw new Errors.NotFoundError(`document not found: ${docId}`)
- }
- if (type !== 'history-ot') {
- throw new Errors.OTTypeMismatchError(type, 'history-ot')
- }
-
- let op = EditOperationBuilder.fromJSON(update.op[0])
- if (version !== update.v) {
- const transformUpdates = await RedisManager.promises.getPreviousDocOps(
- docId,
- update.v,
- version
- )
- for (const transformUpdate of transformUpdates) {
- if (!isHistoryOTEditOperationUpdate(transformUpdate)) {
- throw new Errors.OTTypeMismatchError('sharejs-text-ot', 'history-ot')
- }
-
- if (
- transformUpdate.meta.source &&
- update.dupIfSource?.includes(transformUpdate.meta.source)
- ) {
- update.dup = true
- break
- }
- const other = EditOperationBuilder.fromJSON(transformUpdate.op[0])
- op = EditOperationTransformer.transform(op, other)[0]
- }
- update.op = [op.toJSON()]
- }
-
- if (!update.dup) {
- const file = StringFileData.fromRaw(lines)
- file.edit(op)
- version += 1
- update.meta.ts = Date.now()
- await RedisManager.promises.updateDocument(
- projectId,
- docId,
- file.toRaw(),
- version,
- [update],
- {},
- update.meta
- )
-
- Metrics.inc('history-queue', 1, { status: 'project-history' })
- try {
- const projectOpsLength =
- await ProjectHistoryRedisManager.promises.queueOps(projectId, [
- JSON.stringify({
- ...update,
- meta: {
- ...update.meta,
- pathname,
- },
- }),
- ])
- HistoryManager.recordAndFlushHistoryOps(
- projectId,
- [update],
- projectOpsLength
- )
- profiler.log('recordAndFlushHistoryOps')
- } catch (err) {
- // The full project history can re-sync a project in case
- // updates went missing.
- // Just record the error here and acknowledge the write-op.
- Metrics.inc('history-queue-error')
- }
- }
- RealTimeRedisManager.sendData({
- project_id: projectId,
- doc_id: docId,
- op: update,
- })
-}
-
-/**
- * Apply an update to the given document
- *
- * @param {string} projectId
- * @param {string} docId
- * @param {HistoryOTEditOperationUpdate} update
- */
-async function applyUpdate(projectId, docId, update) {
- const profiler = new Profiler('applyUpdate', {
- project_id: projectId,
- doc_id: docId,
- type: 'history-ot',
- })
-
- try {
- await tryApplyUpdate(projectId, docId, update, profiler)
- } catch (error) {
- RealTimeRedisManager.sendData({
- project_id: projectId,
- doc_id: docId,
- error: error instanceof Error ? error.message : error,
- })
- profiler.log('sendData')
- throw error
- } finally {
- profiler.end()
- }
-}
-
-module.exports = { isHistoryOTEditOperationUpdate, applyUpdate }
diff --git a/services/document-updater/app/js/HttpController.js b/services/document-updater/app/js/HttpController.js
index 0a6ae3b2b4..a3c7b6cd44 100644
--- a/services/document-updater/app/js/HttpController.js
+++ b/services/document-updater/app/js/HttpController.js
@@ -9,7 +9,6 @@ const Metrics = require('./Metrics')
const DeleteQueueManager = require('./DeleteQueueManager')
const { getTotalSizeOfLines } = require('./Limits')
const async = require('async')
-const { StringFileData } = require('overleaf-editor-core')
function getDoc(req, res, next) {
let fromVersion
@@ -28,7 +27,7 @@ function getDoc(req, res, next) {
projectId,
docId,
fromVersion,
- (error, lines, version, ops, ranges, pathname, _projectHistoryId, type) => {
+ (error, lines, version, ops, ranges, pathname) => {
timer.done()
if (error) {
return next(error)
@@ -37,11 +36,6 @@ function getDoc(req, res, next) {
if (lines == null || version == null) {
return next(new Errors.NotFoundError('document not found'))
}
- if (!Array.isArray(lines) && req.query.historyOTSupport !== 'true') {
- const file = StringFileData.fromRaw(lines)
- // TODO(24596): tc support for history-ot
- lines = file.getLines()
- }
res.json({
id: docId,
lines,
@@ -50,35 +44,11 @@ function getDoc(req, res, next) {
ranges,
pathname,
ttlInS: RedisManager.DOC_OPS_TTL,
- type,
})
}
)
}
-function getComment(req, res, next) {
- const docId = req.params.doc_id
- const projectId = req.params.project_id
- const commentId = req.params.comment_id
-
- logger.debug({ projectId, docId, commentId }, 'getting comment via http')
-
- DocumentManager.getCommentWithLock(
- projectId,
- docId,
- commentId,
- (error, comment) => {
- if (error) {
- return next(error)
- }
- if (comment == null) {
- return next(new Errors.NotFoundError('comment not found'))
- }
- res.json(comment)
- }
- )
-}
-
// return the doc from redis if present, but don't load it from mongo
function peekDoc(req, res, next) {
const docId = req.params.doc_id
@@ -91,11 +61,6 @@ function peekDoc(req, res, next) {
if (lines == null || version == null) {
return next(new Errors.NotFoundError('document not found'))
}
- if (!Array.isArray(lines) && req.query.historyOTSupport !== 'true') {
- const file = StringFileData.fromRaw(lines)
- // TODO(24596): tc support for history-ot
- lines = file.getLines()
- }
res.json({ id: docId, lines, version })
})
}
@@ -141,22 +106,6 @@ function getProjectDocsAndFlushIfOld(req, res, next) {
)
}
-function getProjectLastUpdatedAt(req, res, next) {
- const projectId = req.params.project_id
- ProjectManager.getProjectDocsTimestamps(projectId, (err, timestamps) => {
- if (err) return next(err)
-
- // Filter out nulls. This can happen when
- // - docs get flushed between the listing and getting the individual docs ts
- // - a doc flush failed half way (doc keys removed, project tracking not updated)
- timestamps = timestamps.filter(ts => !!ts)
-
- timestamps = timestamps.map(ts => parseInt(ts, 10))
- timestamps.sort((a, b) => (a > b ? 1 : -1))
- res.json({ lastUpdatedAt: timestamps.pop() })
- })
-}
-
function clearProjectState(req, res, next) {
const projectId = req.params.project_id
const timer = new Metrics.Timer('http.clearProjectState')
@@ -195,7 +144,6 @@ function setDoc(req, res, next) {
source,
userId,
undoing,
- true,
(error, result) => {
timer.done()
if (error) {
@@ -207,35 +155,6 @@ function setDoc(req, res, next) {
)
}
-function appendToDoc(req, res, next) {
- const docId = req.params.doc_id
- const projectId = req.params.project_id
- const { lines, source, user_id: userId } = req.body
- const timer = new Metrics.Timer('http.appendToDoc')
- DocumentManager.appendToDocWithLock(
- projectId,
- docId,
- lines,
- source,
- userId,
- (error, result) => {
- timer.done()
- if (error instanceof Errors.FileTooLargeError) {
- logger.warn('refusing to append to file, it would become too large')
- return res.sendStatus(422)
- }
- if (error) {
- return next(error)
- }
- logger.debug(
- { projectId, docId, lines, source, userId },
- 'appending to doc via http'
- )
- res.json(result)
- }
- )
-}
-
function flushDocIfLoaded(req, res, next) {
const docId = req.params.doc_id
const projectId = req.params.project_id
@@ -461,13 +380,7 @@ function updateProject(req, res, next) {
function resyncProjectHistory(req, res, next) {
const projectId = req.params.project_id
- const {
- projectHistoryId,
- docs,
- files,
- historyRangesMigration,
- resyncProjectStructureOnly,
- } = req.body
+ const { projectHistoryId, docs, files, historyRangesMigration } = req.body
logger.debug(
{ projectId, docs, files },
@@ -478,9 +391,6 @@ function resyncProjectHistory(req, res, next) {
if (historyRangesMigration) {
opts.historyRangesMigration = historyRangesMigration
}
- if (resyncProjectStructureOnly) {
- opts.resyncProjectStructureOnly = resyncProjectStructureOnly
- }
HistoryManager.resyncProjectHistory(
projectId,
@@ -549,9 +459,7 @@ module.exports = {
getDoc,
peekDoc,
getProjectDocsAndFlushIfOld,
- getProjectLastUpdatedAt,
clearProjectState,
- appendToDoc,
setDoc,
flushDocIfLoaded,
deleteDoc,
@@ -567,5 +475,4 @@ module.exports = {
flushQueuedProjects,
blockProject,
unblockProject,
- getComment,
}
diff --git a/services/document-updater/app/js/Limits.js b/services/document-updater/app/js/Limits.js
index cbd9293042..268ccd3f9b 100644
--- a/services/document-updater/app/js/Limits.js
+++ b/services/document-updater/app/js/Limits.js
@@ -28,19 +28,4 @@ module.exports = {
// since we didn't hit the limit in the loop, the document is within the allowed length
return false
},
-
- /**
- * @param {StringFileRawData} raw
- * @param {number} maxDocLength
- */
- stringFileDataContentIsTooLarge(raw, maxDocLength) {
- let n = raw.content.length
- if (n <= maxDocLength) return false // definitely under the limit, no need to calculate the total size
- for (const tc of raw.trackedChanges ?? []) {
- if (tc.tracking.type !== 'delete') continue
- n -= tc.range.length
- if (n <= maxDocLength) return false // under the limit now, no need to calculate the exact size
- }
- return true
- },
}
diff --git a/services/document-updater/app/js/PersistenceManager.js b/services/document-updater/app/js/PersistenceManager.js
index 6e832f9aa7..484b9bab2a 100644
--- a/services/document-updater/app/js/PersistenceManager.js
+++ b/services/document-updater/app/js/PersistenceManager.js
@@ -1,4 +1,4 @@
-const { promisify } = require('node:util')
+const { promisify } = require('util')
const { promisifyMultiResult } = require('@overleaf/promise-utils')
const Settings = require('@overleaf/settings')
const Errors = require('./Errors')
@@ -95,13 +95,6 @@ function getDoc(projectId, docId, options = {}, _callback) {
status: body.pathname === '' ? 'zero-length' : 'undefined',
})
}
-
- if (body.otMigrationStage > 0) {
- // Use history-ot
- body.lines = { content: body.lines.join('\n') }
- body.ranges = {}
- }
-
callback(
null,
body.lines,
diff --git a/services/document-updater/app/js/Profiler.js b/services/document-updater/app/js/Profiler.js
index aac8a9706e..8daac4ca41 100644
--- a/services/document-updater/app/js/Profiler.js
+++ b/services/document-updater/app/js/Profiler.js
@@ -1,52 +1,68 @@
+/* eslint-disable
+ no-unused-vars,
+*/
+// TODO: This file was created by bulk-decaffeinate.
+// Fix any style issues and re-enable lint.
+/*
+ * decaffeinate suggestions:
+ * DS206: Consider reworking classes to avoid initClass
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+let Profiler
+const Settings = require('@overleaf/settings')
const logger = require('@overleaf/logger')
-function deltaMs(ta, tb) {
+const deltaMs = function (ta, tb) {
const nanoSeconds = (ta[0] - tb[0]) * 1e9 + (ta[1] - tb[1])
const milliSeconds = Math.floor(nanoSeconds * 1e-6)
return milliSeconds
}
-class Profiler {
- LOG_CUTOFF_TIME = 15 * 1000
- LOG_SYNC_CUTOFF_TIME = 1000
-
- constructor(name, args) {
- this.name = name
- this.args = args
- this.t0 = this.t = process.hrtime()
- this.start = new Date()
- this.updateTimes = []
- this.totalSyncTime = 0
- }
-
- log(label, options = {}) {
- const t1 = process.hrtime()
- const dtMilliSec = deltaMs(t1, this.t)
- this.t = t1
- this.totalSyncTime += options.sync ? dtMilliSec : 0
- this.updateTimes.push([label, dtMilliSec]) // timings in ms
- return this // make it chainable
- }
-
- end() {
- const totalTime = deltaMs(this.t, this.t0)
- const exceedsCutoff = totalTime > this.LOG_CUTOFF_TIME
- const exceedsSyncCutoff = this.totalSyncTime > this.LOG_SYNC_CUTOFF_TIME
- if (exceedsCutoff || exceedsSyncCutoff) {
- // log anything greater than cutoffs
- const args = {}
- for (const k in this.args) {
- const v = this.args[k]
- args[k] = v
- }
- args.updateTimes = this.updateTimes
- args.start = this.start
- args.end = new Date()
- args.status = { exceedsCutoff, exceedsSyncCutoff }
- logger.warn(args, this.name)
+module.exports = Profiler = (function () {
+ Profiler = class Profiler {
+ static initClass() {
+ this.prototype.LOG_CUTOFF_TIME = 15 * 1000
+ this.prototype.LOG_SYNC_CUTOFF_TIME = 1000
}
- return totalTime
- }
-}
-module.exports = Profiler
+ constructor(name, args) {
+ this.name = name
+ this.args = args
+ this.t0 = this.t = process.hrtime()
+ this.start = new Date()
+ this.updateTimes = []
+ this.totalSyncTime = 0
+ }
+
+ log(label, options = {}) {
+ const t1 = process.hrtime()
+ const dtMilliSec = deltaMs(t1, this.t)
+ this.t = t1
+ this.totalSyncTime += options.sync ? dtMilliSec : 0
+ this.updateTimes.push([label, dtMilliSec]) // timings in ms
+ return this // make it chainable
+ }
+
+ end(message) {
+ const totalTime = deltaMs(this.t, this.t0)
+ const exceedsCutoff = totalTime > this.LOG_CUTOFF_TIME
+ const exceedsSyncCutoff = this.totalSyncTime > this.LOG_SYNC_CUTOFF_TIME
+ if (exceedsCutoff || exceedsSyncCutoff) {
+ // log anything greater than cutoffs
+ const args = {}
+ for (const k in this.args) {
+ const v = this.args[k]
+ args[k] = v
+ }
+ args.updateTimes = this.updateTimes
+ args.start = this.start
+ args.end = new Date()
+ args.status = { exceedsCutoff, exceedsSyncCutoff }
+ logger.warn(args, this.name)
+ }
+ return totalTime
+ }
+ }
+ Profiler.initClass()
+ return Profiler
+})()
diff --git a/services/document-updater/app/js/ProjectHistoryRedisManager.js b/services/document-updater/app/js/ProjectHistoryRedisManager.js
index 78e9c2ea4c..b84edfe74a 100644
--- a/services/document-updater/app/js/ProjectHistoryRedisManager.js
+++ b/services/document-updater/app/js/ProjectHistoryRedisManager.js
@@ -8,14 +8,13 @@ const rclient = require('@overleaf/redis-wrapper').createClient(
)
const logger = require('@overleaf/logger')
const metrics = require('./Metrics')
-const { docIsTooLarge, stringFileDataContentIsTooLarge } = require('./Limits')
+const { docIsTooLarge } = require('./Limits')
const { addTrackedDeletesToContent, extractOriginOrSource } = require('./Utils')
const HistoryConversions = require('./HistoryConversions')
const OError = require('@overleaf/o-error')
/**
* @import { Ranges } from './types'
- * @import { StringFileRawData } from 'overleaf-editor-core/lib/types'
*/
const ProjectHistoryRedisManager = {
@@ -123,7 +122,6 @@ const ProjectHistoryRedisManager = {
hash: projectUpdate.hash,
metadata: projectUpdate.metadata,
projectHistoryId,
- createdBlob: projectUpdate.createdBlob ?? false,
}
if (ranges) {
projectUpdate.ranges = ranges
@@ -153,13 +151,7 @@ const ProjectHistoryRedisManager = {
return await ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate)
},
- async queueResyncProjectStructure(
- projectId,
- projectHistoryId,
- docs,
- files,
- opts
- ) {
+ async queueResyncProjectStructure(projectId, projectHistoryId, docs, files) {
logger.debug({ projectId, docs, files }, 'queue project structure resync')
const projectUpdate = {
resyncProjectStructure: { docs, files },
@@ -168,9 +160,6 @@ const ProjectHistoryRedisManager = {
ts: new Date(),
},
}
- if (opts.resyncProjectStructureOnly) {
- projectUpdate.resyncProjectStructureOnly = opts.resyncProjectStructureOnly
- }
const jsonUpdate = JSON.stringify(projectUpdate)
return await ProjectHistoryRedisManager.queueOps(projectId, jsonUpdate)
},
@@ -181,7 +170,7 @@ const ProjectHistoryRedisManager = {
* @param {string} projectId
* @param {string} projectHistoryId
* @param {string} docId
- * @param {string[] | StringFileRawData} lines
+ * @param {string[]} lines
* @param {Ranges} ranges
* @param {string[]} resolvedCommentIds
* @param {number} version
@@ -205,8 +194,13 @@ const ProjectHistoryRedisManager = {
'queue doc content resync'
)
+ let content = lines.join('\n')
+ if (historyRangesSupport) {
+ content = addTrackedDeletesToContent(content, ranges.changes ?? [])
+ }
+
const projectUpdate = {
- resyncDocContent: { version },
+ resyncDocContent: { content, version },
projectHistoryId,
path: pathname,
doc: docId,
@@ -215,38 +209,17 @@ const ProjectHistoryRedisManager = {
},
}
- let content = ''
- if (Array.isArray(lines)) {
- content = lines.join('\n')
- if (historyRangesSupport) {
- content = addTrackedDeletesToContent(content, ranges.changes ?? [])
- projectUpdate.resyncDocContent.ranges =
- HistoryConversions.toHistoryRanges(ranges)
- projectUpdate.resyncDocContent.resolvedCommentIds = resolvedCommentIds
- }
- } else {
- content = lines.content
- projectUpdate.resyncDocContent.historyOTRanges = {
- comments: lines.comments,
- trackedChanges: lines.trackedChanges,
- }
+ if (historyRangesSupport) {
+ projectUpdate.resyncDocContent.ranges =
+ HistoryConversions.toHistoryRanges(ranges)
+ projectUpdate.resyncDocContent.resolvedCommentIds = resolvedCommentIds
}
- projectUpdate.resyncDocContent.content = content
const jsonUpdate = JSON.stringify(projectUpdate)
// Do an optimised size check on the docLines using the serialised
// project update length as an upper bound
const sizeBound = jsonUpdate.length
- if (Array.isArray(lines)) {
- if (docIsTooLarge(sizeBound, lines, Settings.max_doc_length)) {
- throw new OError(
- 'blocking resync doc content insert into project history queue: doc is too large',
- { projectId, docId, docSize: sizeBound }
- )
- }
- } else if (
- stringFileDataContentIsTooLarge(lines, Settings.max_doc_length)
- ) {
+ if (docIsTooLarge(sizeBound, lines, Settings.max_doc_length)) {
throw new OError(
'blocking resync doc content insert into project history queue: doc is too large',
{ projectId, docId, docSize: sizeBound }
diff --git a/services/document-updater/app/js/ProjectManager.js b/services/document-updater/app/js/ProjectManager.js
index cdd4c11482..781ed0e168 100644
--- a/services/document-updater/app/js/ProjectManager.js
+++ b/services/document-updater/app/js/ProjectManager.js
@@ -317,7 +317,6 @@ function updateProjectWithLocks(
}
if (
HistoryManager.shouldFlushHistoryOps(
- projectId,
projectOpsLength,
updates.length,
HistoryManager.FLUSH_PROJECT_EVERY_N_OPS
diff --git a/services/document-updater/app/js/RangesManager.js b/services/document-updater/app/js/RangesManager.js
index c146afda60..7dfa462ce4 100644
--- a/services/document-updater/app/js/RangesManager.js
+++ b/services/document-updater/app/js/RangesManager.js
@@ -127,7 +127,7 @@ const RangesManager = {
return { newRanges, rangesWereCollapsed, historyUpdates }
},
- acceptChanges(projectId, docId, changeIds, ranges, lines) {
+ acceptChanges(changeIds, ranges) {
const { changes, comments } = ranges
logger.debug(`accepting ${changeIds.length} changes in ranges`)
const rangesTracker = new RangesTracker(changes, comments)
@@ -339,12 +339,6 @@ function getHistoryOpForInsert(op, comments, changes) {
}
}
- // If it's determined that the op is a tracked delete rejection, we have to
- // calculate its proper history position. If multiple tracked deletes are
- // found at the same position as the insert, the tracked deletes that come
- // before the tracked delete that was actually rejected offset the history
- // position.
- let trackedDeleteRejectionOffset = 0
for (const change of changes) {
if (!isDelete(change.op)) {
// We're only interested in tracked deletes
@@ -355,25 +349,14 @@ function getHistoryOpForInsert(op, comments, changes) {
// Tracked delete is before the op. Move the op forward.
hpos += change.op.d.length
} else if (change.op.p === op.p) {
- // Tracked delete is at the same position as the op.
+ // Tracked delete is at the same position as the op. The insert comes before
+ // the tracked delete so it doesn't move.
if (op.u && change.op.d.startsWith(op.i)) {
// We're undoing and the insert matches the start of the tracked
// delete. RangesManager treats this as a tracked delete rejection. We
// will note this in the op so that project-history can take the
// appropriate action.
trackedDeleteRejection = true
-
- // The history must be updated to take into account all preceding
- // tracked deletes at the same position
- hpos += trackedDeleteRejectionOffset
-
- // No need to continue. All subsequent tracked deletes are after the
- // insert.
- break
- } else {
- // This tracked delete does not match the insert. Note its length in
- // case we find a tracked delete that matches later.
- trackedDeleteRejectionOffset += change.op.d.length
}
} else {
// Tracked delete is after the insert. Tracked deletes are ordered, so
diff --git a/services/document-updater/app/js/RealTimeRedisManager.js b/services/document-updater/app/js/RealTimeRedisManager.js
index 2b67971c5c..9f7465acb8 100644
--- a/services/document-updater/app/js/RealTimeRedisManager.js
+++ b/services/document-updater/app/js/RealTimeRedisManager.js
@@ -20,8 +20,8 @@ const pubsubClient = require('@overleaf/redis-wrapper').createClient(
)
const Keys = Settings.redis.documentupdater.key_schema
const logger = require('@overleaf/logger')
-const os = require('node:os')
-const crypto = require('node:crypto')
+const os = require('os')
+const crypto = require('crypto')
const metrics = require('./Metrics')
const HOST = os.hostname()
@@ -49,7 +49,7 @@ const RealTimeRedisManager = {
MAX_OPS_PER_ITERATION,
-1
)
- multi.exec(function (error, replys) {
+ return multi.exec(function (error, replys) {
if (error != null) {
return callback(error)
}
@@ -80,7 +80,7 @@ const RealTimeRedisManager = {
},
getUpdatesLength(docId, callback) {
- rclient.llen(Keys.pendingUpdates({ doc_id: docId }), callback)
+ return rclient.llen(Keys.pendingUpdates({ doc_id: docId }), callback)
},
sendCanaryAppliedOp({ projectId, docId, op }) {
@@ -132,5 +132,5 @@ const RealTimeRedisManager = {
module.exports = RealTimeRedisManager
module.exports.promises = promisifyAll(RealTimeRedisManager, {
- without: ['sendCanaryAppliedOp', 'sendData'],
+ without: ['sendData'],
})
diff --git a/services/document-updater/app/js/RedisManager.js b/services/document-updater/app/js/RedisManager.js
index 7f86036427..0d2dad1047 100644
--- a/services/document-updater/app/js/RedisManager.js
+++ b/services/document-updater/app/js/RedisManager.js
@@ -7,7 +7,7 @@ const OError = require('@overleaf/o-error')
const { promisifyAll } = require('@overleaf/promise-utils')
const metrics = require('./Metrics')
const Errors = require('./Errors')
-const crypto = require('node:crypto')
+const crypto = require('crypto')
const async = require('async')
const { docIsTooLarge } = require('./Limits')
@@ -48,7 +48,6 @@ const RedisManager = {
timer.done()
_callback(error)
}
- const shareJSTextOT = Array.isArray(docLines)
const docLinesArray = docLines
docLines = JSON.stringify(docLines)
if (docLines.indexOf('\u0000') !== -1) {
@@ -61,10 +60,7 @@ const RedisManager = {
// Do an optimised size check on the docLines using the serialised
// length as an upper bound
const sizeBound = docLines.length
- if (
- shareJSTextOT && // editor-core has a size check in TextOperation.apply and TextOperation.applyToLength.
- docIsTooLarge(sizeBound, docLinesArray, Settings.max_doc_length)
- ) {
+ if (docIsTooLarge(sizeBound, docLinesArray, Settings.max_doc_length)) {
const docSize = docLines.length
const err = new Error('blocking doc insert into redis: doc is too large')
logger.error({ projectId, docId, err, docSize }, err.message)
@@ -465,7 +461,6 @@ const RedisManager = {
if (appliedOps == null) {
appliedOps = []
}
- const shareJSTextOT = Array.isArray(docLines)
RedisManager.getDocVersion(docId, (error, currentVersion) => {
if (error) {
return callback(error)
@@ -505,10 +500,7 @@ const RedisManager = {
// Do an optimised size check on the docLines using the serialised
// length as an upper bound
const sizeBound = newDocLines.length
- if (
- shareJSTextOT && // editor-core has a size check in TextOperation.apply and TextOperation.applyToLength.
- docIsTooLarge(sizeBound, docLines, Settings.max_doc_length)
- ) {
+ if (docIsTooLarge(sizeBound, docLines, Settings.max_doc_length)) {
const err = new Error('blocking doc update: doc is too large')
const docSize = newDocLines.length
logger.error({ projectId, docId, err, docSize }, err.message)
diff --git a/services/document-updater/app/js/ShareJsUpdateManager.js b/services/document-updater/app/js/ShareJsUpdateManager.js
index 933ded1ce1..edac3700b7 100644
--- a/services/document-updater/app/js/ShareJsUpdateManager.js
+++ b/services/document-updater/app/js/ShareJsUpdateManager.js
@@ -16,10 +16,10 @@ const logger = require('@overleaf/logger')
const Settings = require('@overleaf/settings')
const { promisifyAll } = require('@overleaf/promise-utils')
const Keys = require('./UpdateKeys')
-const { EventEmitter } = require('node:events')
-const util = require('node:util')
+const { EventEmitter } = require('events')
+const util = require('util')
const RealTimeRedisManager = require('./RealTimeRedisManager')
-const crypto = require('node:crypto')
+const crypto = require('crypto')
const metrics = require('./Metrics')
const Errors = require('./Errors')
diff --git a/services/document-updater/app/js/UpdateManager.js b/services/document-updater/app/js/UpdateManager.js
index e5df48575e..b23522a2cb 100644
--- a/services/document-updater/app/js/UpdateManager.js
+++ b/services/document-updater/app/js/UpdateManager.js
@@ -14,11 +14,10 @@ const DocumentManager = require('./DocumentManager')
const RangesManager = require('./RangesManager')
const SnapshotManager = require('./SnapshotManager')
const Profiler = require('./Profiler')
-const { isInsert, isDelete, getDocLength, computeDocHash } = require('./Utils')
-const HistoryOTUpdateManager = require('./HistoryOTUpdateManager')
+const { isInsert, isDelete, getDocLength } = require('./Utils')
/**
- * @import { Ranges, Update, HistoryUpdate } from "./types"
+ * @import { DeleteOp, InsertOp, Op, Ranges, Update, HistoryUpdate } from "./types"
*/
const UpdateManager = {
@@ -81,11 +80,7 @@ const UpdateManager = {
profile.log('getPendingUpdatesForDoc')
for (const update of updates) {
- if (HistoryOTUpdateManager.isHistoryOTEditOperationUpdate(update)) {
- await HistoryOTUpdateManager.applyUpdate(projectId, docId, update)
- } else {
- await UpdateManager.applyUpdate(projectId, docId, update)
- }
+ await UpdateManager.applyUpdate(projectId, docId, update)
profile.log('applyUpdate')
}
profile.log('async done').end()
@@ -115,16 +110,12 @@ const UpdateManager = {
pathname,
projectHistoryId,
historyRangesSupport,
- type,
} = await DocumentManager.promises.getDoc(projectId, docId)
profile.log('getDoc')
if (lines == null || version == null) {
throw new Errors.NotFoundError(`document not found: ${docId}`)
}
- if (type !== 'sharejs-text-ot') {
- throw new Errors.OTTypeMismatchError(type, 'sharejs-text-ot')
- }
const previousVersion = version
const incomingUpdateVersion = update.v
@@ -171,7 +162,6 @@ const UpdateManager = {
projectHistoryId,
lines,
ranges,
- updatedDocLines,
historyRangesSupport
)
@@ -300,9 +290,8 @@ const UpdateManager = {
* @param {HistoryUpdate[]} updates
* @param {string} pathname
* @param {string} projectHistoryId
- * @param {string[]} lines - document lines before updates were applied
- * @param {Ranges} ranges - ranges before updates were applied
- * @param {string[]} newLines - document lines after updates were applied
+ * @param {string[]} lines
+ * @param {Ranges} ranges
* @param {boolean} historyRangesSupport
*/
_adjustHistoryUpdatesMetadata(
@@ -311,7 +300,6 @@ const UpdateManager = {
projectHistoryId,
lines,
ranges,
- newLines,
historyRangesSupport
) {
let docLength = getDocLength(lines)
@@ -375,12 +363,6 @@ const UpdateManager = {
delete update.meta.tc
}
}
-
- if (historyRangesSupport && updates.length > 0) {
- const lastUpdate = updates[updates.length - 1]
- lastUpdate.meta ??= {}
- lastUpdate.meta.doc_hash = computeDocHash(newLines)
- }
},
}
diff --git a/services/document-updater/app/js/Utils.js b/services/document-updater/app/js/Utils.js
index a632cf32eb..4e9e60ba06 100644
--- a/services/document-updater/app/js/Utils.js
+++ b/services/document-updater/app/js/Utils.js
@@ -1,5 +1,4 @@
// @ts-check
-const { createHash } = require('node:crypto')
const _ = require('lodash')
/**
@@ -80,27 +79,6 @@ function addTrackedDeletesToContent(content, trackedChanges) {
return result
}
-/**
- * Compute the content hash for a doc
- *
- * This hash is sent to the history to validate updates.
- *
- * @param {string[]} lines
- * @return {string} the doc hash
- */
-function computeDocHash(lines) {
- const hash = createHash('sha1')
- if (lines.length > 0) {
- for (const line of lines.slice(0, lines.length - 1)) {
- hash.update(line)
- hash.update('\n')
- }
- // The last line doesn't end with a newline
- hash.update(lines[lines.length - 1])
- }
- return hash.digest('hex')
-}
-
/**
* checks if the given originOrSource should be treated as a source or origin
* TODO: remove this hack and remove all "source" references
@@ -124,6 +102,5 @@ module.exports = {
isComment,
addTrackedDeletesToContent,
getDocLength,
- computeDocHash,
extractOriginOrSource,
}
diff --git a/services/document-updater/app/js/mongodb.js b/services/document-updater/app/js/mongodb.js
index 6e38993bc4..6c4415adcc 100644
--- a/services/document-updater/app/js/mongodb.js
+++ b/services/document-updater/app/js/mongodb.js
@@ -24,5 +24,5 @@ module.exports = {
db,
ObjectId,
mongoClient,
- healthCheck: require('node:util').callbackify(healthCheck),
+ healthCheck: require('util').callbackify(healthCheck),
}
diff --git a/services/document-updater/app/js/sharejs/server/model.js b/services/document-updater/app/js/sharejs/server/model.js
index a646b22492..370208cc37 100644
--- a/services/document-updater/app/js/sharejs/server/model.js
+++ b/services/document-updater/app/js/sharejs/server/model.js
@@ -22,7 +22,7 @@
// Actual storage is handled by the database wrappers in db/*, wrapped by DocCache
let Model
-const { EventEmitter } = require('node:events')
+const { EventEmitter } = require('events')
const queue = require('./syncqueue')
const types = require('../types')
diff --git a/services/document-updater/app/js/sharejs/types/model.js b/services/document-updater/app/js/sharejs/types/model.js
index af9fd0ad18..d927811895 100644
--- a/services/document-updater/app/js/sharejs/types/model.js
+++ b/services/document-updater/app/js/sharejs/types/model.js
@@ -22,7 +22,7 @@
// Actual storage is handled by the database wrappers in db/*, wrapped by DocCache
let Model
-const { EventEmitter } = require('node:events')
+const { EventEmitter } = require('events')
const queue = require('./syncqueue')
const types = require('../types')
diff --git a/services/document-updater/app/js/types.ts b/services/document-updater/app/js/types.ts
index 851e62d8c8..d635ab31ca 100644
--- a/services/document-updater/app/js/types.ts
+++ b/services/document-updater/app/js/types.ts
@@ -1,17 +1,12 @@
import {
TrackingPropsRawData,
ClearTrackingPropsRawData,
- RawEditOperation,
} from 'overleaf-editor-core/lib/types'
-export type OTType = 'sharejs-text-ot' | 'history-ot'
-
/**
* An update coming from the editor
*/
export type Update = {
- dup?: boolean
- dupIfSource?: string[]
doc: string
op: Op[]
v: number
@@ -23,11 +18,6 @@ export type Update = {
projectHistoryId?: string
}
-export type HistoryOTEditOperationUpdate = Omit & {
- op: RawEditOperation[]
- meta: Update['meta'] & { source: string }
-}
-
export type Op = InsertOp | DeleteOp | CommentOp | RetainOp
export type InsertOp = {
@@ -94,7 +84,6 @@ export type HistoryUpdate = {
pathname?: string
doc_length?: number
history_doc_length?: number
- doc_hash?: string
tc?: boolean
user_id?: string
}
diff --git a/services/document-updater/buildscript.txt b/services/document-updater/buildscript.txt
index 98d10a8e55..3221114171 100644
--- a/services/document-updater/buildscript.txt
+++ b/services/document-updater/buildscript.txt
@@ -4,6 +4,6 @@ document-updater
--env-add=
--env-pass-through=
--esmock-loader=False
---node-version=22.17.0
+--node-version=18.20.2
--public-repo=True
---script-version=4.7.0
+--script-version=4.5.0
diff --git a/services/document-updater/config/settings.defaults.js b/services/document-updater/config/settings.defaults.js
index 9ed59de6c4..d8c2ada6d3 100755
--- a/services/document-updater/config/settings.defaults.js
+++ b/services/document-updater/config/settings.defaults.js
@@ -1,9 +1,3 @@
-const http = require('node:http')
-const https = require('node:https')
-
-http.globalAgent.keepAlive = false
-https.globalAgent.keepAlive = false
-
module.exports = {
internal: {
documentupdater: {
@@ -176,6 +170,10 @@ module.exports = {
},
},
+ sentry: {
+ dsn: process.env.SENTRY_DSN,
+ },
+
publishOnIndividualChannels:
process.env.PUBLISH_ON_INDIVIDUAL_CHANNELS === 'true',
@@ -184,8 +182,4 @@ module.exports = {
smoothingOffset: process.env.SMOOTHING_OFFSET || 1000, // milliseconds
gracefulShutdownDelayInMs:
parseInt(process.env.GRACEFUL_SHUTDOWN_DELAY_SECONDS ?? '10', 10) * 1000,
-
- shortHistoryQueues: (process.env.SHORT_HISTORY_QUEUES || '')
- .split(',')
- .filter(s => !!s),
}
diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml
index c6ec24a84b..332a9710ca 100644
--- a/services/document-updater/docker-compose.ci.yml
+++ b/services/document-updater/docker-compose.ci.yml
@@ -21,22 +21,18 @@ services:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis
QUEUES_REDIS_HOST: redis
- HISTORY_REDIS_HOST: redis
ANALYTICS_QUEUES_REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
- volumes:
- - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
- condition: service_started
+ condition: service_healthy
redis:
condition: service_healthy
user: node
- entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run test:acceptance
@@ -48,21 +44,16 @@ services:
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
redis:
- image: redis:7.4.3
+ image: redis
healthcheck:
test: ping="$$(redis-cli ping)" && [ "$$ping" = 'PONG' ]
interval: 1s
retries: 20
mongo:
- image: mongo:8.0.11
+ image: mongo:6.0.13
command: --replSet overleaf
- volumes:
- - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
- environment:
- MONGO_INITDB_DATABASE: sharelatex
- extra_hosts:
- # Required when using the automatic database setup for initializing the
- # replica set. This override is not needed when running the setup after
- # starting up mongo.
- - mongo:127.0.0.1
+ healthcheck:
+ test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
+ interval: 1s
+ retries: 20
diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml
index c1b23c11c5..83f5d2efcc 100644
--- a/services/document-updater/docker-compose.yml
+++ b/services/document-updater/docker-compose.yml
@@ -6,7 +6,7 @@ version: "2.3"
services:
test_unit:
- image: node:22.17.0
+ image: node:18.20.2
volumes:
- .:/overleaf/services/document-updater
- ../../node_modules:/overleaf/node_modules
@@ -14,58 +14,49 @@ services:
working_dir: /overleaf/services/document-updater
environment:
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
user: node
test_acceptance:
- image: node:22.17.0
+ image: node:18.20.2
volumes:
- .:/overleaf/services/document-updater
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
working_dir: /overleaf/services/document-updater
environment:
ELASTIC_SEARCH_DSN: es:9200
REDIS_HOST: redis
- HISTORY_REDIS_HOST: redis
QUEUES_REDIS_HOST: redis
ANALYTICS_QUEUES_REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
+ LOG_LEVEL: ERROR
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
user: node
depends_on:
mongo:
- condition: service_started
+ condition: service_healthy
redis:
condition: service_healthy
- entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run --silent test:acceptance
redis:
- image: redis:7.4.3
+ image: redis
healthcheck:
test: ping=$$(redis-cli ping) && [ "$$ping" = 'PONG' ]
interval: 1s
retries: 20
mongo:
- image: mongo:8.0.11
+ image: mongo:6.0.13
command: --replSet overleaf
- volumes:
- - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
- environment:
- MONGO_INITDB_DATABASE: sharelatex
- extra_hosts:
- # Required when using the automatic database setup for initializing the
- # replica set. This override is not needed when running the setup after
- # starting up mongo.
- - mongo:127.0.0.1
+ healthcheck:
+ test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
+ interval: 1s
+ retries: 20
diff --git a/services/document-updater/package.json b/services/document-updater/package.json
index 7d892689e9..bf49f53b21 100644
--- a/services/document-updater/package.json
+++ b/services/document-updater/package.json
@@ -12,8 +12,8 @@
"nodemon": "node --watch app.js",
"benchmark:apply": "node benchmarks/apply",
"lint": "eslint --max-warnings 0 --format unix .",
- "format": "prettier --list-different $PWD/'**/*.*js'",
- "format:fix": "prettier --write $PWD/'**/*.*js'",
+ "format": "prettier --list-different $PWD/'**/*.js'",
+ "format:fix": "prettier --write $PWD/'**/*.js'",
"lint:fix": "eslint --fix .",
"types:check": "tsc --noEmit"
},
@@ -30,11 +30,10 @@
"body-parser": "^1.20.3",
"bunyan": "^1.8.15",
"diff-match-patch": "overleaf/diff-match-patch#89805f9c671a77a263fc53461acd62aa7498f688",
- "express": "^4.21.2",
+ "express": "^4.21.0",
"lodash": "^4.17.21",
"minimist": "^1.2.8",
- "mongodb-legacy": "6.1.3",
- "overleaf-editor-core": "*",
+ "mongodb-legacy": "^6.0.1",
"request": "^2.88.2",
"requestretry": "^7.1.0"
},
@@ -42,7 +41,7 @@
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
"cluster-key-slot": "^1.0.5",
- "mocha": "^11.1.0",
+ "mocha": "^10.2.0",
"sandboxed-module": "^2.0.4",
"sinon": "^9.2.4",
"sinon-chai": "^3.7.0",
diff --git a/services/document-updater/scripts/check_redis_mongo_sync_state.js b/services/document-updater/scripts/check_redis_mongo_sync_state.js
index 51db47af4d..0ca2bb4e8b 100644
--- a/services/document-updater/scripts/check_redis_mongo_sync_state.js
+++ b/services/document-updater/scripts/check_redis_mongo_sync_state.js
@@ -1,5 +1,5 @@
-const fs = require('node:fs')
-const Path = require('node:path')
+const fs = require('fs')
+const Path = require('path')
const _ = require('lodash')
const logger = require('@overleaf/logger')
const OError = require('@overleaf/o-error')
@@ -15,7 +15,6 @@ const request = require('requestretry').defaults({
retryDelay: 10,
})
-const ONLY_PROJECT_ID = process.env.ONLY_PROJECT_ID
const AUTO_FIX_VERSION_MISMATCH =
process.env.AUTO_FIX_VERSION_MISMATCH === 'true'
const AUTO_FIX_PARTIALLY_DELETED_DOC_METADATA =
@@ -320,12 +319,10 @@ async function processProject(projectId) {
* @return {Promise<{perIterationOutOfSync: number, done: boolean}>}
*/
async function scanOnce(processed, outOfSync) {
- const projectIds = ONLY_PROJECT_ID
- ? [ONLY_PROJECT_ID]
- : await ProjectFlusher.promises.flushAllProjects({
- limit: LIMIT,
- dryRun: true,
- })
+ const projectIds = await ProjectFlusher.promises.flushAllProjects({
+ limit: LIMIT,
+ dryRun: true,
+ })
let perIterationOutOfSync = 0
for (const projectId of projectIds) {
diff --git a/services/document-updater/scripts/fix_docs_with_empty_pathnames.js b/services/document-updater/scripts/fix_docs_with_empty_pathnames.js
index e3e034157b..abbfb01ba7 100644
--- a/services/document-updater/scripts/fix_docs_with_empty_pathnames.js
+++ b/services/document-updater/scripts/fix_docs_with_empty_pathnames.js
@@ -6,7 +6,7 @@ const rclient = require('@overleaf/redis-wrapper').createClient(
const keys = Settings.redis.documentupdater.key_schema
const ProjectFlusher = require('app/js/ProjectFlusher')
const DocumentManager = require('app/js/DocumentManager')
-const util = require('node:util')
+const util = require('util')
const flushAndDeleteDocWithLock = util.promisify(
DocumentManager.flushAndDeleteDocWithLock
)
diff --git a/services/document-updater/scripts/fix_docs_with_missing_project.js b/services/document-updater/scripts/fix_docs_with_missing_project.js
index d3022127b3..0a9a05eaaa 100644
--- a/services/document-updater/scripts/fix_docs_with_missing_project.js
+++ b/services/document-updater/scripts/fix_docs_with_missing_project.js
@@ -7,7 +7,7 @@ const keys = Settings.redis.documentupdater.key_schema
const ProjectFlusher = require('../app/js/ProjectFlusher')
const DocumentManager = require('../app/js/DocumentManager')
const { mongoClient, db, ObjectId } = require('../app/js/mongodb')
-const util = require('node:util')
+const util = require('util')
const flushAndDeleteDocWithLock = util.promisify(
DocumentManager.flushAndDeleteDocWithLock
)
diff --git a/services/document-updater/scripts/flush_projects_with_no_history_id.js b/services/document-updater/scripts/flush_projects_with_no_history_id.js
deleted file mode 100644
index aa912b4b66..0000000000
--- a/services/document-updater/scripts/flush_projects_with_no_history_id.js
+++ /dev/null
@@ -1,211 +0,0 @@
-// @ts-check
-
-const Settings = require('@overleaf/settings')
-const logger = require('@overleaf/logger')
-const RedisManager = require('../app/js/RedisManager')
-const minimist = require('minimist')
-const { db, ObjectId } = require('../app/js/mongodb')
-const ProjectManager = require('../app/js/ProjectManager')
-const OError = require('@overleaf/o-error')
-
-const docUpdaterKeys = Settings.redis.documentupdater.key_schema
-
-const rclient = RedisManager.rclient
-
-const { verbose, commit, ...args } = minimist(process.argv.slice(2), {
- boolean: ['verbose', 'commit'],
- string: ['batchSize'],
- default: {
- batchSize: '1000',
- },
-})
-
-logger.logger.level(verbose ? 'debug' : 'warn')
-
-const batchSize = parseInt(args.batchSize, 10)
-
-/**
- * @typedef {import('ioredis').Redis} Redis
- */
-
-/**
- *
- * @param {string} key
- * @return {string|void}
- */
-function extractDocId(key) {
- const matches = key.match(/ProjectHistoryId:\{(.*?)\}/)
- if (matches) {
- return matches[1]
- }
-}
-
-/**
- *
- * @param {string} docId
- * @return {Promise<{projectId: string, historyId: string}>}
- */
-async function getHistoryId(docId) {
- const doc = await db.docs.findOne(
- { _id: new ObjectId(docId) },
- { projection: { project_id: 1 }, readPreference: 'secondaryPreferred' }
- )
-
- if (!doc) {
- throw new OError('Doc not present in mongo', { docId })
- }
-
- const project = await db.projects.findOne(
- { _id: doc.project_id },
- {
- projection: { 'overleaf.history': 1 },
- readPreference: 'secondaryPreferred',
- }
- )
-
- if (!project?.overleaf?.history?.id) {
- throw new OError('Project not present in mongo (or has no history id)', {
- docId,
- project,
- doc,
- })
- }
-
- return {
- historyId: project?.overleaf?.history?.id,
- projectId: doc.project_id.toString(),
- }
-}
-
-/**
- * @typedef {Object} UpdateableDoc
- * @property {string} docId
- * @property {string} projectId
- * @property {string} historyId
- */
-
-/**
- *
- * @param {Redis} node
- * @param {Array} docIds
- * @return {Promise>}
- */
-async function findDocsWithMissingHistoryIds(node, docIds) {
- const historyIds = await node.mget(
- docIds.map(docId => docUpdaterKeys.projectHistoryId({ doc_id: docId }))
- )
-
- const results = []
-
- for (const index in docIds) {
- const historyId = historyIds[index]
- const docId = docIds[index]
- if (!historyId) {
- try {
- const { projectId, historyId } = await getHistoryId(docId)
- results.push({ projectId, historyId, docId })
- } catch (error) {
- logger.warn(
- { error },
- 'Error gathering data for doc with missing history id'
- )
- }
- }
- }
- return results
-}
-
-/**
- *
- * @param {Array} updates
- * @return {Promise}
- */
-async function fixAndFlushProjects(updates) {
- for (const update of updates) {
- if (commit) {
- try {
- await rclient.set(
- docUpdaterKeys.projectHistoryId({ doc_id: update.docId }),
- update.historyId
- )
- logger.debug({ ...update }, 'Set history id in redis')
- await ProjectManager.promises.flushAndDeleteProjectWithLocks(
- update.projectId,
- {}
- )
- logger.debug({ ...update }, 'Flushed project')
- } catch (err) {
- logger.error({ err, ...update }, 'Error fixing and flushing project')
- }
- } else {
- logger.debug(
- { ...update },
- 'Would have set history id in redis and flushed'
- )
- }
- }
-}
-
-/**
- *
- * @param {Array} nodes
- * @param {number} batchSize
- * @return {Promise}
- */
-async function scanNodes(nodes, batchSize = 1000) {
- let scanned = 0
-
- for (const node of nodes) {
- const stream = node.scanStream({
- match: docUpdaterKeys.projectHistoryId({ doc_id: '*' }),
- count: batchSize,
- })
-
- for await (const docKeys of stream) {
- if (docKeys.length === 0) {
- continue
- }
- stream.pause()
- scanned += docKeys.length
-
- const docIds = docKeys
- .map((/** @type {string} */ docKey) => extractDocId(docKey))
- .filter(Boolean)
-
- try {
- const updates = await findDocsWithMissingHistoryIds(node, docIds)
- if (updates.length > 0) {
- logger.info({ updates }, 'Found doc(s) with missing history ids')
- await fixAndFlushProjects(updates)
- }
- } catch (error) {
- logger.error({ docKeys }, 'Error processing batch')
- } finally {
- stream.resume()
- }
- }
-
- logger.info({ scanned, server: node.serverInfo.role }, 'Scanned node')
- }
-}
-
-async function main({ batchSize }) {
- const nodes = (typeof rclient.nodes === 'function'
- ? rclient.nodes('master')
- : undefined) || [rclient]
- await scanNodes(nodes, batchSize)
-}
-
-let code = 0
-
-main({ batchSize })
- .then(() => {
- logger.info({}, 'done')
- })
- .catch(error => {
- logger.error({ error }, 'error')
- code = 1
- })
- .finally(() => {
- rclient.quit().then(() => process.exit(code))
- })
diff --git a/services/document-updater/scripts/remove_deleted_docs.js b/services/document-updater/scripts/remove_deleted_docs.js
index a69807119f..1d0844872e 100644
--- a/services/document-updater/scripts/remove_deleted_docs.js
+++ b/services/document-updater/scripts/remove_deleted_docs.js
@@ -7,7 +7,7 @@ const keys = Settings.redis.documentupdater.key_schema
const ProjectFlusher = require('../app/js/ProjectFlusher')
const RedisManager = require('../app/js/RedisManager')
const { mongoClient, db, ObjectId } = require('../app/js/mongodb')
-const util = require('node:util')
+const util = require('util')
const getDoc = util.promisify((projectId, docId, cb) =>
RedisManager.getDoc(projectId, docId, (err, ...args) => cb(err, args))
)
diff --git a/services/document-updater/test/acceptance/js/ApplyingUpdatesToADocTests.js b/services/document-updater/test/acceptance/js/ApplyingUpdatesToADocTests.js
index 39ec6c2ac7..8de7f091a8 100644
--- a/services/document-updater/test/acceptance/js/ApplyingUpdatesToADocTests.js
+++ b/services/document-updater/test/acceptance/js/ApplyingUpdatesToADocTests.js
@@ -16,36 +16,27 @@ const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
describe('Applying updates to a doc', function () {
- beforeEach(function (done) {
- sinon.spy(MockWebApi, 'getDocument')
+ before(function (done) {
this.lines = ['one', 'two', 'three']
this.version = 42
this.op = {
i: 'one and a half\n',
p: 4,
}
- this.project_id = DocUpdaterClient.randomId()
- this.doc_id = DocUpdaterClient.randomId()
this.update = {
doc: this.doc_id,
op: [this.op],
v: this.version,
}
- this.historyOTUpdate = {
- doc: this.doc_id,
- op: [{ textOperation: [4, 'one and a half\n', 9] }],
- v: this.version,
- meta: { source: 'random-publicId' },
- }
this.result = ['one', 'one and a half', 'two', 'three']
DocUpdaterApp.ensureRunning(done)
})
- afterEach(function () {
- sinon.restore()
- })
describe('when the document is not loaded', function () {
- beforeEach(function (done) {
+ before(function (done) {
+ this.project_id = DocUpdaterClient.randomId()
+ this.doc_id = DocUpdaterClient.randomId()
+ sinon.spy(MockWebApi, 'getDocument')
this.startTime = Date.now()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
@@ -59,25 +50,15 @@ describe('Applying updates to a doc', function () {
if (error != null) {
throw error
}
- setTimeout(() => {
- rclientProjectHistory.get(
- ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
- project_id: this.project_id,
- }),
- (error, result) => {
- if (error != null) {
- throw error
- }
- result = parseInt(result, 10)
- this.firstOpTimestamp = result
- done()
- }
- )
- }, 200)
+ setTimeout(done, 200)
}
)
})
+ after(function () {
+ MockWebApi.getDocument.restore()
+ })
+
it('should load the document from the web API', function () {
MockWebApi.getDocument
.calledWith(this.project_id, this.doc_id)
@@ -111,44 +92,28 @@ describe('Applying updates to a doc', function () {
)
})
- it('should set the first op timestamp', function () {
- this.firstOpTimestamp.should.be.within(this.startTime, Date.now())
- })
-
- it('should yield last updated time', function (done) {
- DocUpdaterClient.getProjectLastUpdatedAt(
- this.project_id,
- (error, res, body) => {
+ it('should set the first op timestamp', function (done) {
+ rclientProjectHistory.get(
+ ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
+ project_id: this.project_id,
+ }),
+ (error, result) => {
if (error != null) {
throw error
}
- res.statusCode.should.equal(200)
- body.lastUpdatedAt.should.be.within(this.startTime, Date.now())
- done()
- }
- )
- })
-
- it('should yield no last updated time for another project', function (done) {
- DocUpdaterClient.getProjectLastUpdatedAt(
- DocUpdaterClient.randomId(),
- (error, res, body) => {
- if (error != null) {
- throw error
- }
- res.statusCode.should.equal(200)
- body.should.deep.equal({})
+ result = parseInt(result, 10)
+ result.should.be.within(this.startTime, Date.now())
+ this.firstOpTimestamp = result
done()
}
)
})
describe('when sending another update', function () {
- beforeEach(function (done) {
- this.timeout(10000)
- this.second_update = Object.assign({}, this.update)
+ before(function (done) {
+ this.timeout = 10000
+ this.second_update = Object.create(this.update)
this.second_update.v = this.version + 1
- this.secondStartTime = Date.now()
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
@@ -162,24 +127,6 @@ describe('Applying updates to a doc', function () {
)
})
- it('should update the doc', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) done(error)
- doc.lines.should.deep.equal([
- 'one',
- 'one and a half',
- 'one and a half',
- 'two',
- 'three',
- ])
- done()
- }
- )
- })
-
it('should not change the first op timestamp', function (done) {
rclientProjectHistory.get(
ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
@@ -195,357 +142,14 @@ describe('Applying updates to a doc', function () {
}
)
})
-
- it('should yield last updated time', function (done) {
- DocUpdaterClient.getProjectLastUpdatedAt(
- this.project_id,
- (error, res, body) => {
- if (error != null) {
- throw error
- }
- res.statusCode.should.equal(200)
- body.lastUpdatedAt.should.be.within(
- this.secondStartTime,
- Date.now()
- )
- done()
- }
- )
- })
- })
-
- describe('when another client is sending a concurrent update', function () {
- beforeEach(function (done) {
- this.timeout(10000)
- this.otherUpdate = {
- doc: this.doc_id,
- op: [{ p: 8, i: 'two and a half\n' }],
- v: this.version,
- meta: { source: 'other-random-publicId' },
- }
- this.secondStartTime = Date.now()
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- this.otherUpdate,
- error => {
- if (error != null) {
- throw error
- }
- setTimeout(done, 200)
- }
- )
- })
-
- it('should update the doc', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) done(error)
- doc.lines.should.deep.equal([
- 'one',
- 'one and a half',
- 'two',
- 'two and a half',
- 'three',
- ])
- done()
- }
- )
- })
-
- it('should not change the first op timestamp', function (done) {
- rclientProjectHistory.get(
- ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
- project_id: this.project_id,
- }),
- (error, result) => {
- if (error != null) {
- throw error
- }
- result = parseInt(result, 10)
- result.should.equal(this.firstOpTimestamp)
- done()
- }
- )
- })
-
- it('should yield last updated time', function (done) {
- DocUpdaterClient.getProjectLastUpdatedAt(
- this.project_id,
- (error, res, body) => {
- if (error != null) {
- throw error
- }
- res.statusCode.should.equal(200)
- body.lastUpdatedAt.should.be.within(
- this.secondStartTime,
- Date.now()
- )
- done()
- }
- )
- })
- })
- })
-
- describe('when the document is not loaded (history-ot)', function () {
- beforeEach(function (done) {
- this.startTime = Date.now()
- MockWebApi.insertDoc(this.project_id, this.doc_id, {
- lines: this.lines,
- version: this.version,
- otMigrationStage: 1,
- })
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- this.historyOTUpdate,
- error => {
- if (error != null) {
- throw error
- }
- setTimeout(() => {
- rclientProjectHistory.get(
- ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
- project_id: this.project_id,
- }),
- (error, result) => {
- if (error != null) {
- throw error
- }
- result = parseInt(result, 10)
- this.firstOpTimestamp = result
- done()
- }
- )
- }, 200)
- }
- )
- })
-
- it('should load the document from the web API', function () {
- MockWebApi.getDocument
- .calledWith(this.project_id, this.doc_id)
- .should.equal(true)
- })
-
- it('should update the doc', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) done(error)
- doc.lines.should.deep.equal(this.result)
- done()
- }
- )
- })
-
- it('should push the applied updates to the project history changes api', function (done) {
- rclientProjectHistory.lrange(
- ProjectHistoryKeys.projectHistoryOps({ project_id: this.project_id }),
- 0,
- -1,
- (error, updates) => {
- if (error != null) {
- throw error
- }
- JSON.parse(updates[0]).op.should.deep.equal(this.historyOTUpdate.op)
- JSON.parse(updates[0]).meta.pathname.should.equal('/a/b/c.tex')
-
- done()
- }
- )
- })
-
- it('should set the first op timestamp', function () {
- this.firstOpTimestamp.should.be.within(this.startTime, Date.now())
- })
-
- it('should yield last updated time', function (done) {
- DocUpdaterClient.getProjectLastUpdatedAt(
- this.project_id,
- (error, res, body) => {
- if (error != null) {
- throw error
- }
- res.statusCode.should.equal(200)
- body.lastUpdatedAt.should.be.within(this.startTime, Date.now())
- done()
- }
- )
- })
-
- it('should yield no last updated time for another project', function (done) {
- DocUpdaterClient.getProjectLastUpdatedAt(
- DocUpdaterClient.randomId(),
- (error, res, body) => {
- if (error != null) {
- throw error
- }
- res.statusCode.should.equal(200)
- body.should.deep.equal({})
- done()
- }
- )
- })
-
- describe('when sending another update', function () {
- beforeEach(function (done) {
- this.timeout(10000)
- this.second_update = Object.assign({}, this.historyOTUpdate)
- this.second_update.op = [
- {
- textOperation: [4, 'one and a half\n', 24],
- },
- ]
- this.second_update.v = this.version + 1
- this.secondStartTime = Date.now()
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- this.second_update,
- error => {
- if (error != null) {
- throw error
- }
- setTimeout(done, 200)
- }
- )
- })
-
- it('should update the doc', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) done(error)
- doc.lines.should.deep.equal([
- 'one',
- 'one and a half',
- 'one and a half',
- 'two',
- 'three',
- ])
- done()
- }
- )
- })
-
- it('should not change the first op timestamp', function (done) {
- rclientProjectHistory.get(
- ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
- project_id: this.project_id,
- }),
- (error, result) => {
- if (error != null) {
- throw error
- }
- result = parseInt(result, 10)
- result.should.equal(this.firstOpTimestamp)
- done()
- }
- )
- })
-
- it('should yield last updated time', function (done) {
- DocUpdaterClient.getProjectLastUpdatedAt(
- this.project_id,
- (error, res, body) => {
- if (error != null) {
- throw error
- }
- res.statusCode.should.equal(200)
- body.lastUpdatedAt.should.be.within(
- this.secondStartTime,
- Date.now()
- )
- done()
- }
- )
- })
- })
-
- describe('when another client is sending a concurrent update', function () {
- beforeEach(function (done) {
- this.timeout(10000)
- this.otherUpdate = {
- doc: this.doc_id,
- op: [{ textOperation: [8, 'two and a half\n', 5] }],
- v: this.version,
- meta: { source: 'other-random-publicId' },
- }
- this.secondStartTime = Date.now()
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- this.otherUpdate,
- error => {
- if (error != null) {
- throw error
- }
- setTimeout(done, 200)
- }
- )
- })
-
- it('should update the doc', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) done(error)
- doc.lines.should.deep.equal([
- 'one',
- 'one and a half',
- 'two',
- 'two and a half',
- 'three',
- ])
- done()
- }
- )
- })
-
- it('should not change the first op timestamp', function (done) {
- rclientProjectHistory.get(
- ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
- project_id: this.project_id,
- }),
- (error, result) => {
- if (error != null) {
- throw error
- }
- result = parseInt(result, 10)
- result.should.equal(this.firstOpTimestamp)
- done()
- }
- )
- })
-
- it('should yield last updated time', function (done) {
- DocUpdaterClient.getProjectLastUpdatedAt(
- this.project_id,
- (error, res, body) => {
- if (error != null) {
- throw error
- }
- res.statusCode.should.equal(200)
- body.lastUpdatedAt.should.be.within(
- this.secondStartTime,
- Date.now()
- )
- done()
- }
- )
- })
})
})
describe('when the document is loaded', function () {
- beforeEach(function (done) {
+ before(function (done) {
+ this.project_id = DocUpdaterClient.randomId()
+ this.doc_id = DocUpdaterClient.randomId()
+
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
@@ -554,7 +158,7 @@ describe('Applying updates to a doc', function () {
if (error != null) {
throw error
}
- sinon.resetHistory()
+ sinon.spy(MockWebApi, 'getDocument')
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
@@ -569,6 +173,10 @@ describe('Applying updates to a doc', function () {
})
})
+ after(function () {
+ MockWebApi.getDocument.restore()
+ })
+
it('should not need to call the web api', function () {
MockWebApi.getDocument.called.should.equal(false)
})
@@ -600,7 +208,10 @@ describe('Applying updates to a doc', function () {
})
describe('when the document is loaded and is using project-history only', function () {
- beforeEach(function (done) {
+ before(function (done) {
+ this.project_id = DocUpdaterClient.randomId()
+ this.doc_id = DocUpdaterClient.randomId()
+
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
@@ -609,7 +220,7 @@ describe('Applying updates to a doc', function () {
if (error != null) {
throw error
}
- sinon.resetHistory()
+ sinon.spy(MockWebApi, 'getDocument')
DocUpdaterClient.sendUpdate(
this.project_id,
this.doc_id,
@@ -624,6 +235,10 @@ describe('Applying updates to a doc', function () {
})
})
+ after(function () {
+ MockWebApi.getDocument.restore()
+ })
+
it('should update the doc', function (done) {
DocUpdaterClient.getDoc(
this.project_id,
@@ -650,61 +265,11 @@ describe('Applying updates to a doc', function () {
})
})
- describe('when the document is loaded (history-ot)', function () {
- beforeEach(function (done) {
- MockWebApi.insertDoc(this.project_id, this.doc_id, {
- lines: this.lines,
- version: this.version,
- otMigrationStage: 1,
- })
- DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
- if (error != null) {
- throw error
- }
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- this.historyOTUpdate,
- error => {
- if (error != null) {
- throw error
- }
- setTimeout(done, 200)
- }
- )
- })
- })
-
- it('should update the doc', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) return done(error)
- doc.lines.should.deep.equal(this.result)
- done()
- }
- )
- })
-
- it('should push the applied updates to the project history changes api', function (done) {
- rclientProjectHistory.lrange(
- ProjectHistoryKeys.projectHistoryOps({ project_id: this.project_id }),
- 0,
- -1,
- (error, updates) => {
- if (error) return done(error)
- JSON.parse(updates[0]).op.should.deep.equal(this.historyOTUpdate.op)
- JSON.parse(updates[0]).meta.pathname.should.equal('/a/b/c.tex')
- done()
- }
- )
- })
- })
-
describe('when the document has been deleted', function () {
describe('when the ops come in a single linear order', function () {
- beforeEach(function (done) {
+ before(function (done) {
+ this.project_id = DocUpdaterClient.randomId()
+ this.doc_id = DocUpdaterClient.randomId()
const lines = ['', '', '']
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines,
@@ -724,49 +289,54 @@ describe('Applying updates to a doc', function () {
{ doc_id: this.doc_id, v: 10, op: [{ i: 'd', p: 10 }] },
]
this.my_result = ['hello world', '', '']
+ done()
+ })
+
+ it('should be able to continue applying updates when the project has been deleted', function (done) {
+ let update
const actions = []
- for (const update of this.updates.slice(0, 6)) {
- actions.push(callback =>
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- update,
- callback
+ for (update of this.updates.slice(0, 6)) {
+ ;(update => {
+ actions.push(callback =>
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ update,
+ callback
+ )
)
- )
+ })(update)
}
actions.push(callback =>
DocUpdaterClient.deleteDoc(this.project_id, this.doc_id, callback)
)
- for (const update of this.updates.slice(6)) {
- actions.push(callback =>
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- update,
- callback
+ for (update of this.updates.slice(6)) {
+ ;(update => {
+ actions.push(callback =>
+ DocUpdaterClient.sendUpdate(
+ this.project_id,
+ this.doc_id,
+ update,
+ callback
+ )
)
- )
+ })(update)
}
- // process updates
- actions.push(cb =>
- DocUpdaterClient.getDoc(this.project_id, this.doc_id, cb)
- )
-
- async.series(actions, done)
- })
-
- it('should be able to continue applying updates when the project has been deleted', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) return done(error)
- doc.lines.should.deep.equal(this.my_result)
- done()
+ async.series(actions, error => {
+ if (error != null) {
+ throw error
}
- )
+ DocUpdaterClient.getDoc(
+ this.project_id,
+ this.doc_id,
+ (error, res, doc) => {
+ if (error) return done(error)
+ doc.lines.should.deep.equal(this.my_result)
+ done()
+ }
+ )
+ })
})
it('should store the doc ops in the correct order', function (done) {
@@ -788,7 +358,9 @@ describe('Applying updates to a doc', function () {
})
describe('when older ops come in after the delete', function () {
- beforeEach(function (done) {
+ before(function (done) {
+ this.project_id = DocUpdaterClient.randomId()
+ this.doc_id = DocUpdaterClient.randomId()
const lines = ['', '', '']
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines,
@@ -856,9 +428,11 @@ describe('Applying updates to a doc', function () {
})
describe('with a broken update', function () {
- beforeEach(function (done) {
+ before(function (done) {
+ this.project_id = DocUpdaterClient.randomId()
+ this.doc_id = DocUpdaterClient.randomId()
this.broken_update = {
- doc: this.doc_id,
+ doc_id: this.doc_id,
v: this.version,
op: [{ d: 'not the correct content', p: 0 }],
}
@@ -908,162 +482,10 @@ describe('Applying updates to a doc', function () {
})
})
- describe('with a broken update (history-ot)', function () {
- beforeEach(function (done) {
- this.broken_update = {
- doc: this.doc_id,
- v: this.version,
- op: [{ textOperation: [99, -1] }],
- meta: { source: '42' },
- }
- MockWebApi.insertDoc(this.project_id, this.doc_id, {
- lines: this.lines,
- version: this.version,
- otMigrationStage: 1,
- })
-
- DocUpdaterClient.subscribeToAppliedOps(
- (this.messageCallback = sinon.stub())
- )
-
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- this.broken_update,
- error => {
- if (error != null) {
- throw error
- }
- setTimeout(done, 200)
- }
- )
- })
-
- it('should not update the doc', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) return done(error)
- doc.lines.should.deep.equal(this.lines)
- done()
- }
- )
- })
-
- it('should send a message with an error', function () {
- this.messageCallback.called.should.equal(true)
- const [channel, message] = this.messageCallback.args[0]
- channel.should.equal('applied-ops')
- JSON.parse(message).should.deep.include({
- project_id: this.project_id,
- doc_id: this.doc_id,
- error:
- "The operation's base length must be equal to the string's length.",
- })
- })
- })
-
- describe('when mixing ot types (sharejs-text-ot -> history-ot)', function () {
- beforeEach(function (done) {
- MockWebApi.insertDoc(this.project_id, this.doc_id, {
- lines: this.lines,
- version: this.version,
- otMigrationStage: 0,
- })
-
- DocUpdaterClient.subscribeToAppliedOps(
- (this.messageCallback = sinon.stub())
- )
-
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- this.historyOTUpdate,
- error => {
- if (error != null) {
- throw error
- }
- setTimeout(done, 200)
- }
- )
- })
-
- it('should not update the doc', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) return done(error)
- doc.lines.should.deep.equal(this.lines)
- done()
- }
- )
- })
-
- it('should send a message with an error', function () {
- this.messageCallback.called.should.equal(true)
- const [channel, message] = this.messageCallback.args[0]
- channel.should.equal('applied-ops')
- JSON.parse(message).should.deep.include({
- project_id: this.project_id,
- doc_id: this.doc_id,
- error: 'ot type mismatch',
- })
- })
- })
-
- describe('when mixing ot types (history-ot -> sharejs-text-ot)', function () {
- beforeEach(function (done) {
- MockWebApi.insertDoc(this.project_id, this.doc_id, {
- lines: this.lines,
- version: this.version,
- otMigrationStage: 1,
- })
-
- DocUpdaterClient.subscribeToAppliedOps(
- (this.messageCallback = sinon.stub())
- )
-
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- this.update,
- error => {
- if (error != null) {
- throw error
- }
- setTimeout(done, 200)
- }
- )
- })
-
- it('should not update the doc', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) return done(error)
- doc.lines.should.deep.equal(this.lines)
- done()
- }
- )
- })
-
- it('should send a message with an error', function () {
- this.messageCallback.called.should.equal(true)
- const [channel, message] = this.messageCallback.args[0]
- channel.should.equal('applied-ops')
- JSON.parse(message).should.deep.include({
- project_id: this.project_id,
- doc_id: this.doc_id,
- error: 'ot type mismatch',
- })
- })
- })
-
describe('when there is no version in Mongo', function () {
- beforeEach(function (done) {
+ before(function (done) {
+ this.project_id = DocUpdaterClient.randomId()
+ this.doc_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
})
@@ -1100,7 +522,9 @@ describe('Applying updates to a doc', function () {
})
describe('when the sending duplicate ops', function () {
- beforeEach(function (done) {
+ before(function (done) {
+ this.project_id = DocUpdaterClient.randomId()
+ this.doc_id = DocUpdaterClient.randomId()
MockWebApi.insertDoc(this.project_id, this.doc_id, {
lines: this.lines,
version: this.version,
@@ -1182,88 +606,12 @@ describe('Applying updates to a doc', function () {
})
})
- describe('when sending duplicate ops (history-ot)', function () {
- beforeEach(function (done) {
- MockWebApi.insertDoc(this.project_id, this.doc_id, {
- lines: this.lines,
- version: this.version,
- otMigrationStage: 1,
- })
-
- DocUpdaterClient.subscribeToAppliedOps(
- (this.messageCallback = sinon.stub())
- )
-
- // One user delete 'one', the next turns it into 'once'. The second becomes a NOP.
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- {
- doc: this.doc_id,
- op: [{ textOperation: [4, 'one and a half\n', 9] }],
- v: this.version,
- meta: {
- source: 'ikHceq3yfAdQYzBo4-xZ',
- },
- },
- error => {
- if (error != null) {
- throw error
- }
- setTimeout(() => {
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- {
- doc: this.doc_id,
- op: [
- {
- textOperation: [4, 'one and a half\n', 9],
- },
- ],
- v: this.version,
- dupIfSource: ['ikHceq3yfAdQYzBo4-xZ'],
- meta: {
- source: 'ikHceq3yfAdQYzBo4-xZ',
- },
- },
- error => {
- if (error != null) {
- throw error
- }
- setTimeout(done, 200)
- }
- )
- }, 200)
- }
- )
- })
-
- it('should update the doc', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) return done(error)
- doc.lines.should.deep.equal(this.result)
- done()
- }
- )
- })
-
- it('should return a message about duplicate ops', function () {
- this.messageCallback.calledTwice.should.equal(true)
- this.messageCallback.args[0][0].should.equal('applied-ops')
- expect(JSON.parse(this.messageCallback.args[0][1]).op.dup).to.be.undefined
- this.messageCallback.args[1][0].should.equal('applied-ops')
- expect(JSON.parse(this.messageCallback.args[1][1]).op.dup).to.equal(true)
- })
- })
-
describe('when sending updates for a non-existing doc id', function () {
- beforeEach(function (done) {
+ before(function (done) {
+ this.project_id = DocUpdaterClient.randomId()
+ this.doc_id = DocUpdaterClient.randomId()
this.non_existing = {
- doc: this.doc_id,
+ doc_id: this.doc_id,
v: this.version,
op: [{ d: 'content', p: 0 }],
}
diff --git a/services/document-updater/test/acceptance/js/CheckRedisMongoSyncStateTests.js b/services/document-updater/test/acceptance/js/CheckRedisMongoSyncStateTests.js
index ebbc015a58..cd4f07bde7 100644
--- a/services/document-updater/test/acceptance/js/CheckRedisMongoSyncStateTests.js
+++ b/services/document-updater/test/acceptance/js/CheckRedisMongoSyncStateTests.js
@@ -1,12 +1,12 @@
const MockWebApi = require('./helpers/MockWebApi')
const DocUpdaterClient = require('./helpers/DocUpdaterClient')
const DocUpdaterApp = require('./helpers/DocUpdaterApp')
-const { promisify } = require('node:util')
-const { exec } = require('node:child_process')
+const { promisify } = require('util')
+const { exec } = require('child_process')
const { expect } = require('chai')
const Settings = require('@overleaf/settings')
-const fs = require('node:fs')
-const Path = require('node:path')
+const fs = require('fs')
+const Path = require('path')
const MockDocstoreApi = require('./helpers/MockDocstoreApi')
const sinon = require('sinon')
diff --git a/services/document-updater/test/acceptance/js/SettingADocumentTests.js b/services/document-updater/test/acceptance/js/SettingADocumentTests.js
index e1bc54dc90..5b0c4ab281 100644
--- a/services/document-updater/test/acceptance/js/SettingADocumentTests.js
+++ b/services/document-updater/test/acceptance/js/SettingADocumentTests.js
@@ -196,167 +196,6 @@ describe('Setting a document', function () {
})
})
- describe('when the updated doc exists in the doc updater (history-ot)', function () {
- before(function (done) {
- numberOfReceivedUpdates = 0
- this.project_id = DocUpdaterClient.randomId()
- this.doc_id = DocUpdaterClient.randomId()
- this.historyOTUpdate = {
- doc: this.doc_id,
- op: [{ textOperation: [4, 'one and a half\n', 9] }],
- v: this.version,
- meta: { source: 'random-publicId' },
- }
- MockWebApi.insertDoc(this.project_id, this.doc_id, {
- lines: this.lines,
- version: this.version,
- otMigrationStage: 1,
- })
- DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
- if (error) {
- throw error
- }
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- this.historyOTUpdate,
- error => {
- if (error) {
- throw error
- }
- setTimeout(() => {
- DocUpdaterClient.setDocLines(
- this.project_id,
- this.doc_id,
- this.newLines,
- this.source,
- this.user_id,
- false,
- (error, res, body) => {
- if (error) {
- return done(error)
- }
- this.statusCode = res.statusCode
- this.body = body
- done()
- }
- )
- }, 200)
- }
- )
- })
- })
-
- after(function () {
- MockProjectHistoryApi.flushProject.resetHistory()
- MockWebApi.setDocument.resetHistory()
- })
-
- it('should return a 200 status code', function () {
- this.statusCode.should.equal(200)
- })
-
- it('should emit two updates (from sendUpdate and setDocLines)', function () {
- expect(numberOfReceivedUpdates).to.equal(2)
- })
-
- it('should send the updated doc lines and version to the web api', function () {
- MockWebApi.setDocument
- .calledWith(this.project_id, this.doc_id, this.newLines)
- .should.equal(true)
- })
-
- it('should update the lines in the doc updater', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) {
- return done(error)
- }
- doc.lines.should.deep.equal(this.newLines)
- done()
- }
- )
- })
-
- it('should bump the version in the doc updater', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) {
- return done(error)
- }
- doc.version.should.equal(this.version + 2)
- done()
- }
- )
- })
-
- it('should leave the document in redis', function (done) {
- docUpdaterRedis.get(
- Keys.docLines({ doc_id: this.doc_id }),
- (error, lines) => {
- if (error) {
- throw error
- }
- expect(JSON.parse(lines)).to.deep.equal({
- content: this.newLines.join('\n'),
- })
- done()
- }
- )
- })
-
- it('should return the mongo rev in the json response', function () {
- this.body.should.deep.equal({ rev: '123' })
- })
-
- describe('when doc has the same contents', function () {
- beforeEach(function (done) {
- numberOfReceivedUpdates = 0
- DocUpdaterClient.setDocLines(
- this.project_id,
- this.doc_id,
- this.newLines,
- this.source,
- this.user_id,
- false,
- (error, res, body) => {
- if (error) {
- return done(error)
- }
- this.statusCode = res.statusCode
- this.body = body
- done()
- }
- )
- })
-
- it('should not bump the version in doc updater', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, doc) => {
- if (error) {
- return done(error)
- }
- doc.version.should.equal(this.version + 2)
- done()
- }
- )
- })
-
- it('should not emit any updates', function (done) {
- setTimeout(() => {
- expect(numberOfReceivedUpdates).to.equal(0)
- done()
- }, 100) // delay by 100ms: make sure we do not check too early!
- })
- })
- })
-
describe('when the updated doc does not exist in the doc updater', function () {
before(function (done) {
this.project_id = DocUpdaterClient.randomId()
@@ -686,285 +525,4 @@ describe('Setting a document', function () {
})
})
})
-
- describe('with track changes (history-ot)', function () {
- const lines = ['one', 'one and a half', 'two', 'three']
- const userId = DocUpdaterClient.randomId()
- const ts = new Date().toISOString()
- beforeEach(function (done) {
- numberOfReceivedUpdates = 0
- this.newLines = ['one', 'two', 'three']
- this.project_id = DocUpdaterClient.randomId()
- this.doc_id = DocUpdaterClient.randomId()
- this.historyOTUpdate = {
- doc: this.doc_id,
- op: [
- {
- textOperation: [
- 4,
- {
- r: 'one and a half\n'.length,
- tracking: {
- type: 'delete',
- userId,
- ts,
- },
- },
- 9,
- ],
- },
- ],
- v: this.version,
- meta: { source: 'random-publicId' },
- }
- MockWebApi.insertDoc(this.project_id, this.doc_id, {
- lines,
- version: this.version,
- otMigrationStage: 1,
- })
- DocUpdaterClient.preloadDoc(this.project_id, this.doc_id, error => {
- if (error) {
- throw error
- }
- DocUpdaterClient.sendUpdate(
- this.project_id,
- this.doc_id,
- this.historyOTUpdate,
- error => {
- if (error) {
- throw error
- }
- DocUpdaterClient.waitForPendingUpdates(
- this.project_id,
- this.doc_id,
- done
- )
- }
- )
- })
- })
-
- afterEach(function () {
- MockProjectHistoryApi.flushProject.resetHistory()
- MockWebApi.setDocument.resetHistory()
- })
- it('should record tracked changes', function (done) {
- docUpdaterRedis.get(
- Keys.docLines({ doc_id: this.doc_id }),
- (error, data) => {
- if (error) {
- throw error
- }
- expect(JSON.parse(data)).to.deep.equal({
- content: lines.join('\n'),
- trackedChanges: [
- {
- range: {
- pos: 4,
- length: 15,
- },
- tracking: {
- ts,
- type: 'delete',
- userId,
- },
- },
- ],
- })
- done()
- }
- )
- })
-
- it('should apply the change', function (done) {
- DocUpdaterClient.getDoc(
- this.project_id,
- this.doc_id,
- (error, res, data) => {
- if (error) {
- throw error
- }
- expect(data.lines).to.deep.equal(this.newLines)
- done()
- }
- )
- })
- const cases = [
- {
- name: 'when resetting the content',
- lines,
- want: {
- content: 'one\none and a half\none and a half\ntwo\nthree',
- trackedChanges: [
- {
- range: {
- pos: 'one and a half\n'.length + 4,
- length: 15,
- },
- tracking: {
- ts,
- type: 'delete',
- userId,
- },
- },
- ],
- },
- },
- {
- name: 'when adding content before a tracked delete',
- lines: ['one', 'INSERT', 'two', 'three'],
- want: {
- content: 'one\nINSERT\none and a half\ntwo\nthree',
- trackedChanges: [
- {
- range: {
- pos: 'INSERT\n'.length + 4,
- length: 15,
- },
- tracking: {
- ts,
- type: 'delete',
- userId,
- },
- },
- ],
- },
- },
- {
- name: 'when adding content after a tracked delete',
- lines: ['one', 'two', 'INSERT', 'three'],
- want: {
- content: 'one\none and a half\ntwo\nINSERT\nthree',
- trackedChanges: [
- {
- range: {
- pos: 4,
- length: 15,
- },
- tracking: {
- ts,
- type: 'delete',
- userId,
- },
- },
- ],
- },
- },
- {
- name: 'when deleting content before a tracked delete',
- lines: ['two', 'three'],
- want: {
- content: 'one and a half\ntwo\nthree',
- trackedChanges: [
- {
- range: {
- pos: 0,
- length: 15,
- },
- tracking: {
- ts,
- type: 'delete',
- userId,
- },
- },
- ],
- },
- },
- {
- name: 'when deleting content after a tracked delete',
- lines: ['one', 'two'],
- want: {
- content: 'one\none and a half\ntwo',
- trackedChanges: [
- {
- range: {
- pos: 4,
- length: 15,
- },
- tracking: {
- ts,
- type: 'delete',
- userId,
- },
- },
- ],
- },
- },
- {
- name: 'when deleting content immediately after a tracked delete',
- lines: ['one', 'three'],
- want: {
- content: 'one\none and a half\nthree',
- trackedChanges: [
- {
- range: {
- pos: 4,
- length: 15,
- },
- tracking: {
- ts,
- type: 'delete',
- userId,
- },
- },
- ],
- },
- },
- {
- name: 'when deleting content across a tracked delete',
- lines: ['onethree'],
- want: {
- content: 'oneone and a half\nthree',
- trackedChanges: [
- {
- range: {
- pos: 3,
- length: 15,
- },
- tracking: {
- ts,
- type: 'delete',
- userId,
- },
- },
- ],
- },
- },
- ]
-
- for (const { name, lines, want } of cases) {
- describe(name, function () {
- beforeEach(function (done) {
- DocUpdaterClient.setDocLines(
- this.project_id,
- this.doc_id,
- lines,
- this.source,
- userId,
- false,
- (error, res, body) => {
- if (error) {
- return done(error)
- }
- this.statusCode = res.statusCode
- this.body = body
- done()
- }
- )
- })
- it('should update accordingly', function (done) {
- docUpdaterRedis.get(
- Keys.docLines({ doc_id: this.doc_id }),
- (error, data) => {
- if (error) {
- throw error
- }
- expect(JSON.parse(data)).to.deep.equal(want)
- done()
- }
- )
- })
- })
- }
- })
})
diff --git a/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js b/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js
index d34996ca7c..33c6882138 100644
--- a/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js
+++ b/services/document-updater/test/acceptance/js/helpers/DocUpdaterApp.js
@@ -9,6 +9,7 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const app = require('../../../../app')
+require('@overleaf/logger').logger.level('fatal')
module.exports = {
running: false,
diff --git a/services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js b/services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js
index 0a4ec8922e..4ed4f929de 100644
--- a/services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js
+++ b/services/document-updater/test/acceptance/js/helpers/DocUpdaterClient.js
@@ -119,18 +119,6 @@ module.exports = DocUpdaterClient = {
)
},
- getProjectLastUpdatedAt(projectId, callback) {
- request.get(
- `http://127.0.0.1:3003/project/${projectId}/last_updated_at`,
- (error, res, body) => {
- if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
- body = JSON.parse(body)
- }
- callback(error, res, body)
- }
- )
- },
-
preloadDoc(projectId, docId, callback) {
DocUpdaterClient.getDoc(projectId, docId, callback)
},
diff --git a/services/document-updater/test/setup.js b/services/document-updater/test/setup.js
index 8ba17d922f..f7d47e7b75 100644
--- a/services/document-updater/test/setup.js
+++ b/services/document-updater/test/setup.js
@@ -31,14 +31,8 @@ SandboxedModule.configure({
requires: {
'@overleaf/logger': stubs.logger,
'mongodb-legacy': require('mongodb-legacy'), // for ObjectId comparisons
- 'overleaf-editor-core': require('overleaf-editor-core'), // does not play nice with sandbox
},
globals: { Buffer, JSON, Math, console, process },
- sourceTransformers: {
- removeNodePrefix: function (source) {
- return source.replace(/require\(['"]node:/g, "require('")
- },
- },
})
// Mocha hooks
diff --git a/services/document-updater/test/stress/js/run.js b/services/document-updater/test/stress/js/run.js
index 1bda73c1aa..4d2614eac4 100644
--- a/services/document-updater/test/stress/js/run.js
+++ b/services/document-updater/test/stress/js/run.js
@@ -15,7 +15,7 @@
*/
const DocUpdaterClient = require('../../acceptance/js/helpers/DocUpdaterClient')
// MockWebApi = require "../../acceptance/js/helpers/MockWebApi"
-const assert = require('node:assert')
+const assert = require('assert')
const async = require('async')
const insert = function (string, pos, content) {
diff --git a/services/document-updater/test/unit/js/DocumentManager/DocumentManagerTests.js b/services/document-updater/test/unit/js/DocumentManager/DocumentManagerTests.js
index 1816579103..a0c379a5be 100644
--- a/services/document-updater/test/unit/js/DocumentManager/DocumentManagerTests.js
+++ b/services/document-updater/test/unit/js/DocumentManager/DocumentManagerTests.js
@@ -49,16 +49,10 @@ describe('DocumentManager', function () {
applyUpdate: sinon.stub().resolves(),
},
}
- this.HistoryOTUpdateManager = {
- applyUpdate: sinon.stub().resolves(),
- }
this.RangesManager = {
acceptChanges: sinon.stub(),
deleteComment: sinon.stub(),
}
- this.Settings = {
- max_doc_length: 2 * 1024 * 1024, // 2mb
- }
this.DocumentManager = SandboxedModule.require(modulePath, {
requires: {
@@ -69,10 +63,8 @@ describe('DocumentManager', function () {
'./Metrics': this.Metrics,
'./DiffCodec': this.DiffCodec,
'./UpdateManager': this.UpdateManager,
- './HistoryOTUpdateManager': this.HistoryOTUpdateManager,
'./RangesManager': this.RangesManager,
'./Errors': Errors,
- '@overleaf/settings': this.Settings,
},
})
this.project_id = 'project-id-123'
@@ -226,7 +218,6 @@ describe('DocumentManager', function () {
ranges: this.ranges,
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
- type: 'sharejs-text-ot',
})
this.RedisManager.promises.getPreviousDocOps.resolves(this.ops)
this.result = await this.DocumentManager.promises.getDocAndRecentOps(
@@ -256,7 +247,6 @@ describe('DocumentManager', function () {
ranges: this.ranges,
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
- type: 'sharejs-text-ot',
})
})
})
@@ -269,7 +259,6 @@ describe('DocumentManager', function () {
ranges: this.ranges,
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
- type: 'sharejs-text-ot',
})
this.RedisManager.promises.getPreviousDocOps.resolves(this.ops)
this.result = await this.DocumentManager.promises.getDocAndRecentOps(
@@ -297,7 +286,6 @@ describe('DocumentManager', function () {
ranges: this.ranges,
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
- type: 'sharejs-text-ot',
})
})
})
@@ -341,7 +329,6 @@ describe('DocumentManager', function () {
unflushedTime: this.unflushedTime,
alreadyLoaded: true,
historyRangesSupport: this.historyRangesSupport,
- type: 'sharejs-text-ot',
})
})
})
@@ -409,7 +396,6 @@ describe('DocumentManager', function () {
unflushedTime: null,
alreadyLoaded: false,
historyRangesSupport: this.historyRangesSupport,
- type: 'sharejs-text-ot',
})
})
})
@@ -460,8 +446,7 @@ describe('DocumentManager', function () {
this.beforeLines,
this.source,
this.user_id,
- false,
- true
+ false
)
})
@@ -495,8 +480,7 @@ describe('DocumentManager', function () {
this.beforeLines,
this.source,
this.user_id,
- false,
- true
+ false
)
})
@@ -529,8 +513,7 @@ describe('DocumentManager', function () {
this.afterLines,
this.source,
this.user_id,
- false,
- true
+ false
)
})
@@ -599,8 +582,7 @@ describe('DocumentManager', function () {
this.afterLines,
this.source,
this.user_id,
- false,
- true
+ false
)
})
@@ -636,8 +618,7 @@ describe('DocumentManager', function () {
null,
this.source,
this.user_id,
- false,
- true
+ false
)
).to.be.rejectedWith('No lines were provided to setDoc')
})
@@ -661,7 +642,6 @@ describe('DocumentManager', function () {
this.afterLines,
this.source,
this.user_id,
- true,
true
)
})
@@ -670,77 +650,6 @@ describe('DocumentManager', function () {
this.ops.map(op => op.u.should.equal(true))
})
})
-
- describe('with the external flag', function () {
- beforeEach(async function () {
- this.undoing = false
- // Copy ops so we don't interfere with other tests
- this.ops = [
- { i: 'foo', p: 4 },
- { d: 'bar', p: 42 },
- ]
- this.DiffCodec.diffAsShareJsOp.returns(this.ops)
- await this.DocumentManager.promises.setDoc(
- this.project_id,
- this.doc_id,
- this.afterLines,
- this.source,
- this.user_id,
- this.undoing,
- true
- )
- })
-
- it('should add the external type to update metadata', function () {
- this.UpdateManager.promises.applyUpdate
- .calledWith(this.project_id, this.doc_id, {
- doc: this.doc_id,
- v: this.version,
- op: this.ops,
- meta: {
- type: 'external',
- source: this.source,
- user_id: this.user_id,
- },
- })
- .should.equal(true)
- })
- })
-
- describe('without the external flag', function () {
- beforeEach(async function () {
- this.undoing = false
- // Copy ops so we don't interfere with other tests
- this.ops = [
- { i: 'foo', p: 4 },
- { d: 'bar', p: 42 },
- ]
- this.DiffCodec.diffAsShareJsOp.returns(this.ops)
- await this.DocumentManager.promises.setDoc(
- this.project_id,
- this.doc_id,
- this.afterLines,
- this.source,
- this.user_id,
- this.undoing,
- false
- )
- })
-
- it('should not add the external type to update metadata', function () {
- this.UpdateManager.promises.applyUpdate
- .calledWith(this.project_id, this.doc_id, {
- doc: this.doc_id,
- v: this.version,
- op: this.ops,
- meta: {
- source: this.source,
- user_id: this.user_id,
- },
- })
- .should.equal(true)
- })
- })
})
})
@@ -782,8 +691,6 @@ describe('DocumentManager', function () {
it('should apply the accept change to the ranges', function () {
this.RangesManager.acceptChanges.should.have.been.calledWith(
- this.project_id,
- this.doc_id,
[this.change_id],
this.ranges
)
@@ -815,12 +722,7 @@ describe('DocumentManager', function () {
it('should apply the accept change to the ranges', function () {
this.RangesManager.acceptChanges
- .calledWith(
- this.project_id,
- this.doc_id,
- this.change_ids,
- this.ranges
- )
+ .calledWith(this.change_ids, this.ranges)
.should.equal(true)
})
})
@@ -845,77 +747,6 @@ describe('DocumentManager', function () {
})
})
- describe('getComment', function () {
- beforeEach(function () {
- this.ranges.comments = [
- {
- id: 'mock-comment-id-1',
- },
- {
- id: 'mock-comment-id-2',
- },
- ]
- this.DocumentManager.promises.getDoc = sinon.stub().resolves({
- lines: this.lines,
- version: this.version,
- ranges: this.ranges,
- })
- })
-
- describe('when comment exists', function () {
- beforeEach(async function () {
- await expect(
- this.DocumentManager.promises.getComment(
- this.project_id,
- this.doc_id,
- 'mock-comment-id-1'
- )
- ).to.eventually.deep.equal({
- comment: { id: 'mock-comment-id-1' },
- })
- })
-
- it("should get the document's current ranges", function () {
- this.DocumentManager.promises.getDoc
- .calledWith(this.project_id, this.doc_id)
- .should.equal(true)
- })
- })
-
- describe('when comment doesnt exists', function () {
- beforeEach(async function () {
- await expect(
- this.DocumentManager.promises.getComment(
- this.project_id,
- this.doc_id,
- 'mock-comment-id-x'
- )
- ).to.be.rejectedWith(Errors.NotFoundError)
- })
-
- it("should get the document's current ranges", function () {
- this.DocumentManager.promises.getDoc
- .calledWith(this.project_id, this.doc_id)
- .should.equal(true)
- })
- })
-
- describe('when the doc is not found', function () {
- beforeEach(async function () {
- this.DocumentManager.promises.getDoc = sinon
- .stub()
- .resolves({ lines: null, version: null, ranges: null })
- await expect(
- this.DocumentManager.promises.acceptChanges(
- this.project_id,
- this.doc_id,
- [this.change_id]
- )
- ).to.be.rejectedWith(Errors.NotFoundError)
- })
- })
- })
-
describe('deleteComment', function () {
beforeEach(function () {
this.comment_id = 'mock-comment-id'
@@ -1298,68 +1129,4 @@ describe('DocumentManager', function () {
})
})
})
-
- describe('appendToDoc', function () {
- describe('sucessfully', function () {
- beforeEach(async function () {
- this.lines = ['one', 'two', 'three']
- this.DocumentManager.promises.setDoc = sinon
- .stub()
- .resolves({ rev: '123' })
- this.DocumentManager.promises.getDoc = sinon.stub().resolves({
- lines: this.lines,
- })
- this.result = await this.DocumentManager.promises.appendToDoc(
- this.project_id,
- this.doc_id,
- ['four', 'five', 'six'],
- this.source,
- this.user_id
- )
- })
-
- it('should call setDoc with concatenated lines', function () {
- this.DocumentManager.promises.setDoc
- .calledWith(
- this.project_id,
- this.doc_id,
- ['one', 'two', 'three', 'four', 'five', 'six'],
- this.source,
- this.user_id,
- false,
- false
- )
- .should.equal(true)
- })
-
- it('should return output from setDoc', function () {
- this.result.should.deep.equal({ rev: '123' })
- })
- })
-
- describe('when doc would become too big', function () {
- beforeEach(async function () {
- this.Settings.max_doc_length = 100
- this.lines = ['one', 'two', 'three']
- this.DocumentManager.promises.setDoc = sinon
- .stub()
- .resolves({ rev: '123' })
- this.DocumentManager.promises.getDoc = sinon.stub().resolves({
- lines: this.lines,
- })
- })
-
- it('should fail with FileTooLarge error', async function () {
- expect(
- this.DocumentManager.promises.appendToDoc(
- this.project_id,
- this.doc_id,
- ['x'.repeat(1000)],
- this.source,
- this.user_id
- )
- ).to.eventually.be.rejectedWith(Errors.FileTooLargeError)
- })
- })
- })
})
diff --git a/services/document-updater/test/unit/js/HistoryManager/HistoryManagerTests.js b/services/document-updater/test/unit/js/HistoryManager/HistoryManagerTests.js
index 2a5fb29b6d..ce98c4eaa8 100644
--- a/services/document-updater/test/unit/js/HistoryManager/HistoryManagerTests.js
+++ b/services/document-updater/test/unit/js/HistoryManager/HistoryManagerTests.js
@@ -3,7 +3,7 @@
*/
const SandboxedModule = require('sandboxed-module')
const sinon = require('sinon')
-const modulePath = require('node:path').join(
+const modulePath = require('path').join(
__dirname,
'../../../../app/js/HistoryManager'
)
@@ -14,7 +14,6 @@ describe('HistoryManager', function () {
requires: {
request: (this.request = {}),
'@overleaf/settings': (this.Settings = {
- shortHistoryQueues: [],
apis: {
project_history: {
url: 'http://project_history.example.com',
@@ -119,7 +118,7 @@ describe('HistoryManager', function () {
beforeEach(function () {
this.HistoryManager.shouldFlushHistoryOps = sinon.stub()
this.HistoryManager.shouldFlushHistoryOps
- .withArgs(this.project_id, this.project_ops_length)
+ .withArgs(this.project_ops_length)
.returns(true)
this.HistoryManager.recordAndFlushHistoryOps(
@@ -140,7 +139,7 @@ describe('HistoryManager', function () {
beforeEach(function () {
this.HistoryManager.shouldFlushHistoryOps = sinon.stub()
this.HistoryManager.shouldFlushHistoryOps
- .withArgs(this.project_id, this.project_ops_length)
+ .withArgs(this.project_ops_length)
.returns(false)
this.HistoryManager.recordAndFlushHistoryOps(
@@ -158,7 +157,6 @@ describe('HistoryManager', function () {
describe('shouldFlushHistoryOps', function () {
it('should return false if the number of ops is not known', function () {
this.HistoryManager.shouldFlushHistoryOps(
- this.project_id,
null,
['a', 'b', 'c'].length,
1
@@ -170,7 +168,6 @@ describe('HistoryManager', function () {
// Previously we were on 11 ops
// We didn't pass over a multiple of 5
this.HistoryManager.shouldFlushHistoryOps(
- this.project_id,
14,
['a', 'b', 'c'].length,
5
@@ -181,7 +178,6 @@ describe('HistoryManager', function () {
// Previously we were on 12 ops
// We've reached a new multiple of 5
this.HistoryManager.shouldFlushHistoryOps(
- this.project_id,
15,
['a', 'b', 'c'].length,
5
@@ -193,22 +189,11 @@ describe('HistoryManager', function () {
// Previously we were on 16 ops
// We didn't pass over a multiple of 5
this.HistoryManager.shouldFlushHistoryOps(
- this.project_id,
17,
['a', 'b', 'c'].length,
5
).should.equal(true)
})
-
- it('should return true if the project has a short queue', function () {
- this.Settings.shortHistoryQueues = [this.project_id]
- this.HistoryManager.shouldFlushHistoryOps(
- this.project_id,
- 14,
- ['a', 'b', 'c'].length,
- 5
- ).should.equal(true)
- })
})
})
@@ -232,75 +217,34 @@ describe('HistoryManager', function () {
.stub()
.yields()
this.DocumentManager.resyncDocContentsWithLock = sinon.stub().yields()
+ this.HistoryManager.resyncProjectHistory(
+ this.project_id,
+ this.projectHistoryId,
+ this.docs,
+ this.files,
+ this.callback
+ )
})
- describe('full sync', function () {
- beforeEach(function () {
- this.HistoryManager.resyncProjectHistory(
+ it('should queue a project structure reync', function () {
+ this.ProjectHistoryRedisManager.queueResyncProjectStructure
+ .calledWith(
this.project_id,
this.projectHistoryId,
this.docs,
- this.files,
- {},
- this.callback
+ this.files
)
- })
-
- it('should queue a project structure reync', function () {
- this.ProjectHistoryRedisManager.queueResyncProjectStructure
- .calledWith(
- this.project_id,
- this.projectHistoryId,
- this.docs,
- this.files
- )
- .should.equal(true)
- })
-
- it('should queue doc content reyncs', function () {
- this.DocumentManager.resyncDocContentsWithLock
- .calledWith(this.project_id, this.docs[0].doc, this.docs[0].path)
- .should.equal(true)
- })
-
- it('should call the callback', function () {
- this.callback.called.should.equal(true)
- })
+ .should.equal(true)
})
- describe('resyncProjectStructureOnly=true', function () {
- beforeEach(function () {
- this.HistoryManager.resyncProjectHistory(
- this.project_id,
- this.projectHistoryId,
- this.docs,
- this.files,
- { resyncProjectStructureOnly: true },
- this.callback
- )
- })
+ it('should queue doc content reyncs', function () {
+ this.DocumentManager.resyncDocContentsWithLock
+ .calledWith(this.project_id, this.docs[0].doc, this.docs[0].path)
+ .should.equal(true)
+ })
- it('should queue a project structure reync', function () {
- this.ProjectHistoryRedisManager.queueResyncProjectStructure
- .calledWith(
- this.project_id,
- this.projectHistoryId,
- this.docs,
- this.files,
- { resyncProjectStructureOnly: true }
- )
- .should.equal(true)
- })
-
- it('should not queue doc content reyncs', function () {
- this.DocumentManager.resyncDocContentsWithLock.called.should.equal(
- false
- )
- })
-
- it('should call the callback', function () {
- this.callback.called.should.equal(true)
- })
+ it('should call the callback', function () {
+ this.callback.called.should.equal(true)
})
})
})
diff --git a/services/document-updater/test/unit/js/HttpController/HttpControllerTests.js b/services/document-updater/test/unit/js/HttpController/HttpControllerTests.js
index 333da10d15..c422b8c2c0 100644
--- a/services/document-updater/test/unit/js/HttpController/HttpControllerTests.js
+++ b/services/document-updater/test/unit/js/HttpController/HttpControllerTests.js
@@ -26,7 +26,6 @@ describe('HttpController', function () {
this.Metrics.Timer.prototype.done = sinon.stub()
this.project_id = 'project-id-123'
- this.projectHistoryId = '123'
this.doc_id = 'doc-id-123'
this.source = 'editor'
this.next = sinon.stub()
@@ -66,9 +65,7 @@ describe('HttpController', function () {
this.version,
[],
this.ranges,
- this.pathname,
- this.projectHistoryId,
- 'sharejs-text-ot'
+ this.pathname
)
this.HttpController.getDoc(this.req, this.res, this.next)
})
@@ -80,16 +77,17 @@ describe('HttpController', function () {
})
it('should return the doc as JSON', function () {
- this.res.json.should.have.been.calledWith({
- id: this.doc_id,
- lines: this.lines,
- version: this.version,
- ops: [],
- ranges: this.ranges,
- pathname: this.pathname,
- ttlInS: 42,
- type: 'sharejs-text-ot',
- })
+ this.res.json
+ .calledWith({
+ id: this.doc_id,
+ lines: this.lines,
+ version: this.version,
+ ops: [],
+ ranges: this.ranges,
+ pathname: this.pathname,
+ ttlInS: 42,
+ })
+ .should.equal(true)
})
it('should log the request', function () {
@@ -117,9 +115,7 @@ describe('HttpController', function () {
this.version,
this.ops,
this.ranges,
- this.pathname,
- this.projectHistoryId,
- 'sharejs-text-ot'
+ this.pathname
)
this.req.query = { fromVersion: `${this.fromVersion}` }
this.HttpController.getDoc(this.req, this.res, this.next)
@@ -132,16 +128,17 @@ describe('HttpController', function () {
})
it('should return the doc as JSON', function () {
- this.res.json.should.have.been.calledWith({
- id: this.doc_id,
- lines: this.lines,
- version: this.version,
- ops: this.ops,
- ranges: this.ranges,
- pathname: this.pathname,
- ttlInS: 42,
- type: 'sharejs-text-ot',
- })
+ this.res.json
+ .calledWith({
+ id: this.doc_id,
+ lines: this.lines,
+ version: this.version,
+ ops: this.ops,
+ ranges: this.ranges,
+ pathname: this.pathname,
+ ttlInS: 42,
+ })
+ .should.equal(true)
})
it('should log the request', function () {
@@ -187,65 +184,6 @@ describe('HttpController', function () {
})
})
- describe('getComment', function () {
- beforeEach(function () {
- this.ranges = {
- changes: 'mock',
- comments: [
- {
- id: 'comment-id-1',
- },
- {
- id: 'comment-id-2',
- },
- ],
- }
- this.req = {
- params: {
- project_id: this.project_id,
- doc_id: this.doc_id,
- comment_id: this.comment_id,
- },
- query: {},
- body: {},
- }
- })
-
- beforeEach(function () {
- this.DocumentManager.getCommentWithLock = sinon
- .stub()
- .callsArgWith(3, null, this.ranges.comments[0])
- this.HttpController.getComment(this.req, this.res, this.next)
- })
-
- it('should get the comment', function () {
- this.DocumentManager.getCommentWithLock
- .calledWith(this.project_id, this.doc_id, this.comment_id)
- .should.equal(true)
- })
-
- it('should return the comment as JSON', function () {
- this.res.json
- .calledWith({
- id: 'comment-id-1',
- })
- .should.equal(true)
- })
-
- it('should log the request', function () {
- this.logger.debug
- .calledWith(
- {
- projectId: this.project_id,
- docId: this.doc_id,
- commentId: this.comment_id,
- },
- 'getting comment via http'
- )
- .should.equal(true)
- })
- })
-
describe('setDoc', function () {
beforeEach(function () {
this.lines = ['one', 'two', 'three']
@@ -271,7 +209,7 @@ describe('HttpController', function () {
beforeEach(function () {
this.DocumentManager.setDocWithLock = sinon
.stub()
- .callsArgWith(7, null, { rev: '123' })
+ .callsArgWith(6, null, { rev: '123' })
this.HttpController.setDoc(this.req, this.res, this.next)
})
@@ -283,8 +221,7 @@ describe('HttpController', function () {
this.lines,
this.source,
this.user_id,
- this.undoing,
- true
+ this.undoing
)
.should.equal(true)
})
@@ -318,7 +255,7 @@ describe('HttpController', function () {
beforeEach(function () {
this.DocumentManager.setDocWithLock = sinon
.stub()
- .callsArgWith(7, new Error('oops'))
+ .callsArgWith(6, new Error('oops'))
this.HttpController.setDoc(this.req, this.res, this.next)
})
@@ -1166,96 +1103,4 @@ describe('HttpController', function () {
})
})
})
-
- describe('appendToDoc', function () {
- beforeEach(function () {
- this.lines = ['one', 'two', 'three']
- this.source = 'dropbox'
- this.user_id = 'user-id-123'
- this.req = {
- headers: {},
- params: {
- project_id: this.project_id,
- doc_id: this.doc_id,
- },
- query: {},
- body: {
- lines: this.lines,
- source: this.source,
- user_id: this.user_id,
- undoing: (this.undoing = true),
- },
- }
- })
-
- describe('successfully', function () {
- beforeEach(function () {
- this.DocumentManager.appendToDocWithLock = sinon
- .stub()
- .callsArgWith(5, null, { rev: '123' })
- this.HttpController.appendToDoc(this.req, this.res, this.next)
- })
-
- it('should append to the doc', function () {
- this.DocumentManager.appendToDocWithLock
- .calledWith(
- this.project_id,
- this.doc_id,
- this.lines,
- this.source,
- this.user_id
- )
- .should.equal(true)
- })
-
- it('should return a json response with the document rev from web', function () {
- this.res.json.calledWithMatch({ rev: '123' }).should.equal(true)
- })
-
- it('should log the request', function () {
- this.logger.debug
- .calledWith(
- {
- docId: this.doc_id,
- projectId: this.project_id,
- lines: this.lines,
- source: this.source,
- userId: this.user_id,
- },
- 'appending to doc via http'
- )
- .should.equal(true)
- })
-
- it('should time the request', function () {
- this.Metrics.Timer.prototype.done.called.should.equal(true)
- })
- })
-
- describe('when an errors occurs', function () {
- beforeEach(function () {
- this.DocumentManager.appendToDocWithLock = sinon
- .stub()
- .callsArgWith(5, new Error('oops'))
- this.HttpController.appendToDoc(this.req, this.res, this.next)
- })
-
- it('should call next with the error', function () {
- this.next.calledWith(sinon.match.instanceOf(Error)).should.equal(true)
- })
- })
-
- describe('when the payload is too large', function () {
- beforeEach(function () {
- this.DocumentManager.appendToDocWithLock = sinon
- .stub()
- .callsArgWith(5, new Errors.FileTooLargeError())
- this.HttpController.appendToDoc(this.req, this.res, this.next)
- })
-
- it('should send back a 422 response', function () {
- this.res.sendStatus.calledWith(422).should.equal(true)
- })
- })
- })
})
diff --git a/services/document-updater/test/unit/js/Limits/LimitsTests.js b/services/document-updater/test/unit/js/Limits/LimitsTests.js
index 11ca38746a..34a5c13c26 100644
--- a/services/document-updater/test/unit/js/Limits/LimitsTests.js
+++ b/services/document-updater/test/unit/js/Limits/LimitsTests.js
@@ -81,88 +81,4 @@ describe('Limits', function () {
})
})
})
-
- describe('stringFileDataContentIsTooLarge', function () {
- it('should handle small docs', function () {
- expect(
- this.Limits.stringFileDataContentIsTooLarge({ content: '' }, 123)
- ).to.equal(false)
- })
- it('should handle docs at the limit', function () {
- expect(
- this.Limits.stringFileDataContentIsTooLarge(
- { content: 'x'.repeat(123) },
- 123
- )
- ).to.equal(false)
- })
- it('should handle docs above the limit', function () {
- expect(
- this.Limits.stringFileDataContentIsTooLarge(
- { content: 'x'.repeat(123 + 1) },
- 123
- )
- ).to.equal(true)
- })
- it('should handle docs above the limit and below with tracked-deletes removed', function () {
- expect(
- this.Limits.stringFileDataContentIsTooLarge(
- {
- content: 'x'.repeat(123 + 1),
- trackedChanges: [
- {
- range: { pos: 1, length: 1 },
- tracking: {
- type: 'delete',
- ts: '2025-06-16T14:31:44.910Z',
- userId: 'user-id',
- },
- },
- ],
- },
- 123
- )
- ).to.equal(false)
- })
- it('should handle docs above the limit and above with tracked-deletes removed', function () {
- expect(
- this.Limits.stringFileDataContentIsTooLarge(
- {
- content: 'x'.repeat(123 + 2),
- trackedChanges: [
- {
- range: { pos: 1, length: 1 },
- tracking: {
- type: 'delete',
- ts: '2025-06-16T14:31:44.910Z',
- userId: 'user-id',
- },
- },
- ],
- },
- 123
- )
- ).to.equal(true)
- })
- it('should handle docs above the limit and with tracked-inserts', function () {
- expect(
- this.Limits.stringFileDataContentIsTooLarge(
- {
- content: 'x'.repeat(123 + 1),
- trackedChanges: [
- {
- range: { pos: 1, length: 1 },
- tracking: {
- type: 'insert',
- ts: '2025-06-16T14:31:44.910Z',
- userId: 'user-id',
- },
- },
- ],
- },
- 123
- )
- ).to.equal(true)
- })
- })
})
diff --git a/services/document-updater/test/unit/js/LockManager/CheckingTheLock.js b/services/document-updater/test/unit/js/LockManager/CheckingTheLock.js
index 575ed90702..eb5b44532a 100644
--- a/services/document-updater/test/unit/js/LockManager/CheckingTheLock.js
+++ b/services/document-updater/test/unit/js/LockManager/CheckingTheLock.js
@@ -10,8 +10,8 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
-const assert = require('node:assert')
-const path = require('node:path')
+const assert = require('assert')
+const path = require('path')
const modulePath = path.join(__dirname, '../../../../app/js/LockManager.js')
const projectId = 1234
const docId = 5678
diff --git a/services/document-updater/test/unit/js/LockManager/ReleasingTheLock.js b/services/document-updater/test/unit/js/LockManager/ReleasingTheLock.js
index a39a3b4fff..0413e268d6 100644
--- a/services/document-updater/test/unit/js/LockManager/ReleasingTheLock.js
+++ b/services/document-updater/test/unit/js/LockManager/ReleasingTheLock.js
@@ -11,8 +11,8 @@
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
*/
const sinon = require('sinon')
-const assert = require('node:assert')
-const path = require('node:path')
+const assert = require('assert')
+const path = require('path')
const modulePath = path.join(__dirname, '../../../../app/js/LockManager.js')
const projectId = 1234
const docId = 5678
diff --git a/services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js b/services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js
index ad6c121dfb..0f5df2e29f 100644
--- a/services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js
+++ b/services/document-updater/test/unit/js/ProjectHistoryRedisManager/ProjectHistoryRedisManagerTests.js
@@ -15,7 +15,6 @@ describe('ProjectHistoryRedisManager', function () {
this.Limits = {
docIsTooLarge: sinon.stub().returns(false),
- stringFileDataContentIsTooLarge: sinon.stub().returns(false),
}
this.ProjectHistoryRedisManager = SandboxedModule.require(modulePath, {
@@ -62,18 +61,22 @@ describe('ProjectHistoryRedisManager', function () {
})
it('should queue an update', function () {
- this.multi.rpush.should.have.been.calledWithExactly(
- `ProjectHistory:Ops:${this.project_id}`,
- this.ops[0],
- this.ops[1]
- )
+ this.multi.rpush
+ .calledWithExactly(
+ `ProjectHistory:Ops:${this.project_id}`,
+ this.ops[0],
+ this.ops[1]
+ )
+ .should.equal(true)
})
it('should set the queue timestamp if not present', function () {
- this.multi.setnx.should.have.been.calledWithExactly(
- `ProjectHistory:FirstOpTimestamp:${this.project_id}`,
- Date.now()
- )
+ this.multi.setnx
+ .calledWithExactly(
+ `ProjectHistory:FirstOpTimestamp:${this.project_id}`,
+ Date.now()
+ )
+ .should.equal(true)
})
})
@@ -115,10 +118,9 @@ describe('ProjectHistoryRedisManager', function () {
file: this.file_id,
}
- this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
- this.project_id,
- JSON.stringify(update)
- )
+ this.ProjectHistoryRedisManager.promises.queueOps
+ .calledWithExactly(this.project_id, JSON.stringify(update))
+ .should.equal(true)
})
})
@@ -160,14 +162,12 @@ describe('ProjectHistoryRedisManager', function () {
},
version: this.version,
projectHistoryId: this.projectHistoryId,
- createdBlob: false,
doc: this.doc_id,
}
- this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
- this.project_id,
- JSON.stringify(update)
- )
+ this.ProjectHistoryRedisManager.promises.queueOps
+ .calledWithExactly(this.project_id, JSON.stringify(update))
+ .should.equal(true)
})
it('should queue an update with file metadata', async function () {
@@ -207,7 +207,6 @@ describe('ProjectHistoryRedisManager', function () {
hash: '1337',
metadata,
projectHistoryId: this.projectHistoryId,
- createdBlob: false,
file: fileId,
}
@@ -292,7 +291,6 @@ describe('ProjectHistoryRedisManager', function () {
},
version: this.version,
projectHistoryId: this.projectHistoryId,
- createdBlob: false,
ranges: historyCompatibleRanges,
doc: this.doc_id,
}
@@ -345,14 +343,12 @@ describe('ProjectHistoryRedisManager', function () {
},
version: this.version,
projectHistoryId: this.projectHistoryId,
- createdBlob: false,
doc: this.doc_id,
}
- this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
- this.project_id,
- JSON.stringify(update)
- )
+ this.ProjectHistoryRedisManager.promises.queueOps
+ .calledWithExactly(this.project_id, JSON.stringify(update))
+ .should.equal(true)
})
it('should not forward ranges if history ranges support is undefined', async function () {
@@ -398,77 +394,12 @@ describe('ProjectHistoryRedisManager', function () {
},
version: this.version,
projectHistoryId: this.projectHistoryId,
- createdBlob: false,
doc: this.doc_id,
}
- this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
- this.project_id,
- JSON.stringify(update)
- )
- })
-
- it('should pass "false" as the createdBlob field if not provided', async function () {
- await this.ProjectHistoryRedisManager.promises.queueAddEntity(
- this.project_id,
- this.projectHistoryId,
- 'doc',
- this.doc_id,
- this.user_id,
- this.rawUpdate,
- this.source
- )
-
- const update = {
- pathname: this.pathname,
- docLines: this.docLines,
- meta: {
- user_id: this.user_id,
- ts: new Date(),
- source: this.source,
- },
- version: this.version,
- projectHistoryId: this.projectHistoryId,
- createdBlob: false,
- doc: this.doc_id,
- }
-
- this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
- this.project_id,
- JSON.stringify(update)
- )
- })
-
- it('should pass through the value of the createdBlob field', async function () {
- this.rawUpdate.createdBlob = true
- await this.ProjectHistoryRedisManager.promises.queueAddEntity(
- this.project_id,
- this.projectHistoryId,
- 'doc',
- this.doc_id,
- this.user_id,
- this.rawUpdate,
- this.source
- )
-
- const update = {
- pathname: this.pathname,
- docLines: this.docLines,
- meta: {
- user_id: this.user_id,
- ts: new Date(),
- source: this.source,
- },
- version: this.version,
- projectHistoryId: this.projectHistoryId,
- createdBlob: true,
- doc: this.doc_id,
- }
-
- this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
- this.project_id,
- JSON.stringify(update)
- )
+ this.ProjectHistoryRedisManager.promises.queueOps
+ .calledWithExactly(this.project_id, JSON.stringify(update))
+ .should.equal(true)
})
})
@@ -496,8 +427,8 @@ describe('ProjectHistoryRedisManager', function () {
beforeEach(async function () {
this.update = {
resyncDocContent: {
- version: this.version,
content: 'one\ntwo',
+ version: this.version,
},
projectHistoryId: this.projectHistoryId,
path: this.pathname,
@@ -519,18 +450,19 @@ describe('ProjectHistoryRedisManager', function () {
})
it('should check if the doc is too large', function () {
- this.Limits.docIsTooLarge.should.have.been.calledWith(
- JSON.stringify(this.update).length,
- this.lines,
- this.settings.max_doc_length
- )
+ this.Limits.docIsTooLarge
+ .calledWith(
+ JSON.stringify(this.update).length,
+ this.lines,
+ this.settings.max_doc_length
+ )
+ .should.equal(true)
})
it('should queue an update', function () {
- this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
- this.project_id,
- JSON.stringify(this.update)
- )
+ this.ProjectHistoryRedisManager.promises.queueOps
+ .calledWithExactly(this.project_id, JSON.stringify(this.update))
+ .should.equal(true)
})
})
@@ -553,8 +485,9 @@ describe('ProjectHistoryRedisManager', function () {
})
it('should not queue an update if the doc is too large', function () {
- this.ProjectHistoryRedisManager.promises.queueOps.should.not.have.been
- .called
+ this.ProjectHistoryRedisManager.promises.queueOps.called.should.equal(
+ false
+ )
})
})
@@ -562,10 +495,10 @@ describe('ProjectHistoryRedisManager', function () {
beforeEach(async function () {
this.update = {
resyncDocContent: {
+ content: 'onedeleted\ntwo',
version: this.version,
ranges: this.ranges,
resolvedCommentIds: this.resolvedCommentIds,
- content: 'onedeleted\ntwo',
},
projectHistoryId: this.projectHistoryId,
path: this.pathname,
@@ -602,76 +535,9 @@ describe('ProjectHistoryRedisManager', function () {
})
it('should queue an update', function () {
- this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
- this.project_id,
- JSON.stringify(this.update)
- )
- })
- })
-
- describe('history-ot', function () {
- beforeEach(async function () {
- this.lines = {
- content: 'onedeleted\ntwo',
- comments: [{ id: 'id1', ranges: [{ pos: 0, length: 3 }] }],
- trackedChanges: [
- {
- range: { pos: 3, length: 7 },
- tracking: {
- type: 'delete',
- userId: 'user-id',
- ts: '2025-06-16T14:31:44.910Z',
- },
- },
- ],
- }
- this.update = {
- resyncDocContent: {
- version: this.version,
- historyOTRanges: {
- comments: this.lines.comments,
- trackedChanges: this.lines.trackedChanges,
- },
- content: this.lines.content,
- },
- projectHistoryId: this.projectHistoryId,
- path: this.pathname,
- doc: this.doc_id,
- meta: { ts: new Date() },
- }
-
- await this.ProjectHistoryRedisManager.promises.queueResyncDocContent(
- this.project_id,
- this.projectHistoryId,
- this.doc_id,
- this.lines,
- this.ranges,
- this.resolvedCommentIds,
- this.version,
- this.pathname,
- true
- )
- })
-
- it('should include tracked deletes in the update', function () {
- this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
- this.project_id,
- JSON.stringify(this.update)
- )
- })
-
- it('should check the doc length without tracked deletes', function () {
- this.Limits.stringFileDataContentIsTooLarge.should.have.been.calledWith(
- this.lines,
- this.settings.max_doc_length
- )
- })
-
- it('should queue an update', function () {
- this.ProjectHistoryRedisManager.promises.queueOps.should.have.been.calledWithExactly(
- this.project_id,
- JSON.stringify(this.update)
- )
+ this.ProjectHistoryRedisManager.promises.queueOps
+ .calledWithExactly(this.project_id, JSON.stringify(this.update))
+ .should.equal(true)
})
})
})
diff --git a/services/document-updater/test/unit/js/RangesManager/RangesManagerTests.js b/services/document-updater/test/unit/js/RangesManager/RangesManagerTests.js
index 4053aafb01..67ac6cc175 100644
--- a/services/document-updater/test/unit/js/RangesManager/RangesManagerTests.js
+++ b/services/document-updater/test/unit/js/RangesManager/RangesManagerTests.js
@@ -323,44 +323,6 @@ describe('RangesManager', function () {
})
})
- describe('tracked delete rejections with multiple tracked deletes at the same position', function () {
- beforeEach(function () {
- // original text is "one [two ][three ][four ]five"
- // [] denotes tracked deletes
- this.ranges = {
- changes: makeRanges([
- { d: 'two ', p: 4 },
- { d: 'three ', p: 4 },
- { d: 'four ', p: 4 },
- ]),
- }
- this.updates = makeUpdates([{ i: 'three ', p: 4, u: true }])
- this.newDocLines = ['one three five']
- this.result = this.RangesManager.applyUpdate(
- this.project_id,
- this.doc_id,
- this.ranges,
- this.updates,
- this.newDocLines,
- { historyRangesSupport: true }
- )
- })
-
- it('should insert the text at the right history position', function () {
- expect(this.result.historyUpdates.map(x => x.op)).to.deep.equal([
- [
- {
- i: 'three ',
- p: 4,
- hpos: 8,
- u: true,
- trackedDeleteRejection: true,
- },
- ],
- ])
- })
- })
-
describe('deletes over tracked changes', function () {
beforeEach(function () {
// original text is "on[1]e [22](three) f[333]ou[4444]r [55555]five"
@@ -696,7 +658,6 @@ describe('RangesManager', function () {
{ i: 'amet', p: 40 },
]),
}
- this.lines = ['lorem xxx', 'ipsum yyy', 'dolor zzz', 'sit wwwww', 'amet']
this.removeChangeIdsSpy = sinon.spy(
this.RangesTracker.prototype,
'removeChangeIds'
@@ -707,11 +668,8 @@ describe('RangesManager', function () {
beforeEach(function () {
this.change_ids = [this.ranges.changes[1].id]
this.result = this.RangesManager.acceptChanges(
- this.project_id,
- this.doc_id,
this.change_ids,
- this.ranges,
- this.lines
+ this.ranges
)
})
@@ -756,11 +714,8 @@ describe('RangesManager', function () {
this.ranges.changes[4].id,
]
this.result = this.RangesManager.acceptChanges(
- this.project_id,
- this.doc_id,
this.change_ids,
- this.ranges,
- this.lines
+ this.ranges
)
})
diff --git a/services/document-updater/test/unit/js/RedisManager/RedisManagerTests.js b/services/document-updater/test/unit/js/RedisManager/RedisManagerTests.js
index 125dd3d08c..e23964eaba 100644
--- a/services/document-updater/test/unit/js/RedisManager/RedisManagerTests.js
+++ b/services/document-updater/test/unit/js/RedisManager/RedisManagerTests.js
@@ -2,7 +2,7 @@ const sinon = require('sinon')
const { expect } = require('chai')
const SandboxedModule = require('sandboxed-module')
const Errors = require('../../../../app/js/Errors')
-const crypto = require('node:crypto')
+const crypto = require('crypto')
const tk = require('timekeeper')
const MODULE_PATH = '../../../../app/js/RedisManager.js'
diff --git a/services/document-updater/test/unit/js/ShareJsUpdateManager/ShareJsUpdateManagerTests.js b/services/document-updater/test/unit/js/ShareJsUpdateManager/ShareJsUpdateManagerTests.js
index e699439a74..347ac7dea6 100644
--- a/services/document-updater/test/unit/js/ShareJsUpdateManager/ShareJsUpdateManagerTests.js
+++ b/services/document-updater/test/unit/js/ShareJsUpdateManager/ShareJsUpdateManagerTests.js
@@ -12,7 +12,7 @@
const sinon = require('sinon')
const modulePath = '../../../../app/js/ShareJsUpdateManager.js'
const SandboxedModule = require('sandboxed-module')
-const crypto = require('node:crypto')
+const crypto = require('crypto')
describe('ShareJsUpdateManager', function () {
beforeEach(function () {
diff --git a/services/document-updater/test/unit/js/UpdateManager/UpdateManagerTests.js b/services/document-updater/test/unit/js/UpdateManager/UpdateManagerTests.js
index 912707e01d..16ee0b12e1 100644
--- a/services/document-updater/test/unit/js/UpdateManager/UpdateManagerTests.js
+++ b/services/document-updater/test/unit/js/UpdateManager/UpdateManagerTests.js
@@ -1,4 +1,5 @@
-const { createHash } = require('node:crypto')
+// @ts-check
+
const sinon = require('sinon')
const { expect } = require('chai')
const SandboxedModule = require('sandboxed-module')
@@ -331,7 +332,6 @@ describe('UpdateManager', function () {
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
historyRangesSupport: false,
- type: 'sharejs-text-ot',
})
this.RangesManager.applyUpdate.returns({
newRanges: this.updated_ranges,
@@ -399,9 +399,7 @@ describe('UpdateManager', function () {
this.historyUpdates,
this.pathname,
this.projectHistoryId,
- this.lines,
- this.ranges,
- this.updatedDocLines
+ this.lines
)
})
@@ -503,7 +501,6 @@ describe('UpdateManager', function () {
pathname: this.pathname,
projectHistoryId: this.projectHistoryId,
historyRangesSupport: true,
- type: 'sharejs-text-ot',
})
await this.UpdateManager.promises.applyUpdate(
this.project_id,
@@ -529,7 +526,6 @@ describe('UpdateManager', function () {
describe('_adjustHistoryUpdatesMetadata', function () {
beforeEach(function () {
this.lines = ['some', 'test', 'data']
- this.updatedDocLines = ['after', 'updates']
this.historyUpdates = [
{
v: 42,
@@ -574,7 +570,6 @@ describe('UpdateManager', function () {
this.pathname,
this.projectHistoryId,
this.lines,
- this.updatedDocLines,
this.ranges,
false
)
@@ -637,7 +632,6 @@ describe('UpdateManager', function () {
this.projectHistoryId,
this.lines,
this.ranges,
- this.updatedDocLines,
true
)
this.historyUpdates.should.deep.equal([
@@ -691,7 +685,6 @@ describe('UpdateManager', function () {
meta: {
pathname: this.pathname,
doc_length: 21, // 23 - 'so'
- doc_hash: stringHash(this.updatedDocLines.join('\n')),
history_doc_length: 28, // 30 - 'so'
},
},
@@ -706,7 +699,6 @@ describe('UpdateManager', function () {
this.projectHistoryId,
[],
{},
- ['foobar'],
false
)
this.historyUpdates.should.deep.equal([
@@ -830,9 +822,3 @@ describe('UpdateManager', function () {
})
})
})
-
-function stringHash(s) {
- const hash = createHash('sha1')
- hash.update(s)
- return hash.digest('hex')
-}
diff --git a/services/document-updater/test/unit/js/UtilsTests.js b/services/document-updater/test/unit/js/UtilsTests.js
index 5d0f03ca64..553b90159a 100644
--- a/services/document-updater/test/unit/js/UtilsTests.js
+++ b/services/document-updater/test/unit/js/UtilsTests.js
@@ -1,6 +1,5 @@
// @ts-check
-const { createHash } = require('node:crypto')
const { expect } = require('chai')
const Utils = require('../../../app/js/Utils')
@@ -25,30 +24,4 @@ describe('Utils', function () {
expect(result).to.equal('the quick brown fox jumps over the lazy dog')
})
})
-
- describe('computeDocHash', function () {
- it('computes the hash for an empty doc', function () {
- const actual = Utils.computeDocHash([])
- const expected = stringHash('')
- expect(actual).to.equal(expected)
- })
-
- it('computes the hash for a single-line doc', function () {
- const actual = Utils.computeDocHash(['hello'])
- const expected = stringHash('hello')
- expect(actual).to.equal(expected)
- })
-
- it('computes the hash for a multiline doc', function () {
- const actual = Utils.computeDocHash(['hello', 'there', 'world'])
- const expected = stringHash('hello\nthere\nworld')
- expect(actual).to.equal(expected)
- })
- })
})
-
-function stringHash(s) {
- const hash = createHash('sha1')
- hash.update(s)
- return hash.digest('hex')
-}
diff --git a/services/filestore/.gitignore b/services/filestore/.gitignore
index 1772191882..a2f4b5afb2 100644
--- a/services/filestore/.gitignore
+++ b/services/filestore/.gitignore
@@ -1,3 +1,54 @@
+compileFolder
+
+Compiled source #
+###################
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.so
+
+# Packages #
+############
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# Logs and databases #
+######################
+*.log
+*.sql
+*.sqlite
+
+# OS generated files #
+######################
+.DS_Store?
+ehthumbs.db
+Icon?
+Thumbs.db
+
+/node_modules/*
+data/*/*
+
+**/*.map
+cookies.txt
uploads/*
+
user_files/*
template_files/*
+
+**.swp
+
+/log.json
+hash_folder
+
+# managed by dev-environment$ bin/update_build_scripts
+.npmrc
diff --git a/services/filestore/.nvmrc b/services/filestore/.nvmrc
index fc37597bcc..123b052798 100644
--- a/services/filestore/.nvmrc
+++ b/services/filestore/.nvmrc
@@ -1 +1 @@
-22.17.0
+18.20.2
diff --git a/services/filestore/Dockerfile b/services/filestore/Dockerfile
index 33de01c80f..6593f60161 100644
--- a/services/filestore/Dockerfile
+++ b/services/filestore/Dockerfile
@@ -2,7 +2,7 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:22.17.0 AS base
+FROM node:18.20.2 AS base
WORKDIR /overleaf/services/filestore
COPY services/filestore/install_deps.sh /overleaf/services/filestore/
diff --git a/services/filestore/Makefile b/services/filestore/Makefile
index 69d7f85bf4..941f989d64 100644
--- a/services/filestore/Makefile
+++ b/services/filestore/Makefile
@@ -32,30 +32,12 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
-
-SHELLCHECK_OPTS = \
- --shell=bash \
- --external-sources
-SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
-SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
-
-shellcheck:
- @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
- koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
-
-shellcheck_fix:
- @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
- diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
- if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
- elif [ -n "$$diff" ]; then echo "$$file"; \
- else echo "\033[2m$$file\033[0m"; fi \
- done
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
format:
$(RUN_LINTING) format
@@ -81,7 +63,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck shellcheck test_unit test_acceptance
+test: format lint typecheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -148,7 +130,6 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
- shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/filestore/app.js b/services/filestore/app.js
index 178e8c7ff0..74b5fff246 100644
--- a/services/filestore/app.js
+++ b/services/filestore/app.js
@@ -1,7 +1,7 @@
// Metrics must be initialized before importing anything else
require('@overleaf/metrics/initialize')
-const Events = require('node:events')
+const Events = require('events')
const Metrics = require('@overleaf/metrics')
const logger = require('@overleaf/logger')
@@ -23,6 +23,10 @@ const app = express()
app.use(RequestLogger.middleware)
+if (settings.sentry && settings.sentry.dsn) {
+ logger.initializeErrorReporting(settings.sentry.dsn)
+}
+
Metrics.open_sockets.monitor(true)
Metrics.memory.monitor(logger)
if (Metrics.event_loop) {
@@ -50,73 +54,64 @@ app.use((req, res, next) => {
Metrics.injectMetricsRoute(app)
-if (settings.filestore.stores.user_files) {
- app.head(
- '/project/:project_id/file/:file_id',
- keyBuilder.userFileKeyMiddleware,
- fileController.getFileHead
- )
- app.get(
- '/project/:project_id/file/:file_id',
- keyBuilder.userFileKeyMiddleware,
- fileController.getFile
- )
- app.post(
- '/project/:project_id/file/:file_id',
- keyBuilder.userFileKeyMiddleware,
- fileController.insertFile
- )
- app.put(
- '/project/:project_id/file/:file_id',
- keyBuilder.userFileKeyMiddleware,
- bodyParser.json(),
- fileController.copyFile
- )
- app.delete(
- '/project/:project_id/file/:file_id',
- keyBuilder.userFileKeyMiddleware,
- fileController.deleteFile
- )
- app.delete(
- '/project/:project_id',
- keyBuilder.userProjectKeyMiddleware,
- fileController.deleteProject
- )
+app.head(
+ '/project/:project_id/file/:file_id',
+ keyBuilder.userFileKeyMiddleware,
+ fileController.getFileHead
+)
+app.get(
+ '/project/:project_id/file/:file_id',
+ keyBuilder.userFileKeyMiddleware,
+ fileController.getFile
+)
+app.post(
+ '/project/:project_id/file/:file_id',
+ keyBuilder.userFileKeyMiddleware,
+ fileController.insertFile
+)
+app.put(
+ '/project/:project_id/file/:file_id',
+ keyBuilder.userFileKeyMiddleware,
+ bodyParser.json(),
+ fileController.copyFile
+)
+app.delete(
+ '/project/:project_id/file/:file_id',
+ keyBuilder.userFileKeyMiddleware,
+ fileController.deleteFile
+)
+app.delete(
+ '/project/:project_id',
+ keyBuilder.userProjectKeyMiddleware,
+ fileController.deleteProject
+)
- app.get(
- '/project/:project_id/size',
- keyBuilder.userProjectKeyMiddleware,
- fileController.directorySize
- )
-}
+app.get(
+ '/project/:project_id/size',
+ keyBuilder.userProjectKeyMiddleware,
+ fileController.directorySize
+)
-if (settings.filestore.stores.template_files) {
- app.head(
- '/template/:template_id/v/:version/:format',
- keyBuilder.templateFileKeyMiddleware,
- fileController.getFileHead
- )
- app.get(
- '/template/:template_id/v/:version/:format',
- keyBuilder.templateFileKeyMiddleware,
- fileController.getFile
- )
- app.get(
- '/template/:template_id/v/:version/:format/:sub_type',
- keyBuilder.templateFileKeyMiddleware,
- fileController.getFile
- )
- app.post(
- '/template/:template_id/v/:version/:format',
- keyBuilder.templateFileKeyMiddleware,
- fileController.insertFile
- )
- app.delete(
- '/template/:template_id/v/:version/:format',
- keyBuilder.templateFileKeyMiddleware,
- fileController.deleteFile
- )
-}
+app.head(
+ '/template/:template_id/v/:version/:format',
+ keyBuilder.templateFileKeyMiddleware,
+ fileController.getFileHead
+)
+app.get(
+ '/template/:template_id/v/:version/:format',
+ keyBuilder.templateFileKeyMiddleware,
+ fileController.getFile
+)
+app.get(
+ '/template/:template_id/v/:version/:format/:sub_type',
+ keyBuilder.templateFileKeyMiddleware,
+ fileController.getFile
+)
+app.post(
+ '/template/:template_id/v/:version/:format',
+ keyBuilder.templateFileKeyMiddleware,
+ fileController.insertFile
+)
app.get(
'/bucket/:bucket/key/*',
@@ -171,10 +166,7 @@ function handleShutdownSignal(signal) {
// stop accepting new connections, the callback is called when existing connections have finished
server.close(() => {
logger.info({ signal }, 'server closed')
- // exit after a short delay so logs can be flushed
- setTimeout(() => {
- process.exit()
- }, 100)
+ process.exit()
})
// close idle http keep-alive connections
server.closeIdleConnections()
@@ -186,7 +178,7 @@ function handleShutdownSignal(signal) {
setTimeout(() => {
process.exit()
}, 100)
- }, settings.gracefulShutdownDelayInMs)
+ }, settings.delayShutdownMs)
}
process.on('SIGTERM', handleShutdownSignal)
diff --git a/services/filestore/app/js/FileController.js b/services/filestore/app/js/FileController.js
index 127bbcc20f..a97869258a 100644
--- a/services/filestore/app/js/FileController.js
+++ b/services/filestore/app/js/FileController.js
@@ -1,8 +1,9 @@
+const PersistorManager = require('./PersistorManager')
const FileHandler = require('./FileHandler')
const metrics = require('@overleaf/metrics')
const parseRange = require('range-parser')
const Errors = require('./Errors')
-const { pipeline } = require('node:stream')
+const { pipeline } = require('stream')
const maxSizeInBytes = 1024 * 1024 * 1024 // 1GB
@@ -138,17 +139,17 @@ function copyFile(req, res, next) {
})
req.requestLogger.setMessage('copying file')
- FileHandler.copyObject(bucket, `${oldProjectId}/${oldFileId}`, key, err => {
- if (err) {
- if (err instanceof Errors.NotFoundError) {
- res.sendStatus(404)
- } else {
- next(err)
+ PersistorManager.copyObject(bucket, `${oldProjectId}/${oldFileId}`, key)
+ .then(() => res.sendStatus(200))
+ .catch(err => {
+ if (err) {
+ if (err instanceof Errors.NotFoundError) {
+ res.sendStatus(404)
+ } else {
+ next(err)
+ }
}
- } else {
- res.sendStatus(200)
- }
- })
+ })
}
function deleteFile(req, res, next) {
diff --git a/services/filestore/app/js/FileConverter.js b/services/filestore/app/js/FileConverter.js
index bfc34314e9..cb21a482e3 100644
--- a/services/filestore/app/js/FileConverter.js
+++ b/services/filestore/app/js/FileConverter.js
@@ -1,11 +1,11 @@
const metrics = require('@overleaf/metrics')
const Settings = require('@overleaf/settings')
-const { callbackify } = require('node:util')
+const { callbackify } = require('util')
const safeExec = require('./SafeExec').promises
const { ConversionError } = require('./Errors')
-const APPROVED_FORMATS = ['png', 'jpg']
+const APPROVED_FORMATS = ['png']
const FOURTY_SECONDS = 40 * 1000
const KILL_SIGNAL = 'SIGTERM'
@@ -34,14 +34,16 @@ async function convert(sourcePath, requestedFormat) {
}
async function thumbnail(sourcePath) {
- const width = '548x'
- return await _convert(sourcePath, 'jpg', [
+ const width = '260x'
+ return await convert(sourcePath, 'png', [
'convert',
'-flatten',
'-background',
'white',
'-density',
'300',
+ '-define',
+ `pdf:fit-page=${width}`,
`${sourcePath}[0]`,
'-resize',
width,
@@ -49,14 +51,16 @@ async function thumbnail(sourcePath) {
}
async function preview(sourcePath) {
- const width = '794x'
- return await _convert(sourcePath, 'jpg', [
+ const width = '548x'
+ return await convert(sourcePath, 'png', [
'convert',
'-flatten',
'-background',
'white',
'-density',
'300',
+ '-define',
+ `pdf:fit-page=${width}`,
`${sourcePath}[0]`,
'-resize',
width,
diff --git a/services/filestore/app/js/FileHandler.js b/services/filestore/app/js/FileHandler.js
index 0c092c85cd..b52418f07d 100644
--- a/services/filestore/app/js/FileHandler.js
+++ b/services/filestore/app/js/FileHandler.js
@@ -1,7 +1,7 @@
const Settings = require('@overleaf/settings')
-const { callbackify } = require('node:util')
-const fs = require('node:fs')
-let PersistorManager = require('./PersistorManager')
+const { callbackify } = require('util')
+const fs = require('fs')
+const PersistorManager = require('./PersistorManager')
const LocalFileWriter = require('./LocalFileWriter')
const FileConverter = require('./FileConverter')
const KeyBuilder = require('./KeyBuilder')
@@ -10,7 +10,6 @@ const { ConversionError, InvalidParametersError } = require('./Errors')
const metrics = require('@overleaf/metrics')
module.exports = {
- copyObject: callbackify(copyObject),
insertFile: callbackify(insertFile),
deleteFile: callbackify(deleteFile),
deleteProject: callbackify(deleteProject),
@@ -19,7 +18,6 @@ module.exports = {
getFileSize: callbackify(getFileSize),
getDirectorySize: callbackify(getDirectorySize),
promises: {
- copyObject,
getFile,
getRedirectUrl,
insertFile,
@@ -30,16 +28,6 @@ module.exports = {
},
}
-if (process.env.NODE_ENV === 'test') {
- module.exports._TESTONLYSwapPersistorManager = _PersistorManager => {
- PersistorManager = _PersistorManager
- }
-}
-
-async function copyObject(bucket, sourceKey, destinationKey) {
- await PersistorManager.copyObject(bucket, sourceKey, destinationKey)
-}
-
async function insertFile(bucket, key, stream) {
const convertedKey = KeyBuilder.getConvertedFolderKey(key)
if (!convertedKey.match(/^[0-9a-f]{24}\/([0-9a-f]{24}|v\/[0-9]+\/[a-z]+)/i)) {
@@ -49,6 +37,9 @@ async function insertFile(bucket, key, stream) {
convertedKey,
})
}
+ if (Settings.enableConversions) {
+ await PersistorManager.deleteDirectory(bucket, convertedKey)
+ }
await PersistorManager.sendStream(bucket, key, stream)
}
@@ -62,10 +53,7 @@ async function deleteFile(bucket, key) {
})
}
const jobs = [PersistorManager.deleteObject(bucket, key)]
- if (
- Settings.enableConversions &&
- bucket === Settings.filestore.stores.template_files
- ) {
+ if (Settings.enableConversions) {
jobs.push(PersistorManager.deleteDirectory(bucket, convertedKey))
}
await Promise.all(jobs)
@@ -130,7 +118,7 @@ async function getFileSize(bucket, key) {
}
async function getDirectorySize(bucket, projectId) {
- return await PersistorManager.directorySize(bucket, projectId)
+ return PersistorManager.directorySize(bucket, projectId)
}
async function _getConvertedFile(bucket, key, opts) {
@@ -150,9 +138,7 @@ async function _getConvertedFileAndCache(bucket, key, convertedKey, opts) {
let convertedFsPath
try {
convertedFsPath = await _convertFile(bucket, key, opts)
- if (convertedFsPath.toLowerCase().endsWith(".png")) {
- await ImageOptimiser.promises.compressPng(convertedFsPath)
- }
+ await ImageOptimiser.promises.compressPng(convertedFsPath)
await PersistorManager.sendFile(bucket, convertedKey, convertedFsPath)
} catch (err) {
LocalFileWriter.deleteFile(convertedFsPath, () => {})
diff --git a/services/filestore/app/js/HealthCheckController.js b/services/filestore/app/js/HealthCheckController.js
index e9b739a971..fb296513b5 100644
--- a/services/filestore/app/js/HealthCheckController.js
+++ b/services/filestore/app/js/HealthCheckController.js
@@ -1,9 +1,9 @@
-const fs = require('node:fs')
-const path = require('node:path')
+const fs = require('fs')
+const path = require('path')
const Settings = require('@overleaf/settings')
const { WritableBuffer } = require('@overleaf/stream-utils')
-const { promisify } = require('node:util')
-const Stream = require('node:stream')
+const { promisify } = require('util')
+const Stream = require('stream')
const pipeline = promisify(Stream.pipeline)
const fsCopy = promisify(fs.copyFile)
diff --git a/services/filestore/app/js/ImageOptimiser.js b/services/filestore/app/js/ImageOptimiser.js
index 6ed29e1c6d..a46a2857e1 100644
--- a/services/filestore/app/js/ImageOptimiser.js
+++ b/services/filestore/app/js/ImageOptimiser.js
@@ -1,6 +1,6 @@
const logger = require('@overleaf/logger')
const metrics = require('@overleaf/metrics')
-const { callbackify } = require('node:util')
+const { callbackify } = require('util')
const safeExec = require('./SafeExec').promises
module.exports = {
diff --git a/services/filestore/app/js/LocalFileWriter.js b/services/filestore/app/js/LocalFileWriter.js
index fe55bdc138..c5b8ea41f1 100644
--- a/services/filestore/app/js/LocalFileWriter.js
+++ b/services/filestore/app/js/LocalFileWriter.js
@@ -1,8 +1,8 @@
-const fs = require('node:fs')
-const crypto = require('node:crypto')
-const path = require('node:path')
-const Stream = require('node:stream')
-const { callbackify, promisify } = require('node:util')
+const fs = require('fs')
+const crypto = require('crypto')
+const path = require('path')
+const Stream = require('stream')
+const { callbackify, promisify } = require('util')
const metrics = require('@overleaf/metrics')
const Settings = require('@overleaf/settings')
const { WriteError } = require('./Errors')
diff --git a/services/filestore/app/js/SafeExec.js b/services/filestore/app/js/SafeExec.js
index 16ebcf126b..63177b8057 100644
--- a/services/filestore/app/js/SafeExec.js
+++ b/services/filestore/app/js/SafeExec.js
@@ -1,5 +1,5 @@
const lodashOnce = require('lodash.once')
-const childProcess = require('node:child_process')
+const childProcess = require('child_process')
const Settings = require('@overleaf/settings')
const { ConversionsDisabledError, FailedCommandError } = require('./Errors')
diff --git a/services/filestore/buildscript.txt b/services/filestore/buildscript.txt
index bd4d2116f6..147e8b4a46 100644
--- a/services/filestore/buildscript.txt
+++ b/services/filestore/buildscript.txt
@@ -2,11 +2,11 @@ filestore
--data-dirs=uploads,user_files,template_files
--dependencies=s3,gcs
--docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker
---env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_STORAGE_CLASS=REDUCED_REDUNDANCY,AWS_S3_USER_FILES_BUCKET_NAME=fake-user-files,AWS_S3_USER_FILES_DEK_BUCKET_NAME=fake-user-files-dek,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake-template-files,GCS_USER_FILES_BUCKET_NAME=fake-gcs-user-files,GCS_TEMPLATE_FILES_BUCKET_NAME=fake-gcs-template-files
+--env-add=ENABLE_CONVERSIONS="true",USE_PROM_METRICS="true",AWS_S3_USER_FILES_BUCKET_NAME=fake_user_files,AWS_S3_TEMPLATE_FILES_BUCKET_NAME=fake_template_files,GCS_USER_FILES_BUCKET_NAME=fake_userfiles,GCS_TEMPLATE_FILES_BUCKET_NAME=fake_templatefiles
--env-pass-through=
--esmock-loader=False
---node-version=22.17.0
+--node-version=18.20.2
--public-repo=True
---script-version=4.7.0
+--script-version=4.5.0
--test-acceptance-shards=SHARD_01_,SHARD_02_,SHARD_03_
--use-large-ci-runner=True
diff --git a/services/filestore/config/settings.defaults.js b/services/filestore/config/settings.defaults.js
index 9a08bb197e..08fa66408f 100644
--- a/services/filestore/config/settings.defaults.js
+++ b/services/filestore/config/settings.defaults.js
@@ -1,4 +1,4 @@
-const Path = require('node:path')
+const Path = require('path')
// environment variables renamed for consistency
// use AWS_ACCESS_KEY_ID-style going forward
@@ -73,10 +73,6 @@ const settings = {
stores: {
user_files: process.env.USER_FILES_BUCKET_NAME,
template_files: process.env.TEMPLATE_FILES_BUCKET_NAME,
-
- // allow signed links to be generated for these buckets
- project_blobs: process.env.OVERLEAF_EDITOR_PROJECT_BLOBS_BUCKET,
- global_blobs: process.env.OVERLEAF_EDITOR_BLOBS_BUCKET,
},
fallback: process.env.FALLBACK_BACKEND
@@ -103,6 +99,10 @@ const settings = {
enableConversions: process.env.ENABLE_CONVERSIONS === 'true',
+ sentry: {
+ dsn: process.env.SENTRY_DSN,
+ },
+
gracefulShutdownDelayInMs:
parseInt(process.env.GRACEFUL_SHUTDOWN_DELAY_SECONDS ?? '30', 10) * 1000,
}
diff --git a/services/filestore/docker-compose.ci.yml b/services/filestore/docker-compose.ci.yml
index fdf860b511..aaa11bea6b 100644
--- a/services/filestore/docker-compose.ci.yml
+++ b/services/filestore/docker-compose.ci.yml
@@ -21,12 +21,10 @@ services:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
- AWS_S3_ENDPOINT: https://minio:9000
+ AWS_S3_ENDPOINT: http://s3:9090
AWS_S3_PATH_STYLE: 'true'
- AWS_ACCESS_KEY_ID: OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY: OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
- MINIO_ROOT_USER: MINIO_ROOT_USER
- MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
+ AWS_ACCESS_KEY_ID: fake
+ AWS_SECRET_ACCESS_KEY: fake
GCS_API_ENDPOINT: http://gcs:9090
GCS_PROJECT_ID: fake
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
@@ -35,21 +33,13 @@ services:
NODE_OPTIONS: "--unhandled-rejections=strict"
ENABLE_CONVERSIONS: "true"
USE_PROM_METRICS: "true"
- AWS_S3_USER_FILES_STORAGE_CLASS: REDUCED_REDUNDANCY
- AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files
- AWS_S3_USER_FILES_DEK_BUCKET_NAME: fake-user-files-dek
- AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files
- GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files
- GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files
- volumes:
- - ./test/acceptance/certs:/certs
+ AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
+ AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
+ GCS_USER_FILES_BUCKET_NAME: fake_userfiles
+ GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles
depends_on:
- certs:
- condition: service_completed_successfully
- minio:
- condition: service_started
- minio_setup:
- condition: service_completed_successfully
+ s3:
+ condition: service_healthy
gcs:
condition: service_healthy
user: node
@@ -63,137 +53,14 @@ services:
- ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
- certs:
- image: node:22.17.0
- volumes:
- - ./test/acceptance/certs:/certs
- working_dir: /certs
- entrypoint: sh
- command:
- - '-cex'
- - |
- if [ ! -f ./certgen ]; then
- wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)"
- chmod +x ./certgen
- fi
- if [ ! -f private.key ] || [ ! -f public.crt ]; then
- ./certgen -host minio
- fi
-
- minio:
- image: minio/minio:RELEASE.2024-10-13T13-34-11Z
- command: server /data
- volumes:
- - ./test/acceptance/certs:/root/.minio/certs
+ s3:
+ image: adobe/s3mock:2.4.14
environment:
- MINIO_ROOT_USER: MINIO_ROOT_USER
- MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
- depends_on:
- certs:
- condition: service_completed_successfully
-
- minio_setup:
- depends_on:
- certs:
- condition: service_completed_successfully
- minio:
- condition: service_started
- image: minio/mc:RELEASE.2024-10-08T09-37-26Z
- volumes:
- - ./test/acceptance/certs:/root/.mc/certs/CAs
- entrypoint: sh
- command:
- - '-cex'
- - |
- sleep 1
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
- || sleep 3 && \
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
- || sleep 3 && \
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
- || sleep 3 && \
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
- mc mb --ignore-existing s3/fake-user-files
- mc mb --ignore-existing s3/fake-user-files-dek
- mc mb --ignore-existing s3/fake-template-files
- mc admin user add s3 \
- OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID \
- OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
-
- echo '
- {
- "Version": "2012-10-17",
- "Statement": [
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::fake-user-files"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::fake-user-files/*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::fake-user-files-dek"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::fake-user-files-dek/*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::fake-template-files"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::fake-template-files/*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::random-bucket-*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::random-bucket-*"
- }
- ]
- }' > policy-filestore.json
-
- mc admin policy create s3 overleaf-filestore policy-filestore.json
- mc admin policy attach s3 overleaf-filestore \
- --user=OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
+ - initialBuckets=fake_user_files,fake_template_files,bucket
+ healthcheck:
+ test: wget --quiet --output-document=/dev/null http://localhost:9090
+ interval: 1s
+ retries: 20
gcs:
image: fsouza/fake-gcs-server:1.45.2
command: ["--port=9090", "--scheme=http"]
diff --git a/services/filestore/docker-compose.yml b/services/filestore/docker-compose.yml
index 971d35b708..427384e452 100644
--- a/services/filestore/docker-compose.yml
+++ b/services/filestore/docker-compose.yml
@@ -17,7 +17,6 @@ services:
working_dir: /overleaf/services/filestore
environment:
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
@@ -32,176 +31,44 @@ services:
- .:/overleaf/services/filestore
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- - ./test/acceptance/certs:/certs
working_dir: /overleaf/services/filestore
environment:
ELASTIC_SEARCH_DSN: es:9200
MONGO_HOST: mongo
POSTGRES_HOST: postgres
- AWS_S3_ENDPOINT: https://minio:9000
+ AWS_S3_ENDPOINT: http://s3:9090
AWS_S3_PATH_STYLE: 'true'
- AWS_ACCESS_KEY_ID: OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY: OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
- MINIO_ROOT_USER: MINIO_ROOT_USER
- MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
+ AWS_ACCESS_KEY_ID: fake
+ AWS_SECRET_ACCESS_KEY: fake
GCS_API_ENDPOINT: http://gcs:9090
GCS_PROJECT_ID: fake
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
+ LOG_LEVEL: ERROR
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
ENABLE_CONVERSIONS: "true"
USE_PROM_METRICS: "true"
- AWS_S3_USER_FILES_STORAGE_CLASS: REDUCED_REDUNDANCY
- AWS_S3_USER_FILES_BUCKET_NAME: fake-user-files
- AWS_S3_USER_FILES_DEK_BUCKET_NAME: fake-user-files-dek
- AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake-template-files
- GCS_USER_FILES_BUCKET_NAME: fake-gcs-user-files
- GCS_TEMPLATE_FILES_BUCKET_NAME: fake-gcs-template-files
+ AWS_S3_USER_FILES_BUCKET_NAME: fake_user_files
+ AWS_S3_TEMPLATE_FILES_BUCKET_NAME: fake_template_files
+ GCS_USER_FILES_BUCKET_NAME: fake_userfiles
+ GCS_TEMPLATE_FILES_BUCKET_NAME: fake_templatefiles
user: node
depends_on:
- certs:
- condition: service_completed_successfully
- minio:
- condition: service_started
- minio_setup:
- condition: service_completed_successfully
+ s3:
+ condition: service_healthy
gcs:
condition: service_healthy
command: npm run --silent test:acceptance
- certs:
- image: node:22.17.0
- volumes:
- - ./test/acceptance/certs:/certs
- working_dir: /certs
- entrypoint: sh
- command:
- - '-cex'
- - |
- if [ ! -f ./certgen ]; then
- wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)"
- chmod +x ./certgen
- fi
- if [ ! -f private.key ] || [ ! -f public.crt ]; then
- ./certgen -host minio
- fi
-
- minio:
- image: minio/minio:RELEASE.2024-10-13T13-34-11Z
- command: server /data
- volumes:
- - ./test/acceptance/certs:/root/.minio/certs
+ s3:
+ image: adobe/s3mock:2.4.14
environment:
- MINIO_ROOT_USER: MINIO_ROOT_USER
- MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
- depends_on:
- certs:
- condition: service_completed_successfully
-
- minio_setup:
- depends_on:
- certs:
- condition: service_completed_successfully
- minio:
- condition: service_started
- image: minio/mc:RELEASE.2024-10-08T09-37-26Z
- volumes:
- - ./test/acceptance/certs:/root/.mc/certs/CAs
- entrypoint: sh
- command:
- - '-cex'
- - |
- sleep 1
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
- || sleep 3 && \
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
- || sleep 3 && \
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
- || sleep 3 && \
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
- mc mb --ignore-existing s3/fake-user-files
- mc mb --ignore-existing s3/fake-user-files-dek
- mc mb --ignore-existing s3/fake-template-files
- mc admin user add s3 \
- OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID \
- OVERLEAF_FILESTORE_S3_SECRET_ACCESS_KEY
-
- echo '
- {
- "Version": "2012-10-17",
- "Statement": [
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::fake-user-files"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::fake-user-files/*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::fake-user-files-dek"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::fake-user-files-dek/*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::fake-template-files"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::fake-template-files/*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::random-bucket-*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::random-bucket-*"
- }
- ]
- }' > policy-filestore.json
-
- mc admin policy create s3 overleaf-filestore policy-filestore.json
- mc admin policy attach s3 overleaf-filestore \
- --user=OVERLEAF_FILESTORE_S3_ACCESS_KEY_ID
+ - initialBuckets=fake_user_files,fake_template_files,bucket
+ healthcheck:
+ test: wget --quiet --output-document=/dev/null http://localhost:9090
+ interval: 1s
+ retries: 20
gcs:
image: fsouza/fake-gcs-server:1.45.2
command: ["--port=9090", "--scheme=http"]
diff --git a/services/filestore/package.json b/services/filestore/package.json
index 4b9043aed7..e680f54b2b 100644
--- a/services/filestore/package.json
+++ b/services/filestore/package.json
@@ -11,8 +11,8 @@
"start": "node app.js",
"nodemon": "node --watch app.js",
"lint": "eslint --max-warnings 0 --format unix .",
- "format": "prettier --list-different $PWD/'**/*.*js'",
- "format:fix": "prettier --write $PWD/'**/*.*js'",
+ "format": "prettier --list-different $PWD/'**/*.js'",
+ "format:fix": "prettier --write $PWD/'**/*.js'",
"test:acceptance:_run": "mocha --recursive --reporter spec --timeout 15000 --exit $@ test/acceptance/js",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
"lint:fix": "eslint --fix .",
@@ -27,7 +27,7 @@
"@overleaf/stream-utils": "^0.1.0",
"body-parser": "^1.20.3",
"bunyan": "^1.8.15",
- "express": "^4.21.2",
+ "express": "^4.21.0",
"glob": "^7.1.6",
"lodash.once": "^4.1.1",
"node-fetch": "^2.7.0",
@@ -36,14 +36,17 @@
},
"devDependencies": {
"@google-cloud/storage": "^6.10.1",
+ "aws-sdk": "^2.718.0",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
- "mocha": "^11.1.0",
- "mongodb": "6.12.0",
+ "disrequire": "^1.1.0",
+ "mocha": "^10.2.0",
+ "mongodb": "^6.1.0",
"sandboxed-module": "2.0.4",
"sinon": "9.0.2",
"sinon-chai": "^3.7.0",
"streamifier": "^0.1.1",
+ "timekeeper": "^2.2.0",
"typescript": "^5.0.4"
}
}
diff --git a/services/filestore/test/acceptance/certs/.gitignore b/services/filestore/test/acceptance/certs/.gitignore
deleted file mode 100644
index d6b7ef32c8..0000000000
--- a/services/filestore/test/acceptance/certs/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-*
-!.gitignore
diff --git a/services/filestore/test/acceptance/deps/healthcheck.sh b/services/filestore/test/acceptance/deps/healthcheck.sh
index 675c205be6..cd19cea637 100755
--- a/services/filestore/test/acceptance/deps/healthcheck.sh
+++ b/services/filestore/test/acceptance/deps/healthcheck.sh
@@ -1,9 +1,9 @@
#!/bin/sh
# health check to allow 404 status code as valid
-STATUSCODE=$(curl --silent --output /dev/null --write-out "%{http_code}" "$1")
+STATUSCODE=$(curl --silent --output /dev/null --write-out "%{http_code}" $1)
# will be 000 on non-http error (e.g. connection failure)
-if test "$STATUSCODE" -ge 500 || test "$STATUSCODE" -lt 200; then
+if test $STATUSCODE -ge 500 || test $STATUSCODE -lt 200; then
exit 1
fi
exit 0
diff --git a/services/filestore/test/acceptance/js/FilestoreApp.js b/services/filestore/test/acceptance/js/FilestoreApp.js
index 61e9a29b7d..924f852eea 100644
--- a/services/filestore/test/acceptance/js/FilestoreApp.js
+++ b/services/filestore/test/acceptance/js/FilestoreApp.js
@@ -1,31 +1,64 @@
-const ObjectPersistor = require('@overleaf/object-persistor')
+const logger = require('@overleaf/logger')
const Settings = require('@overleaf/settings')
-const { promisify } = require('node:util')
-const App = require('../../../app')
-const FileHandler = require('../../../app/js/FileHandler')
+const fs = require('fs')
+const Path = require('path')
+const { promisify } = require('util')
+const disrequire = require('disrequire')
+const AWS = require('aws-sdk')
+
+logger.logger.level('info')
+
+const fsReaddir = promisify(fs.readdir)
+const sleep = promisify(setTimeout)
class FilestoreApp {
+ constructor() {
+ this.running = false
+ this.initing = false
+ }
+
async runServer() {
- if (!this.server) {
- await new Promise((resolve, reject) => {
- this.server = App.listen(
- Settings.internal.filestore.port,
- '127.0.0.1',
- err => {
- if (err) {
- return reject(err)
- }
- resolve()
- }
- )
- })
+ if (this.running) {
+ return
}
- this.persistor = ObjectPersistor({
- ...Settings.filestore,
- paths: Settings.path,
+ if (this.initing) {
+ return await this.waitForInit()
+ }
+ this.initing = true
+
+ this.app = await FilestoreApp.requireApp()
+
+ await new Promise((resolve, reject) => {
+ this.server = this.app.listen(
+ Settings.internal.filestore.port,
+ '127.0.0.1',
+ err => {
+ if (err) {
+ return reject(err)
+ }
+ resolve()
+ }
+ )
})
- FileHandler._TESTONLYSwapPersistorManager(this.persistor)
+
+ if (Settings.filestore.backend === 's3') {
+ try {
+ await FilestoreApp.waitForS3()
+ } catch (err) {
+ await this.stop()
+ throw err
+ }
+ }
+
+ this.initing = false
+ this.persistor = require('../../../app/js/PersistorManager')
+ }
+
+ async waitForInit() {
+ while (this.initing) {
+ await sleep(1000)
+ }
}
async stop() {
@@ -37,6 +70,54 @@ class FilestoreApp {
delete this.server
}
}
+
+ static async waitForS3() {
+ let tries = 0
+ if (!Settings.filestore.s3.endpoint) {
+ return
+ }
+
+ const s3 = new AWS.S3({
+ accessKeyId: Settings.filestore.s3.key,
+ secretAccessKey: Settings.filestore.s3.secret,
+ endpoint: Settings.filestore.s3.endpoint,
+ s3ForcePathStyle: true,
+ signatureVersion: 'v4',
+ })
+
+ while (true) {
+ try {
+ return await s3
+ .putObject({
+ Key: 'startup',
+ Body: '42',
+ Bucket: Settings.filestore.stores.user_files,
+ })
+ .promise()
+ } catch (err) {
+ // swallow errors, as we may experience them until fake-s3 is running
+ if (tries === 9) {
+ // throw just before hitting the 10s test timeout
+ throw err
+ }
+ tries++
+ await sleep(1000)
+ }
+ }
+ }
+
+ static async requireApp() {
+ // unload the app, as we may be doing this on multiple runs with
+ // different settings, which affect startup in some cases
+ const files = await fsReaddir(Path.resolve(__dirname, '../../../app/js'))
+ files.forEach(file => {
+ disrequire(Path.resolve(__dirname, '../../../app/js', file))
+ })
+ disrequire(Path.resolve(__dirname, '../../../app'))
+ disrequire('@overleaf/object-persistor')
+
+ return require('../../../app')
+ }
}
module.exports = FilestoreApp
diff --git a/services/filestore/test/acceptance/js/FilestoreTests.js b/services/filestore/test/acceptance/js/FilestoreTests.js
index 28f90d49b6..441a179c59 100644
--- a/services/filestore/test/acceptance/js/FilestoreTests.js
+++ b/services/filestore/test/acceptance/js/FilestoreTests.js
@@ -1,18 +1,19 @@
const chai = require('chai')
const { expect } = chai
-const fs = require('node:fs')
-const Stream = require('node:stream')
+const fs = require('fs')
const Settings = require('@overleaf/settings')
-const Path = require('node:path')
+const Path = require('path')
const FilestoreApp = require('./FilestoreApp')
const TestHelper = require('./TestHelper')
const fetch = require('node-fetch')
-const { promisify } = require('node:util')
+const S3 = require('aws-sdk/clients/s3')
+const { promisify } = require('util')
const { Storage } = require('@google-cloud/storage')
const streamifier = require('streamifier')
chai.use(require('chai-as-promised'))
const { ObjectId } = require('mongodb')
-const ChildProcess = require('node:child_process')
+const tk = require('timekeeper')
+const ChildProcess = require('child_process')
const fsWriteFile = promisify(fs.writeFile)
const fsStat = promisify(fs.stat)
@@ -31,26 +32,7 @@ process.on('unhandledRejection', e => {
// store settings for multiple backends, so that we can test each one.
// fs will always be available - add others if they are configured
-const {
- BackendSettings,
- s3Config,
- s3SSECConfig,
- AWS_S3_USER_FILES_STORAGE_CLASS,
-} = require('./TestConfig')
-const {
- AlreadyWrittenError,
- NotFoundError,
- NotImplementedError,
- NoKEKMatchedError,
-} = require('@overleaf/object-persistor/src/Errors')
-const {
- PerProjectEncryptedS3Persistor,
- RootKeyEncryptionKey,
-} = require('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor')
-const { S3Persistor } = require('@overleaf/object-persistor/src/S3Persistor')
-const crypto = require('node:crypto')
-const { WritableBuffer } = require('@overleaf/stream-utils')
-const { gzipSync } = require('node:zlib')
+const BackendSettings = require('./TestConfig')
describe('Filestore', function () {
this.timeout(1000 * 10)
@@ -95,10 +77,8 @@ describe('Filestore', function () {
}
// redefine the test suite for every available backend
- for (const [backendVariantWithShardNumber, backendSettings] of Object.entries(
- BackendSettings
- )) {
- describe(backendVariantWithShardNumber, function () {
+ Object.keys(BackendSettings).forEach(backend => {
+ describe(backend, function () {
let app,
previousEgress,
previousIngress,
@@ -106,9 +86,6 @@ describe('Filestore', function () {
projectId,
otherProjectId
- const dataEncryptionKeySize =
- backendSettings.backend === 's3SSEC' ? 32 : 0
-
const BUCKET_NAMES = [
process.env.GCS_USER_FILES_BUCKET_NAME,
process.env.GCS_TEMPLATE_FILES_BUCKET_NAME,
@@ -116,15 +93,15 @@ describe('Filestore', function () {
`${process.env.GCS_TEMPLATE_FILES_BUCKET_NAME}-deleted`,
]
- before('start filestore with new settings', async function () {
+ before(async function () {
// create the app with the relevant filestore settings
- Settings.filestore = backendSettings
+ Settings.filestore = BackendSettings[backend]
app = new FilestoreApp()
await app.runServer()
})
- if (backendSettings.gcs) {
- before('create gcs buckets', async function () {
+ if (BackendSettings[backend].gcs) {
+ before(async function () {
// create test buckets for gcs
const storage = new Storage(Settings.filestore.gcs.endpoint)
for (const bucketName of BUCKET_NAMES) {
@@ -132,7 +109,7 @@ describe('Filestore', function () {
}
})
- after('delete gcs buckets', async function () {
+ after(async function () {
// tear down all the gcs buckets
const storage = new Storage(Settings.filestore.gcs.endpoint)
for (const bucketName of BUCKET_NAMES) {
@@ -143,14 +120,15 @@ describe('Filestore', function () {
})
}
- after('stop filestore app', async function () {
+ after(async function () {
+ await msleep(3000)
await app.stop()
})
- beforeEach('fetch previous egress metric', async function () {
+ beforeEach(async function () {
// retrieve previous metrics from the app
- if (['s3', 's3SSEC', 'gcs'].includes(Settings.filestore.backend)) {
- metricPrefix = Settings.filestore.backend.replace('SSEC', '')
+ if (['s3', 'gcs'].includes(Settings.filestore.backend)) {
+ metricPrefix = Settings.filestore.backend
previousEgress = await TestHelper.getMetric(
filestoreUrl,
`${metricPrefix}_egress`
@@ -174,7 +152,7 @@ describe('Filestore', function () {
const localFileReadPath =
'/tmp/filestore_acceptance_tests_file_read.txt'
- beforeEach('upload file', async function () {
+ beforeEach(async function () {
fileId = new ObjectId().toString()
fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
constantFileContent = [
@@ -186,15 +164,14 @@ describe('Filestore', function () {
await fsWriteFile(localFileReadPath, constantFileContent)
const readStream = fs.createReadStream(localFileReadPath)
- const res = await fetch(fileUrl, { method: 'POST', body: readStream })
- if (!res.ok) throw new Error(res.statusText)
+ await fetch(fileUrl, { method: 'POST', body: readStream })
})
- beforeEach('retrieve previous ingress metric', async function () {
+ beforeEach(async function retrievePreviousIngressMetrics() {
// The upload request can bump the ingress metric.
// The content hash validation might require a full download
// in case the ETag field of the upload response is not a md5 sum.
- if (['s3', 's3SSEC', 'gcs'].includes(Settings.filestore.backend)) {
+ if (['s3', 'gcs'].includes(Settings.filestore.backend)) {
previousIngress = await TestHelper.getMetric(
filestoreUrl,
`${metricPrefix}_ingress`
@@ -235,9 +212,7 @@ describe('Filestore', function () {
})
it('should not leak a socket', async function () {
- const res = await fetch(fileUrl)
- if (!res.ok) throw new Error(res.statusText)
- await res.text()
+ await fetch(fileUrl)
await expectNoSockets()
})
@@ -294,61 +269,24 @@ describe('Filestore', function () {
expect(body).to.equal(newContent)
})
- describe('IfNoneMatch', function () {
- if (backendSettings.backend === 'fs') {
- it('should refuse to handle IfNoneMatch', async function () {
- await expect(
- app.persistor.sendStream(
- Settings.filestore.stores.user_files,
- `${projectId}/${fileId}`,
- fs.createReadStream(localFileReadPath),
- { ifNoneMatch: '*' }
- )
- ).to.be.rejectedWith(NotImplementedError)
- })
- } else {
- it('should reject sendStream on the same key with IfNoneMatch', async function () {
- await expect(
- app.persistor.sendStream(
- Settings.filestore.stores.user_files,
- `${projectId}/${fileId}`,
- fs.createReadStream(localFileReadPath),
- { ifNoneMatch: '*' }
- )
- ).to.be.rejectedWith(AlreadyWrittenError)
- })
- it('should allow sendStream on a different key with IfNoneMatch', async function () {
- await app.persistor.sendStream(
- Settings.filestore.stores.user_files,
- `${projectId}/${fileId}-other`,
- fs.createReadStream(localFileReadPath),
- { ifNoneMatch: '*' }
- )
- })
- }
- })
-
- if (backendSettings.backend !== 'fs') {
+ if (['S3Persistor', 'GcsPersistor'].includes(backend)) {
it('should record an egress metric for the upload', async function () {
const metric = await TestHelper.getMetric(
filestoreUrl,
`${metricPrefix}_egress`
)
- expect(metric - previousEgress).to.equal(
- constantFileContent.length + dataEncryptionKeySize
- )
+ expect(metric - previousEgress).to.equal(constantFileContent.length)
})
it('should record an ingress metric when downloading the file', async function () {
const response = await fetch(fileUrl)
expect(response.ok).to.be.true
- await response.text()
const metric = await TestHelper.getMetric(
filestoreUrl,
`${metricPrefix}_ingress`
)
expect(metric - previousIngress).to.equal(
- constantFileContent.length + dataEncryptionKeySize
+ constantFileContent.length
)
})
@@ -357,12 +295,11 @@ describe('Filestore', function () {
headers: { Range: 'bytes=0-8' },
})
expect(response.ok).to.be.true
- await response.text()
const metric = await TestHelper.getMetric(
filestoreUrl,
`${metricPrefix}_ingress`
)
- expect(metric - previousIngress).to.equal(9 + dataEncryptionKeySize)
+ expect(metric - previousIngress).to.equal(9)
})
}
})
@@ -392,7 +329,7 @@ describe('Filestore', function () {
].join('\n'),
]
- before('create local files', async function () {
+ before(async function () {
return await Promise.all([
fsWriteFile(localFileReadPaths[0], constantFileContents[0]),
fsWriteFile(localFileReadPaths[1], constantFileContents[1]),
@@ -400,7 +337,7 @@ describe('Filestore', function () {
])
})
- beforeEach('upload two files', async function () {
+ beforeEach(async function () {
projectUrl = `${filestoreUrl}/project/${projectId}`
otherProjectUrl = `${filestoreUrl}/project/${otherProjectId}`
fileIds = [
@@ -474,10 +411,9 @@ describe('Filestore', function () {
})
describe('with a large file', function () {
- this.timeout(1000 * 20)
let fileId, fileUrl, largeFileContent, error
- beforeEach('upload large file', async function () {
+ beforeEach(async function () {
fileId = new ObjectId().toString()
fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
@@ -485,8 +421,7 @@ describe('Filestore', function () {
largeFileContent += Math.random()
const readStream = streamifier.createReadStream(largeFileContent)
- const res = await fetch(fileUrl, { method: 'POST', body: readStream })
- if (!res.ok) throw new Error(res.statusText)
+ await fetch(fileUrl, { method: 'POST', body: readStream })
})
it('should be able to get the file back', async function () {
@@ -514,25 +449,27 @@ describe('Filestore', function () {
})
})
- if (
- (backendSettings.backend === 's3' && !backendSettings.fallback) ||
- (backendSettings.backend === 'gcs' &&
- backendSettings.fallback?.backend === 's3')
- ) {
+ if (backend === 'S3Persistor' || backend === 'FallbackGcsToS3Persistor') {
describe('with a file in a specific bucket', function () {
let constantFileContent, fileId, fileUrl, bucketName
- beforeEach('upload file into random bucket', async function () {
+ beforeEach(async function () {
constantFileContent = `This is a file in a different S3 bucket ${Math.random()}`
fileId = new ObjectId().toString()
- bucketName = `random-bucket-${new ObjectId().toString()}`
+ bucketName = new ObjectId().toString()
fileUrl = `${filestoreUrl}/bucket/${bucketName}/key/${fileId}`
- const s3 = new S3Persistor({
- ...s3Config(),
- key: process.env.MINIO_ROOT_USER,
- secret: process.env.MINIO_ROOT_PASSWORD,
- })._getClientForBucket(bucketName)
+ const s3ClientSettings = {
+ credentials: {
+ accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+ secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+ },
+ endpoint: process.env.AWS_S3_ENDPOINT,
+ sslEnabled: false,
+ s3ForcePathStyle: true,
+ }
+
+ const s3 = new S3(s3ClientSettings)
await s3
.createBucket({
Bucket: bucketName,
@@ -555,23 +492,25 @@ describe('Filestore', function () {
})
}
- if (backendSettings.backend === 'gcs') {
+ if (backend === 'GcsPersistor') {
describe('when deleting a file in GCS', function () {
- let fileId, fileUrl, content, error, dateBefore, dateAfter
+ let fileId, fileUrl, content, error, date
- beforeEach('upload and delete file', async function () {
+ beforeEach(async function () {
+ date = new Date()
+ tk.freeze(date)
fileId = new ObjectId()
fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
content = '_wombat_' + Math.random()
const readStream = streamifier.createReadStream(content)
- let res = await fetch(fileUrl, { method: 'POST', body: readStream })
- if (!res.ok) throw new Error(res.statusText)
- dateBefore = new Date()
- res = await fetch(fileUrl, { method: 'DELETE' })
- dateAfter = new Date()
- if (!res.ok) throw new Error(res.statusText)
+ await fetch(fileUrl, { method: 'POST', body: readStream })
+ await fetch(fileUrl, { method: 'DELETE' })
+ })
+
+ afterEach(function () {
+ tk.reset()
})
it('should not throw an error', function () {
@@ -579,16 +518,10 @@ describe('Filestore', function () {
})
it('should copy the file to the deleted-files bucket', async function () {
- let date = dateBefore
- const keys = []
- while (date <= dateAfter) {
- keys.push(`${projectId}/${fileId}-${date.toISOString()}`)
- date = new Date(date.getTime() + 1)
- }
- await TestHelper.expectPersistorToHaveSomeFile(
+ await TestHelper.expectPersistorToHaveFile(
app.persistor,
`${Settings.filestore.stores.user_files}-deleted`,
- keys,
+ `${projectId}/${fileId}-${date.toISOString()}`,
content
)
})
@@ -603,7 +536,7 @@ describe('Filestore', function () {
})
}
- if (backendSettings.fallback) {
+ if (BackendSettings[backend].fallback) {
describe('with a fallback', function () {
let constantFileContent,
fileId,
@@ -612,7 +545,7 @@ describe('Filestore', function () {
bucket,
fallbackBucket
- beforeEach('prepare fallback', function () {
+ beforeEach(function () {
constantFileContent = `This is yet more file content ${Math.random()}`
fileId = new ObjectId().toString()
fileKey = `${projectId}/${fileId}`
@@ -623,7 +556,7 @@ describe('Filestore', function () {
})
describe('with a file in the fallback bucket', function () {
- beforeEach('upload into fallback', async function () {
+ beforeEach(async function () {
await TestHelper.uploadStringToPersistor(
app.persistor.fallbackPersistor,
fallbackBucket,
@@ -650,7 +583,7 @@ describe('Filestore', function () {
})
describe('when copyOnMiss is disabled', function () {
- beforeEach('swap copyOnMiss=false', function () {
+ beforeEach(function () {
app.persistor.settings.copyOnMiss = false
})
@@ -663,7 +596,6 @@ describe('Filestore', function () {
it('should not copy the file to the primary', async function () {
const response = await fetch(fileUrl)
expect(response.ok).to.be.true
- await response.text()
await TestHelper.expectPersistorNotToHaveFile(
app.persistor.primaryPersistor,
@@ -674,7 +606,7 @@ describe('Filestore', function () {
})
describe('when copyOnMiss is enabled', function () {
- beforeEach('swap copyOnMiss=true', function () {
+ beforeEach(function () {
app.persistor.settings.copyOnMiss = true
})
@@ -687,7 +619,6 @@ describe('Filestore', function () {
it('copies the file to the primary', async function () {
const response = await fetch(fileUrl)
expect(response.ok).to.be.true
- await response.text()
// wait for the file to copy in the background
await msleep(1000)
@@ -703,7 +634,7 @@ describe('Filestore', function () {
describe('when copying a file', function () {
let newFileId, newFileUrl, newFileKey, opts
- beforeEach('prepare to copy file', function () {
+ beforeEach(function () {
const newProjectID = new ObjectId().toString()
newFileId = new ObjectId().toString()
newFileUrl = `${filestoreUrl}/project/${newProjectID}/file/${newFileId}`
@@ -724,7 +655,7 @@ describe('Filestore', function () {
})
describe('when copyOnMiss is false', function () {
- beforeEach('copy with copyOnMiss=false', async function () {
+ beforeEach(async function () {
app.persistor.settings.copyOnMiss = false
const response = await fetch(newFileUrl, opts)
@@ -770,7 +701,7 @@ describe('Filestore', function () {
})
describe('when copyOnMiss is true', function () {
- beforeEach('copy with copyOnMiss=false', async function () {
+ beforeEach(async function () {
app.persistor.settings.copyOnMiss = true
const response = await fetch(newFileUrl, opts)
@@ -819,14 +750,10 @@ describe('Filestore', function () {
})
describe('when sending a file', function () {
- beforeEach('upload file', async function () {
+ beforeEach(async function () {
const readStream =
streamifier.createReadStream(constantFileContent)
- const res = await fetch(fileUrl, {
- method: 'POST',
- body: readStream,
- })
- if (!res.ok) throw new Error(res.statusText)
+ await fetch(fileUrl, { method: 'POST', body: readStream })
})
it('should store the file on the primary', async function () {
@@ -849,7 +776,7 @@ describe('Filestore', function () {
describe('when deleting a file', function () {
describe('when the file exists on the primary', function () {
- beforeEach('upload into primary', async function () {
+ beforeEach(async function () {
await TestHelper.uploadStringToPersistor(
app.persistor.primaryPersistor,
bucket,
@@ -867,7 +794,7 @@ describe('Filestore', function () {
})
describe('when the file exists on the fallback', function () {
- beforeEach('upload into fallback', async function () {
+ beforeEach(async function () {
await TestHelper.uploadStringToPersistor(
app.persistor.fallbackPersistor,
fallbackBucket,
@@ -885,23 +812,20 @@ describe('Filestore', function () {
})
describe('when the file exists on both the primary and the fallback', function () {
- beforeEach(
- 'upload into both primary and fallback',
- async function () {
- await TestHelper.uploadStringToPersistor(
- app.persistor.primaryPersistor,
- bucket,
- fileKey,
- constantFileContent
- )
- await TestHelper.uploadStringToPersistor(
- app.persistor.fallbackPersistor,
- fallbackBucket,
- fileKey,
- constantFileContent
- )
- }
- )
+ beforeEach(async function () {
+ await TestHelper.uploadStringToPersistor(
+ app.persistor.primaryPersistor,
+ bucket,
+ fileKey,
+ constantFileContent
+ )
+ await TestHelper.uploadStringToPersistor(
+ app.persistor.fallbackPersistor,
+ fallbackBucket,
+ fileKey,
+ constantFileContent
+ )
+ })
it('should delete the files', async function () {
const response1 = await fetch(fileUrl, { method: 'DELETE' })
@@ -930,14 +854,13 @@ describe('Filestore', function () {
'../../fixtures/test.pdf'
)
- beforeEach('upload test.pdf', async function () {
+ beforeEach(async function () {
fileId = new ObjectId().toString()
fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
const stat = await fsStat(localFileReadPath)
localFileSize = stat.size
const readStream = fs.createReadStream(localFileReadPath)
- const res = await fetch(fileUrl, { method: 'POST', body: readStream })
- if (!res.ok) throw new Error(res.statusText)
+ await fetch(fileUrl, { method: 'POST', body: readStream })
})
it('should be able get the file back', async function () {
@@ -946,15 +869,13 @@ describe('Filestore', function () {
expect(body.substring(0, 8)).to.equal('%PDF-1.5')
})
- if (backendSettings.backend !== 'fs') {
+ if (['S3Persistor', 'GcsPersistor'].includes(backend)) {
it('should record an egress metric for the upload', async function () {
const metric = await TestHelper.getMetric(
filestoreUrl,
`${metricPrefix}_egress`
)
- expect(metric - previousEgress).to.equal(
- localFileSize + dataEncryptionKeySize
- )
+ expect(metric - previousEgress).to.equal(localFileSize)
})
}
@@ -962,20 +883,18 @@ describe('Filestore', function () {
this.timeout(1000 * 20)
let previewFileUrl
- beforeEach('prepare previewFileUrl for preview', function () {
+ beforeEach(function () {
previewFileUrl = `${fileUrl}?style=preview`
})
it('should not time out', async function () {
const response = await fetch(previewFileUrl)
expect(response.status).to.equal(200)
- await response.arrayBuffer()
})
it('should respond with image data', async function () {
// note: this test relies of the imagemagick conversion working
const response = await fetch(previewFileUrl)
- expect(response.status).to.equal(200)
const body = await response.text()
expect(body.length).to.be.greaterThan(400)
expect(body.substr(1, 3)).to.equal('PNG')
@@ -986,23 +905,20 @@ describe('Filestore', function () {
this.timeout(1000 * 20)
let previewFileUrl
- beforeEach('prepare previewFileUrl for cacheWarn', function () {
+ beforeEach(function () {
previewFileUrl = `${fileUrl}?style=preview&cacheWarm=true`
})
it('should not time out', async function () {
const response = await fetch(previewFileUrl)
expect(response.status).to.equal(200)
- await response.arrayBuffer()
})
it('should not leak sockets', async function () {
const response1 = await fetch(previewFileUrl)
expect(response1.status).to.equal(200)
- // do not read the response body, should be destroyed immediately
const response2 = await fetch(previewFileUrl)
expect(response2.status).to.equal(200)
- // do not read the response body, should be destroyed immediately
await expectNoSockets()
})
@@ -1014,551 +930,6 @@ describe('Filestore', function () {
})
})
})
-
- describe('with server side encryption', function () {
- if (backendSettings.backend !== 's3SSEC') return
-
- before('sanity check top-level variable', function () {
- expect(dataEncryptionKeySize).to.equal(32)
- })
-
- let fileId1,
- fileId2,
- fileKey1,
- fileKey2,
- fileKeyOtherProject,
- fileUrl1,
- fileUrl2
- beforeEach('prepare ids', function () {
- fileId1 = new ObjectId().toString()
- fileId2 = new ObjectId().toString()
- fileKey1 = `${projectId}/${fileId1}`
- fileKey2 = `${projectId}/${fileId2}`
- fileKeyOtherProject = `${new ObjectId().toString()}/${new ObjectId().toString()}`
- fileUrl1 = `${filestoreUrl}/project/${projectId}/file/${fileId1}`
- fileUrl2 = `${filestoreUrl}/project/${projectId}/file/${fileId2}`
- })
-
- beforeEach('ensure DEK is missing', async function () {
- // Cannot use test helper expectPersistorNotToHaveFile here, we need to use the KEK.
- await expect(
- app.persistor.getDataEncryptionKeySize(
- backendSettings.stores.user_files,
- fileKey1
- )
- ).to.rejectedWith(NotFoundError)
- })
-
- async function createRandomContent(url, suffix = '') {
- const content = Math.random().toString() + suffix
- const res = await fetch(url, {
- method: 'POST',
- body: Stream.Readable.from([content]),
- })
- if (!res.ok) throw new Error(res.statusText)
- return async () => {
- const res = await fetch(url, { method: 'GET' })
- if (!res.ok) throw new Error(res.statusText)
- expect(await res.text()).to.equal(content)
- }
- }
-
- it('should create a DEK when asked explicitly', async function () {
- await app.persistor.generateDataEncryptionKey(
- backendSettings.stores.user_files,
- fileKey1
- )
- expect(
- await app.persistor.getDataEncryptionKeySize(
- backendSettings.stores.user_files,
- fileKey1
- )
- ).to.equal(32)
- })
-
- it('should create a DEK from writes', async function () {
- await createRandomContent(fileUrl1)
- expect(
- await app.persistor.getDataEncryptionKeySize(
- backendSettings.stores.user_files,
- fileKey1
- )
- ).to.equal(32)
- })
-
- it('should not create a DEK from reads', async function () {
- const res = await fetch(fileUrl1, {
- method: 'GET',
- })
- if (res.status !== 404) throw new Error(`${res.status} should be 404`)
-
- // Cannot use test helper expectPersistorNotToHaveFile here, we need to use the KEK.
- await expect(
- app.persistor.getDataEncryptionKeySize(
- backendSettings.stores.user_files,
- fileKey1
- )
- ).to.rejectedWith(NotFoundError)
- })
-
- it('should never overwrite a data encryption key', async function () {
- const checkGET = await createRandomContent(fileUrl1)
-
- await expect(
- app.persistor.generateDataEncryptionKey(
- backendSettings.stores.user_files,
- fileKey1
- )
- ).to.rejectedWith(AlreadyWrittenError)
-
- await checkGET()
- })
-
- it('should re-use the data encryption key after a write', async function () {
- const checkGET1 = await createRandomContent(fileUrl1, '1')
- const checkGET2 = await createRandomContent(fileUrl2, '2')
- await checkGET1()
- await checkGET2()
- })
-
- describe('kek rotation', function () {
- const newKEK = new RootKeyEncryptionKey(
- crypto.generateKeySync('aes', { length: 256 }).export(),
- Buffer.alloc(32)
- )
- const oldKEK = new RootKeyEncryptionKey(
- crypto.generateKeySync('aes', { length: 256 }).export(),
- Buffer.alloc(32)
- )
- const migrationStep0 = new PerProjectEncryptedS3Persistor({
- ...s3SSECConfig(),
- automaticallyRotateDEKEncryption: false,
- async getRootKeyEncryptionKeys() {
- return [oldKEK] // only old key
- },
- })
- const migrationStep1 = new PerProjectEncryptedS3Persistor({
- ...s3SSECConfig(),
- automaticallyRotateDEKEncryption: false,
- async getRootKeyEncryptionKeys() {
- return [oldKEK, newKEK] // new key as fallback
- },
- })
- const migrationStep2 = new PerProjectEncryptedS3Persistor({
- ...s3SSECConfig(),
- automaticallyRotateDEKEncryption: true, // <- different compared to partiallyRotated
- async getRootKeyEncryptionKeys() {
- return [newKEK, oldKEK] // old keys as fallback
- },
- })
- const migrationStep3 = new PerProjectEncryptedS3Persistor({
- ...s3SSECConfig(),
- automaticallyRotateDEKEncryption: true,
- async getRootKeyEncryptionKeys() {
- return [newKEK] // only new key
- },
- })
-
- async function checkWrites(
- fileKey,
- writer,
- readersSuccess,
- readersFailed
- ) {
- const content = Math.random().toString()
- await writer.sendStream(
- Settings.filestore.stores.user_files,
- fileKey,
- Stream.Readable.from([content])
- )
-
- for (const persistor of readersSuccess) {
- await TestHelper.expectPersistorToHaveFile(
- persistor,
- backendSettings.stores.user_files,
- fileKey,
- content
- )
- }
-
- for (const persistor of readersFailed) {
- await expect(
- TestHelper.expectPersistorToHaveFile(
- persistor,
- backendSettings.stores.user_files,
- fileKey,
- content
- )
- ).to.be.rejectedWith(NoKEKMatchedError)
- }
- }
-
- const stages = [
- {
- name: 'stage 0 - [old]',
- prev: migrationStep0,
- cur: migrationStep0,
- fail: [migrationStep3],
- },
- {
- name: 'stage 1 - [old,new]',
- prev: migrationStep0,
- cur: migrationStep1,
- fail: [],
- },
- {
- name: 'stage 2 - [new,old]',
- prev: migrationStep1,
- cur: migrationStep2,
- fail: [],
- },
- {
- name: 'stage 3 - [new]',
- prev: migrationStep2,
- cur: migrationStep3,
- fail: [migrationStep0],
- },
- ]
-
- for (const { name, prev, cur, fail } of stages) {
- describe(name, function () {
- this.timeout(1000 * 30)
-
- it('can read old writes', async function () {
- await checkWrites(fileKey1, prev, [prev, cur], fail)
- await checkWrites(fileKey2, prev, [prev, cur], fail) // check again after access
- await checkWrites(fileKeyOtherProject, prev, [prev, cur], fail)
- })
- it('can read new writes', async function () {
- await checkWrites(fileKey1, prev, [prev, cur], fail)
- await checkWrites(fileKey2, cur, [prev, cur], fail) // check again after access
- await checkWrites(fileKeyOtherProject, cur, [prev, cur], fail)
- })
- })
- }
-
- describe('full migration', function () {
- it('can read old writes if rotated in sequence', async function () {
- await checkWrites(
- fileKey1,
- migrationStep0,
- [
- migrationStep0,
- migrationStep1,
- migrationStep2, // migrates
- migrationStep3,
- ],
- []
- )
- })
- it('cannot read/write if not rotated', async function () {
- await checkWrites(
- fileKey1,
- migrationStep0,
- [migrationStep0],
- [migrationStep3]
- )
- })
- })
- })
-
- /** @type {import('aws-sdk/clients/s3')} */
- let s3Client
- before('create s3 client', function () {
- s3Client = new S3Persistor(s3Config())._getClientForBucket('')
- })
-
- async function checkDEKStorage({
- dekBucketKeys = [],
- userFilesBucketKeys = [],
- }) {
- await createRandomContent(fileUrl1)
-
- const { Contents: dekEntries } = await s3Client
- .listObjectsV2({
- Bucket: process.env.AWS_S3_USER_FILES_DEK_BUCKET_NAME,
- Prefix: `${projectId}/`,
- })
- .promise()
- expect(dekEntries).to.have.length(dekBucketKeys.length)
- // Order is not predictable, use members
- expect(dekEntries.map(o => o.Key)).to.have.members(dekBucketKeys)
-
- const { Contents: userFilesEntries } = await s3Client
- .listObjectsV2({
- Bucket: backendSettings.stores.user_files,
- Prefix: `${projectId}/`,
- })
- .promise()
- expect(userFilesEntries).to.have.length(userFilesBucketKeys.length)
- // Order is not predictable, use members
- expect(userFilesEntries.map(o => o.Key)).to.have.members(
- userFilesBucketKeys
- )
- }
-
- it('should use a custom bucket for DEKs', async function () {
- await checkDEKStorage({
- dekBucketKeys: [`${projectId}/dek`],
- userFilesBucketKeys: [fileKey1],
- })
- })
-
- describe('deleteDirectory', function () {
- let checkGET1, checkGET2
- beforeEach('create files', async function () {
- checkGET1 = await createRandomContent(fileUrl1, '1')
- checkGET2 = await createRandomContent(fileUrl2, '2')
- })
- it('should refuse to delete top-level prefix', async function () {
- await expect(
- app.persistor.deleteDirectory(
- Settings.filestore.stores.user_files,
- projectId.slice(0, 3)
- )
- ).to.be.rejectedWith('not a project-folder')
- expect(
- await app.persistor.checkIfObjectExists(
- Settings.filestore.stores.user_files,
- fileKey1
- )
- ).to.equal(true)
- await checkGET1()
- expect(
- await app.persistor.checkIfObjectExists(
- Settings.filestore.stores.user_files,
- fileKey2
- )
- ).to.equal(true)
- expect(
- await app.persistor.getDataEncryptionKeySize(
- Settings.filestore.stores.user_files,
- fileKey2
- )
- ).to.equal(32)
- await checkGET2()
- })
- it('should delete sub-folder and keep DEK', async function () {
- await app.persistor.deleteDirectory(
- Settings.filestore.stores.user_files,
- fileKey1 // not really a sub-folder, but it will do for this test.
- )
- expect(
- await app.persistor.checkIfObjectExists(
- Settings.filestore.stores.user_files,
- fileKey1
- )
- ).to.equal(false)
- expect(
- await app.persistor.checkIfObjectExists(
- Settings.filestore.stores.user_files,
- fileKey2
- )
- ).to.equal(true)
- expect(
- await app.persistor.getDataEncryptionKeySize(
- Settings.filestore.stores.user_files,
- fileKey2
- )
- ).to.equal(32)
- await checkGET2()
- })
- it('should delete project folder and DEK', async function () {
- await app.persistor.deleteDirectory(
- Settings.filestore.stores.user_files,
- `${projectId}/`
- )
- expect(
- await app.persistor.checkIfObjectExists(
- Settings.filestore.stores.user_files,
- fileKey1
- )
- ).to.equal(false)
- expect(
- await app.persistor.checkIfObjectExists(
- Settings.filestore.stores.user_files,
- fileKey2
- )
- ).to.equal(false)
- await expect(
- app.persistor.getDataEncryptionKeySize(
- Settings.filestore.stores.user_files,
- fileKey2
- )
- ).to.rejectedWith(NotFoundError)
- })
- })
- })
-
- describe('getObjectSize', function () {
- it('should return a number', async function () {
- const buf = Buffer.from('hello')
- const fileId = new ObjectId().toString()
- const fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
- const res = await fetch(fileUrl, {
- method: 'POST',
- body: Stream.Readable.from([buf]),
- })
- if (!res.ok) throw new Error(res.statusText)
- expect(
- await app.persistor.getObjectSize(
- Settings.filestore.stores.user_files,
- `${projectId}/${fileId}`
- )
- ).to.equal(buf.byteLength)
- })
- })
-
- describe('checkIfObjectExists', function () {
- it('should return false when the object does not exist', async function () {
- expect(
- await app.persistor.checkIfObjectExists(
- Settings.filestore.stores.user_files,
- `${projectId}/${new ObjectId().toString()}`
- )
- ).to.equal(false)
- })
- it('should return true when the object exists', async function () {
- const fileId = new ObjectId().toString()
- const fileUrl = `${filestoreUrl}/project/${projectId}/file/${fileId}`
- const res = await fetch(fileUrl, {
- method: 'POST',
- body: Stream.Readable.from(['hello']),
- })
- if (!res.ok) throw new Error(res.statusText)
- expect(
- await app.persistor.checkIfObjectExists(
- Settings.filestore.stores.user_files,
- `${projectId}/${fileId}`
- )
- ).to.equal(true)
- })
- })
-
- if (backendSettings.backend === 's3SSEC') {
- describe('storageClass', function () {
- it('should use the default storage class for dek', async function () {
- const key = `${projectId}/${new ObjectId()}`
- const dekBucket = process.env.AWS_S3_USER_FILES_DEK_BUCKET_NAME
- await app.persistor.sendStream(
- dekBucket,
- key,
- Stream.Readable.from(['hello'])
- )
- expect(
- await app.persistor.getObjectStorageClass(dekBucket, key)
- ).to.equal(undefined)
- })
-
- it('should use the custom storage class for user files', async function () {
- const key = `${projectId}/${new ObjectId()}`
- await app.persistor.sendStream(
- Settings.filestore.stores.user_files,
- key,
- Stream.Readable.from(['hello'])
- )
- const sc = AWS_S3_USER_FILES_STORAGE_CLASS
- expect(sc).to.exist
- expect(
- await app.persistor.getObjectStorageClass(
- Settings.filestore.stores.user_files,
- key
- )
- ).to.equal(sc)
- })
- })
- }
-
- describe('autoGunzip', function () {
- let key
- beforeEach('new key', function () {
- key = `${projectId}/${new ObjectId().toString()}`
- })
- this.timeout(60 * 1000)
- const body = Buffer.alloc(10 * 1024 * 1024, 'hello')
- const gzippedBody = gzipSync(body)
-
- /**
- * @param {string} key
- * @param {Buffer} wantBody
- * @param {boolean} autoGunzip
- * @return {Promise}
- */
- async function checkBodyIsTheSame(key, wantBody, autoGunzip) {
- const s = await app.persistor.getObjectStream(
- Settings.filestore.stores.user_files,
- key,
- { autoGunzip }
- )
- const buf = new WritableBuffer()
- await Stream.promises.pipeline(s, buf)
- expect(buf.getContents()).to.deep.equal(wantBody)
- }
-
- if (backendSettings.backend === 'fs') {
- it('should refuse to handle autoGunzip', async function () {
- await expect(
- app.persistor.getObjectStream(
- Settings.filestore.stores.user_files,
- key,
- { autoGunzip: true }
- )
- ).to.be.rejectedWith(NotImplementedError)
- })
- } else {
- it('should return the raw body with gzip', async function () {
- await app.persistor.sendStream(
- Settings.filestore.stores.user_files,
- key,
- Stream.Readable.from([gzippedBody]),
- { contentEncoding: 'gzip' }
- )
- expect(
- await app.persistor.getObjectSize(
- Settings.filestore.stores.user_files,
- key
- )
- ).to.equal(gzippedBody.byteLength)
- // raw body with autoGunzip=true
- await checkBodyIsTheSame(key, body, true)
- // gzip body without autoGunzip=false
- await checkBodyIsTheSame(key, gzippedBody, false)
- })
- it('should return the raw body without gzip compression', async function () {
- await app.persistor.sendStream(
- Settings.filestore.stores.user_files,
- key,
- Stream.Readable.from([body])
- )
- expect(
- await app.persistor.getObjectSize(
- Settings.filestore.stores.user_files,
- key
- )
- ).to.equal(body.byteLength)
- // raw body with both autoGunzip options
- await checkBodyIsTheSame(key, body, true)
- await checkBodyIsTheSame(key, body, false)
- })
-
- it('should return the gzip body without gzip header', async function () {
- await app.persistor.sendStream(
- Settings.filestore.stores.user_files,
- key,
- Stream.Readable.from([gzippedBody])
- )
- expect(
- await app.persistor.getObjectSize(
- Settings.filestore.stores.user_files,
- key
- )
- ).to.equal(gzippedBody.byteLength)
- // gzip body with both autoGunzip options
- await checkBodyIsTheSame(key, gzippedBody, true)
- await checkBodyIsTheSame(key, gzippedBody, false)
- })
- }
- })
})
- }
+ })
})
diff --git a/services/filestore/test/acceptance/js/TestConfig.js b/services/filestore/test/acceptance/js/TestConfig.js
index 3ad4ba423d..4b72bc971d 100644
--- a/services/filestore/test/acceptance/js/TestConfig.js
+++ b/services/filestore/test/acceptance/js/TestConfig.js
@@ -1,63 +1,22 @@
-const fs = require('node:fs')
-const Path = require('node:path')
-const crypto = require('node:crypto')
-const {
- RootKeyEncryptionKey,
-} = require('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor')
-
-const AWS_S3_USER_FILES_STORAGE_CLASS =
- process.env.AWS_S3_USER_FILES_STORAGE_CLASS
+const fs = require('fs')
+const Path = require('path')
// use functions to get a fresh copy, not a reference, each time
-function s3BaseConfig() {
- return {
- endpoint: process.env.AWS_S3_ENDPOINT,
- pathStyle: true,
- partSize: 100 * 1024 * 1024,
- ca: [fs.readFileSync('/certs/public.crt')],
- }
-}
-
function s3Config() {
return {
key: process.env.AWS_ACCESS_KEY_ID,
secret: process.env.AWS_SECRET_ACCESS_KEY,
- ...s3BaseConfig(),
- }
-}
-
-const S3SSECKeys = [
- new RootKeyEncryptionKey(
- crypto.generateKeySync('aes', { length: 256 }).export(),
- Buffer.alloc(32)
- ),
-]
-
-function s3SSECConfig() {
- return {
- ...s3Config(),
- ignoreErrorsFromDEKReEncryption: false,
- automaticallyRotateDEKEncryption: true,
- dataEncryptionKeyBucketName: process.env.AWS_S3_USER_FILES_DEK_BUCKET_NAME,
- pathToProjectFolder(_bucketName, path) {
- const match = path.match(/^[a-f0-9]{24}\//)
- if (!match) throw new Error('not a project-folder')
- const [projectFolder] = match
- return projectFolder
- },
- async getRootKeyEncryptionKeys() {
- return S3SSECKeys
- },
- storageClass: {
- [process.env.AWS_S3_USER_FILES_BUCKET_NAME]:
- AWS_S3_USER_FILES_STORAGE_CLASS,
- },
+ endpoint: process.env.AWS_S3_ENDPOINT,
+ pathStyle: true,
+ partSize: 100 * 1024 * 1024,
}
}
function s3ConfigDefaultProviderCredentials() {
return {
- ...s3BaseConfig(),
+ endpoint: process.env.AWS_S3_ENDPOINT,
+ pathStyle: true,
+ partSize: 100 * 1024 * 1024,
}
}
@@ -101,7 +60,7 @@ function fallbackStores(primaryConfig, fallbackConfig) {
}
}
-const BackendSettings = {
+module.exports = {
SHARD_01_FSPersistor: {
backend: 'fs',
stores: fsStores(),
@@ -121,11 +80,6 @@ const BackendSettings = {
gcs: gcsConfig(),
stores: gcsStores(),
},
- SHARD_01_PerProjectEncryptedS3Persistor: {
- backend: 's3SSEC',
- s3SSEC: s3SSECConfig(),
- stores: s3Stores(),
- },
SHARD_02_FallbackS3ToFSPersistor: {
backend: 's3',
s3: s3Config(),
@@ -183,10 +137,3 @@ function checkForUnexpectedTestFile() {
}
}
checkForUnexpectedTestFile()
-
-module.exports = {
- AWS_S3_USER_FILES_STORAGE_CLASS,
- BackendSettings,
- s3Config,
- s3SSECConfig,
-}
diff --git a/services/filestore/test/acceptance/js/TestHelper.js b/services/filestore/test/acceptance/js/TestHelper.js
index 384f8aab6f..1b7eae4d0a 100644
--- a/services/filestore/test/acceptance/js/TestHelper.js
+++ b/services/filestore/test/acceptance/js/TestHelper.js
@@ -1,6 +1,5 @@
const streamifier = require('streamifier')
const fetch = require('node-fetch')
-const ObjectPersistor = require('@overleaf/object-persistor')
const { expect } = require('chai')
@@ -8,7 +7,6 @@ module.exports = {
uploadStringToPersistor,
getStringFromPersistor,
expectPersistorToHaveFile,
- expectPersistorToHaveSomeFile,
expectPersistorNotToHaveFile,
streamToString,
getMetric,
@@ -17,14 +15,10 @@ module.exports = {
async function getMetric(filestoreUrl, metric) {
const res = await fetch(`${filestoreUrl}/metrics`)
expect(res.status).to.equal(200)
- const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'gm')
+ const metricRegex = new RegExp(`^${metric}{[^}]+} ([0-9]+)$`, 'm')
const body = await res.text()
- let v = 0
- // Sum up size="lt-128KiB" and size="gte-128KiB"
- for (const [, found] of body.matchAll(metricRegex)) {
- v += parseInt(found, 10) || 0
- }
- return v
+ const found = metricRegex.exec(body)
+ return parseInt(found ? found[1] : 0) || 0
}
function streamToString(stream) {
@@ -52,25 +46,6 @@ async function expectPersistorToHaveFile(persistor, bucket, key, content) {
expect(foundContent).to.equal(content)
}
-async function expectPersistorToHaveSomeFile(persistor, bucket, keys, content) {
- let foundContent
- for (const key of keys) {
- try {
- foundContent = await getStringFromPersistor(persistor, bucket, key)
- break
- } catch (err) {
- if (err instanceof ObjectPersistor.Errors.NotFoundError) {
- continue
- }
- throw err
- }
- }
- if (foundContent === undefined) {
- expect.fail(`Could not find any of the specified keys: ${keys}`)
- }
- expect(foundContent).to.equal(content)
-}
-
async function expectPersistorNotToHaveFile(persistor, bucket, key) {
await expect(
getStringFromPersistor(persistor, bucket, key)
diff --git a/services/filestore/test/setup.js b/services/filestore/test/setup.js
index 744ab9130f..57a462390f 100644
--- a/services/filestore/test/setup.js
+++ b/services/filestore/test/setup.js
@@ -21,11 +21,6 @@ SandboxedModule.configure({
requires: {
'@overleaf/logger': stubs.logger,
},
- sourceTransformers: {
- removeNodePrefix: function (source) {
- return source.replace(/require\(['"]node:/g, "require('")
- },
- },
})
exports.mochaHooks = {
diff --git a/services/filestore/test/unit/js/FileControllerTests.js b/services/filestore/test/unit/js/FileControllerTests.js
index ec562116a0..508db8b153 100644
--- a/services/filestore/test/unit/js/FileControllerTests.js
+++ b/services/filestore/test/unit/js/FileControllerTests.js
@@ -6,7 +6,14 @@ const Errors = require('../../../app/js/Errors')
const modulePath = '../../../app/js/FileController.js'
describe('FileController', function () {
- let FileHandler, LocalFileWriter, FileController, req, res, next, stream
+ let PersistorManager,
+ FileHandler,
+ LocalFileWriter,
+ FileController,
+ req,
+ res,
+ next,
+ stream
const settings = {
s3: {
buckets: {
@@ -25,8 +32,13 @@ describe('FileController', function () {
const error = new Error('incorrect utensil')
beforeEach(function () {
+ PersistorManager = {
+ sendStream: sinon.stub().yields(),
+ copyObject: sinon.stub().resolves(),
+ deleteObject: sinon.stub().yields(),
+ }
+
FileHandler = {
- copyObject: sinon.stub().yields(),
getFile: sinon.stub().yields(null, fileStream),
getFileSize: sinon.stub().yields(null, fileSize),
deleteFile: sinon.stub().yields(),
@@ -45,6 +57,7 @@ describe('FileController', function () {
requires: {
'./LocalFileWriter': LocalFileWriter,
'./FileHandler': FileHandler,
+ './PersistorManager': PersistorManager,
'./Errors': Errors,
stream,
'@overleaf/settings': settings,
@@ -226,7 +239,7 @@ describe('FileController', function () {
})
describe('insertFile', function () {
- it('should send bucket name key and res to FileHandler', function (done) {
+ it('should send bucket name key and res to PersistorManager', function (done) {
res.sendStatus = code => {
expect(FileHandler.insertFile).to.have.been.calledWith(bucket, key, req)
expect(code).to.equal(200)
@@ -250,10 +263,10 @@ describe('FileController', function () {
}
})
- it('should send bucket name and both keys to FileHandler', function (done) {
+ it('should send bucket name and both keys to PersistorManager', function (done) {
res.sendStatus = code => {
code.should.equal(200)
- expect(FileHandler.copyObject).to.have.been.calledWith(
+ expect(PersistorManager.copyObject).to.have.been.calledWith(
bucket,
oldKey,
key
@@ -264,7 +277,7 @@ describe('FileController', function () {
})
it('should send a 404 if the original file was not found', function (done) {
- FileHandler.copyObject.yields(
+ PersistorManager.copyObject.rejects(
new Errors.NotFoundError({ message: 'not found', info: {} })
)
res.sendStatus = code => {
@@ -275,7 +288,7 @@ describe('FileController', function () {
})
it('should send an error if there was an error', function (done) {
- FileHandler.copyObject.yields(error)
+ PersistorManager.copyObject.rejects(error)
FileController.copyFile(req, res, err => {
expect(err).to.equal(error)
done()
diff --git a/services/filestore/test/unit/js/FileHandlerTests.js b/services/filestore/test/unit/js/FileHandlerTests.js
index 12a23667b4..d4899dd20a 100644
--- a/services/filestore/test/unit/js/FileHandlerTests.js
+++ b/services/filestore/test/unit/js/FileHandlerTests.js
@@ -67,11 +67,7 @@ describe('FileHandler', function () {
compressPng: sinon.stub().resolves(),
},
}
- Settings = {
- filestore: {
- stores: { template_files: 'template_files', user_files: 'user_files' },
- },
- }
+ Settings = {}
fs = {
createReadStream: sinon.stub().returns(readStream),
}
@@ -93,7 +89,7 @@ describe('FileHandler', function () {
},
fs,
},
- globals: { console, process },
+ globals: { console },
})
})
@@ -137,6 +133,23 @@ describe('FileHandler', function () {
done()
})
})
+
+ describe('when conversions are enabled', function () {
+ beforeEach(function () {
+ Settings.enableConversions = true
+ })
+
+ it('should delete the convertedKey folder', function (done) {
+ FileHandler.insertFile(bucket, key, stream, err => {
+ expect(err).not.to.exist
+ expect(PersistorManager.deleteDirectory).to.have.been.calledWith(
+ bucket,
+ convertedFolderKey
+ )
+ done()
+ })
+ })
+ })
})
describe('deleteFile', function () {
@@ -182,31 +195,15 @@ describe('FileHandler', function () {
Settings.enableConversions = true
})
- it('should delete the convertedKey folder for template files', function (done) {
- FileHandler.deleteFile(
- Settings.filestore.stores.template_files,
- key,
- err => {
- expect(err).not.to.exist
- expect(PersistorManager.deleteDirectory).to.have.been.calledWith(
- Settings.filestore.stores.template_files,
- convertedFolderKey
- )
- done()
- }
- )
- })
-
- it('should not delete the convertedKey folder for user files', function (done) {
- FileHandler.deleteFile(
- Settings.filestore.stores.user_files,
- key,
- err => {
- expect(err).not.to.exist
- expect(PersistorManager.deleteDirectory).to.not.have.been.called
- done()
- }
- )
+ it('should delete the convertedKey folder', function (done) {
+ FileHandler.deleteFile(bucket, key, err => {
+ expect(err).not.to.exist
+ expect(PersistorManager.deleteDirectory).to.have.been.calledWith(
+ bucket,
+ convertedFolderKey
+ )
+ done()
+ })
})
})
})
diff --git a/services/git-bridge/.gitignore b/services/git-bridge/.gitignore
index f35e2ee038..74a7f43d6e 100644
--- a/services/git-bridge/.gitignore
+++ b/services/git-bridge/.gitignore
@@ -1,6 +1,53 @@
-# Build output
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# Let's not share anything because we're using Maven.
+
+.idea
+*.iml
+
+# User-specific stuff:
+.idea/workspace.xml
+.idea/tasks.xml
+.idea/dictionaries
+.idea/vcs.xml
+.idea/jsLibraryMappings.xml
+
+# Sensitive or high-churn files:
+.idea/dataSources.ids
+.idea/dataSources.xml
+.idea/dataSources.local.xml
+.idea/sqlDataSources.xml
+.idea/dynamic.xml
+.idea/uiDesigner.xml
+
+# Gradle:
+.idea/gradle.xml
+.idea/libraries
+
+# Mongo Explorer plugin:
+.idea/mongoSettings.xml
+
+## File-based project format:
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
/out/
target/
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
# Local configuration files
conf/runtime.json
diff --git a/services/git-bridge/Dockerfile b/services/git-bridge/Dockerfile
index 48579b9494..0d8b1e43e5 100644
--- a/services/git-bridge/Dockerfile
+++ b/services/git-bridge/Dockerfile
@@ -1,17 +1,11 @@
-# Build the a8m/envsubst binary, as it supports default values,
-# which the gnu envsubst (from gettext-base) does not.
-FROM golang:1.24.3-alpine AS envsubst_builder
-
-WORKDIR /build
-
-RUN go install github.com/a8m/envsubst/cmd/envsubst@latest
+# Dockerfile for git-bridge
FROM maven:3-amazoncorretto-21-debian AS base
RUN apt-get update && apt-get install -y make git sqlite3 \
&& rm -rf /var/lib/apt/lists
-COPY --from=envsubst_builder /go/bin/envsubst /opt/envsubst
+COPY vendor/envsubst /opt/envsubst
RUN chmod +x /opt/envsubst
RUN useradd --create-home node
@@ -35,11 +29,16 @@ RUN apk add --update --no-cache bash git sqlite procps htop net-tools jemalloc u
ENV LD_PRELOAD=/usr/lib/libjemalloc.so.2
+# Install Google Cloud Profiler agent
+RUN mkdir -p /opt/cprof && \
+ wget -q -O- https://storage.googleapis.com/cloud-profiler/java/latest/profiler_java_agent.tar.gz \
+ | tar xzv -C /opt/cprof
+
RUN adduser -D node
COPY --from=builder /git-bridge.jar /
-COPY --from=envsubst_builder /go/bin/envsubst /opt/envsubst
+COPY vendor/envsubst /opt/envsubst
RUN chmod +x /opt/envsubst
COPY conf/envsubst_template.json envsubst_template.json
diff --git a/services/git-bridge/README.md b/services/git-bridge/README.md
index eadc2abc4f..13b24cc6d0 100644
--- a/services/git-bridge/README.md
+++ b/services/git-bridge/README.md
@@ -76,10 +76,12 @@ The configuration file is in `.json` format.
"postbackBaseUrl" (string): the postback url,
"serviceName" (string): current name of writeLaTeX
in case it ever changes,
- "oauth2Server" (string): oauth2 server,
- with protocol and
- without trailing slash,
- null or missing if oauth2 shouldn't be used
+ "oauth2" (object): { null or missing if oauth2 shouldn't be used
+ "oauth2ClientID" (string): oauth2 client ID,
+ "oauth2ClientSecret" (string): oauth2 client secret,
+ "oauth2Server" (string): oauth2 server,
+ with protocol and
+ without trailing slash
},
"repoStore" (object, optional): { configure the repo store
"maxFileSize" (long, optional): maximum size of a file, inclusive
diff --git a/services/git-bridge/conf/envsubst_template.json b/services/git-bridge/conf/envsubst_template.json
index 4ede5bab7f..1f52ffbaef 100644
--- a/services/git-bridge/conf/envsubst_template.json
+++ b/services/git-bridge/conf/envsubst_template.json
@@ -3,11 +3,14 @@
"bindIp": "${GIT_BRIDGE_BIND_IP:-0.0.0.0}",
"idleTimeout": ${GIT_BRIDGE_IDLE_TIMEOUT:-30000},
"rootGitDirectory": "${GIT_BRIDGE_ROOT_DIR:-/tmp/wlgb}",
- "allowedCorsOrigins": "${GIT_BRIDGE_ALLOWED_CORS_ORIGINS:-https://localhost}",
"apiBaseUrl": "${GIT_BRIDGE_API_BASE_URL:-https://localhost/api/v0}",
"postbackBaseUrl": "${GIT_BRIDGE_POSTBACK_BASE_URL:-https://localhost}",
"serviceName": "${GIT_BRIDGE_SERVICE_NAME:-Overleaf}",
- "oauth2Server": "${GIT_BRIDGE_OAUTH2_SERVER:-https://localhost}",
+ "oauth2": {
+ "oauth2ClientID": "${GIT_BRIDGE_OAUTH2_CLIENT_ID}",
+ "oauth2ClientSecret": "${GIT_BRIDGE_OAUTH2_CLIENT_SECRET}",
+ "oauth2Server": "${GIT_BRIDGE_OAUTH2_SERVER:-https://localhost}"
+ },
"userPasswordEnabled": ${GIT_BRIDGE_USER_PASSWORD_ENABLED:-false},
"repoStore": {
"maxFileNum": ${GIT_BRIDGE_REPOSTORE_MAX_FILE_NUM:-2000},
diff --git a/services/git-bridge/conf/example_config.json b/services/git-bridge/conf/example_config.json
index 76b82eb6a0..bfad73f461 100644
--- a/services/git-bridge/conf/example_config.json
+++ b/services/git-bridge/conf/example_config.json
@@ -3,11 +3,14 @@
"bindIp": "127.0.0.1",
"idleTimeout": 30000,
"rootGitDirectory": "/tmp/wlgb",
- "allowedCorsOrigins": "https://localhost",
"apiBaseUrl": "https://localhost/api/v0",
"postbackBaseUrl": "https://localhost",
"serviceName": "Overleaf",
- "oauth2Server": "https://localhost",
+ "oauth2": {
+ "oauth2ClientID": "asdf",
+ "oauth2ClientSecret": "asdf",
+ "oauth2Server": "https://localhost"
+ },
"repoStore": {
"maxFileNum": 2000,
"maxFileSize": 52428800
diff --git a/services/git-bridge/conf/local.json b/services/git-bridge/conf/local.json
index c4de48d819..03ce4febe4 100644
--- a/services/git-bridge/conf/local.json
+++ b/services/git-bridge/conf/local.json
@@ -3,11 +3,14 @@
"bindIp": "0.0.0.0",
"idleTimeout": 30000,
"rootGitDirectory": "/tmp/wlgb",
- "allowedCorsOrigins": "http://v2.overleaf.test",
"apiBaseUrl": "http://v2.overleaf.test:3000/api/v0",
"postbackBaseUrl": "http://git-bridge:8000",
"serviceName": "Overleaf",
- "oauth2Server": "http://v2.overleaf.test:3000",
+ "oauth2": {
+ "oauth2ClientID": "264c723c925c13590880751f861f13084934030c13b4452901e73bdfab226edc",
+ "oauth2ClientSecret": "e6b2e9eee7ae2bb653823250bb69594a91db0547fe3790a7135acb497108e62d",
+ "oauth2Server": "http://v2.overleaf.test:3000"
+ },
"repoStore": {
"maxFileNum": 2000,
"maxFileSize": 52428800
diff --git a/services/git-bridge/pom.xml b/services/git-bridge/pom.xml
index 3feb4dd860..809676d769 100644
--- a/services/git-bridge/pom.xml
+++ b/services/git-bridge/pom.xml
@@ -16,24 +16,24 @@
2.23
4.13.2
2.8.4
- 9.4.57.v20241219
+ 9.4.51.v20230217
2.9.0
- 3.0.2
- 6.10.1.202505221210-r
+ 2.12.3
+ 6.6.1.202309021850-r
3.41.2.2
2.9.9
- 1.37.0
+ 1.34.1
1.23.0
- 3.17.0
- 1.2.13
+ 3.12.0
+ 1.2.3
5.12.0
5.12.0
- 1.12.780
+ 1.11.274
${jaxb.runtime.version}
2.3.2
4.5.14
- 2.18.0
- 1.27.1
+ 2.10.0
+ 1.24.0
0.10.0
1.70
@@ -206,7 +206,7 @@
com.amazonaws
- aws-java-sdk-s3
+ aws-java-sdk
${aws.java.sdk.version}
diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java
index d5b530100e..cf36916600 100644
--- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java
+++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Config.java
@@ -26,11 +26,10 @@ public class Config implements JSONSource {
config.bindIp,
config.idleTimeout,
config.rootGitDirectory,
- config.allowedCorsOrigins,
config.apiBaseURL,
config.postbackURL,
config.serviceName,
- config.oauth2Server,
+ Oauth2.asSanitised(config.oauth2),
config.userPasswordEnabled,
config.repoStore,
SwapStoreConfig.sanitisedCopy(config.swapStore),
@@ -42,11 +41,10 @@ public class Config implements JSONSource {
private String bindIp;
private int idleTimeout;
private String rootGitDirectory;
- private String[] allowedCorsOrigins;
private String apiBaseURL;
private String postbackURL;
private String serviceName;
- @Nullable private String oauth2Server;
+ @Nullable private Oauth2 oauth2;
private boolean userPasswordEnabled;
@Nullable private RepoStoreConfig repoStore;
@Nullable private SwapStoreConfig swapStore;
@@ -66,11 +64,10 @@ public class Config implements JSONSource {
String bindIp,
int idleTimeout,
String rootGitDirectory,
- String[] allowedCorsOrigins,
String apiBaseURL,
String postbackURL,
String serviceName,
- String oauth2Server,
+ Oauth2 oauth2,
boolean userPasswordEnabled,
RepoStoreConfig repoStore,
SwapStoreConfig swapStore,
@@ -80,11 +77,10 @@ public class Config implements JSONSource {
this.bindIp = bindIp;
this.idleTimeout = idleTimeout;
this.rootGitDirectory = rootGitDirectory;
- this.allowedCorsOrigins = allowedCorsOrigins;
this.apiBaseURL = apiBaseURL;
this.postbackURL = postbackURL;
this.serviceName = serviceName;
- this.oauth2Server = oauth2Server;
+ this.oauth2 = oauth2;
this.userPasswordEnabled = userPasswordEnabled;
this.repoStore = repoStore;
this.swapStore = swapStore;
@@ -105,18 +101,11 @@ public class Config implements JSONSource {
}
this.apiBaseURL = apiBaseURL;
serviceName = getElement(configObject, "serviceName").getAsString();
- final String rawAllowedCorsOrigins =
- getOptionalString(configObject, "allowedCorsOrigins").trim();
- if (rawAllowedCorsOrigins.isEmpty()) {
- allowedCorsOrigins = new String[] {};
- } else {
- allowedCorsOrigins = rawAllowedCorsOrigins.split(",");
- }
postbackURL = getElement(configObject, "postbackBaseUrl").getAsString();
if (!postbackURL.endsWith("/")) {
postbackURL += "/";
}
- oauth2Server = getOptionalString(configObject, "oauth2Server");
+ oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
userPasswordEnabled = getOptionalString(configObject, "userPasswordEnabled").equals("true");
repoStore = new Gson().fromJson(configObject.get("repoStore"), RepoStoreConfig.class);
swapStore = new Gson().fromJson(configObject.get("swapStore"), SwapStoreConfig.class);
@@ -150,10 +139,6 @@ public class Config implements JSONSource {
return this.sqliteHeapLimitBytes;
}
- public String[] getAllowedCorsOrigins() {
- return allowedCorsOrigins;
- }
-
public String getAPIBaseURL() {
return apiBaseURL;
}
@@ -166,12 +151,19 @@ public class Config implements JSONSource {
return postbackURL;
}
+ public boolean isUsingOauth2() {
+ return oauth2 != null;
+ }
+
public boolean isUserPasswordEnabled() {
return userPasswordEnabled;
}
- public String getOauth2Server() {
- return oauth2Server;
+ public Oauth2 getOauth2() {
+ if (!isUsingOauth2()) {
+ throw new AssertionError("Getting oauth2 when not using it");
+ }
+ return oauth2;
}
public Optional getRepoStore() {
diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Oauth2.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Oauth2.java
new file mode 100644
index 0000000000..1db7d3b4d2
--- /dev/null
+++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/application/config/Oauth2.java
@@ -0,0 +1,33 @@
+package uk.ac.ic.wlgitbridge.application.config;
+
+/*
+ * Created by winston on 25/10/15.
+ */
+public class Oauth2 {
+
+ private final String oauth2ClientID;
+ private final String oauth2ClientSecret;
+ private final String oauth2Server;
+
+ public Oauth2(String oauth2ClientID, String oauth2ClientSecret, String oauth2Server) {
+ this.oauth2ClientID = oauth2ClientID;
+ this.oauth2ClientSecret = oauth2ClientSecret;
+ this.oauth2Server = oauth2Server;
+ }
+
+ public String getOauth2ClientID() {
+ return oauth2ClientID;
+ }
+
+ public String getOauth2ClientSecret() {
+ return oauth2ClientSecret;
+ }
+
+ public String getOauth2Server() {
+ return oauth2Server;
+ }
+
+ public static Oauth2 asSanitised(Oauth2 oauth2) {
+ return new Oauth2("", "", oauth2.oauth2Server);
+ }
+}
diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/CORSHandler.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/CORSHandler.java
deleted file mode 100644
index 10d978c352..0000000000
--- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/CORSHandler.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package uk.ac.ic.wlgitbridge.server;
-
-import java.io.IOException;
-import java.util.Set;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import org.eclipse.jetty.server.Request;
-import org.eclipse.jetty.server.handler.AbstractHandler;
-import uk.ac.ic.wlgitbridge.util.Log;
-
-public class CORSHandler extends AbstractHandler {
- private final Set allowedCorsOrigins;
-
- public CORSHandler(String[] allowedCorsOrigins) {
- this.allowedCorsOrigins = Set.of(allowedCorsOrigins);
- }
-
- @Override
- public void handle(
- String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
- throws IOException {
-
- String origin = request.getHeader("Origin");
- if (origin == null) {
- return; // Not a CORS request
- }
-
- final boolean ok = allowedCorsOrigins.contains(origin);
- if (ok) {
- response.setHeader("Access-Control-Allow-Origin", origin);
- response.setHeader("Access-Control-Allow-Credentials", "true");
- response.setHeader("Access-Control-Allow-Methods", "GET, HEAD, PUT, POST, DELETE");
- response.setHeader("Access-Control-Allow-Headers", "Authorization, Content-Type");
- response.setHeader("Access-Control-Max-Age", "86400"); // cache for 24h
- }
- String method = baseRequest.getMethod();
- if ("OPTIONS".equals(method)) {
- Log.debug("OPTIONS <- {}", target);
- baseRequest.setHandled(true);
- if (ok) {
- response.setStatus(200);
- } else {
- response.setStatus(403);
- }
- }
- }
-}
diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
index 57d1b34a7b..30c5039212 100644
--- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
+++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/GitBridgeServer.java
@@ -110,7 +110,6 @@ public class GitBridgeServer {
this.jettyServer.addConnector(connector);
HandlerCollection handlers = new HandlerList();
- handlers.addHandler(new CORSHandler(config.getAllowedCorsOrigins()));
handlers.addHandler(initApiHandler());
handlers.addHandler(initBaseHandler());
handlers.addHandler(initGitHandler(config, repoStore, snapshotApi));
@@ -151,9 +150,9 @@ public class GitBridgeServer {
throws ServletException {
final ServletContextHandler servletContextHandler =
new ServletContextHandler(ServletContextHandler.SESSIONS);
- if (config.getOauth2Server() != null) {
+ if (config.isUsingOauth2()) {
Filter filter =
- new Oauth2Filter(snapshotApi, config.getOauth2Server(), config.isUserPasswordEnabled());
+ new Oauth2Filter(snapshotApi, config.getOauth2(), config.isUserPasswordEnabled());
servletContextHandler.addFilter(
new FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST));
}
diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java
index 586a21ab3f..5bd3904e47 100644
--- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java
+++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/server/Oauth2Filter.java
@@ -13,6 +13,7 @@ import javax.servlet.*;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.codec.binary.Base64;
+import uk.ac.ic.wlgitbridge.application.config.Oauth2;
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
import uk.ac.ic.wlgitbridge.util.Instance;
import uk.ac.ic.wlgitbridge.util.Log;
@@ -27,13 +28,13 @@ public class Oauth2Filter implements Filter {
private final SnapshotApi snapshotApi;
- private final String oauth2Server;
+ private final Oauth2 oauth2;
private final boolean isUserPasswordEnabled;
- public Oauth2Filter(SnapshotApi snapshotApi, String oauth2Server, boolean isUserPasswordEnabled) {
+ public Oauth2Filter(SnapshotApi snapshotApi, Oauth2 oauth2, boolean isUserPasswordEnabled) {
this.snapshotApi = snapshotApi;
- this.oauth2Server = oauth2Server;
+ this.oauth2 = oauth2;
this.isUserPasswordEnabled = isUserPasswordEnabled;
}
@@ -107,7 +108,7 @@ public class Oauth2Filter implements Filter {
// fail later (for example, in the unlikely event that the token
// expired between the two requests). In that case, JGit will
// return a 401 without a custom error message.
- int statusCode = checkAccessToken(this.oauth2Server, password, getClientIp(request));
+ int statusCode = checkAccessToken(oauth2, password, getClientIp(request));
if (statusCode == 429) {
handleRateLimit(projectId, username, request, response);
return;
@@ -237,9 +238,10 @@ public class Oauth2Filter implements Filter {
"your Overleaf Account Settings."));
}
- private int checkAccessToken(String oauth2Server, String accessToken, String clientIp)
+ private int checkAccessToken(Oauth2 oauth2, String accessToken, String clientIp)
throws IOException {
- GenericUrl url = new GenericUrl(oauth2Server + "/oauth/token/info?client_ip=" + clientIp);
+ GenericUrl url =
+ new GenericUrl(oauth2.getOauth2Server() + "/oauth/token/info?client_ip=" + clientIp);
HttpRequest request = Instance.httpRequestFactory.buildGetRequest(url);
HttpHeaders headers = new HttpHeaders();
headers.setAuthorization("Bearer " + accessToken);
diff --git a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java
index 512babf9c7..878adde27d 100644
--- a/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java
+++ b/services/git-bridge/src/main/java/uk/ac/ic/wlgitbridge/util/Tar.java
@@ -5,7 +5,6 @@ import java.io.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.commons.compress.archivers.ArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
@@ -148,7 +147,7 @@ public class Tar {
throws IOException {
Preconditions.checkArgument(dir.isDirectory());
String name = base.relativize(Paths.get(dir.getAbsolutePath())).toString();
- TarArchiveEntry entry = tout.createArchiveEntry(dir, name);
+ ArchiveEntry entry = tout.createArchiveEntry(dir, name);
tout.putArchiveEntry(entry);
tout.closeArchiveEntry();
for (File f : dir.listFiles()) {
@@ -161,7 +160,7 @@ public class Tar {
Preconditions.checkArgument(file.isFile(), "given file" + " is not file: %s", file);
checkFileSize(file.length());
String name = base.relativize(Paths.get(file.getAbsolutePath())).toString();
- TarArchiveEntry entry = tout.createArchiveEntry(file, name);
+ ArchiveEntry entry = tout.createArchiveEntry(file, name);
tout.putArchiveEntry(entry);
try (InputStream in = new FileInputStream(file)) {
IOUtils.copy(in, tout);
diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java
index f706d98edf..e250798652 100644
--- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java
+++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/WLGitBridgeIntegrationTest.java
@@ -465,12 +465,8 @@ public class WLGitBridgeIntegrationTest {
@After
public void tearDown() {
- if (server != null) {
- server.stop();
- }
- if (wlgb != null) {
- wlgb.stop();
- }
+ server.stop();
+ wlgb.stop();
}
private void gitConfig(File dir) throws IOException, InterruptedException {
@@ -1395,80 +1391,6 @@ public class WLGitBridgeIntegrationTest {
assertTrue(f.exists());
}
- @Test
- public void noCors() throws IOException, ExecutionException, InterruptedException {
-
- int gitBridgePort = 33893;
- int mockServerPort = 3893;
-
- server = new MockSnapshotServer(mockServerPort, getResource("/canServePushedFiles").toFile());
- server.start();
- server.setState(states.get("canServePushedFiles").get("state"));
-
- wlgb = new GitBridgeApp(new String[] {makeConfigFile(gitBridgePort, mockServerPort)});
- wlgb.run();
-
- String url = "http://127.0.0.1:" + gitBridgePort + "/status";
- Response response = asyncHttpClient().prepareGet(url).execute().get();
- assertEquals(200, response.getStatusCode());
- assertEquals("ok\n", response.getResponseBody());
- assertNull(response.getHeader("Access-Control-Allow-Origin"));
- }
-
- @Test
- public void cors() throws IOException, ExecutionException, InterruptedException {
-
- int gitBridgePort = 33894;
- int mockServerPort = 3894;
-
- server = new MockSnapshotServer(mockServerPort, getResource("/canServePushedFiles").toFile());
- server.start();
- server.setState(states.get("canServePushedFiles").get("state"));
-
- wlgb = new GitBridgeApp(new String[] {makeConfigFile(gitBridgePort, mockServerPort)});
- wlgb.run();
-
- String url = "http://127.0.0.1:" + gitBridgePort + "/status";
-
- // Success
- Response response =
- asyncHttpClient()
- .prepareOptions(url)
- .setHeader("Origin", "https://localhost")
- .execute()
- .get();
- assertEquals(200, response.getStatusCode());
- assertEquals("", response.getResponseBody());
- assertEquals("https://localhost", response.getHeader("Access-Control-Allow-Origin"));
-
- response =
- asyncHttpClient().prepareGet(url).setHeader("Origin", "https://localhost").execute().get();
- assertEquals(200, response.getStatusCode());
- assertEquals("ok\n", response.getResponseBody());
- assertEquals("https://localhost", response.getHeader("Access-Control-Allow-Origin"));
-
- // Deny
- response =
- asyncHttpClient()
- .prepareOptions(url)
- .setHeader("Origin", "https://not-localhost")
- .execute()
- .get();
- assertEquals(403, response.getStatusCode());
- assertEquals("", response.getResponseBody());
- assertNull(response.getHeader("Access-Control-Allow-Origin"));
-
- response =
- asyncHttpClient()
- .prepareGet(url)
- .setHeader("Origin", "https://not-localhost")
- .execute()
- .get();
- assertEquals(200, response.getStatusCode());
- assertEquals("ok\n", response.getResponseBody());
- assertNull(response.getHeader("Access-Control-Allow-Origin"));
- }
-
private String makeConfigFile(int port, int apiPort) throws IOException {
return makeConfigFile(port, apiPort, null);
}
@@ -1487,7 +1409,6 @@ public class WLGitBridgeIntegrationTest {
+ " \"rootGitDirectory\": \""
+ wlgb.getAbsolutePath()
+ "\",\n"
- + " \"allowedCorsOrigins\": \"https://localhost\",\n"
+ " \"apiBaseUrl\": \"http://127.0.0.1:"
+ apiPort
+ "/api/v0\",\n"
@@ -1495,9 +1416,13 @@ public class WLGitBridgeIntegrationTest {
+ port
+ "\",\n"
+ " \"serviceName\": \"Overleaf\",\n"
- + " \"oauth2Server\": \"http://127.0.0.1:"
+ + " \"oauth2\": {\n"
+ + " \"oauth2ClientID\": \"clientID\",\n"
+ + " \"oauth2ClientSecret\": \"oauth2 client secret\",\n"
+ + " \"oauth2Server\": \"http://127.0.0.1:"
+ apiPort
- + "\"";
+ + "\"\n"
+ + " }";
if (swapCfg != null) {
cfgStr +=
",\n"
@@ -1520,6 +1445,7 @@ public class WLGitBridgeIntegrationTest {
+ ",\n"
+ " \"intervalMillis\": "
+ swapCfg.getIntervalMillis()
+ + "\n"
+ " }\n";
}
cfgStr += "}\n";
diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java
index 8c102dbda3..ddafc621d6 100644
--- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java
+++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/application/config/ConfigTest.java
@@ -23,7 +23,11 @@ public class ConfigTest {
+ " \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
+ " \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
+ " \"serviceName\": \"Overleaf\",\n"
- + " \"oauth2Server\": \"https://www.overleaf.com\"\n"
+ + " \"oauth2\": {\n"
+ + " \"oauth2ClientID\": \"clientID\",\n"
+ + " \"oauth2ClientSecret\": \"oauth2 client secret\",\n"
+ + " \"oauth2Server\": \"https://www.overleaf.com\"\n"
+ + " }\n"
+ "}\n");
Config config = new Config(reader);
assertEquals(80, config.getPort());
@@ -31,7 +35,10 @@ public class ConfigTest {
assertEquals("http://127.0.0.1:60000/api/v0/", config.getAPIBaseURL());
assertEquals("http://127.0.0.1/", config.getPostbackURL());
assertEquals("Overleaf", config.getServiceName());
- assertEquals("https://www.overleaf.com", config.getOauth2Server());
+ assertTrue(config.isUsingOauth2());
+ assertEquals("clientID", config.getOauth2().getOauth2ClientID());
+ assertEquals("oauth2 client secret", config.getOauth2().getOauth2ClientSecret());
+ assertEquals("https://www.overleaf.com", config.getOauth2().getOauth2Server());
}
@Test(expected = AssertionError.class)
@@ -46,7 +53,7 @@ public class ConfigTest {
+ " \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
+ " \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
+ " \"serviceName\": \"Overleaf\",\n"
- + " \"oauth2Server\": null\n"
+ + " \"oauth2\": null\n"
+ "}\n");
Config config = new Config(reader);
assertEquals(80, config.getPort());
@@ -54,7 +61,8 @@ public class ConfigTest {
assertEquals("http://127.0.0.1:60000/api/v0/", config.getAPIBaseURL());
assertEquals("http://127.0.0.1/", config.getPostbackURL());
assertEquals("Overleaf", config.getServiceName());
- assertNull(config.getOauth2Server());
+ assertFalse(config.isUsingOauth2());
+ config.getOauth2();
}
@Test
@@ -69,7 +77,11 @@ public class ConfigTest {
+ " \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
+ " \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
+ " \"serviceName\": \"Overleaf\",\n"
- + " \"oauth2Server\": \"https://www.overleaf.com\"\n"
+ + " \"oauth2\": {\n"
+ + " \"oauth2ClientID\": \"my oauth2 client id\",\n"
+ + " \"oauth2ClientSecret\": \"my oauth2 client secret\",\n"
+ + " \"oauth2Server\": \"https://www.overleaf.com\"\n"
+ + " }\n"
+ "}\n");
Config config = new Config(reader);
String expected =
@@ -78,11 +90,14 @@ public class ConfigTest {
+ " \"bindIp\": \"127.0.0.1\",\n"
+ " \"idleTimeout\": 30000,\n"
+ " \"rootGitDirectory\": \"/var/wlgb/git\",\n"
- + " \"allowedCorsOrigins\": [],\n"
+ " \"apiBaseURL\": \"http://127.0.0.1:60000/api/v0/\",\n"
+ " \"postbackURL\": \"http://127.0.0.1/\",\n"
+ " \"serviceName\": \"Overleaf\",\n"
- + " \"oauth2Server\": \"https://www.overleaf.com\",\n"
+ + " \"oauth2\": {\n"
+ + " \"oauth2ClientID\": \"\",\n"
+ + " \"oauth2ClientSecret\": \"\",\n"
+ + " \"oauth2Server\": \"https://www.overleaf.com\"\n"
+ + " },\n"
+ " \"userPasswordEnabled\": false,\n"
+ " \"repoStore\": null,\n"
+ " \"swapStore\": null,\n"
diff --git a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java
index e27c3488c0..f749dea357 100644
--- a/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java
+++ b/services/git-bridge/src/test/java/uk/ac/ic/wlgitbridge/bridge/BridgeTest.java
@@ -50,7 +50,7 @@ public class BridgeTest {
gcJob = mock(GcJob.class);
bridge =
new Bridge(
- new Config(0, "", 0, "", null, "", "", "", null, false, null, null, null, 0),
+ new Config(0, "", 0, "", "", "", "", null, false, null, null, null, 0),
lock,
repoStore,
dbStore,
diff --git a/services/git-bridge/vendor/envsubst b/services/git-bridge/vendor/envsubst
new file mode 100755
index 0000000000..f7ad8081d0
Binary files /dev/null and b/services/git-bridge/vendor/envsubst differ
diff --git a/services/history-v1/.gitignore b/services/history-v1/.gitignore
new file mode 100644
index 0000000000..edb0f85350
--- /dev/null
+++ b/services/history-v1/.gitignore
@@ -0,0 +1,3 @@
+
+# managed by monorepo$ bin/update_build_scripts
+.npmrc
diff --git a/services/history-v1/.nvmrc b/services/history-v1/.nvmrc
index fc37597bcc..123b052798 100644
--- a/services/history-v1/.nvmrc
+++ b/services/history-v1/.nvmrc
@@ -1 +1 @@
-22.17.0
+18.20.2
diff --git a/services/history-v1/Dockerfile b/services/history-v1/Dockerfile
index 322ab67ff8..cf6d0b3aaf 100644
--- a/services/history-v1/Dockerfile
+++ b/services/history-v1/Dockerfile
@@ -2,11 +2,9 @@
# Instead run bin/update_build_scripts from
# https://github.com/overleaf/internal/
-FROM node:22.17.0 AS base
+FROM node:18.20.2 AS base
WORKDIR /overleaf/services/history-v1
-COPY services/history-v1/install_deps.sh /overleaf/services/history-v1/
-RUN chmod 0755 ./install_deps.sh && ./install_deps.sh
# Google Cloud Storage needs a writable $HOME/.config for resumable uploads
# (see https://googleapis.dev/nodejs/storage/latest/File.html#createWriteStream)
diff --git a/services/history-v1/Makefile b/services/history-v1/Makefile
index 7e62ba1812..6ba72740b0 100644
--- a/services/history-v1/Makefile
+++ b/services/history-v1/Makefile
@@ -32,30 +32,12 @@ HERE=$(shell pwd)
MONOREPO=$(shell cd ../../ && pwd)
# Run the linting commands in the scope of the monorepo.
# Eslint and prettier (plus some configs) are on the root.
-RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:22.17.0 npm run --silent
+RUN_LINTING = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(HERE) node:18.20.2 npm run --silent
RUN_LINTING_CI = docker run --rm --volume $(MONOREPO)/.editorconfig:/overleaf/.editorconfig --volume $(MONOREPO)/.eslintignore:/overleaf/.eslintignore --volume $(MONOREPO)/.eslintrc:/overleaf/.eslintrc --volume $(MONOREPO)/.prettierignore:/overleaf/.prettierignore --volume $(MONOREPO)/.prettierrc:/overleaf/.prettierrc --volume $(MONOREPO)/tsconfig.backend.json:/overleaf/tsconfig.backend.json ci/$(PROJECT_NAME):$(BRANCH_NAME)-$(BUILD_NUMBER) npm run --silent
# Same but from the top of the monorepo
-RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:22.17.0 npm run --silent
-
-SHELLCHECK_OPTS = \
- --shell=bash \
- --external-sources
-SHELLCHECK_COLOR := $(if $(CI),--color=never,--color)
-SHELLCHECK_FILES := { git ls-files "*.sh" -z; git grep -Plz "\A\#\!.*bash"; } | sort -zu
-
-shellcheck:
- @$(SHELLCHECK_FILES) | xargs -0 -r docker run --rm -v $(HERE):/mnt -w /mnt \
- koalaman/shellcheck:stable $(SHELLCHECK_OPTS) $(SHELLCHECK_COLOR)
-
-shellcheck_fix:
- @$(SHELLCHECK_FILES) | while IFS= read -r -d '' file; do \
- diff=$$(docker run --rm -v $(HERE):/mnt -w /mnt koalaman/shellcheck:stable $(SHELLCHECK_OPTS) --format=diff "$$file" 2>/dev/null); \
- if [ -n "$$diff" ] && ! echo "$$diff" | patch -p1 >/dev/null 2>&1; then echo "\033[31m$$file\033[0m"; \
- elif [ -n "$$diff" ]; then echo "$$file"; \
- else echo "\033[2m$$file\033[0m"; fi \
- done
+RUN_LINTING_MONOREPO = docker run --rm -v $(MONOREPO):$(MONOREPO) -w $(MONOREPO) node:18.20.2 npm run --silent
format:
$(RUN_LINTING) format
@@ -81,7 +63,7 @@ typecheck:
typecheck_ci:
$(RUN_LINTING_CI) types:check
-test: format lint typecheck shellcheck test_unit test_acceptance
+test: format lint typecheck test_unit test_acceptance
test_unit:
ifneq (,$(wildcard test/unit))
@@ -116,6 +98,13 @@ test_acceptance_clean:
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
test_acceptance_pre_run:
+ $(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
+ $(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
+ while ! mongosh --eval "db.version()" > /dev/null; do \
+ echo "Waiting for Mongo..."; \
+ sleep 1; \
+ done; \
+ mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
endif
@@ -148,7 +137,6 @@ publish:
lint lint_fix \
build_types typecheck \
lint_ci format_ci typecheck_ci \
- shellcheck shellcheck_fix \
test test_clean test_unit test_unit_clean \
test_acceptance test_acceptance_debug test_acceptance_pre_run \
test_acceptance_run test_acceptance_run_debug test_acceptance_clean \
diff --git a/services/history-v1/api/app/rollout.js b/services/history-v1/api/app/rollout.js
deleted file mode 100644
index 24ca0409f8..0000000000
--- a/services/history-v1/api/app/rollout.js
+++ /dev/null
@@ -1,76 +0,0 @@
-const crypto = require('node:crypto')
-
-class Rollout {
- constructor(config) {
- // The history buffer level is used to determine whether to queue changes
- // in Redis or persist them directly to the chunk store.
- // If defaults to 0 (no queuing) if not set.
- this.historyBufferLevel = config.has('historyBufferLevel')
- ? parseInt(config.get('historyBufferLevel'), 10)
- : 0
- // The forcePersistBuffer flag will ensure the buffer is fully persisted before
- // any persist operation. Set this to true if you want to make the persisted-version
- // in Redis match the endVersion of the latest chunk. This should be set to true
- // when downgrading from a history buffer level that queues changes in Redis
- // without persisting them immediately.
- this.forcePersistBuffer = config.has('forcePersistBuffer')
- ? config.get('forcePersistBuffer') === 'true'
- : false
-
- // Support gradual rollout of the next history buffer level
- // with a percentage of projects using it.
- this.nextHistoryBufferLevel = config.has('nextHistoryBufferLevel')
- ? parseInt(config.get('nextHistoryBufferLevel'), 10)
- : null
- this.nextHistoryBufferLevelRolloutPercentage = config.has(
- 'nextHistoryBufferLevelRolloutPercentage'
- )
- ? parseInt(config.get('nextHistoryBufferLevelRolloutPercentage'), 10)
- : 0
- }
-
- report(logger) {
- logger.info(
- {
- historyBufferLevel: this.historyBufferLevel,
- forcePersistBuffer: this.forcePersistBuffer,
- nextHistoryBufferLevel: this.nextHistoryBufferLevel,
- nextHistoryBufferLevelRolloutPercentage:
- this.nextHistoryBufferLevelRolloutPercentage,
- },
- this.historyBufferLevel > 0 || this.forcePersistBuffer
- ? 'using history buffer'
- : 'history buffer disabled'
- )
- }
-
- /**
- * Get the history buffer level for a project.
- * @param {string} projectId
- * @returns {Object} - An object containing the history buffer level and force persist buffer flag.
- * @property {number} historyBufferLevel - The history buffer level to use for processing changes.
- * @property {boolean} forcePersistBuffer - If true, forces the buffer to be persisted before any operation.
- */
- getHistoryBufferLevelOptions(projectId) {
- if (
- this.nextHistoryBufferLevel > this.historyBufferLevel &&
- this.nextHistoryBufferLevelRolloutPercentage > 0
- ) {
- const hash = crypto.createHash('sha1').update(projectId).digest('hex')
- const percentage = parseInt(hash.slice(0, 8), 16) % 100
- // If the project is in the rollout percentage, we use the next history buffer level.
- if (percentage < this.nextHistoryBufferLevelRolloutPercentage) {
- return {
- historyBufferLevel: this.nextHistoryBufferLevel,
- forcePersistBuffer: this.forcePersistBuffer,
- }
- }
- }
- return {
- historyBufferLevel: this.historyBufferLevel,
- forcePersistBuffer: this.forcePersistBuffer,
- }
- }
-}
-
-module.exports = Rollout
diff --git a/services/history-v1/api/app/security.js b/services/history-v1/api/app/security.js
index 08d6f030dc..c82c3d2683 100644
--- a/services/history-v1/api/app/security.js
+++ b/services/history-v1/api/app/security.js
@@ -105,8 +105,6 @@ function handleJWTAuth(req, authOrSecDef, scopesOrApiKey, next) {
next()
}
-exports.hasValidBasicAuthCredentials = hasValidBasicAuthCredentials
-
/**
* Verify and decode the given JSON Web Token
*/
diff --git a/services/history-v1/api/controllers/project_import.js b/services/history-v1/api/controllers/project_import.js
index 02fb793c87..ec4aa317b0 100644
--- a/services/history-v1/api/controllers/project_import.js
+++ b/services/history-v1/api/controllers/project_import.js
@@ -1,10 +1,6 @@
-// @ts-check
-
'use strict'
-const config = require('config')
-const { expressify } = require('@overleaf/promise-utils')
-
+const BPromise = require('bluebird')
const HTTPStatus = require('http-status')
const core = require('overleaf-editor-core')
@@ -22,18 +18,11 @@ const BatchBlobStore = storage.BatchBlobStore
const BlobStore = storage.BlobStore
const chunkStore = storage.chunkStore
const HashCheckBlobStore = storage.HashCheckBlobStore
-const commitChanges = storage.commitChanges
-const persistBuffer = storage.persistBuffer
-const InvalidChangeError = storage.InvalidChangeError
+const persistChanges = storage.persistChanges
const render = require('./render')
-const Rollout = require('../app/rollout')
-const redisBackend = require('../../storage/lib/chunk_store/redis')
-const rollout = new Rollout(config)
-rollout.report(logger) // display the rollout configuration in the logs
-
-async function importSnapshot(req, res) {
+exports.importSnapshot = function importSnapshot(req, res, next) {
const projectId = req.swagger.params.project_id.value
const rawSnapshot = req.swagger.params.snapshot.value
@@ -42,26 +31,24 @@ async function importSnapshot(req, res) {
try {
snapshot = Snapshot.fromRaw(rawSnapshot)
} catch (err) {
- logger.warn({ err, projectId }, 'failed to import snapshot')
return render.unprocessableEntity(res)
}
- let historyId
- try {
- historyId = await chunkStore.initializeProject(projectId, snapshot)
- } catch (err) {
- if (err instanceof chunkStore.AlreadyInitialized) {
- logger.warn({ err, projectId }, 'already initialized')
- return render.conflict(res)
- } else {
- throw err
- }
- }
-
- res.status(HTTPStatus.OK).json({ projectId: historyId })
+ return chunkStore
+ .initializeProject(projectId, snapshot)
+ .then(function (projectId) {
+ res.status(HTTPStatus.OK).json({ projectId })
+ })
+ .catch(err => {
+ if (err instanceof chunkStore.AlreadyInitialized) {
+ render.conflict(res)
+ } else {
+ next(err)
+ }
+ })
}
-async function importChanges(req, res, next) {
+exports.importChanges = function importChanges(req, res, next) {
const projectId = req.swagger.params.project_id.value
const rawChanges = req.swagger.params.changes.value
const endVersion = req.swagger.params.end_version.value
@@ -72,7 +59,7 @@ async function importChanges(req, res, next) {
try {
changes = rawChanges.map(Change.fromRaw)
} catch (err) {
- logger.warn({ err, projectId }, 'failed to parse changes')
+ logger.error(err)
return render.unprocessableEntity(res)
}
@@ -89,102 +76,65 @@ async function importChanges(req, res, next) {
const batchBlobStore = new BatchBlobStore(blobStore)
const hashCheckBlobStore = new HashCheckBlobStore(blobStore)
- async function loadFiles() {
+ function loadFiles() {
const blobHashes = new Set()
- for (const change of changes) {
- // This populates the set blobHashes with blobs referred to in the change
+ changes.forEach(function findBlobHashesToPreload(change) {
change.findBlobHashes(blobHashes)
- }
-
- await batchBlobStore.preload(Array.from(blobHashes))
-
- for (const change of changes) {
- await change.loadFiles('lazy', batchBlobStore)
- }
- }
-
- async function buildResultSnapshot(resultChunk) {
- const chunk =
- resultChunk ||
- (await chunkStore.loadLatest(projectId, { persistedOnly: true }))
- const snapshot = chunk.getSnapshot()
- snapshot.applyAll(chunk.getChanges())
- const rawSnapshot = await snapshot.store(hashCheckBlobStore)
- return rawSnapshot
- }
-
- await loadFiles()
-
- let result
- try {
- const { historyBufferLevel, forcePersistBuffer } =
- rollout.getHistoryBufferLevelOptions(projectId)
- result = await commitChanges(projectId, changes, limits, endVersion, {
- historyBufferLevel,
- forcePersistBuffer,
})
- } catch (err) {
- if (
- err instanceof Chunk.ConflictingEndVersion ||
- err instanceof TextOperation.UnprocessableError ||
- err instanceof File.NotEditableError ||
- err instanceof FileMap.PathnameError ||
- err instanceof Snapshot.EditMissingFileError ||
- err instanceof chunkStore.ChunkVersionConflictError ||
- err instanceof InvalidChangeError
- ) {
- // If we failed to apply operations, that's probably because they were
- // invalid.
- logger.warn({ err, projectId, endVersion }, 'changes rejected by history')
- return render.unprocessableEntity(res)
- } else if (err instanceof Chunk.NotFoundError) {
- logger.warn({ err, projectId }, 'chunk not found')
- return render.notFound(res)
- } else {
- throw err
+
+ function lazyLoadChangeFiles(change) {
+ return change.loadFiles('lazy', batchBlobStore)
}
+
+ return batchBlobStore
+ .preload(Array.from(blobHashes))
+ .then(function lazyLoadChangeFilesWithBatching() {
+ return BPromise.each(changes, lazyLoadChangeFiles)
+ })
}
- if (returnSnapshot === 'none') {
- res.status(HTTPStatus.CREATED).json({
- resyncNeeded: result.resyncNeeded,
+ function buildResultSnapshot(resultChunk) {
+ return BPromise.resolve(
+ resultChunk || chunkStore.loadLatest(projectId)
+ ).then(function (chunk) {
+ const snapshot = chunk.getSnapshot()
+ snapshot.applyAll(chunk.getChanges())
+ return snapshot.store(hashCheckBlobStore)
})
- } else {
- const rawSnapshot = await buildResultSnapshot(result && result.currentChunk)
- res.status(HTTPStatus.CREATED).json(rawSnapshot)
}
-}
-async function flushChanges(req, res, next) {
- const projectId = req.swagger.params.project_id.value
- // Use the same limits importChanges, since these are passed to persistChanges
- const farFuture = new Date()
- farFuture.setTime(farFuture.getTime() + 7 * 24 * 3600 * 1000)
- const limits = {
- maxChanges: 0,
- minChangeTimestamp: farFuture,
- maxChangeTimestamp: farFuture,
- autoResync: true,
- }
- try {
- await persistBuffer(projectId, limits)
- res.status(HTTPStatus.OK).end()
- } catch (err) {
- if (err instanceof Chunk.NotFoundError) {
- render.notFound(res)
- } else {
- throw err
- }
- }
+ return loadFiles()
+ .then(function () {
+ return persistChanges(projectId, changes, limits, endVersion)
+ })
+ .then(function (result) {
+ if (returnSnapshot === 'none') {
+ res.status(HTTPStatus.CREATED).json({})
+ } else {
+ return buildResultSnapshot(result && result.currentChunk).then(
+ function (rawSnapshot) {
+ res.status(HTTPStatus.CREATED).json(rawSnapshot)
+ }
+ )
+ }
+ })
+ .catch(err => {
+ if (
+ err instanceof Chunk.ConflictingEndVersion ||
+ err instanceof TextOperation.UnprocessableError ||
+ err instanceof File.NotEditableError ||
+ err instanceof FileMap.PathnameError ||
+ err instanceof Snapshot.EditMissingFileError ||
+ err instanceof chunkStore.ChunkVersionConflictError
+ ) {
+ // If we failed to apply operations, that's probably because they were
+ // invalid.
+ logger.error(err)
+ render.unprocessableEntity(res)
+ } else if (err instanceof Chunk.NotFoundError) {
+ render.notFound(res)
+ } else {
+ next(err)
+ }
+ })
}
-
-async function expireProject(req, res, next) {
- const projectId = req.swagger.params.project_id.value
- await redisBackend.expireProject(projectId)
- res.status(HTTPStatus.OK).end()
-}
-
-exports.importSnapshot = expressify(importSnapshot)
-exports.importChanges = expressify(importChanges)
-exports.flushChanges = expressify(flushChanges)
-exports.expireProject = expressify(expireProject)
diff --git a/services/history-v1/api/controllers/projects.js b/services/history-v1/api/controllers/projects.js
index b7f07c4834..3e00035d39 100644
--- a/services/history-v1/api/controllers/projects.js
+++ b/services/history-v1/api/controllers/projects.js
@@ -1,13 +1,12 @@
'use strict'
const _ = require('lodash')
-const Path = require('node:path')
-const Stream = require('node:stream')
+const Path = require('path')
+const Stream = require('stream')
const HTTPStatus = require('http-status')
-const fs = require('node:fs')
-const { promisify } = require('node:util')
+const fs = require('fs')
+const { promisify } = require('util')
const config = require('config')
-const OError = require('@overleaf/o-error')
const logger = require('@overleaf/logger')
const { Chunk, ChunkResponse, Blob } = require('overleaf-editor-core')
@@ -15,7 +14,6 @@ const {
BlobStore,
blobHash,
chunkStore,
- redisBuffer,
HashCheckBlobStore,
ProjectArchive,
zipStore,
@@ -35,7 +33,6 @@ async function initializeProject(req, res, next) {
res.status(HTTPStatus.OK).json({ projectId })
} catch (err) {
if (err instanceof chunkStore.AlreadyInitialized) {
- logger.warn({ err, projectId }, 'failed to initialize')
render.conflict(res)
} else {
throw err
@@ -88,26 +85,6 @@ async function getLatestHistory(req, res, next) {
}
}
-async function getLatestHistoryRaw(req, res, next) {
- const projectId = req.swagger.params.project_id.value
- const readOnly = req.swagger.params.readOnly.value
- try {
- const { startVersion, endVersion, endTimestamp } =
- await chunkStore.getLatestChunkMetadata(projectId, { readOnly })
- res.json({
- startVersion,
- endVersion,
- endTimestamp,
- })
- } catch (err) {
- if (err instanceof Chunk.NotFoundError) {
- render.notFound(res)
- } else {
- throw err
- }
- }
-}
-
async function getHistory(req, res, next) {
const projectId = req.swagger.params.project_id.value
const version = req.swagger.params.version.value
@@ -140,43 +117,6 @@ async function getHistoryBefore(req, res, next) {
}
}
-/**
- * Get all changes since the beginning of history or since a given version
- */
-async function getChanges(req, res, next) {
- const projectId = req.swagger.params.project_id.value
- const since = req.swagger.params.since.value ?? 0
-
- if (since < 0) {
- // Negative values would cause an infinite loop
- return res.status(400).json({
- error: `Version out of bounds: ${since}`,
- })
- }
-
- let chunk
- try {
- chunk = await chunkStore.loadAtVersion(projectId, since, {
- preferNewer: true,
- })
- } catch (err) {
- if (err instanceof Chunk.VersionNotFoundError) {
- return res.status(400).json({
- error: `Version out of bounds: ${since}`,
- })
- }
- throw err
- }
-
- const latestChunkMetadata = await chunkStore.getLatestChunkMetadata(projectId)
-
- // Extract the relevant changes from the chunk that contains the start version
- const changes = chunk.getChanges().slice(since - chunk.getStartVersion())
- const hasMore = latestChunkMetadata.endVersion > chunk.getEndVersion()
-
- res.json({ changes: changes.map(change => change.toRaw()), hasMore })
-}
-
async function getZip(req, res, next) {
const projectId = req.swagger.params.project_id.value
const version = req.swagger.params.version.value
@@ -227,9 +167,7 @@ async function createZip(req, res, next) {
async function deleteProject(req, res, next) {
const projectId = req.swagger.params.project_id.value
const blobStore = new BlobStore(projectId)
-
await Promise.all([
- redisBuffer.hardDeleteProject(projectId),
chunkStore.deleteProjectChunks(projectId),
blobStore.deleteBlobs(),
])
@@ -246,127 +184,44 @@ async function createProjectBlob(req, res, next) {
const sizeLimit = new StreamSizeLimit(maxUploadSize)
await pipeline(req, sizeLimit, fs.createWriteStream(tmpPath))
if (sizeLimit.sizeLimitExceeded) {
- logger.warn(
- { projectId, expectedHash, maxUploadSize },
- 'blob exceeds size threshold'
- )
return render.requestEntityTooLarge(res)
}
const hash = await blobHash.fromFile(tmpPath)
if (hash !== expectedHash) {
- logger.warn({ projectId, hash, expectedHash }, 'Hash mismatch')
+ logger.debug({ hash, expectedHash }, 'Hash mismatch')
return render.conflict(res, 'File hash mismatch')
}
const blobStore = new BlobStore(projectId)
- const newBlob = await blobStore.putFile(tmpPath)
-
- if (config.has('backupStore')) {
- try {
- const { backupBlob } = await import('../../storage/lib/backupBlob.mjs')
- await backupBlob(projectId, newBlob, tmpPath)
- } catch (error) {
- logger.warn({ error, projectId, hash }, 'Failed to backup blob')
- }
- }
+ await blobStore.putFile(tmpPath)
res.status(HTTPStatus.CREATED).end()
})
}
-async function headProjectBlob(req, res) {
- const projectId = req.swagger.params.project_id.value
- const hash = req.swagger.params.hash.value
-
- const blobStore = new BlobStore(projectId)
- const blob = await blobStore.getBlob(hash)
- if (blob) {
- res.set('Content-Length', blob.getByteLength())
- res.status(200).end()
- } else {
- res.status(404).end()
- }
-}
-
-// Support simple, singular ranges starting from zero only, up-to 2MB = 2_000_000, 7 digits
-const RANGE_HEADER = /^bytes=0-(\d{1,7})$/
-
-/**
- * @param {string} header
- * @return {{}|{start: number, end: number}}
- * @private
- */
-function _getRangeOpts(header) {
- if (!header) return {}
- const match = header.match(RANGE_HEADER)
- if (match) {
- const end = parseInt(match[1], 10)
- return { start: 0, end }
- }
- return {}
-}
-
async function getProjectBlob(req, res, next) {
const projectId = req.swagger.params.project_id.value
const hash = req.swagger.params.hash.value
- const opts = _getRangeOpts(req.swagger.params.range.value || '')
const blobStore = new BlobStore(projectId)
logger.debug({ projectId, hash }, 'getProjectBlob started')
try {
let stream
try {
- stream = await blobStore.getStream(hash, opts)
+ stream = await blobStore.getStream(hash)
} catch (err) {
if (err instanceof Blob.NotFoundError) {
- logger.warn({ projectId, hash }, 'Blob not found')
- return res.status(404).end()
+ return render.notFound(res)
} else {
throw err
}
}
res.set('Content-Type', 'application/octet-stream')
- try {
- await pipeline(stream, res)
- } catch (err) {
- if (err?.code === 'ERR_STREAM_PREMATURE_CLOSE') {
- res.end()
- } else {
- throw OError.tag(err, 'error transferring stream', { projectId, hash })
- }
- }
+ await pipeline(stream, res)
} finally {
logger.debug({ projectId, hash }, 'getProjectBlob finished')
}
}
-async function copyProjectBlob(req, res, next) {
- const sourceProjectId = req.swagger.params.copyFrom.value
- const targetProjectId = req.swagger.params.project_id.value
- const blobHash = req.swagger.params.hash.value
- // Check that blob exists in source project
- const sourceBlobStore = new BlobStore(sourceProjectId)
- const targetBlobStore = new BlobStore(targetProjectId)
- const [sourceBlob, targetBlob] = await Promise.all([
- sourceBlobStore.getBlob(blobHash),
- targetBlobStore.getBlob(blobHash),
- ])
- if (!sourceBlob) {
- logger.warn(
- { sourceProjectId, targetProjectId, blobHash },
- 'missing source blob when copying across projects'
- )
- return render.notFound(res)
- }
- // Exit early if the blob exists in the target project.
- // This will also catch global blobs, which always exist.
- if (targetBlob) {
- return res.status(HTTPStatus.NO_CONTENT).end()
- }
- // Otherwise, copy blob from source project to target project
- await sourceBlobStore.copyBlob(sourceBlob, targetProjectId)
- res.status(HTTPStatus.CREATED).end()
-}
-
async function getSnapshotAtVersion(projectId, version) {
const chunk = await chunkStore.loadAtVersion(projectId, version)
const snapshot = chunk.getSnapshot()
@@ -385,15 +240,11 @@ module.exports = {
getLatestHashedContent: expressify(getLatestHashedContent),
getLatestPersistedHistory: expressify(getLatestHistory),
getLatestHistory: expressify(getLatestHistory),
- getLatestHistoryRaw: expressify(getLatestHistoryRaw),
getHistory: expressify(getHistory),
getHistoryBefore: expressify(getHistoryBefore),
- getChanges: expressify(getChanges),
getZip: expressify(getZip),
createZip: expressify(createZip),
deleteProject: expressify(deleteProject),
createProjectBlob: expressify(createProjectBlob),
getProjectBlob: expressify(getProjectBlob),
- headProjectBlob: expressify(headProjectBlob),
- copyProjectBlob: expressify(copyProjectBlob),
}
diff --git a/services/history-v1/api/controllers/stream_size_limit.js b/services/history-v1/api/controllers/stream_size_limit.js
index f3a14959f6..fbb2ab8030 100644
--- a/services/history-v1/api/controllers/stream_size_limit.js
+++ b/services/history-v1/api/controllers/stream_size_limit.js
@@ -1,4 +1,4 @@
-const stream = require('node:stream')
+const stream = require('stream')
/**
* Transform stream that stops passing bytes through after some threshold has
diff --git a/services/history-v1/api/controllers/with_tmp_dir.js b/services/history-v1/api/controllers/with_tmp_dir.js
index 2e0737ba69..ab2279ce17 100644
--- a/services/history-v1/api/controllers/with_tmp_dir.js
+++ b/services/history-v1/api/controllers/with_tmp_dir.js
@@ -1,15 +1,15 @@
-const fs = require('node:fs')
+const fs = require('fs')
const fsExtra = require('fs-extra')
const logger = require('@overleaf/logger')
-const os = require('node:os')
-const path = require('node:path')
+const os = require('os')
+const path = require('path')
/**
* Create a temporary directory before executing a function and cleaning up
* after.
*
* @param {string} prefix - prefix for the temporary directory name
- * @param {(tmpDir: string) => Promise} fn - async function to call
+ * @param {Function} fn - async function to call
*/
async function withTmpDir(prefix, fn) {
const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), prefix))
diff --git a/services/history-v1/api/swagger/index.js b/services/history-v1/api/swagger/index.js
index 3702c6ec07..edfb68f4e4 100644
--- a/services/history-v1/api/swagger/index.js
+++ b/services/history-v1/api/swagger/index.js
@@ -84,19 +84,6 @@ module.exports = {
},
},
},
- ChunkResponseRaw: {
- properties: {
- startVersion: {
- type: 'number',
- },
- endVersion: {
- type: 'number',
- },
- endTimestamp: {
- type: 'string',
- },
- },
- },
History: {
properties: {
snapshot: {
diff --git a/services/history-v1/api/swagger/project_import.js b/services/history-v1/api/swagger/project_import.js
index 043dc70667..60eb47fce4 100644
--- a/services/history-v1/api/swagger/project_import.js
+++ b/services/history-v1/api/swagger/project_import.js
@@ -100,120 +100,9 @@ const importChanges = {
],
}
-const getChanges = {
- 'x-swagger-router-controller': 'projects',
- operationId: 'getChanges',
- tags: ['Project'],
- description: 'Get changes applied to a project',
- parameters: [
- {
- name: 'project_id',
- in: 'path',
- description: 'project id',
- required: true,
- type: 'string',
- },
- {
- name: 'since',
- in: 'query',
- description: 'start version',
- required: false,
- type: 'number',
- },
- ],
- responses: {
- 200: {
- description: 'Success',
- schema: {
- type: 'array',
- items: {
- $ref: '#/definitions/Change',
- },
- },
- },
- },
- security: [
- {
- basic: [],
- },
- ],
-}
-
-const flushChanges = {
- 'x-swagger-router-controller': 'project_import',
- operationId: 'flushChanges',
- tags: ['ProjectImport'],
- description: 'Flush project changes from buffer to the chunk store.',
- parameters: [
- {
- name: 'project_id',
- in: 'path',
- description: 'project id',
- required: true,
- type: 'string',
- },
- ],
- responses: {
- 200: {
- description: 'Success',
- schema: {
- $ref: '#/definitions/Project',
- },
- },
- 404: {
- description: 'Not Found',
- schema: {
- $ref: '#/definitions/Error',
- },
- },
- },
- security: [
- {
- basic: [],
- },
- ],
-}
-
-const expireProject = {
- 'x-swagger-router-controller': 'project_import',
- operationId: 'expireProject',
- tags: ['ProjectImport'],
- description: 'Expire project changes from buffer.',
- parameters: [
- {
- name: 'project_id',
- in: 'path',
- description: 'project id',
- required: true,
- type: 'string',
- },
- ],
- responses: {
- 200: {
- description: 'Success',
- schema: {
- $ref: '#/definitions/Project',
- },
- },
- 404: {
- description: 'Not Found',
- schema: {
- $ref: '#/definitions/Error',
- },
- },
- },
- security: [
- {
- basic: [],
- },
- ],
-}
-
exports.paths = {
'/projects/{project_id}/import': { post: importSnapshot },
'/projects/{project_id}/legacy_import': { post: importSnapshot },
- '/projects/{project_id}/changes': { get: getChanges, post: importChanges },
+ '/projects/{project_id}/changes': { post: importChanges },
'/projects/{project_id}/legacy_changes': { post: importChanges },
- '/projects/{project_id}/flush': { post: flushChanges },
- '/projects/{project_id}/expire': { post: expireProject },
}
diff --git a/services/history-v1/api/swagger/projects.js b/services/history-v1/api/swagger/projects.js
index cd4d2338fa..f4b8ed7352 100644
--- a/services/history-v1/api/swagger/projects.js
+++ b/services/history-v1/api/swagger/projects.js
@@ -9,7 +9,6 @@ exports.paths = {
operationId: 'initializeProject',
tags: ['Project'],
description: 'Initialize project.',
- consumes: ['application/json'],
parameters: [
{
name: 'body',
@@ -70,52 +69,6 @@ exports.paths = {
operationId: 'getProjectBlob',
tags: ['Project'],
description: 'Fetch blob content by its project id and hash.',
- parameters: [
- {
- name: 'project_id',
- in: 'path',
- description: 'project id',
- required: true,
- type: 'string',
- },
- {
- name: 'hash',
- in: 'path',
- description: 'Hexadecimal SHA-1 hash',
- required: true,
- type: 'string',
- pattern: Blob.HEX_HASH_RX_STRING,
- },
- {
- name: 'range',
- in: 'header',
- description: 'HTTP Range header',
- required: false,
- type: 'string',
- },
- ],
- produces: ['application/octet-stream'],
- responses: {
- 200: {
- description: 'Success',
- schema: {
- type: 'file',
- },
- },
- 404: {
- description: 'Not Found',
- schema: {
- $ref: '#/definitions/Error',
- },
- },
- },
- security: [{ jwt: [] }, { token: [] }],
- },
- head: {
- 'x-swagger-router-controller': 'projects',
- operationId: 'headProjectBlob',
- tags: ['Project'],
- description: 'Fetch blob content-length by its project id and hash.',
parameters: [
{
name: 'project_id',
@@ -180,42 +133,6 @@ exports.paths = {
},
},
},
- post: {
- 'x-swagger-router-controller': 'projects',
- operationId: 'copyProjectBlob',
- tags: ['Project'],
- description:
- 'Copies a blob from a source project to a target project when duplicating a project',
- parameters: [
- {
- name: 'project_id',
- in: 'path',
- description: 'target project id',
- required: true,
- type: 'string',
- },
- {
- name: 'hash',
- in: 'path',
- description: 'Hexadecimal SHA-1 hash',
- required: true,
- type: 'string',
- pattern: Blob.HEX_HASH_RX_STRING,
- },
- {
- name: 'copyFrom',
- in: 'query',
- description: 'source project id',
- required: true,
- type: 'string',
- },
- ],
- responses: {
- 201: {
- description: 'Created',
- },
- },
- },
},
'/projects/{project_id}/latest/content': {
get: {
@@ -321,44 +238,6 @@ exports.paths = {
},
},
},
- '/projects/{project_id}/latest/history/raw': {
- get: {
- 'x-swagger-router-controller': 'projects',
- operationId: 'getLatestHistoryRaw',
- tags: ['Project'],
- description: 'Get the metadata of latest sequence of changes.',
- parameters: [
- {
- name: 'project_id',
- in: 'path',
- description: 'project id',
- required: true,
- type: 'string',
- },
- {
- name: 'readOnly',
- in: 'query',
- description: 'use read only database connection',
- required: false,
- type: 'boolean',
- },
- ],
- responses: {
- 200: {
- description: 'Success',
- schema: {
- $ref: '#/definitions/ChunkResponseRaw',
- },
- },
- 404: {
- description: 'Not Found',
- schema: {
- $ref: '#/definitions/Error',
- },
- },
- },
- },
- },
'/projects/{project_id}/latest/persistedHistory': {
get: {
'x-swagger-router-controller': 'projects',
diff --git a/services/history-v1/app.js b/services/history-v1/app.js
index dd991c1a6d..a10d69eba6 100644
--- a/services/history-v1/app.js
+++ b/services/history-v1/app.js
@@ -6,7 +6,7 @@
require('@overleaf/metrics/initialize')
const config = require('config')
-const Events = require('node:events')
+const Events = require('events')
const BPromise = require('bluebird')
const express = require('express')
const helmet = require('helmet')
@@ -19,7 +19,7 @@ const swaggerDoc = require('./api/swagger')
const security = require('./api/app/security')
const healthChecks = require('./api/controllers/health_checks')
const { mongodb, loadGlobalBlobs } = require('./storage')
-const path = require('node:path')
+const path = require('path')
Events.setMaxListeners(20)
const app = express()
@@ -84,29 +84,26 @@ function setupErrorHandling() {
// Handle Swagger errors.
app.use(function (err, req, res, next) {
- const projectId = req.swagger?.params?.project_id?.value
if (res.headersSent) {
return next(err)
}
if (err.code === 'SCHEMA_VALIDATION_FAILED') {
- logger.error({ err, projectId }, err.message)
+ logger.error(err)
return res.status(HTTPStatus.UNPROCESSABLE_ENTITY).json(err.results)
}
if (err.code === 'INVALID_TYPE' || err.code === 'PATTERN') {
- logger.error({ err, projectId }, err.message)
+ logger.error(err)
return res.status(HTTPStatus.UNPROCESSABLE_ENTITY).json({
message: 'invalid type: ' + err.paramName,
})
}
if (err.code === 'ENUM_MISMATCH') {
- logger.warn({ err, projectId }, err.message)
return res.status(HTTPStatus.UNPROCESSABLE_ENTITY).json({
message: 'invalid enum value: ' + err.paramName,
})
}
if (err.code === 'REQUIRED') {
- logger.warn({ err, projectId }, err.message)
return res.status(HTTPStatus.UNPROCESSABLE_ENTITY).json({
message: err.message,
})
@@ -115,8 +112,7 @@ function setupErrorHandling() {
})
app.use(function (err, req, res, next) {
- const projectId = req.swagger?.params?.project_id?.value
- logger.error({ err, projectId }, err.message)
+ logger.error(err)
if (res.headersSent) {
return next(err)
diff --git a/services/history-v1/backup-deletion-app.mjs b/services/history-v1/backup-deletion-app.mjs
deleted file mode 100644
index 81b2b5b8b9..0000000000
--- a/services/history-v1/backup-deletion-app.mjs
+++ /dev/null
@@ -1,81 +0,0 @@
-// @ts-check
-// Metrics must be initialized before importing anything else
-import '@overleaf/metrics/initialize.js'
-import http from 'node:http'
-import { fileURLToPath } from 'node:url'
-import { promisify } from 'node:util'
-import express from 'express'
-import logger from '@overleaf/logger'
-import Metrics from '@overleaf/metrics'
-import { hasValidBasicAuthCredentials } from './api/app/security.js'
-import {
- deleteProjectBackupCb,
- healthCheck,
- healthCheckCb,
- NotReadyToDelete,
-} from './storage/lib/backupDeletion.mjs'
-import { mongodb } from './storage/index.js'
-
-const app = express()
-
-logger.initialize('history-v1-backup-deletion')
-Metrics.open_sockets.monitor()
-Metrics.injectMetricsRoute(app)
-app.use(Metrics.http.monitor(logger))
-Metrics.leaked_sockets.monitor(logger)
-Metrics.event_loop.monitor(logger)
-Metrics.memory.monitor(logger)
-
-function basicAuth(req, res, next) {
- if (hasValidBasicAuthCredentials(req)) return next()
- res.setHeader('WWW-Authenticate', 'Basic realm="Application"')
- res.sendStatus(401)
-}
-
-app.delete('/project/:projectId/backup', basicAuth, (req, res, next) => {
- deleteProjectBackupCb(req.params.projectId, err => {
- if (err) {
- return next(err)
- }
- res.sendStatus(204)
- })
-})
-
-app.get('/status', (req, res) => {
- res.send('history-v1-backup-deletion is up')
-})
-
-app.get('/health_check', (req, res, next) => {
- healthCheckCb(err => {
- if (err) return next(err)
- res.sendStatus(200)
- })
-})
-
-app.use((err, req, res, next) => {
- req.logger.addFields({ err })
- if (err instanceof NotReadyToDelete) {
- req.logger.setLevel('warn')
- return res.status(422).send(err.message)
- }
- req.logger.setLevel('error')
- next(err)
-})
-
-/**
- * @param {number} port
- * @return {Promise}
- */
-export async function startApp(port) {
- await mongodb.client.connect()
- await healthCheck()
- const server = http.createServer(app)
- await promisify(server.listen.bind(server, port))()
- return server
-}
-
-// Run this if we're called directly
-if (process.argv[1] === fileURLToPath(import.meta.url)) {
- const PORT = parseInt(process.env.PORT || '3101', 10)
- await startApp(PORT)
-}
diff --git a/services/history-v1/backup-verifier-app.mjs b/services/history-v1/backup-verifier-app.mjs
deleted file mode 100644
index 856a15dd53..0000000000
--- a/services/history-v1/backup-verifier-app.mjs
+++ /dev/null
@@ -1,117 +0,0 @@
-// @ts-check
-// Metrics must be initialized before importing anything else
-import '@overleaf/metrics/initialize.js'
-import http from 'node:http'
-import { fileURLToPath } from 'node:url'
-import { promisify } from 'node:util'
-import { setTimeout } from 'node:timers/promises'
-import express from 'express'
-import logger from '@overleaf/logger'
-import Metrics from '@overleaf/metrics'
-import { healthCheck } from './backupVerifier/healthCheck.mjs'
-import {
- BackupCorruptedError,
- verifyBlob,
-} from './storage/lib/backupVerifier.mjs'
-import { mongodb } from './storage/index.js'
-import { expressify } from '@overleaf/promise-utils'
-import { Blob } from 'overleaf-editor-core'
-import { loadGlobalBlobs } from './storage/lib/blob_store/index.js'
-import { EventEmitter } from 'node:events'
-import {
- loopRandomProjects,
- setWriteMetrics,
-} from './backupVerifier/ProjectVerifier.mjs'
-
-const app = express()
-
-logger.initialize('history-v1-backup-verifier')
-Metrics.open_sockets.monitor()
-Metrics.injectMetricsRoute(app)
-app.use(Metrics.http.monitor(logger))
-Metrics.leaked_sockets.monitor(logger)
-Metrics.event_loop.monitor(logger)
-Metrics.memory.monitor(logger)
-
-app.get(
- '/history/:historyId/blob/:hash/verify',
- expressify(async (req, res) => {
- const { historyId, hash } = req.params
- try {
- await verifyBlob(historyId, hash)
- res.sendStatus(200)
- } catch (err) {
- logger.warn({ err, historyId, hash }, 'manual verify blob failed')
- if (err instanceof Blob.NotFoundError) {
- res.status(404).send(err.message)
- } else if (err instanceof BackupCorruptedError) {
- res.status(422).send(err.message)
- } else {
- throw err
- }
- }
- })
-)
-
-app.get('/status', (req, res) => {
- res.send('history-v1-backup-verifier is up')
-})
-
-app.get(
- '/health_check',
- expressify(async (req, res) => {
- await healthCheck()
- res.sendStatus(200)
- })
-)
-
-app.use((err, req, res, next) => {
- req.logger.addFields({ err })
- req.logger.setLevel('error')
- next(err)
-})
-
-const shutdownEmitter = new EventEmitter()
-
-shutdownEmitter.once('shutdown', async code => {
- logger.info({ code }, 'shutting down')
- await mongodb.client.close()
- await setTimeout(100)
- process.exit(code)
-})
-
-process.on('SIGTERM', () => {
- shutdownEmitter.emit('shutdown', 0)
-})
-
-process.on('SIGINT', () => {
- shutdownEmitter.emit('shutdown', 0)
-})
-
-/**
- * @param {number} port
- * @param {boolean} enableVerificationLoop
- * @return {Promise}
- */
-export async function startApp(port, enableVerificationLoop = true) {
- await mongodb.client.connect()
- await loadGlobalBlobs()
- await healthCheck()
- const server = http.createServer(app)
- await promisify(server.listen.bind(server, port))()
- enableVerificationLoop && loopRandomProjects(shutdownEmitter)
- return server
-}
-
-setWriteMetrics(true)
-
-// Run this if we're called directly
-if (process.argv[1] === fileURLToPath(import.meta.url)) {
- const PORT = parseInt(process.env.PORT || '3102', 10)
- try {
- await startApp(PORT)
- } catch (error) {
- shutdownEmitter.emit('shutdown', 1)
- logger.error({ error }, 'error starting app')
- }
-}
diff --git a/services/history-v1/backup-worker-app.mjs b/services/history-v1/backup-worker-app.mjs
deleted file mode 100644
index b21e55aafe..0000000000
--- a/services/history-v1/backup-worker-app.mjs
+++ /dev/null
@@ -1,70 +0,0 @@
-// @ts-check
-// Metrics must be initialized before importing anything else
-import '@overleaf/metrics/initialize.js'
-import http from 'node:http'
-import { fileURLToPath } from 'node:url'
-import { promisify } from 'node:util'
-import express from 'express'
-import logger from '@overleaf/logger'
-import Metrics from '@overleaf/metrics'
-import { expressify } from '@overleaf/promise-utils'
-import { drainQueue, healthCheck } from './storage/scripts/backup_worker.mjs'
-const app = express()
-
-logger.initialize('history-v1-backup-worker')
-Metrics.open_sockets.monitor()
-Metrics.injectMetricsRoute(app)
-app.use(Metrics.http.monitor(logger))
-Metrics.leaked_sockets.monitor(logger)
-Metrics.event_loop.monitor(logger)
-Metrics.memory.monitor(logger)
-
-app.get('/status', (req, res) => {
- res.send('history-v1-backup-worker is up')
-})
-
-app.get(
- '/health_check',
- expressify(async (req, res) => {
- await healthCheck()
- res.sendStatus(200)
- })
-)
-
-app.use((err, req, res, next) => {
- req.logger.addFields({ err })
- req.logger.setLevel('error')
- next(err)
-})
-
-async function triggerGracefulShutdown(server, signal) {
- logger.info({ signal }, 'graceful shutdown: started shutdown sequence')
- await drainQueue()
- server.close(function () {
- logger.info({ signal }, 'graceful shutdown: closed server')
- setTimeout(() => {
- process.exit(0)
- }, 1000)
- })
-}
-
-/**
- * @param {number} port
- * @return {Promise}
- */
-export async function startApp(port) {
- await healthCheck()
- const server = http.createServer(app)
- await promisify(server.listen.bind(server, port))()
- const signals = ['SIGINT', 'SIGTERM']
- signals.forEach(signal => {
- process.on(signal, () => triggerGracefulShutdown(server, signal))
- })
- return server
-}
-
-// Run this if we're called directly
-if (process.argv[1] === fileURLToPath(import.meta.url)) {
- const PORT = parseInt(process.env.PORT || '3103', 10)
- await startApp(PORT)
-}
diff --git a/services/history-v1/backupVerifier/ProjectMetrics.mjs b/services/history-v1/backupVerifier/ProjectMetrics.mjs
deleted file mode 100644
index ff37085787..0000000000
--- a/services/history-v1/backupVerifier/ProjectMetrics.mjs
+++ /dev/null
@@ -1,33 +0,0 @@
-import Metrics from '@overleaf/metrics'
-import { objectIdFromDate } from './utils.mjs'
-import { db } from '../storage/lib/mongodb.js'
-
-const projectsCollection = db.collection('projects')
-
-/**
- *
- * @param {Date} beforeTime
- * @return {Promise}
- */
-export async function measurePendingChangesBeforeTime(beforeTime) {
- const pendingChangeCount = await projectsCollection.countDocuments({
- 'overleaf.backup.pendingChangeAt': {
- $lt: beforeTime,
- },
- })
-
- Metrics.gauge('backup_verification_pending_changes', pendingChangeCount)
-}
-
-/**
- *
- * @param {Date} graceTime
- * @return {Promise}
- */
-export async function measureNeverBackedUpProjects(graceTime) {
- const neverBackedUpCount = await projectsCollection.countDocuments({
- 'overleaf.backup.lastBackedUpVersion': null,
- _id: { $lt: objectIdFromDate(graceTime) },
- })
- Metrics.gauge('backup_verification_never_backed_up', neverBackedUpCount)
-}
diff --git a/services/history-v1/backupVerifier/ProjectSampler.mjs b/services/history-v1/backupVerifier/ProjectSampler.mjs
deleted file mode 100644
index 93d9a1a31f..0000000000
--- a/services/history-v1/backupVerifier/ProjectSampler.mjs
+++ /dev/null
@@ -1,79 +0,0 @@
-// @ts-check
-import { objectIdFromDate } from './utils.mjs'
-import { db } from '../storage/lib/mongodb.js'
-import config from 'config'
-
-const projectsCollection = db.collection('projects')
-
-const HAS_PROJECTS_WITHOUT_HISTORY =
- config.get('hasProjectsWithoutHistory') === 'true'
-
-/**
- * @param {Date} start
- * @param {Date} end
- * @param {number} N
- * @yields {string}
- */
-export async function* getProjectsCreatedInDateRangeCursor(start, end, N) {
- yield* getSampleProjectsCursor(N, [
- {
- $match: {
- _id: {
- $gt: objectIdFromDate(start),
- $lte: objectIdFromDate(end),
- },
- },
- },
- ])
-}
-
-export async function* getProjectsUpdatedInDateRangeCursor(start, end, N) {
- yield* getSampleProjectsCursor(N, [
- {
- $match: {
- 'overleaf.history.updatedAt': {
- $gt: start,
- $lte: end,
- },
- },
- },
- ])
-}
-
-/**
- * @typedef {import('mongodb').Document} Document
- */
-
-/**
- *
- * @generator
- * @param {number} N
- * @param {Array} preSampleAggregationStages
- * @yields {string}
- */
-export async function* getSampleProjectsCursor(
- N,
- preSampleAggregationStages = []
-) {
- const cursor = projectsCollection.aggregate([
- ...preSampleAggregationStages,
- { $sample: { size: N } },
- { $project: { 'overleaf.history.id': 1 } },
- ])
-
- let validProjects = 0
- let hasInvalidProject = false
-
- for await (const project of cursor) {
- if (HAS_PROJECTS_WITHOUT_HISTORY && !project.overleaf?.history?.id) {
- hasInvalidProject = true
- continue
- }
- validProjects++
- yield project.overleaf.history.id.toString()
- }
-
- if (validProjects === 0 && hasInvalidProject) {
- yield* getSampleProjectsCursor(N, preSampleAggregationStages)
- }
-}
diff --git a/services/history-v1/backupVerifier/ProjectVerifier.mjs b/services/history-v1/backupVerifier/ProjectVerifier.mjs
deleted file mode 100644
index 1e4086b700..0000000000
--- a/services/history-v1/backupVerifier/ProjectVerifier.mjs
+++ /dev/null
@@ -1,320 +0,0 @@
-// @ts-check
-import { verifyProjectWithErrorContext } from '../storage/lib/backupVerifier.mjs'
-import { promiseMapSettledWithLimit } from '@overleaf/promise-utils'
-import logger from '@overleaf/logger'
-import metrics from '@overleaf/metrics'
-import {
- getSampleProjectsCursor,
- getProjectsCreatedInDateRangeCursor,
- getProjectsUpdatedInDateRangeCursor,
-} from './ProjectSampler.mjs'
-import OError from '@overleaf/o-error'
-import { setTimeout } from 'node:timers/promises'
-
-const MS_PER_30_DAYS = 30 * 24 * 60 * 60 * 1000
-
-const failureCounter = new metrics.prom.Counter({
- name: 'backup_project_verification_failed',
- help: 'Number of projects that failed verification',
- labelNames: ['name'],
-})
-
-const successCounter = new metrics.prom.Counter({
- name: 'backup_project_verification_succeeded',
- help: 'Number of projects that succeeded verification',
-})
-
-let WRITE_METRICS = false
-
-/**
- * @typedef {import('node:events').EventEmitter} EventEmitter
- */
-
-/**
- * Allows writing metrics to be enabled or disabled.
- * @param {Boolean} writeMetrics
- */
-export function setWriteMetrics(writeMetrics) {
- WRITE_METRICS = writeMetrics
-}
-
-/**
- *
- * @param {Error|unknown} error
- * @param {string} historyId
- */
-function handleVerificationError(error, historyId) {
- const name = error instanceof Error ? error.name : 'UnknownError'
- logger.error({ historyId, error, name }, 'error verifying project backup')
-
- WRITE_METRICS && failureCounter.inc({ name })
-
- return name
-}
-
-/**
- *
- * @param {Date} startDate
- * @param {Date} endDate
- * @param {number} interval
- * @returns {Array}
- */
-function splitJobs(startDate, endDate, interval) {
- /** @type {Array} */
- const jobs = []
- while (startDate < endDate) {
- const nextStart = new Date(
- Math.min(startDate.getTime() + interval, endDate.getTime())
- )
- jobs.push({ startDate, endDate: nextStart })
- startDate = nextStart
- }
- return jobs
-}
-
-/**
- *
- * @param {AsyncGenerator} historyIdCursor
- * @param {EventEmitter} [eventEmitter]
- * @param {number} [delay] - Allows a delay between each verification
- * @return {Promise<{verified: number, total: number, errorTypes: *[], hasFailure: boolean}>}
- */
-async function verifyProjectsFromCursor(
- historyIdCursor,
- eventEmitter,
- delay = 0
-) {
- const errorTypes = []
- let verified = 0
- let total = 0
- let receivedShutdownSignal = false
- if (eventEmitter) {
- eventEmitter.once('shutdown', () => {
- receivedShutdownSignal = true
- })
- }
- for await (const historyId of historyIdCursor) {
- if (receivedShutdownSignal) {
- break
- }
- total++
- try {
- await verifyProjectWithErrorContext(historyId)
- logger.debug({ historyId }, 'verified project backup successfully')
- WRITE_METRICS && successCounter.inc()
- verified++
- } catch (error) {
- const errorType = handleVerificationError(error, historyId)
- errorTypes.push(errorType)
- }
- if (delay > 0) {
- await setTimeout(delay)
- }
- }
- return {
- verified,
- total,
- errorTypes,
- hasFailure: errorTypes.length > 0,
- }
-}
-
-/**
- *
- * @param {number} nProjectsToSample
- * @param {EventEmitter} [signal]
- * @param {number} [delay]
- * @return {Promise}
- */
-export async function verifyRandomProjectSample(
- nProjectsToSample,
- signal,
- delay = 0
-) {
- const historyIds = await getSampleProjectsCursor(nProjectsToSample)
- return await verifyProjectsFromCursor(historyIds, signal, delay)
-}
-
-/**
- * Samples projects with history IDs between the specified dates and verifies them.
- *
- * @param {Date} startDate
- * @param {Date} endDate
- * @param {number} projectsPerRange
- * @param {EventEmitter} [signal]
- * @return {Promise}
- */
-async function verifyRange(startDate, endDate, projectsPerRange, signal) {
- logger.info({ startDate, endDate }, 'verifying range')
-
- const results = await verifyProjectsFromCursor(
- getProjectsCreatedInDateRangeCursor(startDate, endDate, projectsPerRange),
- signal
- )
-
- if (results.total === 0) {
- logger.debug(
- { start: startDate, end: endDate },
- 'No projects found in range'
- )
- }
-
- const jobStatus = {
- ...results,
- startDate,
- endDate,
- }
-
- logger.debug(
- { ...jobStatus, errorTypes: Array.from(new Set(jobStatus.errorTypes)) },
- 'Verified range'
- )
- return jobStatus
-}
-
-/**
- * @typedef {Object} VerificationJobSpecification
- * @property {Date} startDate
- * @property {Date} endDate
- */
-
-/**
- * @typedef {import('./types.d.ts').VerificationJobStatus} VerificationJobStatus
- */
-
-/**
- * @typedef {Object} VerifyDateRangeOptions
- * @property {Date} startDate
- * @property {Date} endDate
- * @property {number} [interval]
- * @property {number} [projectsPerRange]
- * @property {number} [concurrency]
- * @property {EventEmitter} [signal]
- */
-
-/**
- *
- * @param {VerifyDateRangeOptions} options
- * @return {Promise}
- */
-export async function verifyProjectsCreatedInDateRange({
- concurrency = 0,
- projectsPerRange = 10,
- startDate,
- endDate,
- interval = MS_PER_30_DAYS,
- signal,
-}) {
- const jobs = splitJobs(startDate, endDate, interval)
- if (jobs.length === 0) {
- throw new OError('Time range could not be split into jobs', {
- start: startDate,
- end: endDate,
- interval,
- })
- }
- const settlements = await promiseMapSettledWithLimit(
- concurrency,
- jobs,
- ({ startDate, endDate }) =>
- verifyRange(startDate, endDate, projectsPerRange, signal)
- )
- return settlements.reduce(
- /**
- *
- * @param {VerificationJobStatus} acc
- * @param settlement
- * @return {VerificationJobStatus}
- */
- (acc, settlement) => {
- if (settlement.status !== 'rejected') {
- if (settlement.value.hasFailure) {
- acc.hasFailure = true
- }
- acc.total += settlement.value.total
- acc.verified += settlement.value.verified
- acc.errorTypes = acc.errorTypes.concat(settlement.value.errorTypes)
- } else {
- logger.error({ ...settlement.reason }, 'Error processing range')
- }
- return acc
- },
- /** @type {VerificationJobStatus} */
- {
- startDate,
- endDate,
- verified: 0,
- total: 0,
- hasFailure: false,
- errorTypes: [],
- }
- )
-}
-
-/**
- * Verifies that projects that have recently gone out of RPO have been updated.
- *
- * @param {Date} startDate
- * @param {Date} endDate
- * @param {number} nProjects
- * @param {EventEmitter} [signal]
- * @return {Promise}
- */
-export async function verifyProjectsUpdatedInDateRange(
- startDate,
- endDate,
- nProjects,
- signal
-) {
- logger.debug(
- { startDate, endDate, nProjects },
- 'Sampling projects updated in date range'
- )
- const results = await verifyProjectsFromCursor(
- getProjectsUpdatedInDateRangeCursor(startDate, endDate, nProjects),
- signal
- )
-
- if (results.total === 0) {
- logger.debug(
- { start: startDate, end: endDate },
- 'No projects updated recently'
- )
- }
-
- const jobStatus = {
- ...results,
- startDate,
- endDate,
- }
-
- logger.debug(
- { ...jobStatus, errorTypes: Array.from(new Set(jobStatus.errorTypes)) },
- 'Verified recently updated projects'
- )
- return jobStatus
-}
-
-/**
- *
- * @param {EventEmitter} signal
- * @return {void}
- */
-export function loopRandomProjects(signal) {
- let shutdown = false
- signal.on('shutdown', function () {
- shutdown = true
- })
- async function loop() {
- do {
- try {
- const result = await verifyRandomProjectSample(100, signal, 2_000)
- logger.debug({ result }, 'verified random project sample')
- } catch (error) {
- logger.error({ error }, 'error verifying random project sample')
- }
- // eslint-disable-next-line no-unmodified-loop-condition
- } while (!shutdown)
- }
- loop()
-}
diff --git a/services/history-v1/backupVerifier/healthCheck.mjs b/services/history-v1/backupVerifier/healthCheck.mjs
deleted file mode 100644
index af998748b5..0000000000
--- a/services/history-v1/backupVerifier/healthCheck.mjs
+++ /dev/null
@@ -1,32 +0,0 @@
-import config from 'config'
-import { verifyProjectWithErrorContext } from '../storage/lib/backupVerifier.mjs'
-import {
- measureNeverBackedUpProjects,
- measurePendingChangesBeforeTime,
-} from './ProjectMetrics.mjs'
-import { getEndDateForRPO, RPO } from './utils.mjs'
-
-/** @type {Array} */
-const HEALTH_CHECK_PROJECTS = JSON.parse(config.get('healthCheckProjects'))
-
-export async function healthCheck() {
- if (!Array.isArray(HEALTH_CHECK_PROJECTS)) {
- throw new Error('expected healthCheckProjects to be an array')
- }
- if (HEALTH_CHECK_PROJECTS.length !== 2) {
- throw new Error('expected 2 healthCheckProjects')
- }
- if (!HEALTH_CHECK_PROJECTS.some(id => id.length === 24)) {
- throw new Error('expected mongo id in healthCheckProjects')
- }
- if (!HEALTH_CHECK_PROJECTS.some(id => id.length < 24)) {
- throw new Error('expected postgres id in healthCheckProjects')
- }
-
- for (const historyId of HEALTH_CHECK_PROJECTS) {
- await verifyProjectWithErrorContext(historyId)
- }
-
- await measurePendingChangesBeforeTime(getEndDateForRPO(2))
- await measureNeverBackedUpProjects(getEndDateForRPO(2))
-}
diff --git a/services/history-v1/backupVerifier/types.d.ts b/services/history-v1/backupVerifier/types.d.ts
deleted file mode 100644
index 7bfa4a85ff..0000000000
--- a/services/history-v1/backupVerifier/types.d.ts
+++ /dev/null
@@ -1,8 +0,0 @@
-export type VerificationJobStatus = {
- verified: number
- total: number
- startDate?: Date
- endDate?: Date
- hasFailure: boolean
- errorTypes: Array
-}
diff --git a/services/history-v1/backupVerifier/utils.mjs b/services/history-v1/backupVerifier/utils.mjs
deleted file mode 100644
index b2d7ed2d3c..0000000000
--- a/services/history-v1/backupVerifier/utils.mjs
+++ /dev/null
@@ -1,35 +0,0 @@
-import { ObjectId } from 'mongodb'
-import config from 'config'
-
-export const RPO = parseInt(config.get('backupRPOInMS'), 10)
-
-/**
- * @param {Date} time
- * @return {ObjectId}
- */
-export function objectIdFromDate(time) {
- return ObjectId.createFromTime(time.getTime() / 1000)
-}
-
-/**
- * @param {number} [factor] - Multiply RPO by this factor, default is 1
- * @return {Date}
- */
-export function getEndDateForRPO(factor = 1) {
- return new Date(Date.now() - RPO * factor)
-}
-
-/**
- * Creates a startDate, endDate pair that checks a period of time before the RPO horizon
- *
- * @param {number} offset - How many seconds we should check
- * @return {{endDate: Date, startDate: Date}}
- */
-export function getDatesBeforeRPO(offset) {
- const now = new Date()
- const endDate = new Date(now.getTime() - RPO)
- return {
- endDate,
- startDate: new Date(endDate.getTime() - offset * 1000),
- }
-}
diff --git a/services/history-v1/benchmarks/blob_store.js b/services/history-v1/benchmarks/blob_store.js
index 9efad8747f..2efb90ffb2 100644
--- a/services/history-v1/benchmarks/blob_store.js
+++ b/services/history-v1/benchmarks/blob_store.js
@@ -1,4 +1,4 @@
-const crypto = require('node:crypto')
+const crypto = require('crypto')
const benny = require('benny')
const { Blob } = require('overleaf-editor-core')
const mongoBackend = require('../storage/lib/blob_store/mongo')
diff --git a/services/history-v1/buildscript.txt b/services/history-v1/buildscript.txt
index 4ce8eb63c7..04430a155f 100644
--- a/services/history-v1/buildscript.txt
+++ b/services/history-v1/buildscript.txt
@@ -1,10 +1,10 @@
history-v1
---dependencies=postgres,gcs,mongo,redis,s3
+--dependencies=postgres,gcs,mongo
--docker-repos=us-east1-docker.pkg.dev/overleaf-ops/ol-docker
--env-add=
--env-pass-through=
--esmock-loader=False
---node-version=22.17.0
+--node-version=18.20.2
--public-repo=False
---script-version=4.7.0
---tsconfig-extra-includes=backup-deletion-app.mjs,backup-verifier-app.mjs,backup-worker-app.mjs,api/**/*,migrations/**/*,storage/**/*
+--script-version=4.5.0
+--tsconfig-extra-includes=api/**/*,migrations/**/*,storage/**/*
diff --git a/services/history-v1/config/custom-environment-variables.json b/services/history-v1/config/custom-environment-variables.json
index 686ca25407..1822d33aa4 100644
--- a/services/history-v1/config/custom-environment-variables.json
+++ b/services/history-v1/config/custom-environment-variables.json
@@ -1,6 +1,5 @@
{
"databaseUrl": "HISTORY_CONNECTION_STRING",
- "databaseUrlReadOnly": "HISTORY_FOLLOWER_CONNECTION_STRING",
"herokuDatabaseUrl": "DATABASE_URL",
"databasePoolMin": "DATABASE_POOL_MIN",
"databasePoolMax": "DATABASE_POOL_MAX",
@@ -9,8 +8,6 @@
"s3": {
"key": "AWS_ACCESS_KEY_ID",
"secret": "AWS_SECRET_ACCESS_KEY",
- "endpoint": "AWS_S3_ENDPOINT",
- "pathStyle": "AWS_S3_PATH_STYLE",
"maxRetries": "S3_MAX_RETRIES",
"httpOptions": {
"timeout": "S3_TIMEOUT"
@@ -33,19 +30,6 @@
"buckets": "PERSISTOR_BUCKET_MAPPING"
}
},
- "backupPersistor": {
- "keyEncryptionKeys": "BACKUP_KEY_ENCRYPTION_KEYS",
- "s3SSEC": {
- "key": "AWS_ACCESS_KEY_ID",
- "secret": "AWS_SECRET_ACCESS_KEY",
- "endpoint": "AWS_S3_ENDPOINT",
- "pathStyle": "AWS_S3_PATH_STYLE",
- "maxRetries": "BACKUP_S3_MAX_RETRIES",
- "httpOptions": {
- "timeout": "BACKUP_S3_TIMEOUT"
- }
- }
- },
"blobStore": {
"globalBucket": "OVERLEAF_EDITOR_BLOBS_BUCKET",
"projectBucket": "OVERLEAF_EDITOR_PROJECT_BLOBS_BUCKET"
@@ -58,16 +42,6 @@
"bucket": "OVERLEAF_EDITOR_ZIPS_BUCKET",
"zipTimeoutMs": "ZIP_STORE_ZIP_TIMEOUT_MS"
},
- "backupStore": {
- "chunksBucket":"BACKUP_OVERLEAF_EDITOR_CHUNKS_BUCKET",
- "deksBucket":"BACKUP_OVERLEAF_EDITOR_DEKS_BUCKET",
- "globalBlobsBucket":"BACKUP_OVERLEAF_EDITOR_GLOBAL_BLOBS_BUCKET",
- "projectBlobsBucket":"BACKUP_OVERLEAF_EDITOR_PROJECT_BLOBS_BUCKET"
- },
- "healthCheckBlobs": "HEALTH_CHECK_BLOBS",
- "healthCheckProjects": "HEALTH_CHECK_PROJECTS",
- "backupRPOInMS": "BACKUP_RPO_IN_MS",
- "minSoftDeletionPeriodDays": "MIN_SOFT_DELETION_PERIOD_DAYS",
"mongo": {
"uri": "MONGO_CONNECTION_STRING"
},
@@ -83,30 +57,5 @@
"clusterWorkers": "CLUSTER_WORKERS",
"maxFileUploadSize": "MAX_FILE_UPLOAD_SIZE",
"httpsOnly": "HTTPS_ONLY",
- "httpRequestTimeout": "HTTP_REQUEST_TIMEOUT",
- "historyBufferLevel": "HISTORY_BUFFER_LEVEL",
- "forcePersistBuffer": "FORCE_PERSIST_BUFFER",
- "nextHistoryBufferLevel": "NEXT_HISTORY_BUFFER_LEVEL",
- "nextHistoryBufferLevelRolloutPercentage": "NEXT_HISTORY_BUFFER_LEVEL_ROLLOUT_PERCENTAGE",
- "redis": {
- "queue": {
- "host": "QUEUES_REDIS_HOST",
- "password": "QUEUES_REDIS_PASSWORD",
- "port": "QUEUES_REDIS_PORT"
- },
- "history": {
- "host": "HISTORY_REDIS_HOST",
- "password": "HISTORY_REDIS_PASSWORD",
- "port": "HISTORY_REDIS_PORT"
- },
- "lock": {
- "host": "REDIS_HOST",
- "password": "REDIS_PASSWORD",
- "port": "REDIS_PORT"
- }
- },
- "projectHistory": {
- "host": "PROJECT_HISTORY_HOST",
- "port": "PROJECT_HISTORY_PORT"
- }
+ "httpRequestTimeout": "HTTP_REQUEST_TIMEOUT"
}
diff --git a/services/history-v1/config/default.json b/services/history-v1/config/default.json
index e7732fe3f7..84fd220789 100644
--- a/services/history-v1/config/default.json
+++ b/services/history-v1/config/default.json
@@ -13,25 +13,12 @@
"deleteConcurrency": "50"
}
},
- "backupPersistor": {
- "backend": "s3SSEC",
- "s3SSEC": {
- "maxRetries": "1",
- "pathStyle": false,
- "httpOptions": {
- "timeout": "120000"
- }
- }
- },
- "backupRPOInMS": "3600000",
"chunkStore": {
"historyStoreConcurrency": "4"
},
"zipStore": {
"zipTimeoutMs": "360000"
},
- "hasProjectsWithoutHistory": false,
- "minSoftDeletionPeriodDays": "90",
"maxDeleteKeys": "1000",
"useDeleteObjects": "true",
"clusterWorkers": "1",
@@ -39,8 +26,5 @@
"databasePoolMin": "2",
"databasePoolMax": "10",
"httpsOnly": "false",
- "httpRequestTimeout": "300000",
- "projectHistory": {
- "port": "3054"
- }
+ "httpRequestTimeout": "300000"
}
diff --git a/services/history-v1/config/development.json b/services/history-v1/config/development.json
index 9cd73c62c1..f1423290b5 100644
--- a/services/history-v1/config/development.json
+++ b/services/history-v1/config/development.json
@@ -23,18 +23,6 @@
"zipStore": {
"bucket": "overleaf-development-zips"
},
- "backupStore": {
- "chunksBucket":"overleaf-development-history-chunks",
- "deksBucket":"overleaf-development-history-deks",
- "globalBlobsBucket":"overleaf-development-history-global-blobs",
- "projectBlobsBucket":"overleaf-development-history-project-blobs"
- },
- "backupPersistor": {
- "keyEncryptionKeys": "[{\"key\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\",\"salt\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\"}]",
- "s3SSEC": {
- "ca": "[\"/certs/public.crt\"]"
- }
- },
"useDeleteObjects": "false",
"mongo": {
"uri": "mongodb://mongo:27017/sharelatex"
diff --git a/services/history-v1/config/production.json b/services/history-v1/config/production.json
index 23f836b1f2..ffcd4415b0 100644
--- a/services/history-v1/config/production.json
+++ b/services/history-v1/config/production.json
@@ -1,5 +1 @@
-{
- "backupPersistor": {
- "tieringStorageClass": "INTELLIGENT_TIERING"
- }
-}
+{ }
diff --git a/services/history-v1/config/test.json b/services/history-v1/config/test.json
index c38e28e564..1e4ddd3a0b 100644
--- a/services/history-v1/config/test.json
+++ b/services/history-v1/config/test.json
@@ -1,6 +1,5 @@
{
"databaseUrl": "postgres://overleaf:overleaf@postgres/overleaf-history-v1-test",
- "databaseUrlReadOnly": "postgres://read_only:password@postgres/overleaf-history-v1-test",
"persistor": {
"backend": "gcs",
"gcs": {
@@ -21,22 +20,6 @@
"zipStore": {
"bucket": "overleaf-test-zips"
},
- "backupStore": {
- "chunksBucket":"overleaf-test-history-chunks",
- "deksBucket":"overleaf-test-history-deks",
- "globalBlobsBucket":"overleaf-test-history-global-blobs",
- "projectBlobsBucket":"overleaf-test-history-project-blobs"
- },
- "backupPersistor": {
- "keyEncryptionKeys": "[{\"key\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\",\"salt\":\"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\"}]",
- "s3SSEC": {
- "ca": "[\"/certs/public.crt\"]"
- },
- "tieringStorageClass": "REDUCED_REDUNDANCY"
- },
- "healthCheckBlobs": "[\"42/f70d7bba4ae1f07682e0358bd7a2068094fc023b\",\"000000000000000000000042/98d5521fe746bc2d11761edab5d0829bee286009\"]",
- "healthCheckProjects": "[\"42\",\"000000000000000000000042\"]",
- "backupRPOInMS": "360000",
"maxDeleteKeys": "3",
"useDeleteObjects": "false",
"mongo": {
diff --git a/services/history-v1/docker-compose.ci.yml b/services/history-v1/docker-compose.ci.yml
index cf6ec3357d..4de133e440 100644
--- a/services/history-v1/docker-compose.ci.yml
+++ b/services/history-v1/docker-compose.ci.yml
@@ -19,44 +19,22 @@ services:
image: ci/$PROJECT_NAME:$BRANCH_NAME-$BUILD_NUMBER
environment:
ELASTIC_SEARCH_DSN: es:9200
- REDIS_HOST: redis
- QUEUES_REDIS_HOST: redis
- HISTORY_REDIS_HOST: redis
- ANALYTICS_QUEUES_REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
- AWS_S3_ENDPOINT: https://minio:9000
- AWS_S3_PATH_STYLE: 'true'
- AWS_ACCESS_KEY_ID: OVERLEAF_HISTORY_S3_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY: OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY
- MINIO_ROOT_USER: MINIO_ROOT_USER
- MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
GCS_API_ENDPOINT: http://gcs:9090
GCS_PROJECT_ID: fake
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
MOCHA_GREP: ${MOCHA_GREP}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
- volumes:
- - ./test/acceptance/certs:/certs
- - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
depends_on:
mongo:
- condition: service_started
- redis:
condition: service_healthy
postgres:
condition: service_healthy
- certs:
- condition: service_completed_successfully
- minio:
- condition: service_started
- minio_setup:
- condition: service_completed_successfully
gcs:
condition: service_healthy
user: node
- entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run test:acceptance
@@ -67,169 +45,24 @@ services:
- ./:/tmp/build/
command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs .
user: root
- redis:
- image: redis:7.4.3
+ mongo:
+ image: mongo:6.0.13
+ command: --replSet overleaf
healthcheck:
- test: ping="$$(redis-cli ping)" && [ "$$ping" = 'PONG' ]
+ test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
interval: 1s
retries: 20
-
- mongo:
- image: mongo:8.0.11
- command: --replSet overleaf
- volumes:
- - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
- environment:
- MONGO_INITDB_DATABASE: sharelatex
- extra_hosts:
- # Required when using the automatic database setup for initializing the
- # replica set. This override is not needed when running the setup after
- # starting up mongo.
- - mongo:127.0.0.1
postgres:
image: postgres:10
environment:
POSTGRES_USER: overleaf
POSTGRES_PASSWORD: overleaf
POSTGRES_DB: overleaf-history-v1-test
- volumes:
- - ./test/acceptance/pg-init/:/docker-entrypoint-initdb.d/
healthcheck:
test: pg_isready --quiet
interval: 1s
retries: 20
- certs:
- image: node:22.17.0
- volumes:
- - ./test/acceptance/certs:/certs
- working_dir: /certs
- entrypoint: sh
- command:
- - '-cex'
- - |
- if [ ! -f ./certgen ]; then
- wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)"
- chmod +x ./certgen
- fi
- if [ ! -f private.key ] || [ ! -f public.crt ]; then
- ./certgen -host minio
- fi
-
- minio:
- image: minio/minio:RELEASE.2024-10-13T13-34-11Z
- command: server /data
- volumes:
- - ./test/acceptance/certs:/root/.minio/certs
- environment:
- MINIO_ROOT_USER: MINIO_ROOT_USER
- MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
- depends_on:
- certs:
- condition: service_completed_successfully
-
- minio_setup:
- depends_on:
- certs:
- condition: service_completed_successfully
- minio:
- condition: service_started
- image: minio/mc:RELEASE.2024-10-08T09-37-26Z
- volumes:
- - ./test/acceptance/certs:/root/.mc/certs/CAs
- entrypoint: sh
- command:
- - '-cex'
- - |
- sleep 1
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
- || sleep 3 && \
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
- || sleep 3 && \
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
- || sleep 3 && \
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
- mc mb --ignore-existing s3/overleaf-test-history-chunks
- mc mb --ignore-existing s3/overleaf-test-history-deks
- mc mb --ignore-existing s3/overleaf-test-history-global-blobs
- mc mb --ignore-existing s3/overleaf-test-history-project-blobs
- mc admin user add s3 \
- OVERLEAF_HISTORY_S3_ACCESS_KEY_ID \
- OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY
- echo '
- {
- "Version": "2012-10-17",
- "Statement": [
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-chunks"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-chunks/*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-deks"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-deks/*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-global-blobs"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-global-blobs/*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-project-blobs"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-project-blobs/*"
- }
- ]
- }' > policy-history.json
-
- mc admin policy create s3 overleaf-history policy-history.json
- mc admin policy attach s3 overleaf-history \
- --user=OVERLEAF_HISTORY_S3_ACCESS_KEY_ID
gcs:
image: fsouza/fake-gcs-server:1.45.2
command: ["--port=9090", "--scheme=http"]
diff --git a/services/history-v1/docker-compose.yml b/services/history-v1/docker-compose.yml
index 3a33882d28..f40ab5b940 100644
--- a/services/history-v1/docker-compose.yml
+++ b/services/history-v1/docker-compose.yml
@@ -6,10 +6,7 @@ version: "2.3"
services:
test_unit:
- build:
- context: ../..
- dockerfile: services/history-v1/Dockerfile
- target: base
+ image: node:18.20.2
volumes:
- .:/overleaf/services/history-v1
- ../../node_modules:/overleaf/node_modules
@@ -17,228 +14,58 @@ services:
working_dir: /overleaf/services/history-v1
environment:
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
command: npm run --silent test:unit
user: node
test_acceptance:
- build:
- context: ../..
- dockerfile: services/history-v1/Dockerfile
- target: base
+ image: node:18.20.2
volumes:
- .:/overleaf/services/history-v1
- ../../node_modules:/overleaf/node_modules
- ../../libraries:/overleaf/libraries
- - ./test/acceptance/certs:/certs
- - ../../bin/shared/wait_for_it:/overleaf/bin/shared/wait_for_it
working_dir: /overleaf/services/history-v1
environment:
ELASTIC_SEARCH_DSN: es:9200
- REDIS_HOST: redis
- HISTORY_REDIS_HOST: redis
- QUEUES_REDIS_HOST: redis
- ANALYTICS_QUEUES_REDIS_HOST: redis
MONGO_HOST: mongo
POSTGRES_HOST: postgres
- AWS_S3_ENDPOINT: https://minio:9000
- AWS_S3_PATH_STYLE: 'true'
- AWS_ACCESS_KEY_ID: OVERLEAF_HISTORY_S3_ACCESS_KEY_ID
- AWS_SECRET_ACCESS_KEY: OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY
- MINIO_ROOT_USER: MINIO_ROOT_USER
- MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
GCS_API_ENDPOINT: http://gcs:9090
GCS_PROJECT_ID: fake
STORAGE_EMULATOR_HOST: http://gcs:9090/storage/v1
MOCHA_GREP: ${MOCHA_GREP}
- LOG_LEVEL: ${LOG_LEVEL:-}
+ LOG_LEVEL: ERROR
NODE_ENV: test
NODE_OPTIONS: "--unhandled-rejections=strict"
user: node
depends_on:
mongo:
- condition: service_started
- redis:
condition: service_healthy
postgres:
condition: service_healthy
- certs:
- condition: service_completed_successfully
- minio:
- condition: service_started
- minio_setup:
- condition: service_completed_successfully
gcs:
condition: service_healthy
- entrypoint: /overleaf/bin/shared/wait_for_it mongo:27017 --timeout=0 --
command: npm run --silent test:acceptance
- redis:
- image: redis:7.4.3
+ mongo:
+ image: mongo:6.0.13
+ command: --replSet overleaf
healthcheck:
- test: ping=$$(redis-cli ping) && [ "$$ping" = 'PONG' ]
+ test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
interval: 1s
retries: 20
- mongo:
- image: mongo:8.0.11
- command: --replSet overleaf
- volumes:
- - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
- environment:
- MONGO_INITDB_DATABASE: sharelatex
- extra_hosts:
- # Required when using the automatic database setup for initializing the
- # replica set. This override is not needed when running the setup after
- # starting up mongo.
- - mongo:127.0.0.1
-
postgres:
image: postgres:10
environment:
POSTGRES_USER: overleaf
POSTGRES_PASSWORD: overleaf
POSTGRES_DB: overleaf-history-v1-test
- volumes:
- - ./test/acceptance/pg-init/:/docker-entrypoint-initdb.d/
healthcheck:
test: pg_isready --host=localhost --quiet
interval: 1s
retries: 20
- certs:
- image: node:22.17.0
- volumes:
- - ./test/acceptance/certs:/certs
- working_dir: /certs
- entrypoint: sh
- command:
- - '-cex'
- - |
- if [ ! -f ./certgen ]; then
- wget -O ./certgen "https://github.com/minio/certgen/releases/download/v1.3.0/certgen-linux-$(dpkg --print-architecture)"
- chmod +x ./certgen
- fi
- if [ ! -f private.key ] || [ ! -f public.crt ]; then
- ./certgen -host minio
- fi
-
- minio:
- image: minio/minio:RELEASE.2024-10-13T13-34-11Z
- command: server /data
- volumes:
- - ./test/acceptance/certs:/root/.minio/certs
- environment:
- MINIO_ROOT_USER: MINIO_ROOT_USER
- MINIO_ROOT_PASSWORD: MINIO_ROOT_PASSWORD
- depends_on:
- certs:
- condition: service_completed_successfully
-
- minio_setup:
- depends_on:
- certs:
- condition: service_completed_successfully
- minio:
- condition: service_started
- image: minio/mc:RELEASE.2024-10-08T09-37-26Z
- volumes:
- - ./test/acceptance/certs:/root/.mc/certs/CAs
- entrypoint: sh
- command:
- - '-cex'
- - |
- sleep 1
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
- || sleep 3 && \
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
- || sleep 3 && \
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD \
- || sleep 3 && \
- mc alias set s3 https://minio:9000 MINIO_ROOT_USER MINIO_ROOT_PASSWORD
- mc mb --ignore-existing s3/overleaf-test-history-chunks
- mc mb --ignore-existing s3/overleaf-test-history-deks
- mc mb --ignore-existing s3/overleaf-test-history-global-blobs
- mc mb --ignore-existing s3/overleaf-test-history-project-blobs
- mc admin user add s3 \
- OVERLEAF_HISTORY_S3_ACCESS_KEY_ID \
- OVERLEAF_HISTORY_S3_SECRET_ACCESS_KEY
- echo '
- {
- "Version": "2012-10-17",
- "Statement": [
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-chunks"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-chunks/*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-deks"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-deks/*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-global-blobs"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-global-blobs/*"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:ListBucket"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-project-blobs"
- },
- {
- "Effect": "Allow",
- "Action": [
- "s3:PutObject",
- "s3:GetObject",
- "s3:DeleteObject"
- ],
- "Resource": "arn:aws:s3:::overleaf-test-history-project-blobs/*"
- }
- ]
- }' > policy-history.json
-
- mc admin policy create s3 overleaf-history policy-history.json
- mc admin policy attach s3 overleaf-history \
- --user=OVERLEAF_HISTORY_S3_ACCESS_KEY_ID
gcs:
image: fsouza/fake-gcs-server:1.45.2
command: ["--port=9090", "--scheme=http"]
diff --git a/services/history-v1/install_deps.sh b/services/history-v1/install_deps.sh
deleted file mode 100755
index 4ce7223b46..0000000000
--- a/services/history-v1/install_deps.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/sh
-
-set -ex
-
-apt-get update
-
-apt-get install jq parallel --yes
-
-rm -rf /var/lib/apt/lists/*
diff --git a/services/history-v1/migrations/20250415210802_add_chunks_closed.js b/services/history-v1/migrations/20250415210802_add_chunks_closed.js
deleted file mode 100644
index b5c1d577f9..0000000000
--- a/services/history-v1/migrations/20250415210802_add_chunks_closed.js
+++ /dev/null
@@ -1,27 +0,0 @@
-// @ts-check
-
-/**
- * @import { Knex } from "knex"
- */
-
-/**
- * @param { Knex } knex
- * @returns { Promise }
- */
-exports.up = async function (knex) {
- await knex.raw(`
- ALTER TABLE chunks
- ADD COLUMN closed BOOLEAN NOT NULL DEFAULT FALSE
- `)
-}
-
-/**
- * @param { Knex } knex
- * @returns { Promise }
- */
-exports.down = async function (knex) {
- await knex.raw(`
- ALTER TABLE chunks
- DROP COLUMN closed
- `)
-}
diff --git a/services/history-v1/package.json b/services/history-v1/package.json
index 4796cafd03..68c3dfd8aa 100644
--- a/services/history-v1/package.json
+++ b/services/history-v1/package.json
@@ -6,14 +6,10 @@
"license": "Proprietary",
"private": true,
"dependencies": {
- "@google-cloud/secret-manager": "^5.6.0",
- "@overleaf/fetch-utils": "*",
"@overleaf/logger": "*",
"@overleaf/metrics": "*",
- "@overleaf/mongo-utils": "*",
"@overleaf/o-error": "*",
"@overleaf/object-persistor": "*",
- "@overleaf/promise-utils": "*",
"@overleaf/redis-wrapper": "*",
"@overleaf/settings": "*",
"@overleaf/stream-utils": "^0.1.0",
@@ -21,12 +17,11 @@
"basic-auth": "^2.0.1",
"bluebird": "^3.7.2",
"body-parser": "^1.20.3",
- "bull": "^4.16.5",
"bunyan": "^1.8.12",
"check-types": "^11.1.2",
"command-line-args": "^3.0.3",
- "config": "^3.3.12",
- "express": "^4.21.2",
+ "config": "^1.19.0",
+ "express": "^4.21.0",
"fs-extra": "^9.0.1",
"generic-pool": "^2.1.1",
"helmet": "^3.22.0",
@@ -34,12 +29,9 @@
"jsonwebtoken": "^9.0.0",
"knex": "^2.4.0",
"lodash": "^4.17.19",
- "mongodb": "6.12.0",
+ "mongodb": "^6.2.0",
"overleaf-editor-core": "*",
- "p-limit": "^6.2.0",
- "p-queue": "^8.1.0",
"pg": "^8.7.1",
- "pg-query-stream": "^4.2.4",
"swagger-tools": "^0.10.4",
"temp": "^0.8.3",
"throng": "^4.0.0",
@@ -50,8 +42,7 @@
"benny": "^3.7.1",
"chai": "^4.3.6",
"chai-as-promised": "^7.1.1",
- "chai-exclude": "^2.1.1",
- "mocha": "^11.1.0",
+ "mocha": "^10.2.0",
"node-fetch": "^2.7.0",
"sinon": "^9.0.2",
"swagger-client": "^3.10.0",
@@ -62,8 +53,8 @@
"start": "node app.js",
"lint": "eslint --max-warnings 0 --format unix .",
"lint:fix": "eslint --fix .",
- "format": "prettier --list-different $PWD/'**/*.*js'",
- "format:fix": "prettier --write $PWD/'**/*.*js'",
+ "format": "prettier --list-different $PWD/'**/*.js'",
+ "format:fix": "prettier --write $PWD/'**/*.js'",
"test:unit": "npm run test:unit:_run -- --grep=$MOCHA_GREP",
"test:acceptance": "npm run test:acceptance:_run -- --grep=$MOCHA_GREP",
"test:unit:_run": "mocha --recursive --reporter spec $@ test/unit/js",
diff --git a/services/history-v1/storage/index.js b/services/history-v1/storage/index.js
index 6bc81f60e8..a0fd471829 100644
--- a/services/history-v1/storage/index.js
+++ b/services/history-v1/storage/index.js
@@ -2,16 +2,11 @@ exports.BatchBlobStore = require('./lib/batch_blob_store')
exports.blobHash = require('./lib/blob_hash')
exports.HashCheckBlobStore = require('./lib/hash_check_blob_store')
exports.chunkStore = require('./lib/chunk_store')
-exports.redisBuffer = require('./lib/chunk_store/redis')
-exports.historyStore = require('./lib/history_store').historyStore
+exports.historyStore = require('./lib/history_store')
exports.knex = require('./lib/knex')
exports.mongodb = require('./lib/mongodb')
-exports.redis = require('./lib/redis')
exports.persistChanges = require('./lib/persist_changes')
exports.persistor = require('./lib/persistor')
-exports.persistBuffer = require('./lib/persist_buffer')
-exports.commitChanges = require('./lib/commit_changes')
-exports.queueChanges = require('./lib/queue_changes')
exports.ProjectArchive = require('./lib/project_archive')
exports.streams = require('./lib/streams')
exports.temp = require('./lib/temp')
@@ -20,9 +15,3 @@ exports.zipStore = require('./lib/zip_store')
const { BlobStore, loadGlobalBlobs } = require('./lib/blob_store')
exports.BlobStore = BlobStore
exports.loadGlobalBlobs = loadGlobalBlobs
-
-const { InvalidChangeError } = require('./lib/errors')
-exports.InvalidChangeError = InvalidChangeError
-
-const { ChunkVersionConflictError } = require('./lib/chunk_store/errors')
-exports.ChunkVersionConflictError = ChunkVersionConflictError
diff --git a/services/history-v1/storage/lib/assert.js b/services/history-v1/storage/lib/assert.js
index 91f24da7e0..6f79086209 100644
--- a/services/history-v1/storage/lib/assert.js
+++ b/services/history-v1/storage/lib/assert.js
@@ -1,7 +1,5 @@
'use strict'
-const OError = require('@overleaf/o-error')
-
const check = require('check-types')
const { Blob } = require('overleaf-editor-core')
@@ -9,58 +7,37 @@ const assert = check.assert
const MONGO_ID_REGEXP = /^[0-9a-f]{24}$/
const POSTGRES_ID_REGEXP = /^[1-9][0-9]{0,9}$/
-const MONGO_OR_POSTGRES_ID_REGEXP = /^([0-9a-f]{24}|[1-9][0-9]{0,9})$/
+const PROJECT_ID_REGEXP = /^([0-9a-f]{24}|[1-9][0-9]{0,9})$/
function transaction(transaction, message) {
assert.function(transaction, message)
}
function blobHash(arg, message) {
- try {
- assert.match(arg, Blob.HEX_HASH_RX, message)
- } catch (error) {
- throw OError.tag(error, message, { arg })
- }
-}
-
-/**
- * A project id is a string that contains either an integer (for projects stored in Postgres) or 24
- * hex digits (for projects stored in Mongo)
- */
-function projectId(arg, message) {
- try {
- assert.match(arg, MONGO_OR_POSTGRES_ID_REGEXP, message)
- } catch (error) {
- throw OError.tag(error, message, { arg })
- }
+ assert.match(arg, Blob.HEX_HASH_RX, message)
}
/**
* A chunk id is a string that contains either an integer (for projects stored in Postgres) or 24
* hex digits (for projects stored in Mongo)
*/
+function projectId(arg, message) {
+ assert.match(arg, PROJECT_ID_REGEXP, message)
+}
+
+/**
+ * A chunk id is either a number (for projects stored in Postgres) or a 24
+ * character string (for projects stored in Mongo)
+ */
function chunkId(arg, message) {
- try {
- assert.match(arg, MONGO_OR_POSTGRES_ID_REGEXP, message)
- } catch (error) {
- throw OError.tag(error, message, { arg })
+ const valid = check.integer(arg) || check.match(arg, MONGO_ID_REGEXP)
+ if (!valid) {
+ throw new TypeError(message)
}
}
function mongoId(arg, message) {
- try {
- assert.match(arg, MONGO_ID_REGEXP, message)
- } catch (error) {
- throw OError.tag(error, message, { arg })
- }
-}
-
-function postgresId(arg, message) {
- try {
- assert.match(arg, POSTGRES_ID_REGEXP, message)
- } catch (error) {
- throw OError.tag(error, message, { arg })
- }
+ assert.match(arg, MONGO_ID_REGEXP)
}
module.exports = {
@@ -70,7 +47,6 @@ module.exports = {
projectId,
chunkId,
mongoId,
- postgresId,
MONGO_ID_REGEXP,
POSTGRES_ID_REGEXP,
}
diff --git a/services/history-v1/storage/lib/backupArchiver.mjs b/services/history-v1/storage/lib/backupArchiver.mjs
deleted file mode 100644
index c6f0e3755d..0000000000
--- a/services/history-v1/storage/lib/backupArchiver.mjs
+++ /dev/null
@@ -1,474 +0,0 @@
-// @ts-check
-import path from 'node:path'
-import projectKey from './project_key.js'
-import {
- chunksBucket,
- backupPersistor,
- projectBlobsBucket,
- globalBlobsBucket as backupGlobalBlobsBucket,
-} from './backupPersistor.mjs'
-import core, { Chunk, History } from 'overleaf-editor-core'
-import {
- GLOBAL_BLOBS,
- makeProjectKey,
- getStringLengthOfFile,
- makeGlobalKey,
-} from './blob_store/index.js'
-import streams from './streams.js'
-import objectPersistor from '@overleaf/object-persistor'
-import OError from '@overleaf/o-error'
-import chunkStore from './chunk_store/index.js'
-import logger from '@overleaf/logger'
-import fs from 'node:fs'
-import { pipeline } from 'node:stream/promises'
-import withTmpDir from '../../api/controllers/with_tmp_dir.js'
-import { loadChunk } from './backupVerifier.mjs'
-import globalBlobPersistor from './persistor.js'
-import config from 'config'
-import { NoKEKMatchedError } from '@overleaf/object-persistor/src/Errors.js'
-
-const globalBlobsBucket = config.get('blobStore.globalBucket')
-
-class BackupBlobStore {
- /**
- *
- * @param {string} historyId
- * @param {string} tmp
- * @param {CachedPerProjectEncryptedS3Persistor} persistor
- * @param {boolean} useBackupGlobalBlobs
- */
- constructor(historyId, tmp, persistor, useBackupGlobalBlobs) {
- this.historyId = historyId
- this.tmp = tmp
- this.blobs = new Map()
- this.persistor = persistor
- this.useBackupGlobalBlobs = useBackupGlobalBlobs
- }
-
- /**
- * Required for BlobStore interface - not supported.
- *
- * @template T
- * @param {string} hash
- * @return {Promise}
- */
- async getObject(hash) {
- try {
- const stream = await this.getStream(hash)
- const buffer = await streams.readStreamToBuffer(stream)
- return JSON.parse(buffer.toString())
- } catch (err) {
- logger.warn({ err, hash }, 'Failed to fetch chunk blob')
- throw err
- }
- }
-
- /**
- *
- * @param {Set} hashes
- * @return {Promise}
- */
- async fetchBlobs(hashes) {
- for await (const hash of hashes) {
- if (this.blobs.has(hash)) return
- const path = `${this.tmp}/${hash}`
- /** @type {core.Blob} */
- let blob
- /** @type {NodeJS.ReadableStream} */
- let blobStream
- if (GLOBAL_BLOBS.has(hash)) {
- try {
- const blobData = await this.fetchGlobalBlob(hash)
- await pipeline(blobData.stream, fs.createWriteStream(path))
- blob = blobData.blob
- } catch (err) {
- logger.warn({ hash, err }, 'Failed to fetch global blob')
- continue
- }
- } else {
- try {
- blobStream = await fetchBlob(this.historyId, hash, this.persistor)
- await pipeline(blobStream, fs.createWriteStream(path))
- blob = await this.makeBlob(hash, path)
- } catch (err) {
- logger.warn({ err, hash }, 'Failed to fetch chunk blob')
- continue
- }
- }
-
- this.blobs.set(hash, blob)
- }
- }
-
- /**
- *
- * @param {string} hash
- * @return {Promise<{ blob: core.Blob, stream: NodeJS.ReadableStream }>}
- */
- async fetchGlobalBlob(hash) {
- const globalBlob = GLOBAL_BLOBS.get(hash)
- if (!globalBlob) {
- throw new Error('blob does not exist or is not a global blob')
- }
- let stream
-
- const key = makeGlobalKey(hash)
-
- if (this.useBackupGlobalBlobs) {
- stream = await this.persistor.getObjectStream(
- backupGlobalBlobsBucket,
- key
- )
- } else {
- stream = await globalBlobPersistor.getObjectStream(globalBlobsBucket, key)
- }
- return { blob: globalBlob.blob, stream }
- }
-
- /**
- *
- * @param {string} hash
- * @param {string} pathname
- * @return {Promise}
- */
- async makeBlob(hash, pathname) {
- const stat = await fs.promises.stat(pathname)
- const byteLength = stat.size
- const stringLength = await getStringLengthOfFile(byteLength, pathname)
- if (stringLength) {
- return new core.Blob(hash, byteLength, stringLength)
- }
- return new core.Blob(hash, byteLength)
- }
-
- /**
- *
- * @param {string} hash
- * @return {Promise}
- */
- async getString(hash) {
- const stream = await this.getStream(hash)
- const buffer = await streams.readStreamToBuffer(stream)
- return buffer.toString()
- }
-
- /**
- *
- * @param {string} hash
- * @return {Promise}
- */
- async getStream(hash) {
- return fs.createReadStream(this.getBlobPathname(hash))
- }
-
- /**
- *
- * @param {string} hash
- * @return {Promise}
- */
- async getBlob(hash) {
- return this.blobs.get(hash)
- }
-
- /**
- *
- * @param {string} hash
- * @return {string}
- */
- getBlobPathname(hash) {
- return path.join(this.tmp, hash)
- }
-}
-
-/**
- * @typedef {(import('@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor.js').CachedPerProjectEncryptedS3Persistor)} CachedPerProjectEncryptedS3Persistor
- */
-
-/**
- * @typedef {(import('archiver').Archiver)} Archiver
- */
-
-/**
- * @typedef {(import('overleaf-editor-core').FileMap)} FileMap
- */
-
-/**
- *
- * @param historyId
- * @return {Promise}
- */
-async function getProjectPersistor(historyId) {
- try {
- return await backupPersistor.forProjectRO(
- projectBlobsBucket,
- makeProjectKey(historyId, '')
- )
- } catch (error) {
- if (error instanceof NoKEKMatchedError) {
- logger.info({}, 'no kek matched')
- }
- throw new BackupPersistorError(
- 'Failed to get project persistor',
- { historyId },
- error instanceof Error ? error : undefined
- )
- }
-}
-
-/**
- *
- * @param persistor
- * @param {string} key
- * @return {Promise<{chunkData: any, buffer: Buffer}>}
- */
-async function loadChunkByKey(persistor, key) {
- try {
- const buf = await streams.gunzipStreamToBuffer(
- await persistor.getObjectStream(chunksBucket, key)
- )
- return { chunkData: JSON.parse(buf.toString('utf-8')), buffer: buf }
- } catch (err) {
- if (err instanceof objectPersistor.Errors.NotFoundError) {
- throw new Chunk.NotPersistedError('chunk not found')
- }
- if (err instanceof Error) {
- throw OError.tag(err, 'Failed to load chunk', { key })
- }
- throw err
- }
-}
-
-/**
- *
- * @param {string} historyId
- * @param {string} hash
- * @param {CachedPerProjectEncryptedS3Persistor} persistor
- * @return {Promise}
- */
-async function fetchBlob(historyId, hash, persistor) {
- const path = makeProjectKey(historyId, hash)
- return await persistor.getObjectStream(projectBlobsBucket, path, {
- autoGunzip: true,
- })
-}
-
-/**
- * @typedef {object} AddChunkOptions
- * @property {string} [prefix] Should include trailing slash (if length > 0)
- * @property {boolean} [useBackupGlobalBlobs]
- */
-
-/**
- *
- * @param {History} history
- * @param {Archiver} archive
- * @param {CachedPerProjectEncryptedS3Persistor} projectCache
- * @param {string} historyId
- * @param {AddChunkOptions} [options]
- * @returns {Promise}
- */
-async function addChunkToArchive(
- history,
- archive,
- projectCache,
- historyId,
- { prefix = '', useBackupGlobalBlobs = false } = {}
-) {
- const chunkBlobs = new Set()
- history.findBlobHashes(chunkBlobs)
-
- await withTmpDir('recovery-blob-', async tmpDir => {
- const blobStore = new BackupBlobStore(
- historyId,
- tmpDir,
- projectCache,
- useBackupGlobalBlobs
- )
- await blobStore.fetchBlobs(chunkBlobs)
-
- await history.loadFiles('lazy', blobStore)
-
- const snapshot = history.getSnapshot()
- snapshot.applyAll(history.getChanges())
-
- const filePaths = snapshot.getFilePathnames()
-
- if (filePaths.length === 0) {
- logger.warn(
- { historyId, projectVersion: snapshot.projectVersion },
- 'No files found in snapshot backup'
- )
- }
- for (const filePath of filePaths) {
- /** @type {core.File | null | undefined} */
- const file = snapshot.getFile(filePath)
- if (!file) {
- logger.error({ filePath }, 'File not found in snapshot')
- continue
- }
-
- try {
- await file.load('eager', blobStore)
- } catch (err) {
- logger.error(
- { filePath, err },
- 'Failed to load file from snapshot, skipping'
- )
- continue
- }
-
- const hash = file.getHash()
-
- /** @type {string | fs.ReadStream | null | undefined} */
- let content = file.getContent({ filterTrackedDeletes: true })
-
- if (content === null) {
- if (!hash) {
- logger.error({ filePath }, 'File does not have a hash')
- continue
- }
- const blob = await blobStore.getBlob(hash)
- if (!blob) {
- logger.error({ filePath }, 'Blob not found in blob store')
- continue
- }
- content = await blobStore.getStream(hash)
- }
- archive.append(content, {
- name: `${prefix}${filePath}`,
- })
- }
- })
-}
-
-/**
- *
- * @param {string} historyId
- * @return {Promise}
- */
-async function findStartVersionOfLatestChunk(historyId) {
- const backend = chunkStore.getBackend(historyId)
- const chunk = await backend.getLatestChunk(historyId, { readOnly: true })
- if (!chunk) {
- throw new Error('Latest chunk could not be loaded')
- }
- return chunk.startVersion
-}
-
-/**
- * Restore a project from the latest snapshot
- *
- * There is an assumption that the database backup has been restored.
- *
- * @param {Archiver} archive
- * @param {string} historyId
- * @param {boolean} [useBackupGlobalBlobs]
- * @return {Promise}
- */
-export async function archiveLatestChunk(
- archive,
- historyId,
- useBackupGlobalBlobs = false
-) {
- logger.info({ historyId, useBackupGlobalBlobs }, 'Archiving latest chunk')
-
- const projectCache = await getProjectPersistor(historyId)
-
- const startVersion = await findStartVersionOfLatestChunk(historyId)
-
- const backedUpChunkRaw = await loadChunk(
- historyId,
- startVersion,
- projectCache
- )
-
- const backedUpChunk = History.fromRaw(backedUpChunkRaw)
-
- await addChunkToArchive(backedUpChunk, archive, projectCache, historyId, {
- useBackupGlobalBlobs,
- })
-
- return archive
-}
-
-/**
- * Fetches all raw blobs from the project and adds them to the archive.
- *
- * @param {string} historyId
- * @param {Archiver} archive
- * @param {CachedPerProjectEncryptedS3Persistor} projectCache
- * @return {Promise}
- */
-async function addRawBlobsToArchive(historyId, archive, projectCache) {
- const key = projectKey.format(historyId)
- const { contents } = await projectCache.listDirectory(projectBlobsBucket, key)
- for (const blobRecord of contents) {
- if (!blobRecord.Key) {
- logger.debug({ blobRecord }, 'no key')
- continue
- }
- const blobKey = blobRecord.Key
- try {
- const stream = await projectCache.getObjectStream(
- projectBlobsBucket,
- blobKey,
- { autoGunzip: true }
- )
- archive.append(stream, {
- name: path.join(historyId, 'blobs', blobKey),
- })
- } catch (err) {
- logger.warn(
- { err, path: blobRecord.Key },
- 'Failed to append blob to archive'
- )
- }
- }
-}
-
-/**
- * Download raw files from the backup.
- *
- * This can work without the database being backed up.
- *
- * It will split the project into chunks per directory and download the blobs alongside the chunk.
- *
- * @param {Archiver} archive
- * @param {string} historyId
- * @param {boolean} [useBackupGlobalBlobs]
- * @return {Promise}
- */
-export async function archiveRawProject(
- archive,
- historyId,
- useBackupGlobalBlobs = false
-) {
- const projectCache = await getProjectPersistor(historyId)
-
- const { contents: chunks } = await projectCache.listDirectory(
- chunksBucket,
- projectKey.format(historyId)
- )
-
- if (chunks.length === 0) {
- throw new Error('No chunks found')
- }
-
- for (const chunkRecord of chunks) {
- if (!chunkRecord.Key) {
- logger.debug({ chunkRecord }, 'no key')
- continue
- }
- const chunkId = chunkRecord.Key.split('/').pop()
- logger.debug({ chunkId, key: chunkRecord.Key }, 'Processing chunk')
-
- const { buffer } = await loadChunkByKey(projectCache, chunkRecord.Key)
-
- archive.append(buffer, {
- name: `${historyId}/chunks/${chunkId}/chunk.json`,
- })
- }
- await addRawBlobsToArchive(historyId, archive, projectCache)
-}
-
-export class BackupPersistorError extends OError {}
diff --git a/services/history-v1/storage/lib/backupBlob.mjs b/services/history-v1/storage/lib/backupBlob.mjs
deleted file mode 100644
index 8ae1a6a901..0000000000
--- a/services/history-v1/storage/lib/backupBlob.mjs
+++ /dev/null
@@ -1,251 +0,0 @@
-// @ts-check
-import { backupPersistor, projectBlobsBucket } from './backupPersistor.mjs'
-import { GLOBAL_BLOBS, makeProjectKey, BlobStore } from './blob_store/index.js'
-import Stream from 'node:stream'
-import fs from 'node:fs'
-import Crypto from 'node:crypto'
-import assert from './assert.js'
-import { backedUpBlobs, projects } from './mongodb.js'
-import { Binary, ObjectId } from 'mongodb'
-import logger from '@overleaf/logger/logging-manager.js'
-import { AlreadyWrittenError } from '@overleaf/object-persistor/src/Errors.js'
-import metrics from '@overleaf/metrics'
-import zLib from 'node:zlib'
-import Path from 'node:path'
-
-const HIGHWATER_MARK = 1024 * 1024
-
-/**
- * @typedef {import("overleaf-editor-core").Blob} Blob
- */
-
-/**
- * @typedef {import("@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor").CachedPerProjectEncryptedS3Persistor} CachedPerProjectEncryptedS3Persistor
- */
-
-/**
- * Increment a metric to record the outcome of a backup operation.
- *
- * @param {"success"|"failure"|"skipped"} status
- * @param {"global"|"already_backed_up"|"none"} reason
- */
-function recordBackupConclusion(status, reason = 'none') {
- metrics.inc('blob_backed_up', 1, { status, reason })
-}
-
-/**
- * Downloads a blob to a specified directory
- *
- * @param {string} historyId - The history ID of the project the blob belongs to
- * @param {Blob} blob - The blob to download
- * @param {string} tmpDir - The directory path where the blob will be downloaded
- * @returns {Promise} The full path where the blob was downloaded
- */
-export async function downloadBlobToDir(historyId, blob, tmpDir) {
- const blobStore = new BlobStore(historyId)
- const blobHash = blob.getHash()
- const src = await blobStore.getStream(blobHash)
- const filePath = Path.join(tmpDir, `${historyId}-${blobHash}`)
- try {
- const dst = fs.createWriteStream(filePath, {
- highWaterMark: HIGHWATER_MARK,
- flags: 'wx',
- })
- await Stream.promises.pipeline(src, dst)
- return filePath
- } catch (error) {
- try {
- await fs.promises.unlink(filePath)
- } catch {}
- throw error
- }
-}
-
-/**
- * Performs the actual upload of the blob to the backup storage.
- *
- * @param {string} historyId - The history ID of the project the blob belongs to
- * @param {Blob} blob - The blob being uploaded
- * @param {string} path - The path to the file to upload (should have been stored on disk already)
- * @return {Promise}
- */
-export async function uploadBlobToBackup(historyId, blob, path, persistor) {
- const md5 = Crypto.createHash('md5')
- const filePathCompressed = path + '.gz'
- let backupSource
- let contentEncoding
- let size
- try {
- if (blob.getStringLength()) {
- backupSource = filePathCompressed
- contentEncoding = 'gzip'
- size = 0
- await Stream.promises.pipeline(
- fs.createReadStream(path, { highWaterMark: HIGHWATER_MARK }),
- zLib.createGzip(),
- async function* (source) {
- for await (const chunk of source) {
- size += chunk.byteLength
- md5.update(chunk)
- yield chunk
- }
- },
- fs.createWriteStream(filePathCompressed, {
- highWaterMark: HIGHWATER_MARK,
- })
- )
- } else {
- backupSource = path
- size = blob.getByteLength()
- await Stream.promises.pipeline(
- fs.createReadStream(path, { highWaterMark: HIGHWATER_MARK }),
- md5
- )
- }
- const key = makeProjectKey(historyId, blob.getHash())
- await persistor.sendStream(
- projectBlobsBucket,
- key,
- fs.createReadStream(backupSource, { highWaterMark: HIGHWATER_MARK }),
- {
- contentEncoding,
- contentType: 'application/octet-stream',
- contentLength: size,
- sourceMd5: md5.digest('hex'),
- ifNoneMatch: '*',
- }
- )
- } finally {
- if (backupSource === filePathCompressed) {
- try {
- await fs.promises.rm(filePathCompressed, { force: true })
- } catch {}
- }
- }
-}
-
-/**
- * Converts a legacy (postgres) historyId to a mongo projectId
- *
- * @param {string} historyId
- * @return {Promise}
- * @private
- */
-async function _convertLegacyHistoryIdToProjectId(historyId) {
- const project = await projects.findOne(
- { 'overleaf.history.id': parseInt(historyId) },
- { projection: { _id: 1 } }
- )
-
- if (!project?._id) {
- throw new Error('Did not find project for history id')
- }
-
- return project?._id?.toString()
-}
-
-/**
- * Records that a blob was backed up for a project.
- *
- * @param {string} projectId - projectId for a project (mongo format)
- * @param {string} hash
- * @return {Promise}
- */
-export async function storeBlobBackup(projectId, hash) {
- await backedUpBlobs.updateOne(
- { _id: new ObjectId(projectId) },
- { $addToSet: { blobs: new Binary(Buffer.from(hash, 'hex')) } },
- { upsert: true }
- )
-}
-
-/**
- * Determine whether a specific blob has been backed up in this project.
- *
- * @param {string} projectId
- * @param {string} hash
- * @return {Promise<*>}
- * @private
- */
-export async function _blobIsBackedUp(projectId, hash) {
- const blobs = await backedUpBlobs.findOne(
- {
- _id: new ObjectId(projectId),
- blobs: new Binary(Buffer.from(hash, 'hex')),
- },
- { projection: { _id: 1 } }
- )
- return blobs?._id
-}
-
-/**
- * Back up a blob to the global storage and record that it was backed up.
- *
- * @param {string} historyId - history ID for a project (can be postgres format or mongo format)
- * @param {Blob} blob - The blob that is being backed up
- * @param {string} tmpPath - The path to a temporary file storing the contents of the blob.
- * @param {CachedPerProjectEncryptedS3Persistor} [persistor] - The persistor to use (optional)
- * @return {Promise}
- */
-export async function backupBlob(historyId, blob, tmpPath, persistor) {
- const hash = blob.getHash()
-
- let projectId = historyId
- if (assert.POSTGRES_ID_REGEXP.test(historyId)) {
- projectId = await _convertLegacyHistoryIdToProjectId(historyId)
- }
-
- const globalBlob = GLOBAL_BLOBS.get(hash)
-
- if (globalBlob && !globalBlob.demoted) {
- recordBackupConclusion('skipped', 'global')
- logger.debug({ projectId, hash }, 'Blob is global - skipping backup')
- return
- }
-
- try {
- if (await _blobIsBackedUp(projectId, hash)) {
- recordBackupConclusion('skipped', 'already_backed_up')
- logger.debug(
- { projectId, hash },
- 'Blob already backed up - skipping backup'
- )
- return
- }
- } catch (error) {
- logger.warn({ error }, 'Failed to check if blob is backed up')
- // We'll try anyway - we'll catch the error if it was backed up
- }
- // If we weren't passed a persistor for this project, create one.
- // This will fetch the key from AWS, so it's prefereable to use
- // the same persistor for all blobs in a project where possible.
- if (!persistor) {
- logger.debug(
- { historyId, hash },
- 'warning: persistor not passed to backupBlob'
- )
- }
- persistor ??= await backupPersistor.forProject(
- projectBlobsBucket,
- makeProjectKey(historyId, '')
- )
- try {
- logger.debug({ projectId, hash }, 'Starting blob backup')
- await uploadBlobToBackup(historyId, blob, tmpPath, persistor)
- await storeBlobBackup(projectId, hash)
- recordBackupConclusion('success')
- } catch (error) {
- if (error instanceof AlreadyWrittenError) {
- logger.debug({ error, projectId, hash }, 'Blob already backed up')
- // record that we backed it up already
- await storeBlobBackup(projectId, hash)
- recordBackupConclusion('failure', 'already_backed_up')
- return
- }
- // eventually queue this for retry - for now this will be fixed by running the script
- recordBackupConclusion('failure')
- logger.warn({ error, projectId, hash }, 'Failed to upload blob to backup')
- } finally {
- logger.debug({ projectId, hash }, 'Ended blob backup')
- }
-}
diff --git a/services/history-v1/storage/lib/backupDeletion.mjs b/services/history-v1/storage/lib/backupDeletion.mjs
deleted file mode 100644
index ef50609753..0000000000
--- a/services/history-v1/storage/lib/backupDeletion.mjs
+++ /dev/null
@@ -1,93 +0,0 @@
-// @ts-check
-import { callbackify } from 'util'
-import { ObjectId } from 'mongodb'
-import config from 'config'
-import OError from '@overleaf/o-error'
-import { db } from './mongodb.js'
-import projectKey from './project_key.js'
-import chunkStore from '../lib/chunk_store/index.js'
-import {
- backupPersistor,
- chunksBucket,
- projectBlobsBucket,
-} from './backupPersistor.mjs'
-
-const MS_PER_DAY = 24 * 60 * 60 * 1000
-const EXPIRE_PROJECTS_AFTER_MS =
- parseInt(config.get('minSoftDeletionPeriodDays'), 10) * MS_PER_DAY
-const deletedProjectsCollection = db.collection('deletedProjects')
-
-/**
- * @param {string} historyId
- * @return {Promise}
- */
-async function projectHasLatestChunk(historyId) {
- const chunk = await chunkStore.getBackend(historyId).getLatestChunk(historyId)
- return chunk != null
-}
-
-export class NotReadyToDelete extends OError {}
-
-/**
- * @param {string} projectId
- * @return {Promise}
- */
-async function deleteProjectBackup(projectId) {
- const deletedProject = await deletedProjectsCollection.findOne(
- { 'deleterData.deletedProjectId': new ObjectId(projectId) },
- {
- projection: {
- 'deleterData.deletedProjectOverleafHistoryId': 1,
- 'deleterData.deletedAt': 1,
- },
- }
- )
- if (!deletedProject) {
- throw new NotReadyToDelete('refusing to delete non-deleted project')
- }
- const expiresAt =
- deletedProject.deleterData.deletedAt.getTime() + EXPIRE_PROJECTS_AFTER_MS
- if (expiresAt > Date.now()) {
- throw new NotReadyToDelete('refusing to delete non-expired project')
- }
-
- const historyId =
- deletedProject.deleterData.deletedProjectOverleafHistoryId?.toString()
- if (!historyId) {
- throw new NotReadyToDelete(
- 'refusing to delete project with unknown historyId'
- )
- }
-
- if (await projectHasLatestChunk(historyId)) {
- throw new NotReadyToDelete(
- 'refusing to delete project with remaining chunks'
- )
- }
-
- const prefix = projectKey.format(historyId) + '/'
- await backupPersistor.deleteDirectory(chunksBucket, prefix)
- await backupPersistor.deleteDirectory(projectBlobsBucket, prefix)
-}
-
-export async function healthCheck() {
- const HEALTH_CHECK_PROJECTS = JSON.parse(config.get('healthCheckProjects'))
- if (HEALTH_CHECK_PROJECTS.length !== 2) {
- throw new Error('expected 2 healthCheckProjects')
- }
- if (!HEALTH_CHECK_PROJECTS.some(id => id.length === 24)) {
- throw new Error('expected mongo id in healthCheckProjects')
- }
- if (!HEALTH_CHECK_PROJECTS.some(id => id.length < 24)) {
- throw new Error('expected postgres id in healthCheckProjects')
- }
-
- for (const historyId of HEALTH_CHECK_PROJECTS) {
- if (!(await projectHasLatestChunk(historyId))) {
- throw new Error(`project has no history: ${historyId}`)
- }
- }
-}
-
-export const healthCheckCb = callbackify(healthCheck)
-export const deleteProjectBackupCb = callbackify(deleteProjectBackup)
diff --git a/services/history-v1/storage/lib/backupGenerator.mjs b/services/history-v1/storage/lib/backupGenerator.mjs
deleted file mode 100644
index d8f1b0e99a..0000000000
--- a/services/history-v1/storage/lib/backupGenerator.mjs
+++ /dev/null
@@ -1,153 +0,0 @@
-/**
- * Provides a generator function to back up project chunks and blobs.
- */
-
-import chunkStore from './chunk_store/index.js'
-
-import {
- GLOBAL_BLOBS, // NOTE: must call loadGlobalBlobs() before using this
- BlobStore,
-} from './blob_store/index.js'
-
-import assert from './assert.js'
-
-async function lookBehindForSeenBlobs(
- projectId,
- chunk,
- lastBackedUpVersion,
- seenBlobs
-) {
- if (chunk.startVersion === 0) {
- return // this is the first chunk, no need to check for blobs in the previous chunk
- }
- if (chunk.startVersion > 0 && lastBackedUpVersion > chunk.startVersion) {
- return // the snapshot in this chunk has already been backed up
- }
- if (
- chunk.startVersion > 0 &&
- lastBackedUpVersion === chunk.startVersion // same as previousChunk.endVersion
- ) {
- // the snapshot in this chunk has not been backed up
- // so we find the set of backed up blobs from the previous chunk
- const previousChunk = await chunkStore.loadAtVersion(
- projectId,
- lastBackedUpVersion,
- { persistedOnly: true }
- )
- const previousChunkHistory = previousChunk.getHistory()
- previousChunkHistory.findBlobHashes(seenBlobs)
- }
-}
-
-/**
- * Records blob hashes that have been previously seen in a chunk's history.
- *
- * @param {Object} chunk - The chunk containing history data
- * @param {number} currentBackedUpVersion - The version number that has been backed up
- * @param {Set} seenBlobs - Set to collect previously seen blob hashes
- * @returns {void}
- */
-function recordPreviouslySeenBlobs(chunk, currentBackedUpVersion, seenBlobs) {
- // We need to look at the chunk and decide how far we have backed up.
- // If we have not backed up this chunk at all, we need to backup the blobs
- // in the snapshot. Otherwise we need to backup the blobs in the changes
- // that have occurred since the last backup.
- const history = chunk.getHistory()
- const startVersion = chunk.getStartVersion()
- if (currentBackedUpVersion === 0) {
- // If we have only backed up version 0 (i.e. the first change)
- // then that includes the initial snapshot, so we consider
- // the blobs of the initial snapshot as seen. If the project
- // has not been backed up at all then currentBackedUpVersion
- // will be undefined.
- history.snapshot.findBlobHashes(seenBlobs)
- } else if (currentBackedUpVersion > startVersion) {
- history.snapshot.findBlobHashes(seenBlobs)
- for (let i = 0; i < currentBackedUpVersion - startVersion; i++) {
- history.changes[i].findBlobHashes(seenBlobs)
- }
- }
-}
-
-/**
- * Collects new blob objects that need to be backed up from a given chunk.
- *
- * @param {Object} chunk - The chunk object containing history data
- * @param {Object} blobStore - Storage interface for retrieving blobs
- * @param {Set} seenBlobs - Set of blob hashes that have already been processed
- * @returns {Promise} Array of blob objects that need to be backed up
- * @throws {Error} If blob retrieval fails
- */
-async function collectNewBlobsForBackup(chunk, blobStore, seenBlobs) {
- /** @type {Set} */
- const blobHashes = new Set()
- const history = chunk.getHistory()
- // Get all the blobs in this chunk, then exclude the seenBlobs and global blobs
- history.findBlobHashes(blobHashes)
- const blobsToBackup = await blobStore.getBlobs(
- [...blobHashes].filter(
- hash =>
- hash &&
- !seenBlobs.has(hash) &&
- (!GLOBAL_BLOBS.has(hash) || GLOBAL_BLOBS.get(hash).demoted)
- )
- )
- return blobsToBackup
-}
-
-/**
- * Asynchronously generates backups for a project based on provided versions.
- * @param {string} projectId - The ID of the project's history to back up.
- * @param {number} lastBackedUpVersion - The last version that was successfully backed up.
- * @yields {AsyncGenerator<{ chunkRecord: object, chunkToBackup: object, chunkBuffer: Buffer, blobsToBackup: object[] }>}
- * Yields chunk records and corresponding data needed for backups.
- */
-export async function* backupGenerator(projectId, lastBackedUpVersion) {
- assert.projectId(projectId, 'bad projectId')
- assert.maybe.integer(lastBackedUpVersion, 'bad lastBackedUpVersion')
-
- const blobStore = new BlobStore(projectId)
-
- /** @type {Set} */
- const seenBlobs = new Set() // records the blobs that are already backed up
-
- const firstPendingVersion =
- lastBackedUpVersion >= 0 ? lastBackedUpVersion + 1 : 0
- let isStartingChunk = true
- let currentBackedUpVersion = lastBackedUpVersion
- const chunkRecordIterator = chunkStore.getProjectChunksFromVersion(
- projectId,
- firstPendingVersion
- )
-
- for await (const chunkRecord of chunkRecordIterator) {
- const { chunk, chunkBuffer } = await chunkStore.loadByChunkRecord(
- projectId,
- chunkRecord
- )
-
- if (isStartingChunk) {
- await lookBehindForSeenBlobs(
- projectId,
- chunkRecord,
- lastBackedUpVersion,
- seenBlobs
- )
- isStartingChunk = false
- }
-
- recordPreviouslySeenBlobs(chunk, currentBackedUpVersion, seenBlobs)
-
- const blobsToBackup = await collectNewBlobsForBackup(
- chunk,
- blobStore,
- seenBlobs
- )
-
- yield { chunkRecord, chunkToBackup: chunk, chunkBuffer, blobsToBackup }
-
- // After we generate a backup of this chunk, mark the backed up blobs as seen
- blobsToBackup.forEach(blob => seenBlobs.add(blob.getHash()))
- currentBackedUpVersion = chunkRecord.endVersion
- }
-}
diff --git a/services/history-v1/storage/lib/backupPersistor.mjs b/services/history-v1/storage/lib/backupPersistor.mjs
deleted file mode 100644
index 8f80e5faaf..0000000000
--- a/services/history-v1/storage/lib/backupPersistor.mjs
+++ /dev/null
@@ -1,121 +0,0 @@
-// @ts-check
-import fs from 'node:fs'
-import Path from 'node:path'
-import _ from 'lodash'
-import config from 'config'
-import { SecretManagerServiceClient } from '@google-cloud/secret-manager'
-import OError from '@overleaf/o-error'
-import {
- PerProjectEncryptedS3Persistor,
- RootKeyEncryptionKey,
-} from '@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor.js'
-import { HistoryStore } from './history_store.js'
-
-const persistorConfig = _.cloneDeep(config.get('backupPersistor'))
-const { chunksBucket, deksBucket, globalBlobsBucket, projectBlobsBucket } =
- config.get('backupStore')
-
-export { chunksBucket, globalBlobsBucket, projectBlobsBucket }
-
-function convertKey(key, convertFn) {
- if (_.has(persistorConfig, key)) {
- _.update(persistorConfig, key, convertFn)
- }
-}
-
-convertKey('s3SSEC.httpOptions.timeout', s => parseInt(s, 10))
-convertKey('s3SSEC.maxRetries', s => parseInt(s, 10))
-convertKey('s3SSEC.pathStyle', s => s === 'true')
-// array of CA, either inlined or on disk
-convertKey('s3SSEC.ca', s =>
- JSON.parse(s).map(ca => (ca.startsWith('/') ? fs.readFileSync(ca) : ca))
-)
-
-/** @type {() => Promise} */
-let getRawRootKeyEncryptionKeys
-
-if ((process.env.NODE_ENV || 'production') === 'production') {
- ;[persistorConfig.s3SSEC.key, persistorConfig.s3SSEC.secret] = (
- await loadFromSecretsManager(
- process.env.BACKUP_AWS_CREDENTIALS || '',
- 'BACKUP_AWS_CREDENTIALS'
- )
- ).split(':')
- getRawRootKeyEncryptionKeys = () =>
- loadFromSecretsManager(
- persistorConfig.keyEncryptionKeys,
- 'BACKUP_KEY_ENCRYPTION_KEYS'
- )
-} else {
- getRawRootKeyEncryptionKeys = () => persistorConfig.keyEncryptionKeys
-}
-
-export const DELETION_ONLY = persistorConfig.keyEncryptionKeys === 'none'
-if (DELETION_ONLY) {
- // For Backup-deleter; should not encrypt or read data; deleting does not need key.
- getRawRootKeyEncryptionKeys = () => new Promise(_resolve => {})
-}
-
-const PROJECT_FOLDER_REGEX =
- /^\d{3}\/\d{3}\/\d{3,}\/|[0-9a-f]{3}\/[0-9a-f]{3}\/[0-9a-f]{18}\/$/
-
-/**
- * @param {string} bucketName
- * @param {string} path
- * @return {string}
- */
-export function pathToProjectFolder(bucketName, path) {
- switch (bucketName) {
- case deksBucket:
- case chunksBucket:
- case projectBlobsBucket:
- const projectFolder = Path.join(...path.split('/').slice(0, 3)) + '/'
- if (!PROJECT_FOLDER_REGEX.test(projectFolder)) {
- throw new OError('invalid project folder', { bucketName, path })
- }
- return projectFolder
- default:
- throw new Error(`${bucketName} does not store per-project files`)
- }
-}
-
-/**
- * @param {string} name
- * @param {string} label
- * @return {Promise}
- */
-async function loadFromSecretsManager(name, label) {
- const client = new SecretManagerServiceClient()
- const [version] = await client.accessSecretVersion({ name })
- if (!version.payload?.data) throw new Error(`empty secret: ${label}`)
- return version.payload.data.toString()
-}
-
-async function getRootKeyEncryptionKeys() {
- return JSON.parse(await getRawRootKeyEncryptionKeys()).map(
- ({ key, salt }) => {
- return new RootKeyEncryptionKey(
- Buffer.from(key, 'base64'),
- Buffer.from(salt, 'base64')
- )
- }
- )
-}
-
-export const backupPersistor = new PerProjectEncryptedS3Persistor({
- ...persistorConfig.s3SSEC,
- disableMultiPartUpload: true,
- dataEncryptionKeyBucketName: deksBucket,
- pathToProjectFolder,
- getRootKeyEncryptionKeys,
- storageClass: {
- [deksBucket]: 'STANDARD',
- [chunksBucket]: persistorConfig.tieringStorageClass,
- [projectBlobsBucket]: persistorConfig.tieringStorageClass,
- },
-})
-
-export const backupHistoryStore = new HistoryStore(
- backupPersistor,
- chunksBucket
-)
diff --git a/services/history-v1/storage/lib/backupVerifier.mjs b/services/history-v1/storage/lib/backupVerifier.mjs
deleted file mode 100644
index 6e767b21ba..0000000000
--- a/services/history-v1/storage/lib/backupVerifier.mjs
+++ /dev/null
@@ -1,220 +0,0 @@
-// @ts-check
-import OError from '@overleaf/o-error'
-import chunkStore from '../lib/chunk_store/index.js'
-import {
- backupPersistor,
- chunksBucket,
- projectBlobsBucket,
-} from './backupPersistor.mjs'
-import { Blob, Chunk, History } from 'overleaf-editor-core'
-import { BlobStore, GLOBAL_BLOBS, makeProjectKey } from './blob_store/index.js'
-import blobHash from './blob_hash.js'
-import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
-import logger from '@overleaf/logger'
-import path from 'node:path'
-import projectKey from './project_key.js'
-import streams from './streams.js'
-import objectPersistor from '@overleaf/object-persistor'
-import { getEndDateForRPO } from '../../backupVerifier/utils.mjs'
-
-/**
- * @typedef {import("@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor.js").CachedPerProjectEncryptedS3Persistor} CachedPerProjectEncryptedS3Persistor
- */
-
-/**
- * @param {string} historyId
- * @param {string} hash
- */
-export async function verifyBlob(historyId, hash) {
- return await verifyBlobs(historyId, [hash])
-}
-
-/**
- *
- * @param {string} historyId
- * @return {Promise}
- */
-async function getProjectPersistor(historyId) {
- try {
- return await backupPersistor.forProjectRO(
- projectBlobsBucket,
- makeProjectKey(historyId, '')
- )
- } catch (err) {
- if (err instanceof NotFoundError) {
- throw new BackupCorruptedError('dek does not exist', {}, err)
- }
- throw err
- }
-}
-
-/**
- * @param {string} historyId
- * @param {Array} hashes
- * @param {CachedPerProjectEncryptedS3Persistor} [projectCache]
- */
-export async function verifyBlobs(historyId, hashes, projectCache) {
- if (hashes.length === 0) throw new Error('bug: empty hashes')
-
- if (!projectCache) {
- projectCache = await getProjectPersistor(historyId)
- }
- const blobStore = new BlobStore(historyId)
- for (const hash of hashes) {
- const path = makeProjectKey(historyId, hash)
- const blob = await blobStore.getBlob(hash)
- if (!blob) throw new Blob.NotFoundError(hash)
- let stream
- try {
- stream = await projectCache.getObjectStream(projectBlobsBucket, path, {
- autoGunzip: true,
- })
- } catch (err) {
- if (err instanceof NotFoundError) {
- throw new BackupCorruptedMissingBlobError('missing blob', {
- path,
- hash,
- })
- }
- throw err
- }
- const backupHash = await blobHash.fromStream(blob.getByteLength(), stream)
- if (backupHash !== hash) {
- throw new BackupCorruptedInvalidBlobError(
- 'hash mismatch for backed up blob',
- {
- path,
- hash,
- backupHash,
- }
- )
- }
- }
-}
-
-/**
- * @param {string} historyId
- * @param {Date} [endTimestamp]
- */
-export async function verifyProjectWithErrorContext(
- historyId,
- endTimestamp = getEndDateForRPO()
-) {
- try {
- await verifyProject(historyId, endTimestamp)
- } catch (err) {
- // @ts-ignore err is Error instance
- throw OError.tag(err, 'verifyProject', { historyId, endTimestamp })
- }
-}
-
-/**
- *
- * @param {string} historyId
- * @param {number} startVersion
- * @param {CachedPerProjectEncryptedS3Persistor} backupPersistorForProject
- * @return {Promise}
- */
-export async function loadChunk(
- historyId,
- startVersion,
- backupPersistorForProject
-) {
- const key = path.join(
- projectKey.format(historyId),
- projectKey.pad(startVersion)
- )
- try {
- const buf = await streams.gunzipStreamToBuffer(
- await backupPersistorForProject.getObjectStream(chunksBucket, key)
- )
- return JSON.parse(buf.toString('utf-8'))
- } catch (err) {
- if (err instanceof objectPersistor.Errors.NotFoundError) {
- throw new Chunk.NotPersistedError(historyId)
- }
- if (err instanceof Error) {
- throw OError.tag(err, 'Failed to load chunk', { historyId, startVersion })
- }
- throw err
- }
-}
-
-/**
- * @param {string} historyId
- * @param {Date} endTimestamp
- */
-export async function verifyProject(historyId, endTimestamp) {
- const backend = chunkStore.getBackend(historyId)
- const [first, last] = await Promise.all([
- backend.getFirstChunkBeforeTimestamp(historyId, endTimestamp),
- backend.getLastActiveChunkBeforeTimestamp(historyId, endTimestamp),
- ])
-
- const chunksRecordsToVerify = [
- {
- chunkId: first.id,
- chunkLabel: 'first',
- },
- ]
- if (first.startVersion !== last.startVersion) {
- chunksRecordsToVerify.push({
- chunkId: last.id,
- chunkLabel: 'last before RPO',
- })
- }
-
- const projectCache = await getProjectPersistor(historyId)
-
- const chunks = await Promise.all(
- chunksRecordsToVerify.map(async chunk => {
- try {
- return History.fromRaw(
- await loadChunk(historyId, chunk.startVersion, projectCache)
- )
- } catch (err) {
- if (err instanceof Chunk.NotPersistedError) {
- throw new BackupRPOViolationChunkNotBackedUpError(
- 'BackupRPOviolation: chunk not backed up',
- chunk
- )
- }
- throw err
- }
- })
- )
- const seenBlobs = new Set()
- const blobsToVerify = []
- for (const chunk of chunks) {
- /** @type {Set} */
- const chunkBlobs = new Set()
- chunk.findBlobHashes(chunkBlobs)
- let hasAddedBlobFromThisChunk = false
- for (const blobHash of chunkBlobs) {
- if (seenBlobs.has(blobHash)) continue // old blob
- if (GLOBAL_BLOBS.has(blobHash)) continue // global blob
- seenBlobs.add(blobHash)
- if (!hasAddedBlobFromThisChunk) {
- blobsToVerify.push(blobHash)
- hasAddedBlobFromThisChunk = true
- }
- }
- }
- if (blobsToVerify.length === 0) {
- logger.debug(
- {
- historyId,
- chunksRecordsToVerify: chunksRecordsToVerify.map(c => c.chunkId),
- },
- 'chunks contain no blobs to verify'
- )
- return
- }
- await verifyBlobs(historyId, blobsToVerify, projectCache)
-}
-
-export class BackupCorruptedError extends OError {}
-export class BackupRPOViolationError extends OError {}
-export class BackupCorruptedMissingBlobError extends BackupCorruptedError {}
-export class BackupCorruptedInvalidBlobError extends BackupCorruptedError {}
-export class BackupRPOViolationChunkNotBackedUpError extends OError {}
diff --git a/services/history-v1/storage/lib/backup_store/index.js b/services/history-v1/storage/lib/backup_store/index.js
deleted file mode 100644
index da7944786a..0000000000
--- a/services/history-v1/storage/lib/backup_store/index.js
+++ /dev/null
@@ -1,212 +0,0 @@
-const { Binary, ObjectId } = require('mongodb')
-const { projects, backedUpBlobs } = require('../mongodb')
-const OError = require('@overleaf/o-error')
-
-// List projects with pending backups older than the specified interval
-function listPendingBackups(timeIntervalMs = 0, limit = null) {
- const cutoffTime = new Date(Date.now() - timeIntervalMs)
- const options = {
- projection: { 'overleaf.backup.pendingChangeAt': 1 },
- sort: { 'overleaf.backup.pendingChangeAt': 1 },
- }
-
- // Apply limit if provided
- if (limit) {
- options.limit = limit
- }
-
- const cursor = projects.find(
- {
- 'overleaf.backup.pendingChangeAt': {
- $exists: true,
- $lt: cutoffTime,
- },
- },
- options
- )
- return cursor
-}
-
-// List projects that have never been backed up and are older than the specified interval
-function listUninitializedBackups(timeIntervalMs = 0, limit = null) {
- const cutoffTimeInSeconds = (Date.now() - timeIntervalMs) / 1000
- const options = {
- projection: { _id: 1 },
- sort: { _id: 1 },
- }
- // Apply limit if provided
- if (limit) {
- options.limit = limit
- }
- const cursor = projects.find(
- {
- 'overleaf.backup.lastBackedUpVersion': null,
- _id: {
- $lt: ObjectId.createFromTime(cutoffTimeInSeconds),
- },
- },
- options
- )
- return cursor
-}
-
-// Retrieve the history ID for a given project without giving direct access to the
-// projects collection.
-
-async function getHistoryId(projectId) {
- const project = await projects.findOne(
- { _id: new ObjectId(projectId) },
- {
- projection: {
- 'overleaf.history.id': 1,
- },
- }
- )
- if (!project) {
- throw new Error('Project not found')
- }
- return project.overleaf.history.id
-}
-
-async function getBackupStatus(projectId) {
- const project = await projects.findOne(
- { _id: new ObjectId(projectId) },
- {
- projection: {
- 'overleaf.history': 1,
- 'overleaf.backup': 1,
- },
- }
- )
- if (!project) {
- throw new Error('Project not found')
- }
- return {
- backupStatus: project.overleaf.backup,
- historyId: `${project.overleaf.history.id}`,
- currentEndVersion: project.overleaf.history.currentEndVersion,
- currentEndTimestamp: project.overleaf.history.currentEndTimestamp,
- }
-}
-
-async function setBackupVersion(
- projectId,
- previousBackedUpVersion,
- currentBackedUpVersion,
- currentBackedUpAt
-) {
- // FIXME: include a check to handle race conditions
- // to make sure only one process updates the version numbers
- const result = await projects.updateOne(
- {
- _id: new ObjectId(projectId),
- 'overleaf.backup.lastBackedUpVersion': previousBackedUpVersion,
- },
- {
- $set: {
- 'overleaf.backup.lastBackedUpVersion': currentBackedUpVersion,
- 'overleaf.backup.lastBackedUpAt': currentBackedUpAt,
- },
- }
- )
- if (result.matchedCount === 0 || result.modifiedCount === 0) {
- throw new OError('Failed to update backup version', {
- previousBackedUpVersion,
- currentBackedUpVersion,
- currentBackedUpAt,
- result,
- })
- }
-}
-
-async function updateCurrentMetadataIfNotSet(projectId, latestChunkMetadata) {
- await projects.updateOne(
- {
- _id: new ObjectId(projectId),
- 'overleaf.history.currentEndVersion': { $exists: false },
- 'overleaf.history.currentEndTimestamp': { $exists: false },
- },
- {
- $set: {
- 'overleaf.history.currentEndVersion': latestChunkMetadata.endVersion,
- 'overleaf.history.currentEndTimestamp':
- latestChunkMetadata.endTimestamp,
- },
- }
- )
-}
-
-/**
- * Updates the pending change timestamp for a project's backup status
- * @param {string} projectId - The ID of the project to update
- * @param {Date} backupStartTime - The timestamp to set for pending changes
- * @returns {Promise}
- *
- * If the project's last backed up version matches the current end version,
- * the pending change timestamp is removed. Otherwise, it's set to the provided
- * backup start time.
- */
-async function updatePendingChangeTimestamp(projectId, backupStartTime) {
- await projects.updateOne({ _id: new ObjectId(projectId) }, [
- {
- $set: {
- 'overleaf.backup.pendingChangeAt': {
- $cond: {
- if: {
- $eq: [
- '$overleaf.backup.lastBackedUpVersion',
- '$overleaf.history.currentEndVersion',
- ],
- },
- then: '$$REMOVE',
- else: backupStartTime,
- },
- },
- },
- },
- ])
-}
-
-async function getBackedUpBlobHashes(projectId) {
- const result = await backedUpBlobs.findOne(
- { _id: new ObjectId(projectId) },
- { projection: { blobs: 1 } }
- )
- if (!result) {
- return new Set()
- }
- const hashes = result.blobs.map(b => b.buffer.toString('hex'))
- return new Set(hashes)
-}
-
-async function unsetBackedUpBlobHashes(projectId, hashes) {
- const binaryHashes = hashes.map(h => new Binary(Buffer.from(h, 'hex')))
- const result = await backedUpBlobs.findOneAndUpdate(
- { _id: new ObjectId(projectId) },
- {
- $pullAll: {
- blobs: binaryHashes,
- },
- },
- { returnDocument: 'after' }
- )
- if (result && result.blobs.length === 0) {
- await backedUpBlobs.deleteOne({
- _id: new ObjectId(projectId),
- blobs: { $size: 0 },
- })
- }
- return result
-}
-
-module.exports = {
- getHistoryId,
- getBackupStatus,
- setBackupVersion,
- updateCurrentMetadataIfNotSet,
- updatePendingChangeTimestamp,
- listPendingBackups,
- listUninitializedBackups,
- getBackedUpBlobHashes,
- unsetBackedUpBlobHashes,
-}
diff --git a/services/history-v1/storage/lib/blob_hash.js b/services/history-v1/storage/lib/blob_hash.js
index 10ac64b87b..5c2edac14f 100644
--- a/services/history-v1/storage/lib/blob_hash.js
+++ b/services/history-v1/storage/lib/blob_hash.js
@@ -2,9 +2,9 @@
'use strict'
const BPromise = require('bluebird')
-const fs = BPromise.promisifyAll(require('node:fs'))
-const crypto = require('node:crypto')
-const { pipeline } = require('node:stream')
+const fs = BPromise.promisifyAll(require('fs'))
+const crypto = require('crypto')
+const { pipeline } = require('stream')
const assert = require('./assert')
function getGitBlobHeader(byteLength) {
@@ -63,7 +63,7 @@ exports.fromString = function blobHashFromString(string) {
* Compute the git blob hash for the content of a file
*
* @param {string} filePath
- * @return {Promise} hexadecimal SHA-1 hash
+ * @return {string} hexadecimal SHA-1 hash
*/
exports.fromFile = function blobHashFromFile(pathname) {
assert.string(pathname, 'blobHash: bad pathname')
diff --git a/services/history-v1/storage/lib/blob_store/index.js b/services/history-v1/storage/lib/blob_store/index.js
index 033e288554..6f5632a595 100644
--- a/services/history-v1/storage/lib/blob_store/index.js
+++ b/services/history-v1/storage/lib/blob_store/index.js
@@ -1,7 +1,7 @@
'use strict'
const config = require('config')
-const fs = require('node:fs')
+const fs = require('fs')
const isValidUtf8 = require('utf-8-validate')
const { ReadableString } = require('@overleaf/stream-utils')
@@ -24,7 +24,6 @@ const logger = require('@overleaf/logger')
/** @import { Readable } from 'stream' */
-/** @type {Map} */
const GLOBAL_BLOBS = new Map()
function makeGlobalKey(hash) {
@@ -80,12 +79,23 @@ function getBackend(projectId) {
}
async function makeBlobForFile(pathname) {
- const { size: byteLength } = await fs.promises.stat(pathname)
- const hash = await blobHash.fromStream(
- byteLength,
- fs.createReadStream(pathname)
- )
- return new Blob(hash, byteLength)
+ async function getByteLengthOfFile() {
+ const stat = await fs.promises.stat(pathname)
+ return stat.size
+ }
+
+ async function getHashOfFile(blob) {
+ const stream = fs.createReadStream(pathname)
+ const hash = await blobHash.fromStream(blob.getByteLength(), stream)
+ return hash
+ }
+
+ const blob = new Blob()
+ const byteLength = await getByteLengthOfFile()
+ blob.setByteLength(byteLength)
+ const hash = await getHashOfFile(blob)
+ blob.setHash(hash)
+ return blob
}
async function getStringLengthOfFile(byteLength, pathname) {
@@ -126,34 +136,6 @@ async function loadGlobalBlobs() {
}
}
-/**
- * Return metadata for all blobs in the given project
- * @param {Array} projectIds
- * @return {Promise<{nBlobs:number, blobs:Map>}>}
- */
-async function getProjectBlobsBatch(projectIds) {
- const mongoProjects = []
- const postgresProjects = []
- for (const projectId of projectIds) {
- if (typeof projectId === 'number') {
- postgresProjects.push(projectId)
- } else {
- mongoProjects.push(projectId)
- }
- }
- const [
- { nBlobs: nBlobsPostgres, blobs: blobsPostgres },
- { nBlobs: nBlobsMongo, blobs: blobsMongo },
- ] = await Promise.all([
- postgresBackend.getProjectBlobsBatch(postgresProjects),
- mongoBackend.getProjectBlobsBatch(mongoProjects),
- ])
- for (const [id, blobs] of blobsPostgres.entries()) {
- blobsMongo.set(id.toString(), blobs)
- }
- return { nBlobs: nBlobsPostgres + nBlobsMongo, blobs: blobsMongo }
-}
-
/**
* @classdesc
* Fetch and store the content of files using content-addressable hashing. The
@@ -206,7 +188,7 @@ class BlobStore {
* temporary file).
*
* @param {string} pathname
- * @return {Promise}
+ * @return {Promise.}
*/
async putFile(pathname) {
assert.string(pathname, 'bad pathname')
@@ -220,28 +202,11 @@ class BlobStore {
pathname
)
newBlob.setStringLength(stringLength)
- await this.putBlob(pathname, newBlob)
+ await uploadBlob(this.projectId, newBlob, fs.createReadStream(pathname))
+ await this.backend.insertBlob(this.projectId, newBlob)
return newBlob
}
- /**
- * Write a new blob, the stringLength must have been added already. It should
- * have been checked that the blob does not exist yet. Consider using
- * {@link putFile} instead of this lower-level method.
- *
- * @param {string} pathname
- * @param {core.Blob} finializedBlob
- * @return {Promise}
- */
- async putBlob(pathname, finializedBlob) {
- await uploadBlob(
- this.projectId,
- finializedBlob,
- fs.createReadStream(pathname)
- )
- await this.backend.insertBlob(this.projectId, finializedBlob)
- }
-
/**
* Stores an object as a JSON string in a blob.
*
@@ -310,15 +275,14 @@ class BlobStore {
* failure, so the caller must be prepared to retry on errors, if appropriate.
*
* @param {string} hash hexadecimal SHA-1 hash
- * @param {Object} opts
* @return {Promise.} a stream to read the file
*/
- async getStream(hash, opts = {}) {
+ async getStream(hash) {
assert.blobHash(hash, 'bad hash')
const { bucket, key } = getBlobLocation(this.projectId, hash)
try {
- const stream = await persistor.getObjectStream(bucket, key, opts)
+ const stream = await persistor.getObjectStream(bucket, key)
return stream
} catch (err) {
if (err instanceof objectPersistor.Errors.NotFoundError) {
@@ -344,11 +308,6 @@ class BlobStore {
return blob
}
- /**
- *
- * @param {Array} hashes
- * @return {Promise<*[]>}
- */
async getBlobs(hashes) {
assert.array(hashes, 'bad hashes')
const nonGlobalHashes = []
@@ -361,9 +320,6 @@ class BlobStore {
nonGlobalHashes.push(hash)
}
}
- if (nonGlobalHashes.length === 0) {
- return blobs // to avoid unnecessary database lookup
- }
const projectBlobs = await this.backend.findBlobs(
this.projectId,
nonGlobalHashes
@@ -372,16 +328,6 @@ class BlobStore {
return blobs
}
- /**
- * Retrieve all blobs associated with the project.
- * @returns {Promise} A promise that resolves to an array of blobs.
- */
-
- async getProjectBlobs() {
- const projectBlobs = await this.backend.getProjectBlobs(this.projectId)
- return projectBlobs
- }
-
/**
* Delete all blobs that belong to the project.
*/
@@ -400,41 +346,6 @@ class BlobStore {
const blob = await this.backend.findBlob(this.projectId, hash)
return blob
}
-
- /**
- * Copy an existing sourceBlob in this project to a target project.
- * @param {Blob} sourceBlob
- * @param {string} targetProjectId
- * @return {Promise}
- */
- async copyBlob(sourceBlob, targetProjectId) {
- assert.instance(sourceBlob, Blob, 'bad sourceBlob')
- assert.projectId(targetProjectId, 'bad targetProjectId')
- const hash = sourceBlob.getHash()
- const sourceProjectId = this.projectId
- const { bucket, key: sourceKey } = getBlobLocation(sourceProjectId, hash)
- const destKey = makeProjectKey(targetProjectId, hash)
- const targetBackend = getBackend(targetProjectId)
- logger.debug({ sourceProjectId, targetProjectId, hash }, 'copyBlob started')
- try {
- await persistor.copyObject(bucket, sourceKey, destKey)
- await targetBackend.insertBlob(targetProjectId, sourceBlob)
- } finally {
- logger.debug(
- { sourceProjectId, targetProjectId, hash },
- 'copyBlob finished'
- )
- }
- }
}
-module.exports = {
- BlobStore,
- getProjectBlobsBatch,
- loadGlobalBlobs,
- makeProjectKey,
- makeGlobalKey,
- makeBlobForFile,
- getStringLengthOfFile,
- GLOBAL_BLOBS,
-}
+module.exports = { BlobStore, loadGlobalBlobs }
diff --git a/services/history-v1/storage/lib/blob_store/mongo.js b/services/history-v1/storage/lib/blob_store/mongo.js
index 9117382148..6bd516addb 100644
--- a/services/history-v1/storage/lib/blob_store/mongo.js
+++ b/services/history-v1/storage/lib/blob_store/mongo.js
@@ -1,4 +1,3 @@
-// @ts-check
/**
* Mongo backend for the blob store.
*
@@ -16,20 +15,15 @@
*/
const { Blob } = require('overleaf-editor-core')
-const { ObjectId, Binary, MongoError, ReadPreference } = require('mongodb')
+const { ObjectId, Binary } = require('mongodb')
const assert = require('../assert')
const mongodb = require('../mongodb')
const MAX_BLOBS_IN_BUCKET = 8
const DUPLICATE_KEY_ERROR_CODE = 11000
-/**
- * @typedef {import('mongodb').ReadPreferenceLike} ReadPreferenceLike
- */
-
/**
* Set up the data structures for a given project.
- * @param {string} projectId
*/
async function initialize(projectId) {
assert.mongoId(projectId, 'bad projectId')
@@ -39,18 +33,14 @@ async function initialize(projectId) {
blobs: {},
})
} catch (err) {
- if (err instanceof MongoError && err.code === DUPLICATE_KEY_ERROR_CODE) {
- return // ignore already initialized case
+ if (err.code !== DUPLICATE_KEY_ERROR_CODE) {
+ throw err
}
- throw err
}
}
/**
* Return blob metadata for the given project and hash.
- * @param {string} projectId
- * @param {string} hash
- * @return {Promise}
*/
async function findBlob(projectId, hash) {
assert.mongoId(projectId, 'bad projectId')
@@ -79,9 +69,6 @@ async function findBlob(projectId, hash) {
/**
* Search in the sharded collection for blob metadata
- * @param {string} projectId
- * @param {string} hash
- * @return {Promise}
*/
async function findBlobSharded(projectId, hash) {
const [shard, bucket] = getShardedBucket(hash)
@@ -94,15 +81,11 @@ async function findBlobSharded(projectId, hash) {
return null
}
const record = result.blobs.find(blob => blob.h.toString('hex') === hash)
- if (!record) return null
return recordToBlob(record)
}
/**
* Read multiple blob metadata records by hexadecimal hashes.
- * @param {string} projectId
- * @param {Array} hashes
- * @return {Promise>}
*/
async function findBlobs(projectId, hashes) {
assert.mongoId(projectId, 'bad projectId')
@@ -152,9 +135,6 @@ async function findBlobs(projectId, hashes) {
/**
* Search in the sharded collection for blob metadata.
- * @param {string} projectId
- * @param {Set} hashSet
- * @return {Promise>}
*/
async function findBlobsSharded(projectId, hashSet) {
// Build a map of buckets by shard key
@@ -201,113 +181,8 @@ async function findBlobsSharded(projectId, hashSet) {
return blobs
}
-/**
- * Return metadata for all blobs in the given project
- */
-async function getProjectBlobs(projectId) {
- assert.mongoId(projectId, 'bad projectId')
-
- const result = await mongodb.blobs.findOne(
- { _id: new ObjectId(projectId) },
- { projection: { _id: 0 } }
- )
-
- if (!result) {
- return []
- }
-
- // Build blobs from the query results
- const blobs = []
- for (const bucket of Object.values(result.blobs)) {
- for (const record of bucket) {
- blobs.push(recordToBlob(record))
- }
- }
-
- // Look for all possible sharded blobs
-
- const minShardedId = makeShardedId(projectId, '0')
- const maxShardedId = makeShardedId(projectId, 'f')
- // @ts-ignore We are using a custom _id here.
- const shardedRecords = mongodb.shardedBlobs.find(
- {
- _id: { $gte: minShardedId, $lte: maxShardedId },
- },
- { projection: { _id: 0 } }
- )
-
- for await (const shardedRecord of shardedRecords) {
- if (shardedRecord.blobs == null) {
- continue
- }
- for (const bucket of Object.values(shardedRecord.blobs)) {
- for (const record of bucket) {
- blobs.push(recordToBlob(record))
- }
- }
- }
-
- return blobs
-}
-
-/**
- * Return metadata for all blobs in the given project
- * @param {Array} projectIds
- * @return {Promise<{ nBlobs: number, blobs: Map> }>}
- */
-async function getProjectBlobsBatch(projectIds) {
- for (const project of projectIds) {
- assert.mongoId(project, 'bad projectId')
- }
- let nBlobs = 0
- const blobs = new Map()
- if (projectIds.length === 0) return { nBlobs, blobs }
-
- // blobs
- {
- const cursor = await mongodb.blobs.find(
- { _id: { $in: projectIds.map(projectId => new ObjectId(projectId)) } },
- { readPreference: ReadPreference.secondaryPreferred }
- )
- for await (const record of cursor) {
- const projectBlobs = Object.values(record.blobs).flat().map(recordToBlob)
- blobs.set(record._id.toString(), projectBlobs)
- nBlobs += projectBlobs.length
- }
- }
-
- // sharded blobs
- {
- // @ts-ignore We are using a custom _id here.
- const cursor = await mongodb.shardedBlobs.find(
- {
- _id: {
- $gte: makeShardedId(projectIds[0], '0'),
- $lte: makeShardedId(projectIds[projectIds.length - 1], 'f'),
- },
- },
- { readPreference: ReadPreference.secondaryPreferred }
- )
- for await (const record of cursor) {
- const recordIdHex = record._id.toString('hex')
- const recordProjectId = recordIdHex.slice(0, 24)
- const projectBlobs = Object.values(record.blobs).flat().map(recordToBlob)
- const found = blobs.get(recordProjectId)
- if (found) {
- found.push(...projectBlobs)
- } else {
- blobs.set(recordProjectId, projectBlobs)
- }
- nBlobs += projectBlobs.length
- }
- }
- return { nBlobs, blobs }
-}
-
/**
* Add a blob's metadata to the blobs collection after it has been uploaded.
- * @param {string} projectId
- * @param {Blob} blob
*/
async function insertBlob(projectId, blob) {
assert.mongoId(projectId, 'bad projectId')
@@ -333,10 +208,6 @@ async function insertBlob(projectId, blob) {
/**
* Add a blob's metadata to the sharded blobs collection.
- * @param {string} projectId
- * @param {string} hash
- * @param {Record} record
- * @return {Promise}
*/
async function insertRecordSharded(projectId, hash, record) {
const [shard, bucket] = getShardedBucket(hash)
@@ -350,7 +221,6 @@ async function insertRecordSharded(projectId, hash, record) {
/**
* Delete all blobs for a given project.
- * @param {string} projectId
*/
async function deleteBlobs(projectId) {
assert.mongoId(projectId, 'bad projectId')
@@ -358,15 +228,12 @@ async function deleteBlobs(projectId) {
const minShardedId = makeShardedId(projectId, '0')
const maxShardedId = makeShardedId(projectId, 'f')
await mongodb.shardedBlobs.deleteMany({
- // @ts-ignore We are using a custom _id here.
_id: { $gte: minShardedId, $lte: maxShardedId },
})
}
/**
* Return the Mongo path to the bucket for the given hash.
- * @param {string} hash
- * @return {string}
*/
function getBucket(hash) {
return `blobs.${hash.slice(0, 3)}`
@@ -375,8 +242,6 @@ function getBucket(hash) {
/**
* Return the shard key and Mongo path to the bucket for the given hash in the
* sharded collection.
- * @param {string} hash
- * @return {[string, string]}
*/
function getShardedBucket(hash) {
const shard = hash.slice(0, 1)
@@ -386,25 +251,13 @@ function getShardedBucket(hash) {
/**
* Create an _id key for the sharded collection.
- * @param {string} projectId
- * @param {string} shard
- * @return {Binary}
*/
function makeShardedId(projectId, shard) {
return new Binary(Buffer.from(`${projectId}0${shard}`, 'hex'))
}
-/**
- * @typedef {Object} Record
- * @property {Binary} h
- * @property {number} b
- * @property {number} [s]
- */
-
/**
* Return the Mongo record for the given blob.
- * @param {Blob} blob
- * @return {Record}
*/
function blobToRecord(blob) {
const hash = blob.getHash()
@@ -419,10 +272,11 @@ function blobToRecord(blob) {
/**
* Create a blob from the given Mongo record.
- * @param {Record} record
- * @return {Blob}
*/
function recordToBlob(record) {
+ if (record == null) {
+ return
+ }
return new Blob(record.h.toString('hex'), record.b, record.s)
}
@@ -430,8 +284,6 @@ module.exports = {
initialize,
findBlob,
findBlobs,
- getProjectBlobs,
- getProjectBlobsBatch,
insertBlob,
deleteBlobs,
}
diff --git a/services/history-v1/storage/lib/blob_store/postgres.js b/services/history-v1/storage/lib/blob_store/postgres.js
index 1cedeec5d7..9e40c255da 100644
--- a/services/history-v1/storage/lib/blob_store/postgres.js
+++ b/services/history-v1/storage/lib/blob_store/postgres.js
@@ -13,8 +13,8 @@ async function initialize(projectId) {
* Return blob metadata for the given project and hash
*/
async function findBlob(projectId, hash) {
- assert.postgresId(projectId, 'bad projectId')
projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
assert.blobHash(hash, 'bad hash')
const binaryHash = hashToBuffer(hash)
@@ -35,8 +35,8 @@ async function findBlob(projectId, hash) {
* @return {Promise.>} no guarantee on order
*/
async function findBlobs(projectId, hashes) {
- assert.postgresId(projectId, 'bad projectId')
projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
assert.array(hashes, 'bad hashes: not array')
hashes.forEach(function (hash) {
assert.blobHash(hash, 'bad hash')
@@ -53,58 +53,12 @@ async function findBlobs(projectId, hashes) {
return blobs
}
-/**
- * Return metadata for all blobs in the given project
- */
-async function getProjectBlobs(projectId) {
- assert.postgresId(projectId, 'bad projectId')
- projectId = parseInt(projectId, 10)
-
- const records = await knex('project_blobs')
- .select('hash_bytes', 'byte_length', 'string_length')
- .where({
- project_id: projectId,
- })
-
- const blobs = records.map(recordToBlob)
- return blobs
-}
-
-/**
- * Return metadata for all blobs in the given project
- * @param {Array} projectIds
- * @return {Promise<{ nBlobs: number, blobs: Map> }>}
- */
-async function getProjectBlobsBatch(projectIds) {
- for (const projectId of projectIds) {
- assert.integer(projectId, 'bad projectId')
- }
- let nBlobs = 0
- const blobs = new Map()
- if (projectIds.length === 0) return { nBlobs, blobs }
-
- const cursor = knex('project_blobs')
- .select('project_id', 'hash_bytes', 'byte_length', 'string_length')
- .whereIn('project_id', projectIds)
- .stream()
- for await (const record of cursor) {
- const found = blobs.get(record.project_id)
- if (found) {
- found.push(recordToBlob(record))
- } else {
- blobs.set(record.project_id, [recordToBlob(record)])
- }
- nBlobs++
- }
- return { nBlobs, blobs }
-}
-
/**
* Add a blob's metadata to the blobs table after it has been uploaded.
*/
async function insertBlob(projectId, blob) {
- assert.postgresId(projectId, 'bad projectId')
projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
await knex('project_blobs')
.insert(blobToRecord(projectId, blob))
@@ -116,8 +70,8 @@ async function insertBlob(projectId, blob) {
* Deletes all blobs for a given project
*/
async function deleteBlobs(projectId) {
- assert.postgresId(projectId, 'bad projectId')
projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
await knex('project_blobs').where('project_id', projectId).delete()
}
@@ -154,8 +108,6 @@ module.exports = {
initialize,
findBlob,
findBlobs,
- getProjectBlobs,
- getProjectBlobsBatch,
insertBlob,
deleteBlobs,
}
diff --git a/services/history-v1/storage/lib/chunk_store/errors.js b/services/history-v1/storage/lib/chunk_store/errors.js
index 75b830f9a0..5f0eba6aac 100644
--- a/services/history-v1/storage/lib/chunk_store/errors.js
+++ b/services/history-v1/storage/lib/chunk_store/errors.js
@@ -1,15 +1,7 @@
const OError = require('@overleaf/o-error')
class ChunkVersionConflictError extends OError {}
-class BaseVersionConflictError extends OError {}
-class JobNotFoundError extends OError {}
-class JobNotReadyError extends OError {}
-class VersionOutOfBoundsError extends OError {}
module.exports = {
ChunkVersionConflictError,
- BaseVersionConflictError,
- JobNotFoundError,
- JobNotReadyError,
- VersionOutOfBoundsError,
}
diff --git a/services/history-v1/storage/lib/chunk_store/index.js b/services/history-v1/storage/lib/chunk_store/index.js
index f387b68d90..eb3c8ba48d 100644
--- a/services/history-v1/storage/lib/chunk_store/index.js
+++ b/services/history-v1/storage/lib/chunk_store/index.js
@@ -1,5 +1,3 @@
-// @ts-check
-
'use strict'
/**
@@ -29,18 +27,10 @@ const { Chunk, History, Snapshot } = require('overleaf-editor-core')
const assert = require('../assert')
const BatchBlobStore = require('../batch_blob_store')
const { BlobStore } = require('../blob_store')
-const { historyStore } = require('../history_store')
+const historyStore = require('../history_store')
const mongoBackend = require('./mongo')
const postgresBackend = require('./postgres')
-const redisBackend = require('./redis')
-const {
- ChunkVersionConflictError,
- VersionOutOfBoundsError,
-} = require('./errors')
-
-/**
- * @import { Change } from 'overleaf-editor-core'
- */
+const { ChunkVersionConflictError } = require('./errors')
const DEFAULT_DELETE_BATCH_SIZE = parseInt(config.get('maxDeleteKeys'), 10)
const DEFAULT_DELETE_TIMEOUT_SECS = 3000 // 50 minutes
@@ -91,120 +81,49 @@ async function lazyLoadHistoryFiles(history, batchBlobStore) {
/**
* Load the latest Chunk stored for a project, including blob metadata.
*
- * @param {string} projectId
- * @param {Object} [opts]
- * @param {boolean} [opts.readOnly]
- * @return {Promise<{id: string, startVersion: number, endVersion: number, endTimestamp: Date}>}
+ * @param {number} projectId
+ * @return {Promise.}
*/
-async function getLatestChunkMetadata(projectId, opts) {
+async function loadLatest(projectId) {
assert.projectId(projectId, 'bad projectId')
const backend = getBackend(projectId)
- const chunkMetadata = await backend.getLatestChunk(projectId, opts)
- if (chunkMetadata == null) {
- throw new Chunk.NotFoundError(projectId)
- }
- return chunkMetadata
-}
-
-/**
- * Load the latest Chunk stored for a project, including blob metadata.
- *
- * @param {string} projectId
- * @param {object} [opts]
- * @param {boolean} [opts.persistedOnly] - only include persisted changes
- * @return {Promise}
- */
-async function loadLatest(projectId, opts = {}) {
- const chunkMetadata = await getLatestChunkMetadata(projectId)
- const rawHistory = await historyStore.loadRaw(projectId, chunkMetadata.id)
- const history = History.fromRaw(rawHistory)
-
- if (!opts.persistedOnly) {
- const nonPersistedChanges = await getChunkExtension(
- projectId,
- chunkMetadata.endVersion
- )
- history.pushChanges(nonPersistedChanges)
- }
-
const blobStore = new BlobStore(projectId)
const batchBlobStore = new BatchBlobStore(blobStore)
+ const chunkRecord = await backend.getLatestChunk(projectId)
+ if (chunkRecord == null) {
+ throw new Chunk.NotFoundError(projectId)
+ }
+
+ const rawHistory = await historyStore.loadRaw(projectId, chunkRecord.id)
+ const history = History.fromRaw(rawHistory)
await lazyLoadHistoryFiles(history, batchBlobStore)
- return new Chunk(history, chunkMetadata.startVersion)
+ return new Chunk(history, chunkRecord.startVersion)
}
/**
* Load the the chunk that contains the given version, including blob metadata.
- *
- * @param {string} projectId
- * @param {number} version
- * @param {object} [opts]
- * @param {boolean} [opts.persistedOnly] - only include persisted changes
- * @param {boolean} [opts.preferNewer] - If the version is at the boundary of
- * two chunks, return the newer chunk.
*/
-async function loadAtVersion(projectId, version, opts = {}) {
+async function loadAtVersion(projectId, version) {
assert.projectId(projectId, 'bad projectId')
assert.integer(version, 'bad version')
const backend = getBackend(projectId)
const blobStore = new BlobStore(projectId)
const batchBlobStore = new BatchBlobStore(blobStore)
- const latestChunkMetadata = await getLatestChunkMetadata(projectId)
- // When loading a chunk for a version there are three cases to consider:
- // 1. If `persistedOnly` is true, we always use the requested version
- // to fetch the chunk.
- // 2. If `persistedOnly` is false and the requested version is in the
- // persisted chunk version range, we use the requested version.
- // 3. If `persistedOnly` is false and the requested version is ahead of
- // the persisted chunk versions, we fetch the latest chunk and see if
- // the non-persisted changes include the requested version.
- const targetChunkVersion = opts.persistedOnly
- ? version
- : Math.min(latestChunkMetadata.endVersion, version)
-
- const chunkRecord = await backend.getChunkForVersion(
- projectId,
- targetChunkVersion,
- {
- preferNewer: opts.preferNewer,
- }
- )
+ const chunkRecord = await backend.getChunkForVersion(projectId, version)
const rawHistory = await historyStore.loadRaw(projectId, chunkRecord.id)
const history = History.fromRaw(rawHistory)
- const startVersion = chunkRecord.endVersion - history.countChanges()
-
- if (!opts.persistedOnly) {
- // Try to extend the chunk with any non-persisted changes that
- // follow the chunk's end version.
- const nonPersistedChanges = await getChunkExtension(
- projectId,
- chunkRecord.endVersion
- )
- history.pushChanges(nonPersistedChanges)
-
- // Check that the changes do actually contain the requested version
- if (version > chunkRecord.endVersion + nonPersistedChanges.length) {
- throw new Chunk.VersionNotFoundError(projectId, version)
- }
- }
-
await lazyLoadHistoryFiles(history, batchBlobStore)
- return new Chunk(history, startVersion)
+ return new Chunk(history, chunkRecord.endVersion - history.countChanges())
}
/**
* Load the chunk that contains the version that was current at the given
* timestamp, including blob metadata.
- *
- * @param {string} projectId
- * @param {Date} timestamp
- * @param {object} [opts]
- * @param {boolean} [opts.persistedOnly] - only include persisted changes
*/
-async function loadAtTimestamp(projectId, timestamp, opts = {}) {
+async function loadAtTimestamp(projectId, timestamp) {
assert.projectId(projectId, 'bad projectId')
assert.date(timestamp, 'bad timestamp')
@@ -215,58 +134,24 @@ async function loadAtTimestamp(projectId, timestamp, opts = {}) {
const chunkRecord = await backend.getChunkForTimestamp(projectId, timestamp)
const rawHistory = await historyStore.loadRaw(projectId, chunkRecord.id)
const history = History.fromRaw(rawHistory)
- const startVersion = chunkRecord.endVersion - history.countChanges()
-
- if (!opts.persistedOnly) {
- const nonPersistedChanges = await getChunkExtension(
- projectId,
- chunkRecord.endVersion
- )
- history.pushChanges(nonPersistedChanges)
- }
-
await lazyLoadHistoryFiles(history, batchBlobStore)
- return new Chunk(history, startVersion)
+ return new Chunk(history, chunkRecord.endVersion - history.countChanges())
}
/**
* Store the chunk and insert corresponding records in the database.
*
- * @param {string} projectId
+ * @param {number} projectId
* @param {Chunk} chunk
- * @param {Date} [earliestChangeTimestamp]
+ * @return {Promise.} for the chunkId of the inserted chunk
*/
-async function create(projectId, chunk, earliestChangeTimestamp) {
+async function create(projectId, chunk) {
assert.projectId(projectId, 'bad projectId')
assert.instance(chunk, Chunk, 'bad chunk')
- assert.maybe.date(earliestChangeTimestamp, 'bad timestamp')
const backend = getBackend(projectId)
- const chunkStart = chunk.getStartVersion()
-
- const opts = {}
- if (chunkStart > 0) {
- const oldChunk = await backend.getChunkForVersion(projectId, chunkStart)
-
- if (oldChunk.endVersion !== chunkStart) {
- throw new ChunkVersionConflictError(
- 'unexpected end version on chunk to be updated',
- {
- projectId,
- expectedVersion: chunkStart,
- actualVersion: oldChunk.endVersion,
- }
- )
- }
-
- opts.oldChunkId = oldChunk.id
- }
- if (earliestChangeTimestamp != null) {
- opts.earliestChangeTimestamp = earliestChangeTimestamp
- }
-
const chunkId = await uploadChunk(projectId, chunk)
- await backend.confirmCreate(projectId, chunk, chunkId, opts)
+ await backend.confirmCreate(projectId, chunk, chunkId)
}
/**
@@ -295,67 +180,29 @@ async function uploadChunk(projectId, chunk) {
* Extend the project's history by replacing the latest chunk with a new
* chunk.
*
- * @param {string} projectId
+ * @param {number} projectId
+ * @param {number} oldEndVersion
* @param {Chunk} newChunk
- * @param {Date} [earliestChangeTimestamp]
* @return {Promise}
*/
-async function update(projectId, newChunk, earliestChangeTimestamp) {
+async function update(projectId, oldEndVersion, newChunk) {
assert.projectId(projectId, 'bad projectId')
+ assert.integer(oldEndVersion, 'bad oldEndVersion')
assert.instance(newChunk, Chunk, 'bad newChunk')
- assert.maybe.date(earliestChangeTimestamp, 'bad timestamp')
const backend = getBackend(projectId)
- const oldChunk = await backend.getChunkForVersion(
- projectId,
- newChunk.getStartVersion(),
- { preferNewer: true }
- )
-
- if (oldChunk.startVersion !== newChunk.getStartVersion()) {
- throw new ChunkVersionConflictError(
- 'unexpected start version on chunk to be updated',
- {
- projectId,
- expectedVersion: newChunk.getStartVersion(),
- actualVersion: oldChunk.startVersion,
- }
- )
- }
-
- if (oldChunk.endVersion > newChunk.getEndVersion()) {
- throw new ChunkVersionConflictError(
- 'chunk update would decrease chunk version',
- {
- projectId,
- currentVersion: oldChunk.endVersion,
- newVersion: newChunk.getEndVersion(),
- }
- )
- }
-
+ const oldChunkId = await getChunkIdForVersion(projectId, oldEndVersion)
const newChunkId = await uploadChunk(projectId, newChunk)
- const opts = {}
- if (earliestChangeTimestamp != null) {
- opts.earliestChangeTimestamp = earliestChangeTimestamp
- }
-
- await backend.confirmUpdate(
- projectId,
- oldChunk.id,
- newChunk,
- newChunkId,
- opts
- )
+ await backend.confirmUpdate(projectId, oldChunkId, newChunk, newChunkId)
}
/**
* Find the chunk ID for a given version of a project.
*
- * @param {string} projectId
+ * @param {number} projectId
* @param {number} version
- * @return {Promise.}
+ * @return {Promise.}
*/
async function getChunkIdForVersion(projectId, version) {
const backend = getBackend(projectId)
@@ -363,19 +210,6 @@ async function getChunkIdForVersion(projectId, version) {
return chunkRecord.id
}
-/**
- * Find the chunk metadata for a given version of a project.
- *
- * @param {string} projectId
- * @param {number} version
- * @return {Promise.<{id: string|number, startVersion: number, endVersion: number}>}
- */
-async function getChunkMetadataForVersion(projectId, version) {
- const backend = getBackend(projectId)
- const chunkRecord = await backend.getChunkForVersion(projectId, version)
- return chunkRecord
-}
-
/**
* Get all of a project's chunk ids
*/
@@ -385,62 +219,6 @@ async function getProjectChunkIds(projectId) {
return chunkIds
}
-/**
- * Get all of a projects chunks directly
- */
-async function getProjectChunks(projectId) {
- const backend = getBackend(projectId)
- const chunkIds = await backend.getProjectChunks(projectId)
- return chunkIds
-}
-
-/**
- * Load the chunk for a given chunk record, including blob metadata.
- */
-async function loadByChunkRecord(projectId, chunkRecord) {
- const blobStore = new BlobStore(projectId)
- const batchBlobStore = new BatchBlobStore(blobStore)
- const { raw: rawHistory, buffer: chunkBuffer } =
- await historyStore.loadRawWithBuffer(projectId, chunkRecord.id)
- const history = History.fromRaw(rawHistory)
- await lazyLoadHistoryFiles(history, batchBlobStore)
- return {
- chunk: new Chunk(history, chunkRecord.endVersion - history.countChanges()),
- chunkBuffer,
- }
-}
-
-/**
- * Asynchronously retrieves project chunks starting from a specific version.
- *
- * This generator function yields chunk records for a given project starting from the specified version (inclusive).
- * It continues to fetch and yield subsequent chunk records until the end version of the latest chunk metadata is reached.
- * If you want to fetch all the chunks *after* a version V, call this function with V+1.
- *
- * @param {string} projectId - The ID of the project.
- * @param {number} version - The starting version to retrieve chunks from.
- * @returns {AsyncGenerator} An async generator that yields chunk records.
- */
-async function* getProjectChunksFromVersion(projectId, version) {
- const backend = getBackend(projectId)
- const latestChunkMetadata = await getLatestChunkMetadata(projectId)
- if (!latestChunkMetadata || version > latestChunkMetadata.endVersion) {
- return
- }
- let chunkRecord = await backend.getChunkForVersion(projectId, version)
- while (chunkRecord != null) {
- yield chunkRecord
- if (chunkRecord.endVersion >= latestChunkMetadata.endVersion) {
- break
- } else {
- chunkRecord = await backend.getChunkForVersion(
- projectId,
- chunkRecord.endVersion + 1
- )
- }
- }
-}
-
/**
* Delete the given chunk from the database.
*
@@ -464,14 +242,10 @@ async function deleteProjectChunks(projectId) {
* Delete a given number of old chunks from both the database
* and from object storage.
*
- * @param {object} options
- * @param {number} [options.batchSize] - number of chunks to delete in each
- * batch
- * @param {number} [options.maxBatches] - maximum number of batches to process
- * @param {number} [options.minAgeSecs] - minimum age of chunks to delete
- * @param {number} [options.timeout] - maximum time to spend deleting chunks
- *
- * @return {Promise} number of chunks deleted
+ * @param {number} count - number of chunks to delete
+ * @param {number} minAgeSecs - how many seconds ago must chunks have been
+ * deleted
+ * @return {Promise}
*/
async function deleteOldChunks(options = {}) {
const batchSize = options.batchSize ?? DEFAULT_DELETE_BATCH_SIZE
@@ -534,31 +308,6 @@ function getBackend(projectId) {
}
}
-/**
- * Gets non-persisted changes that could extend a chunk
- *
- * @param {string} projectId
- * @param {number} chunkEndVersion - end version of the chunk to extend
- *
- * @return {Promise}
- */
-async function getChunkExtension(projectId, chunkEndVersion) {
- try {
- const changes = await redisBackend.getNonPersistedChanges(
- projectId,
- chunkEndVersion
- )
- return changes
- } catch (err) {
- if (err instanceof VersionOutOfBoundsError) {
- // If we can't extend the chunk, simply return an empty list
- return []
- } else {
- throw err
- }
- }
-}
-
class AlreadyInitialized extends OError {
constructor(projectId) {
super('Project is already initialized', { projectId })
@@ -566,21 +315,15 @@ class AlreadyInitialized extends OError {
}
module.exports = {
- getBackend,
initializeProject,
loadLatest,
- getLatestChunkMetadata,
loadAtVersion,
loadAtTimestamp,
- loadByChunkRecord,
create,
update,
destroy,
getChunkIdForVersion,
- getChunkMetadataForVersion,
getProjectChunkIds,
- getProjectChunks,
- getProjectChunksFromVersion,
deleteProjectChunks,
deleteOldChunks,
AlreadyInitialized,
diff --git a/services/history-v1/storage/lib/chunk_store/mongo.js b/services/history-v1/storage/lib/chunk_store/mongo.js
index 49020c6be4..f56131a25b 100644
--- a/services/history-v1/storage/lib/chunk_store/mongo.js
+++ b/services/history-v1/storage/lib/chunk_store/mongo.js
@@ -1,40 +1,21 @@
-// @ts-check
-
-const { ObjectId, ReadPreference, MongoError } = require('mongodb')
+const { ObjectId } = require('mongodb')
const { Chunk } = require('overleaf-editor-core')
const OError = require('@overleaf/o-error')
-const config = require('config')
const assert = require('../assert')
const mongodb = require('../mongodb')
const { ChunkVersionConflictError } = require('./errors')
const DUPLICATE_KEY_ERROR_CODE = 11000
-/**
- * @import { ClientSession } from 'mongodb'
- */
-
/**
* Get the latest chunk's metadata from the database
- * @param {string} projectId
- * @param {Object} [opts]
- * @param {boolean} [opts.readOnly]
*/
-async function getLatestChunk(projectId, opts = {}) {
+async function getLatestChunk(projectId) {
assert.mongoId(projectId, 'bad projectId')
- const { readOnly = false } = opts
const record = await mongodb.chunks.findOne(
- {
- projectId: new ObjectId(projectId),
- state: { $in: ['active', 'closed'] },
- },
- {
- sort: { startVersion: -1 },
- readPreference: readOnly
- ? ReadPreference.secondaryPreferred
- : ReadPreference.primary,
- }
+ { projectId: new ObjectId(projectId), state: 'active' },
+ { sort: { startVersion: -1 } }
)
if (record == null) {
return null
@@ -44,25 +25,19 @@ async function getLatestChunk(projectId, opts = {}) {
/**
* Get the metadata for the chunk that contains the given version.
- *
- * @param {string} projectId
- * @param {number} version
- * @param {object} [opts]
- * @param {boolean} [opts.preferNewer] - If the version is at the boundary of
- * two chunks, return the newer chunk.
*/
-async function getChunkForVersion(projectId, version, opts = {}) {
+async function getChunkForVersion(projectId, version) {
assert.mongoId(projectId, 'bad projectId')
assert.integer(version, 'bad version')
const record = await mongodb.chunks.findOne(
{
projectId: new ObjectId(projectId),
- state: { $in: ['active', 'closed'] },
+ state: 'active',
startVersion: { $lte: version },
endVersion: { $gte: version },
},
- { sort: { startVersion: opts.preferNewer ? -1 : 1 } }
+ { sort: { startVersion: 1 } }
)
if (record == null) {
throw new Chunk.VersionNotFoundError(projectId, version)
@@ -70,35 +45,6 @@ async function getChunkForVersion(projectId, version, opts = {}) {
return chunkFromRecord(record)
}
-/**
- * Get the metadata for the chunk that contains the given version before the endTime.
- */
-async function getFirstChunkBeforeTimestamp(projectId, timestamp) {
- assert.mongoId(projectId, 'bad projectId')
- assert.date(timestamp, 'bad timestamp')
-
- const recordActive = await getChunkForVersion(projectId, 0)
- if (recordActive && recordActive.endTimestamp <= timestamp) {
- return recordActive
- }
-
- // fallback to deleted chunk
- const recordDeleted = await mongodb.chunks.findOne(
- {
- projectId: new ObjectId(projectId),
- state: 'deleted',
- startVersion: 0,
- updatedAt: { $lte: timestamp }, // indexed for state=deleted
- endTimestamp: { $lte: timestamp },
- },
- { sort: { updatedAt: -1 } }
- )
- if (recordDeleted) {
- return chunkFromRecord(recordDeleted)
- }
- throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
-}
-
/**
* Get the metadata for the chunk that contains the version that was current at
* the given timestamp.
@@ -110,7 +56,7 @@ async function getChunkForTimestamp(projectId, timestamp) {
const record = await mongodb.chunks.findOne(
{
projectId: new ObjectId(projectId),
- state: { $in: ['active', 'closed'] },
+ state: 'active',
endTimestamp: { $gte: timestamp },
},
// We use the index on the startVersion for sorting records. This assumes
@@ -131,39 +77,6 @@ async function getChunkForTimestamp(projectId, timestamp) {
return chunkFromRecord(record)
}
-/**
- * Get the metadata for the chunk that contains the version that was current before
- * the given timestamp.
- */
-async function getLastActiveChunkBeforeTimestamp(projectId, timestamp) {
- assert.mongoId(projectId, 'bad projectId')
- assert.date(timestamp, 'bad timestamp')
-
- const record = await mongodb.chunks.findOne(
- {
- projectId: new ObjectId(projectId),
- state: { $in: ['active', 'closed'] },
- $or: [
- {
- endTimestamp: {
- $lte: timestamp,
- },
- },
- {
- endTimestamp: null,
- },
- ],
- },
- // We use the index on the startVersion for sorting records. This assumes
- // that timestamps go up with each version.
- { sort: { startVersion: -1 } }
- )
- if (record == null) {
- throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
- }
- return chunkFromRecord(record)
-}
-
/**
* Get all of a project's chunk ids
*/
@@ -171,33 +84,12 @@ async function getProjectChunkIds(projectId) {
assert.mongoId(projectId, 'bad projectId')
const cursor = mongodb.chunks.find(
- {
- projectId: new ObjectId(projectId),
- state: { $in: ['active', 'closed'] },
- },
+ { projectId: new ObjectId(projectId), state: 'active' },
{ projection: { _id: 1 } }
)
return await cursor.map(record => record._id).toArray()
}
-/**
- * Get all of a projects chunks directly
- */
-async function getProjectChunks(projectId) {
- assert.mongoId(projectId, 'bad projectId')
-
- const cursor = mongodb.chunks
- .find(
- {
- projectId: new ObjectId(projectId),
- state: { $in: ['active', 'closed'] },
- },
- { projection: { state: 0 } }
- )
- .sort({ startVersion: 1 })
- return await cursor.map(chunkFromRecord).toArray()
-}
-
/**
* Insert a pending chunk before sending it to object storage.
*/
@@ -220,141 +112,10 @@ async function insertPendingChunk(projectId, chunk) {
/**
* Record that a new chunk was created.
- *
- * @param {string} projectId
- * @param {Chunk} chunk
- * @param {string} chunkId
- * @param {object} opts
- * @param {Date} [opts.earliestChangeTimestamp]
- * @param {string} [opts.oldChunkId]
*/
-async function confirmCreate(projectId, chunk, chunkId, opts = {}) {
- assert.mongoId(projectId, 'bad projectId')
- assert.instance(chunk, Chunk, 'bad newChunk')
- assert.mongoId(chunkId, 'bad newChunkId')
-
- await mongodb.client.withSession(async session => {
- await session.withTransaction(async () => {
- if (opts.oldChunkId != null) {
- await closeChunk(projectId, opts.oldChunkId, { session })
- }
-
- await activateChunk(projectId, chunkId, { session })
-
- await updateProjectRecord(
- projectId,
- chunk,
- opts.earliestChangeTimestamp,
- { session }
- )
- })
- })
-}
-
-/**
- * Write the metadata to the project record
- */
-async function updateProjectRecord(
- projectId,
- chunk,
- earliestChangeTimestamp,
- mongoOpts = {}
-) {
- if (!config.has('backupStore')) {
- return
- }
- // record the end version against the project
- await mongodb.projects.updateOne(
- {
- 'overleaf.history.id': projectId, // string for Object ids, number for postgres ids
- },
- {
- // always store the latest end version and timestamp for the chunk
- $max: {
- 'overleaf.history.currentEndVersion': chunk.getEndVersion(),
- 'overleaf.history.currentEndTimestamp': chunk.getEndTimestamp(),
- 'overleaf.history.updatedAt': new Date(),
- },
- // store the first pending change timestamp for the chunk, this will
- // be cleared every time a backup is completed.
- $min: {
- 'overleaf.backup.pendingChangeAt':
- earliestChangeTimestamp || chunk.getEndTimestamp() || new Date(),
- },
- },
- mongoOpts
- )
-}
-
-/**
- * @param {number} historyId
- * @return {Promise}
- */
-async function lookupMongoProjectIdFromHistoryId(historyId) {
- const project = await mongodb.projects.findOne(
- // string for Object ids, number for postgres ids
- { 'overleaf.history.id': historyId },
- { projection: { _id: 1 } }
- )
- if (!project) {
- // should not happen: We flush before allowing a project to be soft-deleted.
- throw new OError('mongo project not found by history id', { historyId })
- }
- return project._id.toString()
-}
-
-async function resolveHistoryIdToMongoProjectId(projectId) {
- return projectId
-}
-
-/**
- * Record that a chunk was replaced by a new one.
- *
- * @param {string} projectId
- * @param {string} oldChunkId
- * @param {Chunk} newChunk
- * @param {string} newChunkId
- * @param {object} [opts]
- * @param {Date} [opts.earliestChangeTimestamp]
- */
-async function confirmUpdate(
- projectId,
- oldChunkId,
- newChunk,
- newChunkId,
- opts = {}
-) {
- assert.mongoId(projectId, 'bad projectId')
- assert.mongoId(oldChunkId, 'bad oldChunkId')
- assert.instance(newChunk, Chunk, 'bad newChunk')
- assert.mongoId(newChunkId, 'bad newChunkId')
-
- await mongodb.client.withSession(async session => {
- await session.withTransaction(async () => {
- await deleteActiveChunk(projectId, oldChunkId, { session })
-
- await activateChunk(projectId, newChunkId, { session })
-
- await updateProjectRecord(
- projectId,
- newChunk,
- opts.earliestChangeTimestamp,
- { session }
- )
- })
- })
-}
-
-/**
- * Activate a pending chunk
- *
- * @param {string} projectId
- * @param {string} chunkId
- * @param {object} [opts]
- * @param {ClientSession} [opts.session]
- */
-async function activateChunk(projectId, chunkId, opts = {}) {
+async function confirmCreate(projectId, chunk, chunkId, mongoOpts = {}) {
assert.mongoId(projectId, 'bad projectId')
+ assert.instance(chunk, Chunk, 'bad chunk')
assert.mongoId(chunkId, 'bad chunkId')
let result
@@ -366,10 +127,10 @@ async function activateChunk(projectId, chunkId, opts = {}) {
state: 'pending',
},
{ $set: { state: 'active', updatedAt: new Date() } },
- opts
+ mongoOpts
)
} catch (err) {
- if (err instanceof MongoError && err.code === DUPLICATE_KEY_ERROR_CODE) {
+ if (err.code === DUPLICATE_KEY_ERROR_CODE) {
throw new ChunkVersionConflictError('chunk start version is not unique', {
projectId,
chunkId,
@@ -384,70 +145,30 @@ async function activateChunk(projectId, chunkId, opts = {}) {
}
/**
- * Close a chunk
- *
- * A closed chunk is one that can't be extended anymore.
- *
- * @param {string} projectId
- * @param {string} chunkId
- * @param {object} [opts]
- * @param {ClientSession} [opts.session]
+ * Record that a chunk was replaced by a new one.
*/
-async function closeChunk(projectId, chunkId, opts = {}) {
- const result = await mongodb.chunks.updateOne(
- {
- _id: new ObjectId(chunkId),
- projectId: new ObjectId(projectId),
- state: 'active',
- },
- { $set: { state: 'closed' } },
- opts
- )
+async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
+ assert.mongoId(projectId, 'bad projectId')
+ assert.mongoId(oldChunkId, 'bad oldChunkId')
+ assert.instance(newChunk, Chunk, 'bad newChunk')
+ assert.mongoId(newChunkId, 'bad newChunkId')
- if (result.matchedCount === 0) {
- throw new ChunkVersionConflictError('unable to close chunk', {
- projectId,
- chunkId,
- })
- }
-}
-
-/**
- * Delete an active chunk
- *
- * This is used to delete chunks that are in the process of being extended. It
- * will refuse to delete chunks that are already closed and can therefore not be
- * extended.
- *
- * @param {string} projectId
- * @param {string} chunkId
- * @param {object} [opts]
- * @param {ClientSession} [opts.session]
- */
-async function deleteActiveChunk(projectId, chunkId, opts = {}) {
- const updateResult = await mongodb.chunks.updateOne(
- {
- _id: new ObjectId(chunkId),
- projectId: new ObjectId(projectId),
- state: 'active',
- },
- { $set: { state: 'deleted', updatedAt: new Date() } },
- opts
- )
-
- if (updateResult.matchedCount === 0) {
- throw new ChunkVersionConflictError('unable to delete active chunk', {
- projectId,
- chunkId,
+ const session = mongodb.client.startSession()
+ try {
+ await session.withTransaction(async () => {
+ await deleteChunk(projectId, oldChunkId, { session })
+ await confirmCreate(projectId, newChunk, newChunkId, { session })
})
+ } finally {
+ await session.endSession()
}
}
/**
* Delete a chunk.
*
- * @param {string} projectId
- * @param {string} chunkId
+ * @param {number} projectId
+ * @param {number} chunkId
* @return {Promise}
*/
async function deleteChunk(projectId, chunkId, mongoOpts = {}) {
@@ -468,10 +189,7 @@ async function deleteProjectChunks(projectId) {
assert.mongoId(projectId, 'bad projectId')
await mongodb.chunks.updateMany(
- {
- projectId: new ObjectId(projectId),
- state: { $in: ['active', 'closed'] },
- },
+ { projectId: new ObjectId(projectId), state: 'active' },
{ $set: { state: 'deleted', updatedAt: new Date() } }
)
}
@@ -534,26 +252,19 @@ function chunkFromRecord(record) {
id: record._id.toString(),
startVersion: record.startVersion,
endVersion: record.endVersion,
- endTimestamp: record.endTimestamp,
}
}
module.exports = {
getLatestChunk,
- getFirstChunkBeforeTimestamp,
- getLastActiveChunkBeforeTimestamp,
getChunkForVersion,
getChunkForTimestamp,
getProjectChunkIds,
- getProjectChunks,
insertPendingChunk,
confirmCreate,
confirmUpdate,
- updateProjectRecord,
deleteChunk,
deleteProjectChunks,
getOldChunksBatch,
deleteOldChunks,
- lookupMongoProjectIdFromHistoryId,
- resolveHistoryIdToMongoProjectId,
}
diff --git a/services/history-v1/storage/lib/chunk_store/postgres.js b/services/history-v1/storage/lib/chunk_store/postgres.js
index 8906db38e1..f6eead7354 100644
--- a/services/history-v1/storage/lib/chunk_store/postgres.js
+++ b/services/history-v1/storage/lib/chunk_store/postgres.js
@@ -1,33 +1,19 @@
-// @ts-check
-
const { Chunk } = require('overleaf-editor-core')
const assert = require('../assert')
const knex = require('../knex')
-const knexReadOnly = require('../knex_read_only')
const { ChunkVersionConflictError } = require('./errors')
-const {
- updateProjectRecord,
- lookupMongoProjectIdFromHistoryId,
-} = require('./mongo')
const DUPLICATE_KEY_ERROR_CODE = '23505'
-/**
- * @import { Knex } from 'knex'
- */
-
/**
* Get the latest chunk's metadata from the database
- * @param {string} projectId
- * @param {Object} [opts]
- * @param {boolean} [opts.readOnly]
*/
-async function getLatestChunk(projectId, opts = {}) {
- assert.postgresId(projectId, 'bad projectId')
- const { readOnly = false } = opts
+async function getLatestChunk(projectId) {
+ projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
- const record = await (readOnly ? knexReadOnly : knex)('chunks')
- .where('doc_id', parseInt(projectId, 10))
+ const record = await knex('chunks')
+ .where('doc_id', projectId)
.orderBy('end_version', 'desc')
.first()
if (record == null) {
@@ -38,21 +24,15 @@ async function getLatestChunk(projectId, opts = {}) {
/**
* Get the metadata for the chunk that contains the given version.
- *
- * @param {string} projectId
- * @param {number} version
- * @param {object} [opts]
- * @param {boolean} [opts.preferNewer] - If the version is at the boundary of
- * two chunks, return the newer chunk.
*/
-async function getChunkForVersion(projectId, version, opts = {}) {
- assert.postgresId(projectId, 'bad projectId')
+async function getChunkForVersion(projectId, version) {
+ projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
const record = await knex('chunks')
- .where('doc_id', parseInt(projectId, 10))
- .where('start_version', '<=', version)
+ .where('doc_id', projectId)
.where('end_version', '>=', version)
- .orderBy('end_version', opts.preferNewer ? 'desc' : 'asc')
+ .orderBy('end_version')
.first()
if (!record) {
throw new Chunk.VersionNotFoundError(projectId, version)
@@ -60,73 +40,13 @@ async function getChunkForVersion(projectId, version, opts = {}) {
return chunkFromRecord(record)
}
-/**
- * Get the metadata for the chunk that contains the given version.
- *
- * @param {string} projectId
- * @param {Date} timestamp
- */
-async function getFirstChunkBeforeTimestamp(projectId, timestamp) {
- assert.date(timestamp, 'bad timestamp')
-
- const recordActive = await getChunkForVersion(projectId, 0)
-
- // projectId must be valid if getChunkForVersion did not throw
- if (recordActive && recordActive.endTimestamp <= timestamp) {
- return recordActive
- }
-
- // fallback to deleted chunk
- const recordDeleted = await knex('old_chunks')
- .where('doc_id', parseInt(projectId, 10))
- .where('start_version', '=', 0)
- .where('end_timestamp', '<=', timestamp)
- .orderBy('end_version', 'desc')
- .first()
- if (recordDeleted) {
- return chunkFromRecord(recordDeleted)
- }
- throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
-}
-
/**
* Get the metadata for the chunk that contains the version that was current at
* the given timestamp.
- *
- * @param {string} projectId
- * @param {Date} timestamp
- */
-async function getLastActiveChunkBeforeTimestamp(projectId, timestamp) {
- assert.date(timestamp, 'bad timestamp')
- assert.postgresId(projectId, 'bad projectId')
-
- const query = knex('chunks')
- .where('doc_id', parseInt(projectId, 10))
- .where(function () {
- this.where('end_timestamp', '<=', timestamp).orWhere(
- 'end_timestamp',
- null
- )
- })
- .orderBy('end_version', 'desc', 'last')
-
- const record = await query.first()
-
- if (!record) {
- throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
- }
- return chunkFromRecord(record)
-}
-
-/**
- * Get the metadata for the chunk that contains the version that was current at
- * the given timestamp.
- *
- * @param {string} projectId
- * @param {Date} timestamp
*/
async function getChunkForTimestamp(projectId, timestamp) {
- assert.postgresId(projectId, 'bad projectId')
+ projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
// This query will find the latest chunk after the timestamp (query orders
// in reverse chronological order), OR the latest chunk
@@ -139,11 +59,11 @@ async function getChunkForTimestamp(projectId, timestamp) {
'WHERE doc_id = ? ' +
'ORDER BY end_version desc LIMIT 1' +
')',
- [timestamp, parseInt(projectId, 10)]
+ [timestamp, projectId]
)
const record = await knex('chunks')
- .where('doc_id', parseInt(projectId, 10))
+ .where('doc_id', projectId)
.where(whereAfterEndTimestampOrLatestChunk)
.orderBy('end_version')
.first()
@@ -158,50 +78,29 @@ async function getChunkForTimestamp(projectId, timestamp) {
*/
function chunkFromRecord(record) {
return {
- id: record.id.toString(),
+ id: record.id,
startVersion: record.start_version,
endVersion: record.end_version,
- endTimestamp: record.end_timestamp,
}
}
/**
* Get all of a project's chunk ids
- *
- * @param {string} projectId
*/
async function getProjectChunkIds(projectId) {
- assert.postgresId(projectId, 'bad projectId')
+ projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
- const records = await knex('chunks')
- .select('id')
- .where('doc_id', parseInt(projectId, 10))
+ const records = await knex('chunks').select('id').where('doc_id', projectId)
return records.map(record => record.id)
}
-/**
- * Get all of a projects chunks directly
- *
- * @param {string} projectId
- */
-async function getProjectChunks(projectId) {
- assert.postgresId(projectId, 'bad projectId')
-
- const records = await knex('chunks')
- .select()
- .where('doc_id', parseInt(projectId, 10))
- .orderBy('end_version')
- return records.map(chunkFromRecord)
-}
-
/**
* Insert a pending chunk before sending it to object storage.
- *
- * @param {string} projectId
- * @param {Chunk} chunk
*/
async function insertPendingChunk(projectId, chunk) {
- assert.postgresId(projectId, 'bad projectId')
+ projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
const result = await knex.first(
knex.raw("nextval('chunks_id_seq'::regclass)::integer as chunkid")
@@ -209,119 +108,67 @@ async function insertPendingChunk(projectId, chunk) {
const chunkId = result.chunkid
await knex('pending_chunks').insert({
id: chunkId,
- doc_id: parseInt(projectId, 10),
+ doc_id: projectId,
end_version: chunk.getEndVersion(),
start_version: chunk.getStartVersion(),
end_timestamp: chunk.getEndTimestamp(),
})
- return chunkId.toString()
+ return chunkId
}
/**
* Record that a new chunk was created.
- *
- * @param {string} projectId
- * @param {Chunk} chunk
- * @param {string} chunkId
- * @param {object} opts
- * @param {Date} [opts.earliestChangeTimestamp]
- * @param {string} [opts.oldChunkId]
*/
-async function confirmCreate(projectId, chunk, chunkId, opts = {}) {
- assert.postgresId(projectId, 'bad projectId')
+async function confirmCreate(projectId, chunk, chunkId) {
+ projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
await knex.transaction(async tx => {
- if (opts.oldChunkId != null) {
- await _assertChunkIsNotClosed(tx, projectId, opts.oldChunkId)
- await _closeChunk(tx, projectId, opts.oldChunkId)
- }
await Promise.all([
_deletePendingChunk(tx, projectId, chunkId),
_insertChunk(tx, projectId, chunk, chunkId),
])
- await updateProjectRecord(
- // The history id in Mongo is an integer for Postgres projects
- parseInt(projectId, 10),
- chunk,
- opts.earliestChangeTimestamp
- )
})
}
/**
* Record that a chunk was replaced by a new one.
- *
- * @param {string} projectId
- * @param {string} oldChunkId
- * @param {Chunk} newChunk
- * @param {string} newChunkId
*/
-async function confirmUpdate(
- projectId,
- oldChunkId,
- newChunk,
- newChunkId,
- opts = {}
-) {
- assert.postgresId(projectId, 'bad projectId')
+async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
+ projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
await knex.transaction(async tx => {
- await _assertChunkIsNotClosed(tx, projectId, oldChunkId)
await _deleteChunks(tx, { doc_id: projectId, id: oldChunkId })
await Promise.all([
_deletePendingChunk(tx, projectId, newChunkId),
_insertChunk(tx, projectId, newChunk, newChunkId),
])
- await updateProjectRecord(
- // The history id in Mongo is an integer for Postgres projects
- parseInt(projectId, 10),
- newChunk,
- opts.earliestChangeTimestamp
- )
})
}
-/**
- * Delete a pending chunk
- *
- * @param {Knex} tx
- * @param {string} projectId
- * @param {string} chunkId
- */
async function _deletePendingChunk(tx, projectId, chunkId) {
await tx('pending_chunks')
.where({
- doc_id: parseInt(projectId, 10),
- id: parseInt(chunkId, 10),
+ doc_id: projectId,
+ id: chunkId,
})
.del()
}
-/**
- * Adds an active chunk
- *
- * @param {Knex} tx
- * @param {string} projectId
- * @param {Chunk} chunk
- * @param {string} chunkId
- */
async function _insertChunk(tx, projectId, chunk, chunkId) {
const startVersion = chunk.getStartVersion()
const endVersion = chunk.getEndVersion()
try {
await tx('chunks').insert({
- id: parseInt(chunkId, 10),
- doc_id: parseInt(projectId, 10),
+ id: chunkId,
+ doc_id: projectId,
start_version: startVersion,
end_version: endVersion,
end_timestamp: chunk.getEndTimestamp(),
})
} catch (err) {
- if (
- err instanceof Error &&
- 'code' in err &&
- err.code === DUPLICATE_KEY_ERROR_CODE
- ) {
+ if (err.code === DUPLICATE_KEY_ERROR_CODE) {
throw new ChunkVersionConflictError(
'chunk start or end version is not unique',
{ projectId, chunkId, startVersion, endVersion }
@@ -331,92 +178,35 @@ async function _insertChunk(tx, projectId, chunk, chunkId) {
}
}
-/**
- * Check that a chunk is not closed
- *
- * This is used to synchronize chunk creations and extensions.
- *
- * @param {Knex} tx
- * @param {string} projectId
- * @param {string} chunkId
- */
-async function _assertChunkIsNotClosed(tx, projectId, chunkId) {
- const record = await tx('chunks')
- .forUpdate()
- .select('closed')
- .where('doc_id', parseInt(projectId, 10))
- .where('id', parseInt(chunkId, 10))
- .first()
- if (!record) {
- throw new ChunkVersionConflictError('unable to close chunk: not found', {
- projectId,
- chunkId,
- })
- }
- if (record.closed) {
- throw new ChunkVersionConflictError(
- 'unable to close chunk: already closed',
- {
- projectId,
- chunkId,
- }
- )
- }
-}
-
-/**
- * Close a chunk
- *
- * A closed chunk can no longer be extended.
- *
- * @param {Knex} tx
- * @param {string} projectId
- * @param {string} chunkId
- */
-async function _closeChunk(tx, projectId, chunkId) {
- await tx('chunks')
- .update({ closed: true })
- .where('doc_id', parseInt(projectId, 10))
- .where('id', parseInt(chunkId, 10))
-}
-
/**
* Delete a chunk.
*
- * @param {string} projectId
- * @param {string} chunkId
+ * @param {number} projectId
+ * @param {number} chunkId
+ * @return {Promise}
*/
async function deleteChunk(projectId, chunkId) {
- assert.postgresId(projectId, 'bad projectId')
+ projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
assert.integer(chunkId, 'bad chunkId')
- await _deleteChunks(knex, {
- doc_id: parseInt(projectId, 10),
- id: parseInt(chunkId, 10),
- })
+ await _deleteChunks(knex, { doc_id: projectId, id: chunkId })
}
/**
* Delete all of a project's chunks
- *
- * @param {string} projectId
*/
async function deleteProjectChunks(projectId) {
- assert.postgresId(projectId, 'bad projectId')
+ projectId = parseInt(projectId, 10)
+ assert.integer(projectId, 'bad projectId')
await knex.transaction(async tx => {
- await _deleteChunks(knex, { doc_id: parseInt(projectId, 10) })
+ await _deleteChunks(knex, { doc_id: projectId })
})
}
-/**
- * Delete many chunks
- *
- * @param {Knex} tx
- * @param {any} whereClause
- */
async function _deleteChunks(tx, whereClause) {
- const rows = await tx('chunks').where(whereClause).del().returning('*')
+ const rows = await tx('chunks').returning('*').where(whereClause).del()
if (rows.length === 0) {
return
}
@@ -434,9 +224,6 @@ async function _deleteChunks(tx, whereClause) {
/**
* Get a batch of old chunks for deletion
- *
- * @param {number} count
- * @param {number} minAgeSecs
*/
async function getOldChunksBatch(count, minAgeSecs) {
const maxDeletedAt = new Date(Date.now() - minAgeSecs * 1000)
@@ -447,22 +234,15 @@ async function getOldChunksBatch(count, minAgeSecs) {
.limit(count)
return records.map(oldChunk => ({
projectId: oldChunk.doc_id.toString(),
- chunkId: oldChunk.chunk_id.toString(),
+ chunkId: oldChunk.chunk_id,
}))
}
/**
* Delete a batch of old chunks from the database
- *
- * @param {string[]} chunkIds
*/
async function deleteOldChunks(chunkIds) {
- await knex('old_chunks')
- .whereIn(
- 'chunk_id',
- chunkIds.map(id => parseInt(id, 10))
- )
- .del()
+ await knex('old_chunks').whereIn('chunk_id', chunkIds).del()
}
/**
@@ -475,18 +255,11 @@ async function generateProjectId() {
return record.doc_id.toString()
}
-async function resolveHistoryIdToMongoProjectId(projectId) {
- return await lookupMongoProjectIdFromHistoryId(parseInt(projectId, 10))
-}
-
module.exports = {
getLatestChunk,
- getFirstChunkBeforeTimestamp,
- getLastActiveChunkBeforeTimestamp,
getChunkForVersion,
getChunkForTimestamp,
getProjectChunkIds,
- getProjectChunks,
insertPendingChunk,
confirmCreate,
confirmUpdate,
@@ -495,5 +268,4 @@ module.exports = {
getOldChunksBatch,
deleteOldChunks,
generateProjectId,
- resolveHistoryIdToMongoProjectId,
}
diff --git a/services/history-v1/storage/lib/chunk_store/redis.js b/services/history-v1/storage/lib/chunk_store/redis.js
deleted file mode 100644
index b8a79b498d..0000000000
--- a/services/history-v1/storage/lib/chunk_store/redis.js
+++ /dev/null
@@ -1,854 +0,0 @@
-// @ts-check
-
-const metrics = require('@overleaf/metrics')
-const OError = require('@overleaf/o-error')
-const { Change, Snapshot } = require('overleaf-editor-core')
-const redis = require('../redis')
-const rclient = redis.rclientHistory
-const {
- BaseVersionConflictError,
- JobNotFoundError,
- JobNotReadyError,
- VersionOutOfBoundsError,
-} = require('./errors')
-
-const MAX_PERSISTED_CHANGES = 100 // Maximum number of persisted changes to keep in the buffer for clients that need to catch up.
-const PROJECT_TTL_MS = 3600 * 1000 // Amount of time a project can stay inactive before it gets expired
-const MAX_PERSIST_DELAY_MS = 300 * 1000 // Maximum amount of time before a change is persisted
-const RETRY_DELAY_MS = 120 * 1000 // Time before a claimed job is considered stale and a worker can retry it.
-
-const keySchema = {
- head({ projectId }) {
- return `head:{${projectId}}`
- },
- headVersion({ projectId }) {
- return `head-version:{${projectId}}`
- },
- persistedVersion({ projectId }) {
- return `persisted-version:{${projectId}}`
- },
- expireTime({ projectId }) {
- return `expire-time:{${projectId}}`
- },
- persistTime({ projectId }) {
- return `persist-time:{${projectId}}`
- },
- changes({ projectId }) {
- return `changes:{${projectId}}`
- },
-}
-
-rclient.defineCommand('get_head_snapshot', {
- numberOfKeys: 2,
- lua: `
- local headSnapshotKey = KEYS[1]
- local headVersionKey = KEYS[2]
-
- -- Check if the head version exists. If not, consider it a cache miss.
- local version = redis.call('GET', headVersionKey)
- if not version then
- return nil
- end
-
- -- Retrieve the snapshot value
- local snapshot = redis.call('GET', headSnapshotKey)
- return {snapshot, version}
- `,
-})
-
-/**
- * Retrieves the head snapshot from Redis storage
- * @param {string} projectId - The unique identifier of the project
- * @returns {Promise<{version: number, snapshot: Snapshot}|null>} A Promise that resolves to an object containing the version and Snapshot,
- * or null if retrieval fails or cache miss
- * @throws {Error} If Redis operations fail
- */
-async function getHeadSnapshot(projectId) {
- try {
- const result = await rclient.get_head_snapshot(
- keySchema.head({ projectId }),
- keySchema.headVersion({ projectId })
- )
- if (!result) {
- metrics.inc('chunk_store.redis.get_head_snapshot', 1, {
- status: 'cache-miss',
- })
- return null // cache-miss
- }
- const snapshot = Snapshot.fromRaw(JSON.parse(result[0]))
- const version = parseInt(result[1], 10)
- metrics.inc('chunk_store.redis.get_head_snapshot', 1, {
- status: 'success',
- })
- return { version, snapshot }
- } catch (err) {
- metrics.inc('chunk_store.redis.get_head_snapshot', 1, { status: 'error' })
- throw err
- }
-}
-
-rclient.defineCommand('queue_changes', {
- numberOfKeys: 5,
- lua: `
- local headSnapshotKey = KEYS[1]
- local headVersionKey = KEYS[2]
- local changesKey = KEYS[3]
- local expireTimeKey = KEYS[4]
- local persistTimeKey = KEYS[5]
-
- local baseVersion = tonumber(ARGV[1])
- local head = ARGV[2]
- local persistTime = tonumber(ARGV[3])
- local expireTime = tonumber(ARGV[4])
- local onlyIfExists = ARGV[5]
- local changesIndex = 6 -- Changes start here
-
- local headVersion = tonumber(redis.call('GET', headVersionKey))
-
- -- Check if updates should only be queued if the project already exists (used for gradual rollouts)
- if not headVersion and onlyIfExists == 'true' then
- return 'ignore'
- end
-
- -- Check that the supplied baseVersion matches the head version
- -- If headVersion is nil, it means the project does not exist yet and will be created.
- if headVersion and headVersion ~= baseVersion then
- return 'conflict'
- end
-
- -- Check if there are any changes to queue
- if #ARGV < changesIndex then
- return 'no_changes_provided'
- end
-
- -- Store the changes
- -- RPUSH changesKey change1 change2 ...
- redis.call('RPUSH', changesKey, unpack(ARGV, changesIndex, #ARGV))
-
- -- Update head snapshot only if changes were successfully pushed
- redis.call('SET', headSnapshotKey, head)
-
- -- Update the head version
- local numChanges = #ARGV - changesIndex + 1
- local newHeadVersion = baseVersion + numChanges
- redis.call('SET', headVersionKey, newHeadVersion)
-
- -- Update the persist time if the new time is sooner
- local currentPersistTime = tonumber(redis.call('GET', persistTimeKey))
- if not currentPersistTime or persistTime < currentPersistTime then
- redis.call('SET', persistTimeKey, persistTime)
- end
-
- -- Update the expire time
- redis.call('SET', expireTimeKey, expireTime)
-
- return 'ok'
- `,
-})
-
-/**
- * Atomically queues changes to the project history in Redis if the baseVersion matches.
- * Updates head snapshot, version, persist time, and expire time.
- *
- * @param {string} projectId - The project identifier.
- * @param {Snapshot} headSnapshot - The new head snapshot after applying changes.
- * @param {number} baseVersion - The expected current head version.
- * @param {Change[]} changes - An array of Change objects to queue.
- * @param {object} [opts]
- * @param {number} [opts.persistTime] - Timestamp (ms since epoch) when the
- * oldest change in the buffer should be persisted.
- * @param {number} [opts.expireTime] - Timestamp (ms since epoch) when the
- * project buffer should expire if inactive.
- * @param {boolean} [opts.onlyIfExists] - If true, only queue changes if the
- * project already exists in Redis, otherwise ignore.
- * @returns {Promise} Resolves on success to either 'ok' or 'ignore'.
- * @throws {BaseVersionConflictError} If the baseVersion does not match the current head version in Redis.
- * @throws {Error} If changes array is empty or if Redis operations fail.
- */
-async function queueChanges(
- projectId,
- headSnapshot,
- baseVersion,
- changes,
- opts = {}
-) {
- if (!changes || changes.length === 0) {
- throw new Error('Cannot queue empty changes array')
- }
-
- const persistTime = opts.persistTime ?? Date.now() + MAX_PERSIST_DELAY_MS
- const expireTime = opts.expireTime ?? Date.now() + PROJECT_TTL_MS
- const onlyIfExists = Boolean(opts.onlyIfExists)
-
- try {
- const keys = [
- keySchema.head({ projectId }),
- keySchema.headVersion({ projectId }),
- keySchema.changes({ projectId }),
- keySchema.expireTime({ projectId }),
- keySchema.persistTime({ projectId }),
- ]
-
- const args = [
- baseVersion.toString(),
- JSON.stringify(headSnapshot.toRaw()),
- persistTime.toString(),
- expireTime.toString(),
- onlyIfExists.toString(), // Only queue changes if the snapshot already exists
- ...changes.map(change => JSON.stringify(change.toRaw())), // Serialize changes
- ]
-
- const status = await rclient.queue_changes(keys, args)
- metrics.inc('chunk_store.redis.queue_changes', 1, { status })
- if (status === 'ok') {
- return status
- }
- if (status === 'ignore') {
- return status // skip changes when project does not exist and onlyIfExists is true
- }
- if (status === 'conflict') {
- throw new BaseVersionConflictError('base version mismatch', {
- projectId,
- baseVersion,
- })
- } else {
- throw new Error(`unexpected result queuing changes: ${status}`)
- }
- } catch (err) {
- if (err instanceof BaseVersionConflictError) {
- // Re-throw conflict errors directly
- throw err
- }
- metrics.inc('chunk_store.redis.queue_changes', 1, { status: 'error' })
- throw err
- }
-}
-
-rclient.defineCommand('get_state', {
- numberOfKeys: 6, // Number of keys defined in keySchema
- lua: `
- local headSnapshotKey = KEYS[1]
- local headVersionKey = KEYS[2]
- local persistedVersionKey = KEYS[3]
- local expireTimeKey = KEYS[4]
- local persistTimeKey = KEYS[5]
- local changesKey = KEYS[6]
-
- local headSnapshot = redis.call('GET', headSnapshotKey)
- local headVersion = redis.call('GET', headVersionKey)
- local persistedVersion = redis.call('GET', persistedVersionKey)
- local expireTime = redis.call('GET', expireTimeKey)
- local persistTime = redis.call('GET', persistTimeKey)
- local changes = redis.call('LRANGE', changesKey, 0, -1) -- Get all changes in the list
-
- return {headSnapshot, headVersion, persistedVersion, expireTime, persistTime, changes}
- `,
-})
-
-/**
- * Retrieves the entire state associated with a project from Redis atomically.
- * @param {string} projectId - The unique identifier of the project.
- * @returns {Promise} A Promise that resolves to an object containing the project state,
- * or null if the project state does not exist (e.g., head version is missing).
- * @throws {Error} If Redis operations fail.
- */
-async function getState(projectId) {
- const keys = [
- keySchema.head({ projectId }),
- keySchema.headVersion({ projectId }),
- keySchema.persistedVersion({ projectId }),
- keySchema.expireTime({ projectId }),
- keySchema.persistTime({ projectId }),
- keySchema.changes({ projectId }),
- ]
-
- // Pass keys individually, not as an array
- const result = await rclient.get_state(...keys)
-
- const [
- rawHeadSnapshot,
- rawHeadVersion,
- rawPersistedVersion,
- rawExpireTime,
- rawPersistTime,
- rawChanges,
- ] = result
-
- // Safely parse values, providing defaults or nulls if necessary
- const headSnapshot = rawHeadSnapshot
- ? JSON.parse(rawHeadSnapshot)
- : rawHeadSnapshot
- const headVersion = rawHeadVersion ? parseInt(rawHeadVersion, 10) : null // Should always exist if result is not null
- const persistedVersion = rawPersistedVersion
- ? parseInt(rawPersistedVersion, 10)
- : null
- const expireTime = rawExpireTime ? parseInt(rawExpireTime, 10) : null
- const persistTime = rawPersistTime ? parseInt(rawPersistTime, 10) : null
- const changes = rawChanges ? rawChanges.map(JSON.parse) : null
-
- return {
- headSnapshot,
- headVersion,
- persistedVersion,
- expireTime,
- persistTime,
- changes,
- }
-}
-
-rclient.defineCommand('get_changes_since_version', {
- numberOfKeys: 2,
- lua: `
- local headVersionKey = KEYS[1]
- local changesKey = KEYS[2]
-
- local requestedVersion = tonumber(ARGV[1])
-
- -- Check if head version exists
- local headVersion = tonumber(redis.call('GET', headVersionKey))
- if not headVersion then
- return {'not_found'}
- end
-
- -- If requested version equals head version, return empty array
- if requestedVersion == headVersion then
- return {'ok', {}}
- end
-
- -- If requested version is greater than head version, return error
- if requestedVersion > headVersion then
- return {'out_of_bounds'}
- end
-
- -- Get length of changes list
- local changesCount = redis.call('LLEN', changesKey)
-
- -- Check if requested version is too old (changes already removed from buffer)
- if requestedVersion < (headVersion - changesCount) then
- return {'out_of_bounds'}
- end
-
- -- Calculate the starting index, using negative indexing to count backwards
- -- from the end of the list
- local startIndex = requestedVersion - headVersion
-
- -- Get changes using LRANGE
- local changes = redis.call('LRANGE', changesKey, startIndex, -1)
-
- return {'ok', changes}
- `,
-})
-
-/**
- * Retrieves changes since a specific version for a project from Redis.
- *
- * @param {string} projectId - The unique identifier of the project.
- * @param {number} version - The version number to retrieve changes since.
- * @returns {Promise<{status: string, changes?: Array}>} A Promise that resolves to an object containing:
- * - status: 'OK', 'NOT_FOUND', or 'OUT_OF_BOUNDS'
- * - changes: Array of Change objects (only when status is 'OK')
- * @throws {Error} If Redis operations fail.
- */
-async function getChangesSinceVersion(projectId, version) {
- try {
- const keys = [
- keySchema.headVersion({ projectId }),
- keySchema.changes({ projectId }),
- ]
-
- const args = [version.toString()]
-
- const result = await rclient.get_changes_since_version(keys, args)
- const status = result[0]
-
- if (status === 'ok') {
- // If status is OK, parse the changes
- const changes = result[1]
- ? result[1].map(rawChange =>
- typeof rawChange === 'string' ? JSON.parse(rawChange) : rawChange
- )
- : []
-
- metrics.inc('chunk_store.redis.get_changes_since_version', 1, {
- status: 'success',
- })
- return { status, changes }
- } else {
- // For other statuses, just return the status
- metrics.inc('chunk_store.redis.get_changes_since_version', 1, {
- status,
- })
- return { status }
- }
- } catch (err) {
- metrics.inc('chunk_store.redis.get_changes_since_version', 1, {
- status: 'error',
- })
- throw err
- }
-}
-
-rclient.defineCommand('get_non_persisted_changes', {
- numberOfKeys: 3,
- lua: `
- local headVersionKey = KEYS[1]
- local persistedVersionKey = KEYS[2]
- local changesKey = KEYS[3]
- local baseVersion = tonumber(ARGV[1])
- local maxChanges = tonumber(ARGV[2])
-
- -- Check if head version exists
- local headVersion = tonumber(redis.call('GET', headVersionKey))
- if not headVersion then
- return {'not_found'}
- end
-
- -- Check if persisted version exists
- local persistedVersion = tonumber(redis.call('GET', persistedVersionKey))
- if not persistedVersion then
- local changesCount = tonumber(redis.call('LLEN', changesKey))
- persistedVersion = headVersion - changesCount
- end
-
- if baseVersion < persistedVersion or baseVersion > headVersion then
- return {'out_of_bounds'}
- elseif baseVersion == headVersion then
- return {'ok', {}}
- else
- local numChanges = headVersion - baseVersion
-
- local endIndex, expectedChanges
- if maxChanges > 0 and maxChanges < numChanges then
- -- return only the first maxChanges changes; the end index is inclusive
- endIndex = -numChanges + maxChanges - 1
- expectedChanges = maxChanges
- else
- endIndex = -1
- expectedChanges = numChanges
- end
-
- local changes = redis.call('LRANGE', changesKey, -numChanges, endIndex)
-
- if #changes < expectedChanges then
- -- We didn't get as many changes as we expected
- return {'out_of_bounds'}
- end
-
- return {'ok', changes}
- end
- `,
-})
-
-/**
- * Retrieves non-persisted changes for a project from Redis.
- *
- * @param {string} projectId - The unique identifier of the project.
- * @param {number} baseVersion - The version on top of which the changes should
- * be applied.
- * @param {object} [opts]
- * @param {number} [opts.maxChanges] - The maximum number of changes to return.
- * Defaults to 0, meaning no limit.
- * @returns {Promise} Changes that can be applied on top of
- * baseVersion. An empty array means that the project doesn't have
- * changes to persist. A null value means that the non-persisted
- * changes can't be applied to the given base version.
- *
- * @throws {Error} If Redis operations fail.
- */
-async function getNonPersistedChanges(projectId, baseVersion, opts = {}) {
- let result
- try {
- result = await rclient.get_non_persisted_changes(
- keySchema.headVersion({ projectId }),
- keySchema.persistedVersion({ projectId }),
- keySchema.changes({ projectId }),
- baseVersion.toString(),
- opts.maxChanges ?? 0
- )
- } catch (err) {
- metrics.inc('chunk_store.redis.get_non_persisted_changes', 1, {
- status: 'error',
- })
- throw err
- }
-
- const status = result[0]
- metrics.inc('chunk_store.redis.get_non_persisted_changes', 1, {
- status,
- })
-
- if (status === 'ok') {
- return result[1].map(json => Change.fromRaw(JSON.parse(json)))
- } else if (status === 'not_found') {
- return []
- } else if (status === 'out_of_bounds') {
- throw new VersionOutOfBoundsError(
- "Non-persisted changes can't be applied to base version",
- { projectId, baseVersion }
- )
- } else {
- throw new OError('unknown status for get_non_persisted_changes', {
- projectId,
- baseVersion,
- status,
- })
- }
-}
-
-rclient.defineCommand('set_persisted_version', {
- numberOfKeys: 4,
- lua: `
- local headVersionKey = KEYS[1]
- local persistedVersionKey = KEYS[2]
- local persistTimeKey = KEYS[3]
- local changesKey = KEYS[4]
-
- local newPersistedVersion = tonumber(ARGV[1])
- local maxPersistedChanges = tonumber(ARGV[2])
-
- -- Check if head version exists
- local headVersion = tonumber(redis.call('GET', headVersionKey))
- if not headVersion then
- return 'not_found'
- end
-
- -- Get current persisted version
- local persistedVersion = tonumber(redis.call('GET', persistedVersionKey))
- if persistedVersion and persistedVersion > newPersistedVersion then
- return 'too_low'
- end
-
- -- Refuse to set a persisted version that is higher than the head version
- if newPersistedVersion > headVersion then
- return 'too_high'
- end
-
- -- Set the persisted version
- redis.call('SET', persistedVersionKey, newPersistedVersion)
-
- -- Clear the persist time if the persisted version now matches the head version
- if newPersistedVersion == headVersion then
- redis.call('DEL', persistTimeKey)
- end
-
- -- Calculate the starting index, to keep only maxPersistedChanges beyond the persisted version
- -- Using negative indexing to count backwards from the end of the list
- local startIndex = newPersistedVersion - headVersion - maxPersistedChanges
-
- -- Trim the changes list to keep only the specified number of changes beyond persisted version
- if startIndex < 0 then
- redis.call('LTRIM', changesKey, startIndex, -1)
- end
-
- return 'ok'
- `,
-})
-
-/**
- * Sets the persisted version for a project in Redis and trims the changes list.
- *
- * @param {string} projectId - The unique identifier of the project.
- * @param {number} persistedVersion - The version number to set as persisted.
- * @returns {Promise} A Promise that resolves to 'OK' or 'NOT_FOUND'.
- * @throws {Error} If Redis operations fail.
- */
-async function setPersistedVersion(projectId, persistedVersion) {
- try {
- const keys = [
- keySchema.headVersion({ projectId }),
- keySchema.persistedVersion({ projectId }),
- keySchema.persistTime({ projectId }),
- keySchema.changes({ projectId }),
- ]
-
- const args = [persistedVersion.toString(), MAX_PERSISTED_CHANGES.toString()]
-
- const status = await rclient.set_persisted_version(keys, args)
-
- metrics.inc('chunk_store.redis.set_persisted_version', 1, {
- status,
- })
-
- if (status === 'too_high') {
- throw new VersionOutOfBoundsError(
- 'Persisted version cannot be higher than head version',
- { projectId, persistedVersion }
- )
- }
-
- return status
- } catch (err) {
- metrics.inc('chunk_store.redis.set_persisted_version', 1, {
- status: 'error',
- })
- throw err
- }
-}
-
-rclient.defineCommand('hard_delete_project', {
- numberOfKeys: 6,
- lua: `
- local headKey = KEYS[1]
- local headVersionKey = KEYS[2]
- local persistedVersionKey = KEYS[3]
- local expireTimeKey = KEYS[4]
- local persistTimeKey = KEYS[5]
- local changesKey = KEYS[6]
- -- Delete all keys associated with the project
- redis.call('DEL',
- headKey,
- headVersionKey,
- persistedVersionKey,
- expireTimeKey,
- persistTimeKey,
- changesKey
- )
- return 'ok'
- `,
-})
-
-/** Hard delete a project from Redis by removing all keys associated with it.
- * This is only to be used when a project is **permanently** deleted.
- * DO NOT USE THIS FOR ANY OTHER PURPOSES AS IT WILL REMOVE NON-PERSISTED CHANGES.
- * @param {string} projectId - The unique identifier of the project to delete.
- * @returns {Promise} A Promise that resolves to 'ok' on success.
- * @throws {Error} If Redis operations fail.
- */
-async function hardDeleteProject(projectId) {
- try {
- const status = await rclient.hard_delete_project(
- keySchema.head({ projectId }),
- keySchema.headVersion({ projectId }),
- keySchema.persistedVersion({ projectId }),
- keySchema.expireTime({ projectId }),
- keySchema.persistTime({ projectId }),
- keySchema.changes({ projectId })
- )
- metrics.inc('chunk_store.redis.hard_delete_project', 1, { status })
- return status
- } catch (err) {
- metrics.inc('chunk_store.redis.hard_delete_project', 1, { status: 'error' })
- throw err
- }
-}
-
-rclient.defineCommand('set_expire_time', {
- numberOfKeys: 2,
- lua: `
- local expireTimeKey = KEYS[1]
- local headVersionKey = KEYS[2]
- local expireTime = tonumber(ARGV[1])
-
- -- Only set the expire time if the project is loaded in Redis
- local headVersion = redis.call('GET', headVersionKey)
- if headVersion then
- redis.call('SET', expireTimeKey, expireTime)
- end
- `,
-})
-
-/**
- * Sets the expire version for a project in Redis
- *
- * @param {string} projectId
- * @param {number} expireTime - Timestamp (ms since epoch) when the project
- * buffer should expire if inactive
- */
-async function setExpireTime(projectId, expireTime) {
- try {
- await rclient.set_expire_time(
- keySchema.expireTime({ projectId }),
- keySchema.headVersion({ projectId }),
- expireTime.toString()
- )
- metrics.inc('chunk_store.redis.set_expire_time', 1, { status: 'success' })
- } catch (err) {
- metrics.inc('chunk_store.redis.set_expire_time', 1, { status: 'error' })
- throw err
- }
-}
-
-rclient.defineCommand('expire_project', {
- numberOfKeys: 6,
- lua: `
- local headKey = KEYS[1]
- local headVersionKey = KEYS[2]
- local changesKey = KEYS[3]
- local persistedVersionKey = KEYS[4]
- local persistTimeKey = KEYS[5]
- local expireTimeKey = KEYS[6]
-
- local headVersion = tonumber(redis.call('GET', headVersionKey))
- if not headVersion then
- return 'not-found'
- end
-
- local persistedVersion = tonumber(redis.call('GET', persistedVersionKey))
- if not persistedVersion or persistedVersion ~= headVersion then
- return 'not-persisted'
- end
-
- redis.call('DEL',
- headKey,
- headVersionKey,
- changesKey,
- persistedVersionKey,
- persistTimeKey,
- expireTimeKey
- )
- return 'success'
- `,
-})
-
-async function expireProject(projectId) {
- try {
- const status = await rclient.expire_project(
- keySchema.head({ projectId }),
- keySchema.headVersion({ projectId }),
- keySchema.changes({ projectId }),
- keySchema.persistedVersion({ projectId }),
- keySchema.persistTime({ projectId }),
- keySchema.expireTime({ projectId })
- )
- metrics.inc('chunk_store.redis.expire_project', 1, {
- status,
- })
- return status
- } catch (err) {
- metrics.inc('chunk_store.redis.expire_project', 1, {
- status: 'error',
- })
- throw err
- }
-}
-
-rclient.defineCommand('claim_job', {
- numberOfKeys: 1,
- lua: `
- local jobTimeKey = KEYS[1]
- local currentTime = tonumber(ARGV[1])
- local retryDelay = tonumber(ARGV[2])
-
- local jobTime = tonumber(redis.call('GET', jobTimeKey))
- if not jobTime then
- return {'no-job'}
- end
-
- local msUntilReady = jobTime - currentTime
- if msUntilReady <= 0 then
- local retryTime = currentTime + retryDelay
- redis.call('SET', jobTimeKey, retryTime)
- return {'ok', retryTime}
- else
- return {'wait', msUntilReady}
- end
- `,
-})
-
-rclient.defineCommand('close_job', {
- numberOfKeys: 1,
- lua: `
- local jobTimeKey = KEYS[1]
- local expectedJobTime = tonumber(ARGV[1])
-
- local jobTime = tonumber(redis.call('GET', jobTimeKey))
- if jobTime and jobTime == expectedJobTime then
- redis.call('DEL', jobTimeKey)
- end
- `,
-})
-
-/**
- * Claim an expire job
- *
- * @param {string} projectId
- * @return {Promise}
- */
-async function claimExpireJob(projectId) {
- return await claimJob(keySchema.expireTime({ projectId }))
-}
-
-/**
- * Claim a persist job
- *
- * @param {string} projectId
- * @return {Promise}
- */
-async function claimPersistJob(projectId) {
- return await claimJob(keySchema.persistTime({ projectId }))
-}
-
-/**
- * Claim a persist or expire job
- *
- * @param {string} jobKey - the Redis key containing the time at which the job
- * is ready
- * @return {Promise}
- */
-async function claimJob(jobKey) {
- let result, status
- try {
- result = await rclient.claim_job(jobKey, Date.now(), RETRY_DELAY_MS)
- status = result[0]
- metrics.inc('chunk_store.redis.claim_job', 1, { status })
- } catch (err) {
- metrics.inc('chunk_store.redis.claim_job', 1, { status: 'error' })
- throw err
- }
-
- if (status === 'ok') {
- return new Job(jobKey, parseInt(result[1], 10))
- } else if (status === 'wait') {
- throw new JobNotReadyError('job not ready', {
- jobKey,
- retryTime: result[1],
- })
- } else if (status === 'no-job') {
- throw new JobNotFoundError('job not found', { jobKey })
- } else {
- throw new OError('unknown status for claim_job', { jobKey, status })
- }
-}
-
-/**
- * Handle for a claimed job
- */
-class Job {
- /**
- * @param {string} redisKey
- * @param {number} claimTimestamp
- */
- constructor(redisKey, claimTimestamp) {
- this.redisKey = redisKey
- this.claimTimestamp = claimTimestamp
- }
-
- async close() {
- try {
- await rclient.close_job(this.redisKey, this.claimTimestamp.toString())
- metrics.inc('chunk_store.redis.close_job', 1, { status: 'success' })
- } catch (err) {
- metrics.inc('chunk_store.redis.close_job', 1, { status: 'error' })
- throw err
- }
- }
-}
-
-module.exports = {
- getHeadSnapshot,
- queueChanges,
- getState,
- getChangesSinceVersion,
- getNonPersistedChanges,
- setPersistedVersion,
- hardDeleteProject,
- setExpireTime,
- expireProject,
- claimExpireJob,
- claimPersistJob,
- MAX_PERSISTED_CHANGES,
- MAX_PERSIST_DELAY_MS,
- PROJECT_TTL_MS,
- RETRY_DELAY_MS,
- keySchema,
-}
diff --git a/services/history-v1/storage/lib/commit_changes.js b/services/history-v1/storage/lib/commit_changes.js
deleted file mode 100644
index 5749e5fc0e..0000000000
--- a/services/history-v1/storage/lib/commit_changes.js
+++ /dev/null
@@ -1,159 +0,0 @@
-// @ts-check
-
-'use strict'
-
-const metrics = require('@overleaf/metrics')
-const redisBackend = require('./chunk_store/redis')
-const logger = require('@overleaf/logger')
-const queueChanges = require('./queue_changes')
-const persistChanges = require('./persist_changes')
-const persistBuffer = require('./persist_buffer')
-
-/**
- * @typedef {import('overleaf-editor-core').Change} Change
- */
-
-/**
- * Handle incoming changes by processing them according to the specified options.
- * @param {string} projectId
- * @param {Change[]} changes
- * @param {Object} limits
- * @param {number} endVersion
- * @param {Object} options
- * @param {number} [options.historyBufferLevel] - The history buffer level to use for processing changes.
- * @param {Boolean} [options.forcePersistBuffer] - If true, forces the buffer to be persisted before any operation.
- * @return {Promise.}
- */
-
-async function commitChanges(
- projectId,
- changes,
- limits,
- endVersion,
- options = {}
-) {
- const { historyBufferLevel, forcePersistBuffer } = options
-
- // Force the buffer to be persisted if specified.
- if (forcePersistBuffer) {
- try {
- const status = await redisBackend.expireProject(projectId) // clear the project from Redis if it is persisted, returns 'not-persisted' if it was not persisted
- if (status === 'not-persisted') {
- await persistBuffer(projectId, limits)
- await redisBackend.expireProject(projectId) // clear the project from Redis after persisting
- metrics.inc('persist_buffer_force', 1, { status: 'persisted' })
- }
- } catch (err) {
- metrics.inc('persist_buffer_force', 1, { status: 'error' })
- logger.error(
- { err, projectId },
- 'failed to persist buffer before committing changes'
- )
- }
- }
-
- metrics.inc('commit_changes', 1, {
- history_buffer_level: historyBufferLevel || 0,
- })
-
- // Now handle the changes based on the configured history buffer level.
- switch (historyBufferLevel) {
- case 4: // Queue changes and only persist them in the background
- await queueChanges(projectId, changes, endVersion)
- return {}
- case 3: // Queue changes and immediately persist with persistBuffer
- await queueChanges(projectId, changes, endVersion)
- return await persistBuffer(projectId, limits)
- case 2: // Equivalent to queueChangesInRedis:true
- await queueChangesFake(projectId, changes, endVersion)
- return await persistChanges(projectId, changes, limits, endVersion)
- case 1: // Queue changes with fake persist only for projects in redis already
- await queueChangesFakeOnlyIfExists(projectId, changes, endVersion)
- return await persistChanges(projectId, changes, limits, endVersion)
- case 0: // Persist changes directly to the chunk store
- return await persistChanges(projectId, changes, limits, endVersion)
- default:
- throw new Error(`Invalid history buffer level: ${historyBufferLevel}`)
- }
-}
-
-/**
- * Queues a set of changes in redis as if they had been persisted, ignoring any errors.
- * @param {string} projectId
- * @param {Change[]} changes
- * @param {number} endVersion
- * @param {Object} [options]
- * @param {boolean} [options.onlyIfExists] - If true, only queue changes if the project
- * already exists in Redis.
- */
-
-async function queueChangesFake(projectId, changes, endVersion, options = {}) {
- try {
- await queueChanges(projectId, changes, endVersion)
- await fakePersistRedisChanges(projectId, changes, endVersion)
- } catch (err) {
- logger.error({ err }, 'Chunk buffer verification failed')
- }
-}
-
-/**
- * Queues changes in Redis, simulating persistence, but only if the project already exists.
- * @param {string} projectId - The ID of the project.
- * @param {Change[]} changes - An array of changes to be queued.
- * @param {number} endVersion - The expected version of the project before these changes are applied.
- */
-
-async function queueChangesFakeOnlyIfExists(projectId, changes, endVersion) {
- await queueChangesFake(projectId, changes, endVersion, {
- onlyIfExists: true,
- })
-}
-
-/**
- * Simulates the persistence of changes by verifying a given set of changes against
- * what is currently stored as non-persisted in Redis, and then updates the
- * persisted version number in Redis.
- *
- * @async
- * @param {string} projectId - The ID of the project.
- * @param {Change[]} changesToPersist - An array of changes that are expected to be
- * persisted. These are used for verification
- * against the changes currently in Redis.
- * @param {number} baseVersion - The base version number from which to calculate
- * the new persisted version.
- * @returns {Promise} A promise that resolves when the persisted version
- * in Redis has been updated.
- */
-async function fakePersistRedisChanges(
- projectId,
- changesToPersist,
- baseVersion
-) {
- const nonPersistedChanges = await redisBackend.getNonPersistedChanges(
- projectId,
- baseVersion
- )
-
- if (
- serializeChanges(nonPersistedChanges) === serializeChanges(changesToPersist)
- ) {
- metrics.inc('persist_redis_changes_verification', 1, { status: 'match' })
- } else {
- logger.warn({ projectId }, 'mismatch of non-persisted changes from Redis')
- metrics.inc('persist_redis_changes_verification', 1, {
- status: 'mismatch',
- })
- }
-
- const persistedVersion = baseVersion + nonPersistedChanges.length
- await redisBackend.setPersistedVersion(projectId, persistedVersion)
-}
-
-/**
- * @param {Change[]} changes
- */
-function serializeChanges(changes) {
- return JSON.stringify(changes.map(change => change.toRaw()))
-}
-
-module.exports = commitChanges
diff --git a/services/history-v1/storage/lib/content_hash.js b/services/history-v1/storage/lib/content_hash.js
deleted file mode 100644
index a381babc04..0000000000
--- a/services/history-v1/storage/lib/content_hash.js
+++ /dev/null
@@ -1,18 +0,0 @@
-// @ts-check
-
-const { createHash } = require('node:crypto')
-
-/**
- * Compute a SHA-1 hash of the content
- *
- * This is used to validate incoming updates.
- *
- * @param {string} content
- */
-function getContentHash(content) {
- const hash = createHash('sha-1')
- hash.update(content)
- return hash.digest('hex')
-}
-
-module.exports = { getContentHash }
diff --git a/services/history-v1/storage/lib/errors.js b/services/history-v1/storage/lib/errors.js
deleted file mode 100644
index 626536b079..0000000000
--- a/services/history-v1/storage/lib/errors.js
+++ /dev/null
@@ -1,5 +0,0 @@
-const OError = require('@overleaf/o-error')
-
-class InvalidChangeError extends OError {}
-
-module.exports = { InvalidChangeError }
diff --git a/services/history-v1/storage/lib/history_store.js b/services/history-v1/storage/lib/history_store.js
index e51bdc25c5..0e7c11ffb5 100644
--- a/services/history-v1/storage/lib/history_store.js
+++ b/services/history-v1/storage/lib/history_store.js
@@ -1,13 +1,10 @@
-// @ts-check
'use strict'
+const BPromise = require('bluebird')
const core = require('overleaf-editor-core')
const config = require('config')
-const path = require('node:path')
-const Stream = require('node:stream')
-const { promisify } = require('node:util')
-const zlib = require('node:zlib')
+const path = require('path')
const OError = require('@overleaf/o-error')
const objectPersistor = require('@overleaf/object-persistor')
@@ -20,48 +17,26 @@ const streams = require('./streams')
const Chunk = core.Chunk
-const gzip = promisify(zlib.gzip)
-const gunzip = promisify(zlib.gunzip)
+const BUCKET = config.get('chunkStore.bucket')
class LoadError extends OError {
- /**
- * @param {string} projectId
- * @param {string} chunkId
- * @param {any} cause
- */
- constructor(projectId, chunkId, cause) {
- super(
- 'HistoryStore: failed to load chunk history',
- { projectId, chunkId },
- cause
- )
+ constructor(projectId, chunkId) {
+ super('HistoryStore: failed to load chunk history', { projectId, chunkId })
this.projectId = projectId
this.chunkId = chunkId
}
}
+HistoryStore.LoadError = LoadError
class StoreError extends OError {
- /**
- * @param {string} projectId
- * @param {string} chunkId
- * @param {any} cause
- */
- constructor(projectId, chunkId, cause) {
- super(
- 'HistoryStore: failed to store chunk history',
- { projectId, chunkId },
- cause
- )
+ constructor(projectId, chunkId) {
+ super('HistoryStore: failed to store chunk history', { projectId, chunkId })
this.projectId = projectId
this.chunkId = chunkId
}
}
+HistoryStore.StoreError = StoreError
-/**
- * @param {string} projectId
- * @param {string} chunkId
- * @return {string}
- */
function getKey(projectId, chunkId) {
return path.join(projectKey.format(projectId), projectKey.pad(chunkId))
}
@@ -78,125 +53,86 @@ function getKey(projectId, chunkId) {
*
* @class
*/
-class HistoryStore {
- #persistor
- #bucket
- constructor(persistor, bucket) {
- this.#persistor = persistor
- this.#bucket = bucket
- }
+function HistoryStore() {}
- /**
- * Load the raw object for a History.
- *
- * @param {string} projectId
- * @param {string} chunkId
- * @return {Promise}
- */
- async loadRaw(projectId, chunkId) {
- assert.projectId(projectId, 'bad projectId')
- assert.chunkId(chunkId, 'bad chunkId')
+/**
+ * Load the raw object for a History.
+ *
+ * @param {number} projectId
+ * @param {number} chunkId
+ * @return {Promise.}
+ */
+HistoryStore.prototype.loadRaw = function historyStoreLoadRaw(
+ projectId,
+ chunkId
+) {
+ assert.projectId(projectId, 'bad projectId')
+ assert.chunkId(chunkId, 'bad chunkId')
- const key = getKey(projectId, chunkId)
+ const key = getKey(projectId, chunkId)
- logger.debug({ projectId, chunkId }, 'loadRaw started')
- try {
- const buf = await streams.gunzipStreamToBuffer(
- await this.#persistor.getObjectStream(this.#bucket, key)
- )
- return JSON.parse(buf.toString('utf-8'))
- } catch (err) {
+ logger.debug({ projectId, chunkId }, 'loadRaw started')
+ return BPromise.resolve()
+ .then(() => persistor.getObjectStream(BUCKET, key))
+ .then(streams.gunzipStreamToBuffer)
+ .then(buffer => JSON.parse(buffer))
+ .catch(err => {
if (err instanceof objectPersistor.Errors.NotFoundError) {
throw new Chunk.NotPersistedError(projectId)
}
- throw new LoadError(projectId, chunkId, err)
- } finally {
- logger.debug({ projectId, chunkId }, 'loadRaw finished')
- }
- }
-
- async loadRawWithBuffer(projectId, chunkId) {
- assert.projectId(projectId, 'bad projectId')
- assert.chunkId(chunkId, 'bad chunkId')
-
- const key = getKey(projectId, chunkId)
-
- logger.debug({ projectId, chunkId }, 'loadBuffer started')
- try {
- const buf = await streams.readStreamToBuffer(
- await this.#persistor.getObjectStream(this.#bucket, key)
- )
- const unzipped = await gunzip(buf)
- return {
- buffer: buf,
- raw: JSON.parse(unzipped.toString('utf-8')),
- }
- } catch (err) {
- if (err instanceof objectPersistor.Errors.NotFoundError) {
- throw new Chunk.NotPersistedError(projectId)
- }
- throw new LoadError(projectId, chunkId, err)
- } finally {
- logger.debug({ projectId, chunkId }, 'loadBuffer finished')
- }
- }
-
- /**
- * Compress and store a {@link History}.
- *
- * @param {string} projectId
- * @param {string} chunkId
- * @param {import('overleaf-editor-core/lib/types').RawHistory} rawHistory
- */
- async storeRaw(projectId, chunkId, rawHistory) {
- assert.projectId(projectId, 'bad projectId')
- assert.chunkId(chunkId, 'bad chunkId')
- assert.object(rawHistory, 'bad rawHistory')
-
- const key = getKey(projectId, chunkId)
-
- logger.debug({ projectId, chunkId }, 'storeRaw started')
-
- const buf = await gzip(JSON.stringify(rawHistory))
- try {
- await this.#persistor.sendStream(
- this.#bucket,
- key,
- Stream.Readable.from([buf]),
- {
- contentType: 'application/json',
- contentEncoding: 'gzip',
- contentLength: buf.byteLength,
- }
- )
- } catch (err) {
- throw new StoreError(projectId, chunkId, err)
- } finally {
- logger.debug({ projectId, chunkId }, 'storeRaw finished')
- }
- }
-
- /**
- * Delete multiple chunks from bucket. Expects an Array of objects with
- * projectId and chunkId properties
- * @param {Array<{projectId: string,chunkId:string}>} chunks
- */
- async deleteChunks(chunks) {
- logger.debug({ chunks }, 'deleteChunks started')
- try {
- await Promise.all(
- chunks.map(chunk => {
- const key = getKey(chunk.projectId, chunk.chunkId)
- return this.#persistor.deleteObject(this.#bucket, key)
- })
- )
- } finally {
- logger.debug({ chunks }, 'deleteChunks finished')
- }
- }
+ throw new HistoryStore.LoadError(projectId, chunkId).withCause(err)
+ })
+ .finally(() => logger.debug({ projectId, chunkId }, 'loadRaw finished'))
}
-module.exports = {
- HistoryStore,
- historyStore: new HistoryStore(persistor, config.get('chunkStore.bucket')),
+/**
+ * Compress and store a {@link History}.
+ *
+ * @param {number} projectId
+ * @param {number} chunkId
+ * @param {Object} rawHistory
+ * @return {Promise}
+ */
+HistoryStore.prototype.storeRaw = function historyStoreStoreRaw(
+ projectId,
+ chunkId,
+ rawHistory
+) {
+ assert.projectId(projectId, 'bad projectId')
+ assert.chunkId(chunkId, 'bad chunkId')
+ assert.object(rawHistory, 'bad rawHistory')
+
+ const key = getKey(projectId, chunkId)
+
+ logger.debug({ projectId, chunkId }, 'storeRaw started')
+ return BPromise.resolve()
+ .then(() => streams.gzipStringToStream(JSON.stringify(rawHistory)))
+ .then(stream =>
+ persistor.sendStream(BUCKET, key, stream, {
+ contentType: 'application/json',
+ contentEncoding: 'gzip',
+ })
+ )
+ .catch(err => {
+ throw new HistoryStore.StoreError(projectId, chunkId).withCause(err)
+ })
+ .finally(() => logger.debug({ projectId, chunkId }, 'storeRaw finished'))
}
+
+/**
+ * Delete multiple chunks from bucket. Expects an Array of objects with
+ * projectId and chunkId properties
+ * @param {Array} chunks
+ * @return {Promise}
+ */
+HistoryStore.prototype.deleteChunks = function historyDeleteChunks(chunks) {
+ logger.debug({ chunks }, 'deleteChunks started')
+ return BPromise.all(
+ chunks.map(chunk => {
+ const key = getKey(chunk.projectId, chunk.chunkId)
+ return persistor.deleteObject(BUCKET, key)
+ })
+ ).finally(() => logger.debug({ chunks }, 'deleteChunks finished'))
+}
+
+module.exports = new HistoryStore()
diff --git a/services/history-v1/storage/lib/knex.js b/services/history-v1/storage/lib/knex.js
index 7000fe034c..5cdc85e2ab 100644
--- a/services/history-v1/storage/lib/knex.js
+++ b/services/history-v1/storage/lib/knex.js
@@ -1,8 +1,6 @@
-// @ts-check
-
'use strict'
const env = process.env.NODE_ENV || 'development'
const knexfile = require('../../knexfile')
-module.exports = require('knex').default(knexfile[env])
+module.exports = require('knex')(knexfile[env])
diff --git a/services/history-v1/storage/lib/knex_read_only.js b/services/history-v1/storage/lib/knex_read_only.js
deleted file mode 100644
index a78c4689a4..0000000000
--- a/services/history-v1/storage/lib/knex_read_only.js
+++ /dev/null
@@ -1,19 +0,0 @@
-'use strict'
-
-const config = require('config')
-const knexfile = require('../../knexfile')
-
-const env = process.env.NODE_ENV || 'development'
-
-if (config.databaseUrlReadOnly) {
- module.exports = require('knex')({
- ...knexfile[env],
- pool: {
- ...knexfile[env].pool,
- min: 0,
- },
- connection: config.databaseUrlReadOnly,
- })
-} else {
- module.exports = require('./knex')
-}
diff --git a/services/history-v1/storage/lib/mongodb.js b/services/history-v1/storage/lib/mongodb.js
index e887bc25a5..53b1837a8f 100644
--- a/services/history-v1/storage/lib/mongodb.js
+++ b/services/history-v1/storage/lib/mongodb.js
@@ -10,21 +10,7 @@ const chunks = db.collection('projectHistoryChunks')
const blobs = db.collection('projectHistoryBlobs')
const globalBlobs = db.collection('projectHistoryGlobalBlobs')
const shardedBlobs = db.collection('projectHistoryShardedBlobs')
-const projects = db.collection('projects')
-// Temporary collection for tracking progress of backed up old blobs (without a hash).
-// The initial sync process will be able to skip over these.
-// Schema: _id: projectId, blobs: [Binary]
-const backedUpBlobs = db.collection('projectHistoryBackedUpBlobs')
Metrics.mongodb.monitor(client)
-module.exports = {
- client,
- db,
- chunks,
- blobs,
- globalBlobs,
- projects,
- shardedBlobs,
- backedUpBlobs,
-}
+module.exports = { client, db, chunks, blobs, globalBlobs, shardedBlobs }
diff --git a/services/history-v1/storage/lib/persist_buffer.js b/services/history-v1/storage/lib/persist_buffer.js
deleted file mode 100644
index 68b71e148f..0000000000
--- a/services/history-v1/storage/lib/persist_buffer.js
+++ /dev/null
@@ -1,237 +0,0 @@
-// @ts-check
-'use strict'
-
-const logger = require('@overleaf/logger')
-const metrics = require('@overleaf/metrics')
-const OError = require('@overleaf/o-error')
-const assert = require('./assert')
-const chunkStore = require('./chunk_store')
-const { BlobStore } = require('./blob_store')
-const BatchBlobStore = require('./batch_blob_store')
-const persistChanges = require('./persist_changes')
-const resyncProject = require('./resync_project')
-const redisBackend = require('./chunk_store/redis')
-
-const PERSIST_BATCH_SIZE = 50
-
-/**
- * Persist the changes from Redis buffer to the main storage
- *
- * Algorithm Outline:
- * 1. Get the latest chunk's endVersion from the database
- * 2. Get non-persisted changes from Redis that are after this endVersion.
- * 3. If no such changes, exit.
- * 4. Load file blobs for these Redis changes.
- * 5. Run the persistChanges() algorithm to store these changes into a new chunk(s) in GCS.
- * - This must not decrease the endVersion. If changes were processed, it must advance.
- * 6. Set the new persisted version (endVersion of the latest persisted chunk) in Redis.
- *
- * @param {string} projectId
- * @param {Object} limits
- * @throws {Error | OError} If a critical error occurs during persistence.
- */
-async function persistBuffer(projectId, limits) {
- assert.projectId(projectId)
- logger.debug({ projectId }, 'starting persistBuffer operation')
-
- // 1. Get the latest chunk's endVersion from GCS/main store
- let endVersion
- const latestChunkMetadata = await chunkStore.getLatestChunkMetadata(projectId)
-
- if (latestChunkMetadata) {
- endVersion = latestChunkMetadata.endVersion
- } else {
- endVersion = 0 // No chunks found, start from version 0
- logger.debug({ projectId }, 'no existing chunks found in main storage')
- }
- const originalEndVersion = endVersion
-
- logger.debug({ projectId, endVersion }, 'got latest persisted chunk')
-
- // Process changes in batches
- let numberOfChangesPersisted = 0
- let currentChunk = null
- let resyncNeeded = false
- let resyncChangesWerePersisted = false
- while (true) {
- // 2. Get non-persisted changes from Redis
- const changesToPersist = await redisBackend.getNonPersistedChanges(
- projectId,
- endVersion,
- { maxChanges: PERSIST_BATCH_SIZE }
- )
-
- if (changesToPersist.length === 0) {
- break
- }
-
- logger.debug(
- {
- projectId,
- endVersion,
- count: changesToPersist.length,
- },
- 'found changes in Redis to persist'
- )
-
- // 4. Load file blobs for these Redis changes. Errors will propagate.
- const blobStore = new BlobStore(projectId)
- const batchBlobStore = new BatchBlobStore(blobStore)
-
- const blobHashes = new Set()
- for (const change of changesToPersist) {
- change.findBlobHashes(blobHashes)
- }
- if (blobHashes.size > 0) {
- await batchBlobStore.preload(Array.from(blobHashes))
- }
- for (const change of changesToPersist) {
- await change.loadFiles('lazy', blobStore)
- }
-
- // 5. Run the persistChanges() algorithm. Errors will propagate.
- logger.debug(
- {
- projectId,
- endVersion,
- changeCount: changesToPersist.length,
- },
- 'calling persistChanges'
- )
-
- const persistResult = await persistChanges(
- projectId,
- changesToPersist,
- limits,
- endVersion
- )
-
- if (!persistResult || !persistResult.currentChunk) {
- metrics.inc('persist_buffer', 1, { status: 'no-chunk-error' })
- throw new OError(
- 'persistChanges did not produce a new chunk for non-empty changes',
- {
- projectId,
- endVersion,
- changeCount: changesToPersist.length,
- }
- )
- }
-
- currentChunk = persistResult.currentChunk
- const newEndVersion = currentChunk.getEndVersion()
-
- if (newEndVersion <= endVersion) {
- metrics.inc('persist_buffer', 1, { status: 'chunk-version-mismatch' })
- throw new OError(
- 'persisted chunk endVersion must be greater than current persisted chunk end version for non-empty changes',
- {
- projectId,
- newEndVersion,
- endVersion,
- changeCount: changesToPersist.length,
- }
- )
- }
-
- logger.debug(
- {
- projectId,
- oldVersion: endVersion,
- newVersion: newEndVersion,
- },
- 'successfully persisted changes from Redis to main storage'
- )
-
- // 6. Set the persisted version in Redis. Errors will propagate.
- const status = await redisBackend.setPersistedVersion(
- projectId,
- newEndVersion
- )
-
- if (status !== 'ok') {
- metrics.inc('persist_buffer', 1, { status: 'error-on-persisted-version' })
- throw new OError('failed to update persisted version in Redis', {
- projectId,
- newEndVersion,
- status,
- })
- }
-
- logger.debug(
- { projectId, newEndVersion },
- 'updated persisted version in Redis'
- )
- numberOfChangesPersisted += persistResult.numberOfChangesPersisted
- endVersion = newEndVersion
-
- // Check if a resync might be needed
- if (persistResult.resyncNeeded) {
- resyncNeeded = true
- }
-
- if (
- changesToPersist.some(
- change => change.getOrigin()?.getKind() === 'history-resync'
- )
- ) {
- resyncChangesWerePersisted = true
- }
-
- if (persistResult.numberOfChangesPersisted < PERSIST_BATCH_SIZE) {
- // We reached the end of available changes
- break
- }
- }
-
- if (numberOfChangesPersisted === 0) {
- logger.debug(
- { projectId, endVersion },
- 'no new changes in Redis buffer to persist'
- )
- metrics.inc('persist_buffer', 1, { status: 'no_changes' })
- // No changes to persist, update the persisted version in Redis
- // to match the current endVersion. This shouldn't be needed
- // unless a worker failed to update the persisted version.
- await redisBackend.setPersistedVersion(projectId, endVersion)
- } else {
- logger.debug(
- { projectId, finalPersistedVersion: endVersion },
- 'persistBuffer operation completed successfully'
- )
- metrics.inc('persist_buffer', 1, { status: 'persisted' })
- }
-
- if (limits.autoResync && resyncNeeded) {
- if (resyncChangesWerePersisted) {
- // To avoid an infinite loop, do not resync if the current batch of
- // changes contains a history resync.
- logger.warn(
- { projectId },
- 'content hash validation failed while persisting a history resync, skipping additional resync'
- )
- } else {
- const backend = chunkStore.getBackend(projectId)
- const mongoProjectId =
- await backend.resolveHistoryIdToMongoProjectId(projectId)
- await resyncProject(mongoProjectId)
- }
- }
-
- if (currentChunk == null) {
- const { chunk } = await chunkStore.loadByChunkRecord(
- projectId,
- latestChunkMetadata
- )
- currentChunk = chunk
- }
-
- return {
- numberOfChangesPersisted,
- originalEndVersion,
- currentChunk,
- resyncNeeded,
- }
-}
-
-module.exports = persistBuffer
diff --git a/services/history-v1/storage/lib/persist_changes.js b/services/history-v1/storage/lib/persist_changes.js
index d2ca00053f..b661a4818c 100644
--- a/services/history-v1/storage/lib/persist_changes.js
+++ b/services/history-v1/storage/lib/persist_changes.js
@@ -1,9 +1,8 @@
-// @ts-check
-
+/** @module */
'use strict'
const _ = require('lodash')
-const logger = require('@overleaf/logger')
+const BPromise = require('bluebird')
const core = require('overleaf-editor-core')
const Chunk = core.Chunk
@@ -11,9 +10,6 @@ const History = core.History
const assert = require('./assert')
const chunkStore = require('./chunk_store')
-const { BlobStore } = require('./blob_store')
-const { InvalidChangeError } = require('./errors')
-const { getContentHash } = require('./content_hash')
function countChangeBytes(change) {
// Note: This is not quite accurate, because the raw change may contain raw
@@ -52,35 +48,28 @@ Timer.prototype.elapsed = function () {
* endVersion may be better suited to the metadata record.
*
* @param {string} projectId
- * @param {core.Change[]} allChanges
+ * @param {Array.} allChanges
* @param {Object} limits
* @param {number} clientEndVersion
* @return {Promise.}
*/
-async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
+module.exports = function persistChanges(
+ projectId,
+ allChanges,
+ limits,
+ clientEndVersion
+) {
assert.projectId(projectId)
assert.array(allChanges)
assert.maybe.object(limits)
assert.integer(clientEndVersion)
- const blobStore = new BlobStore(projectId)
-
- const earliestChangeTimestamp =
- allChanges.length > 0 ? allChanges[0].getTimestamp() : null
-
let currentChunk
-
- /**
- * currentSnapshot tracks the latest change that we're applying; we use it to
- * check that the changes we are persisting are valid.
- *
- * @type {core.Snapshot}
- */
+ // currentSnapshot tracks the latest change that we're applying; we use it to
+ // check that the changes we are persisting are valid.
let currentSnapshot
-
let originalEndVersion
let changesToPersist
- let resyncNeeded = false
limits = limits || {}
_.defaults(limits, {
@@ -99,135 +88,57 @@ async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
}
}
- /**
- * Add changes to a chunk until the chunk is full
- *
- * The chunk is full if it reaches a certain number of changes or a certain
- * size in bytes
- *
- * @param {core.Chunk} chunk
- * @param {core.Change[]} changes
- */
- async function fillChunk(chunk, changes) {
+ function fillChunk(chunk, changes) {
let totalBytes = totalChangeBytes(chunk.getChanges())
let changesPushed = false
while (changes.length > 0) {
- if (chunk.getChanges().length >= limits.maxChunkChanges) {
- break
- }
-
- const change = changes[0]
- const changeBytes = countChangeBytes(change)
-
- if (totalBytes + changeBytes > limits.maxChunkChangeBytes) {
- break
- }
-
- for (const operation of change.iterativelyApplyTo(currentSnapshot, {
- strict: true,
- })) {
- await validateContentHash(operation)
- }
-
- chunk.pushChanges([change])
- changes.shift()
+ if (chunk.getChanges().length >= limits.maxChunkChanges) break
+ const changeBytes = countChangeBytes(changes[0])
+ if (totalBytes + changeBytes > limits.maxChunkChangeBytes) break
+ const changesToFill = changes.splice(0, 1)
+ currentSnapshot.applyAll(changesToFill, { strict: true })
+ chunk.pushChanges(changesToFill)
totalBytes += changeBytes
changesPushed = true
}
return changesPushed
}
- /**
- * Check that the operation is valid and can be incorporated to the history.
- *
- * For now, this checks content hashes when they are provided.
- *
- * @param {core.Operation} operation
- */
- async function validateContentHash(operation) {
- if (operation instanceof core.EditFileOperation) {
- const editOperation = operation.getOperation()
- if (
- editOperation instanceof core.TextOperation &&
- editOperation.contentHash != null
- ) {
- const path = operation.getPathname()
- const file = currentSnapshot.getFile(path)
- if (file == null) {
- throw new InvalidChangeError('file not found for hash validation', {
- projectId,
- path,
- })
- }
- await file.load('eager', blobStore)
- const content = file.getContent({ filterTrackedDeletes: true })
- const expectedHash = editOperation.contentHash
- const actualHash = content != null ? getContentHash(content) : null
- logger.debug({ expectedHash, actualHash }, 'validating content hash')
- if (actualHash !== expectedHash) {
- // only log a warning on the first mismatch in each persistChanges call
- if (!resyncNeeded) {
- logger.warn(
- { projectId, path, expectedHash, actualHash },
- 'content hash mismatch'
- )
- }
- resyncNeeded = true
- }
-
- // Remove the content hash from the change before storing it in the chunk.
- // It was only useful for validation.
- editOperation.contentHash = null
+ function extendLastChunkIfPossible() {
+ return chunkStore.loadLatest(projectId).then(function (latestChunk) {
+ currentChunk = latestChunk
+ originalEndVersion = latestChunk.getEndVersion()
+ if (originalEndVersion !== clientEndVersion) {
+ throw new Chunk.ConflictingEndVersion(
+ clientEndVersion,
+ originalEndVersion
+ )
}
- }
- }
- async function loadLatestChunk() {
- const latestChunk = await chunkStore.loadLatest(projectId, {
- persistedOnly: true,
- })
-
- currentChunk = latestChunk
- originalEndVersion = latestChunk.getEndVersion()
- if (originalEndVersion !== clientEndVersion) {
- throw new Chunk.ConflictingEndVersion(
- clientEndVersion,
- originalEndVersion
- )
- }
-
- currentSnapshot = latestChunk.getSnapshot().clone()
- currentSnapshot.applyAll(currentChunk.getChanges())
- }
-
- async function extendLastChunkIfPossible() {
- const timer = new Timer()
- const changesPushed = await fillChunk(currentChunk, changesToPersist)
- if (!changesPushed) {
- return
- }
-
- checkElapsedTime(timer)
-
- await chunkStore.update(projectId, currentChunk, earliestChangeTimestamp)
- }
-
- async function createNewChunksAsNeeded() {
- while (changesToPersist.length > 0) {
- const endVersion = currentChunk.getEndVersion()
- const history = new History(currentSnapshot.clone(), [])
- const chunk = new Chunk(history, endVersion)
+ currentSnapshot = latestChunk.getSnapshot().clone()
const timer = new Timer()
+ currentSnapshot.applyAll(latestChunk.getChanges())
- const changesPushed = await fillChunk(chunk, changesToPersist)
- if (changesPushed) {
- checkElapsedTime(timer)
- currentChunk = chunk
- await chunkStore.create(projectId, chunk, earliestChangeTimestamp)
- } else {
- throw new Error('failed to fill empty chunk')
- }
+ if (!fillChunk(currentChunk, changesToPersist)) return
+ checkElapsedTime(timer)
+
+ return chunkStore.update(projectId, originalEndVersion, currentChunk)
+ })
+ }
+
+ function createNewChunksAsNeeded() {
+ if (changesToPersist.length === 0) return
+
+ const endVersion = currentChunk.getEndVersion()
+ const history = new History(currentSnapshot.clone(), [])
+ const chunk = new Chunk(history, endVersion)
+ const timer = new Timer()
+ if (fillChunk(chunk, changesToPersist)) {
+ checkElapsedTime(timer)
+ currentChunk = chunk
+ return chunkStore.create(projectId, chunk).then(createNewChunksAsNeeded)
}
+ throw new Error('failed to fill empty chunk')
}
function isOlderThanMinChangeTimestamp(change) {
@@ -246,20 +157,15 @@ async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
if (anyTooOld || tooManyChanges || tooManyBytes) {
changesToPersist = oldChanges
const numberOfChangesToPersist = oldChanges.length
-
- await loadLatestChunk()
- await extendLastChunkIfPossible()
- await createNewChunksAsNeeded()
-
- return {
- numberOfChangesPersisted: numberOfChangesToPersist,
- originalEndVersion,
- currentChunk,
- resyncNeeded,
- }
- } else {
- return null
+ return extendLastChunkIfPossible()
+ .then(createNewChunksAsNeeded)
+ .then(function () {
+ return {
+ numberOfChangesPersisted: numberOfChangesToPersist,
+ originalEndVersion,
+ currentChunk,
+ }
+ })
}
+ return BPromise.resolve(null)
}
-
-module.exports = persistChanges
diff --git a/services/history-v1/storage/lib/project_archive.js b/services/history-v1/storage/lib/project_archive.js
index 8a8e93f1c9..d6680ae33c 100644
--- a/services/history-v1/storage/lib/project_archive.js
+++ b/services/history-v1/storage/lib/project_archive.js
@@ -8,8 +8,8 @@
const Archive = require('archiver')
const BPromise = require('bluebird')
-const fs = require('node:fs')
-const { pipeline } = require('node:stream')
+const fs = require('fs')
+const { pipeline } = require('stream')
const core = require('overleaf-editor-core')
@@ -49,7 +49,7 @@ class ProjectArchive {
/**
* @constructor
* @param {Snapshot} snapshot
- * @param {number} [timeout] in ms
+ * @param {?number} timeout in ms
* @classdesc
* Writes the project snapshot to a zip file.
*/
diff --git a/services/history-v1/storage/lib/project_key.js b/services/history-v1/storage/lib/project_key.js
index 03fb2a5141..e84f024e06 100644
--- a/services/history-v1/storage/lib/project_key.js
+++ b/services/history-v1/storage/lib/project_key.js
@@ -1,6 +1,5 @@
-// Keep in sync with services/web/app/src/Features/History/project_key.js
const _ = require('lodash')
-const path = require('node:path')
+const path = require('path')
//
// The advice in http://docs.aws.amazon.com/AmazonS3/latest/dev/
diff --git a/services/history-v1/storage/lib/queue_changes.js b/services/history-v1/storage/lib/queue_changes.js
deleted file mode 100644
index 6b8d4b22b4..0000000000
--- a/services/history-v1/storage/lib/queue_changes.js
+++ /dev/null
@@ -1,75 +0,0 @@
-// @ts-check
-
-'use strict'
-
-const redisBackend = require('./chunk_store/redis')
-const { BlobStore } = require('./blob_store')
-const chunkStore = require('./chunk_store')
-const core = require('overleaf-editor-core')
-const Chunk = core.Chunk
-
-/**
- * Queues an incoming set of changes after validating them against the current snapshot.
- *
- * @async
- * @function queueChanges
- * @param {string} projectId - The project to queue changes for.
- * @param {Array} changesToQueue - An array of change objects to be applied and queued.
- * @param {number} endVersion - The expected version of the project before these changes are applied.
- * This is used for optimistic concurrency control.
- * @param {Object} [opts] - Additional options for queuing changes.
- * @throws {Chunk.ConflictingEndVersion} If the provided `endVersion` does not match the
- * current version of the project.
- * @returns {Promise} A promise that resolves with the status returned by the
- * `redisBackend.queueChanges` operation.
- */
-async function queueChanges(projectId, changesToQueue, endVersion, opts) {
- const result = await redisBackend.getHeadSnapshot(projectId)
- let currentSnapshot = null
- let currentVersion = null
- if (result) {
- // If we have a snapshot in redis, we can use it to check the current state
- // of the project and apply changes to it.
- currentSnapshot = result.snapshot
- currentVersion = result.version
- } else {
- // Otherwise, load the latest chunk from the chunk store.
- const latestChunk = await chunkStore.loadLatest(projectId, {
- persistedOnly: true,
- })
- // Throw an error if no latest chunk is found, indicating the project has not been initialised.
- if (!latestChunk) {
- throw new Chunk.NotFoundError(projectId)
- }
- currentSnapshot = latestChunk.getSnapshot()
- currentSnapshot.applyAll(latestChunk.getChanges())
- currentVersion = latestChunk.getEndVersion()
- }
-
- // Ensure the endVersion matches the current version of the project.
- if (endVersion !== currentVersion) {
- throw new Chunk.ConflictingEndVersion(endVersion, currentVersion)
- }
-
- // Compute the new hollow snapshot to be saved to redis.
- const hollowSnapshot = currentSnapshot
- const blobStore = new BlobStore(projectId)
- await hollowSnapshot.loadFiles('hollow', blobStore)
- // Clone the changes to avoid modifying the original ones when computing the hollow snapshot.
- const hollowChanges = changesToQueue.map(change => change.clone())
- for (const change of hollowChanges) {
- await change.loadFiles('hollow', blobStore)
- }
- hollowSnapshot.applyAll(hollowChanges, { strict: true })
- const baseVersion = currentVersion
- const status = await redisBackend.queueChanges(
- projectId,
- hollowSnapshot,
- baseVersion,
- changesToQueue,
- opts
- )
- return status
-}
-
-module.exports = queueChanges
diff --git a/services/history-v1/storage/lib/redis.js b/services/history-v1/storage/lib/redis.js
deleted file mode 100644
index 9b00cc0a26..0000000000
--- a/services/history-v1/storage/lib/redis.js
+++ /dev/null
@@ -1,19 +0,0 @@
-const config = require('config')
-const redis = require('@overleaf/redis-wrapper')
-
-const historyRedisOptions = config.get('redis.history')
-const rclientHistory = redis.createClient(historyRedisOptions)
-
-const lockRedisOptions = config.get('redis.history')
-const rclientLock = redis.createClient(lockRedisOptions)
-
-async function disconnect() {
- await Promise.all([rclientHistory.disconnect(), rclientLock.disconnect()])
-}
-
-module.exports = {
- rclientHistory,
- rclientLock,
- redis,
- disconnect,
-}
diff --git a/services/history-v1/storage/lib/resync_project.js b/services/history-v1/storage/lib/resync_project.js
deleted file mode 100644
index 3ec680bb5b..0000000000
--- a/services/history-v1/storage/lib/resync_project.js
+++ /dev/null
@@ -1,14 +0,0 @@
-// @ts-check
-
-const config = require('config')
-const { fetchNothing } = require('@overleaf/fetch-utils')
-
-const PROJECT_HISTORY_URL = `http://${config.projectHistory.host}:${config.projectHistory.port}`
-
-async function resyncProject(projectId) {
- await fetchNothing(`${PROJECT_HISTORY_URL}/project/${projectId}/resync`, {
- method: 'POST',
- })
-}
-
-module.exports = resyncProject
diff --git a/services/history-v1/storage/lib/scan.js b/services/history-v1/storage/lib/scan.js
deleted file mode 100644
index d55f5362c1..0000000000
--- a/services/history-v1/storage/lib/scan.js
+++ /dev/null
@@ -1,202 +0,0 @@
-// @ts-check
-
-'use strict'
-
-const logger = require('@overleaf/logger')
-const { JobNotFoundError, JobNotReadyError } = require('./chunk_store/errors')
-const BATCH_SIZE = 1000 // Default batch size for SCAN
-
-/**
- * Asynchronously scans a Redis instance or cluster for keys matching a pattern.
- *
- * This function handles both standalone Redis instances and Redis clusters.
- * For clusters, it iterates over all master nodes. It yields keys in batches
- * as they are found by the SCAN command.
- *
- * @param {object} redisClient - The Redis client instance (from @overleaf/redis-wrapper).
- * @param {string} pattern - The pattern to match keys against (e.g., 'user:*').
- * @param {number} [count=BATCH_SIZE] - Optional hint for Redis SCAN count per iteration.
- * @yields {string[]} A batch of matching keys.
- */
-async function* scanRedisCluster(redisClient, pattern, count = BATCH_SIZE) {
- const nodes = redisClient.nodes ? redisClient.nodes('master') : [redisClient]
-
- for (const node of nodes) {
- let cursor = '0'
- do {
- // redisClient from @overleaf/redis-wrapper uses ioredis style commands
- const [nextCursor, keys] = await node.scan(
- cursor,
- 'MATCH',
- pattern,
- 'COUNT',
- count
- )
- cursor = nextCursor
- if (keys.length > 0) {
- yield keys
- }
- } while (cursor !== '0')
- }
-}
-
-/**
- * Extracts the content within the first pair of curly braces {} from a string.
- * This is used to extract a user ID or project ID from a Redis key.
- *
- * @param {string} key - The input string containing content within curly braces.
- * @returns {string | null} The extracted content (the key ID) if found, otherwise null.
- */
-function extractKeyId(key) {
- const match = key.match(/\{(.*?)\}/)
- if (match && match[1]) {
- return match[1]
- }
- return null
-}
-
-/**
- * Fetches timestamps for a list of project IDs based on a given key name.
- *
- * @param {string[]} projectIds - Array of project identifiers.
- * @param {object} rclient - The Redis client instance.
- * @param {string} keyName - The base name for the Redis keys storing the timestamps (e.g., "expire-time", "persist-time").
- * @param {number} currentTime - The current time (timestamp in milliseconds) to compare against.
- * @returns {Promise>}
- * A promise that resolves to an array of objects, each containing a projectId and
- * its corresponding timestampValue, for due projects only.
- */
-async function fetchOverdueProjects(projectIds, rclient, keyName, currentTime) {
- if (!projectIds || projectIds.length === 0) {
- return []
- }
- const timestampKeys = projectIds.map(id => `${keyName}:{${id}}`)
- const timestamps = await rclient.mget(timestampKeys)
-
- const dueProjects = []
- for (let i = 0; i < projectIds.length; i++) {
- const projectId = projectIds[i]
- const timestampValue = timestamps[i]
-
- if (timestampValue !== null) {
- const timestamp = parseInt(timestampValue, 10)
- if (!isNaN(timestamp) && currentTime > timestamp) {
- dueProjects.push({ projectId, timestampValue })
- }
- }
- }
- return dueProjects
-}
-
-/**
- * Scans Redis for keys matching a pattern derived from keyName, identifies items that are "due" based on a timestamp,
- * and performs a specified action on them.
- *
- * @param {object} rclient - The Redis client instance.
- * @param {string} taskName - A descriptive name for the task (used in logging).
- * @param {string} keyName - The base name for the Redis keys (e.g., "expire-time", "persist-time").
- * The function will derive the key prefix as `${keyName}:` and scan pattern as `${keyName}:{*}`.
- * @param {function(string): Promise} actionFn - An async function that takes a projectId and performs an action.
- * @param {boolean} DRY_RUN - If true, logs actions that would be taken without performing them.
- * @returns {Promise<{scannedKeyCount: number, processedKeyCount: number}>} Counts of scanned and processed keys.
- */
-async function scanAndProcessDueItems(
- rclient,
- taskName,
- keyName,
- actionFn,
- DRY_RUN
-) {
- let scannedKeyCount = 0
- let processedKeyCount = 0
- const START_TIME = Date.now()
- const logContext = { taskName, dryRun: DRY_RUN }
-
- const scanPattern = `${keyName}:{*}`
-
- if (DRY_RUN) {
- logger.info(logContext, `Starting ${taskName} scan in DRY RUN mode`)
- } else {
- logger.info(logContext, `Starting ${taskName} scan`)
- }
-
- for await (const keysBatch of scanRedisCluster(rclient, scanPattern)) {
- scannedKeyCount += keysBatch.length
- const projectIds = keysBatch.map(extractKeyId).filter(id => id != null)
-
- if (projectIds.length === 0) {
- continue
- }
-
- const currentTime = Date.now()
- const overdueProjects = await fetchOverdueProjects(
- projectIds,
- rclient,
- keyName,
- currentTime
- )
-
- for (const project of overdueProjects) {
- const { projectId } = project
- if (DRY_RUN) {
- logger.info(
- { ...logContext, projectId },
- `[Dry Run] Would perform ${taskName} for project`
- )
- } else {
- try {
- await actionFn(projectId)
- logger.debug(
- { ...logContext, projectId },
- `Successfully performed ${taskName} for project`
- )
- } catch (err) {
- if (err instanceof JobNotReadyError) {
- // the project has been touched since the job was created
- logger.info(
- { ...logContext, projectId },
- `Job not ready for ${taskName} for project`
- )
- } else if (err instanceof JobNotFoundError) {
- // the project has been expired already by another worker
- logger.info(
- { ...logContext, projectId },
- `Job not found for ${taskName} for project`
- )
- } else {
- logger.error(
- { ...logContext, projectId, err },
- `Error performing ${taskName} for project`
- )
- }
- continue
- }
- }
- processedKeyCount++
-
- if (processedKeyCount % 1000 === 0 && processedKeyCount > 0) {
- logger.info(
- { ...logContext, scannedKeyCount, processedKeyCount },
- `${taskName} scan progress`
- )
- }
- }
- }
-
- logger.info(
- {
- ...logContext,
- scannedKeyCount,
- processedKeyCount,
- elapsedTimeInSeconds: Math.floor((Date.now() - START_TIME) / 1000),
- },
- `${taskName} scan complete`
- )
- return { scannedKeyCount, processedKeyCount }
-}
-
-module.exports = {
- scanRedisCluster,
- extractKeyId,
- scanAndProcessDueItems,
-}
diff --git a/services/history-v1/storage/lib/streams.js b/services/history-v1/storage/lib/streams.js
index e60e5aa725..9e216c3f97 100644
--- a/services/history-v1/storage/lib/streams.js
+++ b/services/history-v1/storage/lib/streams.js
@@ -1,4 +1,3 @@
-// @ts-check
/**
* Promises are promises and streams are streams, and ne'er the twain shall
* meet.
@@ -6,20 +5,52 @@
*/
'use strict'
-const Stream = require('node:stream')
-const zlib = require('node:zlib')
-const { WritableBuffer } = require('@overleaf/stream-utils')
+const BPromise = require('bluebird')
+const zlib = require('zlib')
+const { WritableBuffer, ReadableString } = require('@overleaf/stream-utils')
+const { pipeline } = require('stream')
+
+/**
+ * Pipe a read stream to a write stream. The promise resolves when the write
+ * stream finishes.
+ *
+ * @function
+ * @param {stream.Readable} readStream
+ * @param {stream.Writable} writeStream
+ * @return {Promise}
+ */
+function promisePipe(readStream, writeStream) {
+ return new BPromise(function (resolve, reject) {
+ pipeline(readStream, writeStream, function (err) {
+ if (err) {
+ reject(err)
+ } else {
+ resolve()
+ }
+ })
+ })
+}
+
+exports.promisePipe = promisePipe
/**
* Create a promise for the result of reading a stream to a buffer.
*
- * @param {Stream.Readable} readStream
- * @return {Promise}
+ * @function
+ * @param {stream.Readable} readStream
+ * @return {Promise.}
*/
-async function readStreamToBuffer(readStream) {
- const bufferStream = new WritableBuffer()
- await Stream.promises.pipeline(readStream, bufferStream)
- return bufferStream.contents()
+function readStreamToBuffer(readStream) {
+ return new BPromise(function (resolve, reject) {
+ const bufferStream = new WritableBuffer()
+ pipeline(readStream, bufferStream, function (err) {
+ if (err) {
+ reject(err)
+ } else {
+ resolve(bufferStream.contents())
+ }
+ })
+ })
}
exports.readStreamToBuffer = readStreamToBuffer
@@ -27,14 +58,43 @@ exports.readStreamToBuffer = readStreamToBuffer
/**
* Create a promise for the result of un-gzipping a stream to a buffer.
*
- * @param {NodeJS.ReadableStream} readStream
- * @return {Promise}
+ * @function
+ * @param {stream.Readable} readStream
+ * @return {Promise.}
*/
-async function gunzipStreamToBuffer(readStream) {
+function gunzipStreamToBuffer(readStream) {
const gunzip = zlib.createGunzip()
const bufferStream = new WritableBuffer()
- await Stream.promises.pipeline(readStream, gunzip, bufferStream)
- return bufferStream.contents()
+ return new BPromise(function (resolve, reject) {
+ pipeline(readStream, gunzip, bufferStream, function (err) {
+ if (err) {
+ reject(err)
+ } else {
+ resolve(bufferStream.contents())
+ }
+ })
+ })
}
exports.gunzipStreamToBuffer = gunzipStreamToBuffer
+
+/**
+ * Create a write stream that gzips the given string.
+ *
+ * @function
+ * @param {string} string
+ * @return {Promise.}
+ */
+function gzipStringToStream(string) {
+ return new BPromise(function (resolve, reject) {
+ zlib.gzip(Buffer.from(string), function (error, result) {
+ if (error) {
+ reject(error)
+ } else {
+ resolve(new ReadableString(result))
+ }
+ })
+ })
+}
+
+exports.gzipStringToStream = gzipStringToStream
diff --git a/services/history-v1/storage/lib/temp.js b/services/history-v1/storage/lib/temp.js
index 1aab3c1df4..719e0767a6 100644
--- a/services/history-v1/storage/lib/temp.js
+++ b/services/history-v1/storage/lib/temp.js
@@ -7,7 +7,7 @@
*/
const BPromise = require('bluebird')
-const fs = BPromise.promisifyAll(require('node:fs'))
+const fs = BPromise.promisifyAll(require('fs'))
const temp = BPromise.promisifyAll(require('temp'))
exports.open = function (affixes) {
diff --git a/services/history-v1/storage/lib/zip_store.js b/services/history-v1/storage/lib/zip_store.js
index 0741829303..b3cf7c2663 100644
--- a/services/history-v1/storage/lib/zip_store.js
+++ b/services/history-v1/storage/lib/zip_store.js
@@ -2,8 +2,8 @@
const BPromise = require('bluebird')
const config = require('config')
-const fs = require('node:fs')
-const path = require('node:path')
+const fs = require('fs')
+const path = require('path')
const OError = require('@overleaf/o-error')
const objectPersistor = require('@overleaf/object-persistor')
diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs
deleted file mode 100644
index 2e12328e5c..0000000000
--- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs
+++ /dev/null
@@ -1,1158 +0,0 @@
-// @ts-check
-import Events from 'node:events'
-import fs from 'node:fs'
-import Path from 'node:path'
-import { performance } from 'node:perf_hooks'
-import Stream from 'node:stream'
-import { setTimeout } from 'node:timers/promises'
-import { ObjectId } from 'mongodb'
-import pLimit from 'p-limit'
-import logger from '@overleaf/logger'
-import {
- batchedUpdate,
- objectIdFromInput,
- renderObjectId,
-} from '@overleaf/mongo-utils/batchedUpdate.js'
-import OError from '@overleaf/o-error'
-import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
-import {
- BlobStore,
- GLOBAL_BLOBS,
- loadGlobalBlobs,
- getProjectBlobsBatch,
- getStringLengthOfFile,
- makeBlobForFile,
-} from '../lib/blob_store/index.js'
-import { db } from '../lib/mongodb.js'
-import commandLineArgs from 'command-line-args'
-import readline from 'node:readline'
-
-// Silence warning.
-Events.setMaxListeners(20)
-
-// Enable caching for ObjectId.toString()
-ObjectId.cacheHexString = true
-
-/**
- * @typedef {import("overleaf-editor-core").Blob} Blob
- * @typedef {import("perf_hooks").EventLoopUtilization} EventLoopUtilization
- * @typedef {import("mongodb").Collection} Collection
- * @typedef {import("mongodb").Collection} ProjectsCollection
- * @typedef {import("mongodb").Collection<{project:Project}>} DeletedProjectsCollection
- * @typedef {import("@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor").CachedPerProjectEncryptedS3Persistor} CachedPerProjectEncryptedS3Persistor
- */
-
-/**
- * @typedef {Object} FileRef
- * @property {ObjectId} _id
- * @property {string} hash
- */
-
-/**
- * @typedef {Object} Folder
- * @property {Array} folders
- * @property {Array} fileRefs
- */
-
-/**
- * @typedef {Object} DeletedFileRef
- * @property {ObjectId} _id
- * @property {ObjectId} projectId
- * @property {string} hash
- */
-
-/**
- * @typedef {Object} Project
- * @property {ObjectId} _id
- * @property {Array} rootFolder
- * @property {{history: {id: (number|string)}}} overleaf
- */
-
-/**
- * @typedef {Object} QueueEntry
- * @property {ProjectContext} ctx
- * @property {string} cacheKey
- * @property {string} [fileId]
- * @property {string} path
- * @property {string} [hash]
- * @property {Blob} [blob]
- */
-
-/**
- * @return {{PROJECT_IDS_FROM: string, PROCESS_HASHED_FILES: boolean, LOGGING_IDENTIFIER: string, BATCH_RANGE_START: string, PROCESS_BLOBS: boolean, BATCH_RANGE_END: string, PROCESS_NON_DELETED_PROJECTS: boolean, PROCESS_DELETED_PROJECTS: boolean}}
- */
-function parseArgs() {
- const PUBLIC_LAUNCH_DATE = new Date('2012-01-01T00:00:00Z')
- const args = commandLineArgs([
- { name: 'processNonDeletedProjects', type: String, defaultValue: 'false' },
- { name: 'processDeletedProjects', type: String, defaultValue: 'false' },
- { name: 'processHashedFiles', type: String, defaultValue: 'false' },
- { name: 'processBlobs', type: String, defaultValue: 'true' },
- { name: 'projectIdsFrom', type: String, defaultValue: '' },
- {
- name: 'BATCH_RANGE_START',
- type: String,
- defaultValue: PUBLIC_LAUNCH_DATE.toISOString(),
- },
- {
- name: 'BATCH_RANGE_END',
- type: String,
- defaultValue: new Date().toISOString(),
- },
- { name: 'LOGGING_IDENTIFIER', type: String, defaultValue: '' },
- ])
- /**
- * commandLineArgs cannot handle --foo=false, so go the long way
- * @param {string} name
- * @return {boolean}
- */
- function boolVal(name) {
- const v = args[name]
- if (['true', 'false'].includes(v)) return v === 'true'
- throw new Error(`expected "true" or "false" for boolean option ${name}`)
- }
- const BATCH_RANGE_START = objectIdFromInput(
- args['BATCH_RANGE_START']
- ).toString()
- const BATCH_RANGE_END = objectIdFromInput(args['BATCH_RANGE_END']).toString()
- return {
- PROCESS_NON_DELETED_PROJECTS: boolVal('processNonDeletedProjects'),
- PROCESS_DELETED_PROJECTS: boolVal('processDeletedProjects'),
- PROCESS_BLOBS: boolVal('processBlobs'),
- PROCESS_HASHED_FILES: boolVal('processHashedFiles'),
- BATCH_RANGE_START,
- BATCH_RANGE_END,
- LOGGING_IDENTIFIER: args['LOGGING_IDENTIFIER'] || BATCH_RANGE_START,
- PROJECT_IDS_FROM: args['projectIdsFrom'],
- }
-}
-
-const {
- PROCESS_NON_DELETED_PROJECTS,
- PROCESS_DELETED_PROJECTS,
- PROCESS_BLOBS,
- PROCESS_HASHED_FILES,
- BATCH_RANGE_START,
- BATCH_RANGE_END,
- LOGGING_IDENTIFIER,
- PROJECT_IDS_FROM,
-} = parseArgs()
-
-// We need to handle the start and end differently as ids of deleted projects are created at time of deletion.
-if (process.env.BATCH_RANGE_START || process.env.BATCH_RANGE_END) {
- throw new Error('use --BATCH_RANGE_START and --BATCH_RANGE_END')
-}
-
-// Concurrency for downloading from GCS and updating hashes in mongo
-const CONCURRENCY = parseInt(process.env.CONCURRENCY || '100', 10)
-const CONCURRENT_BATCHES = parseInt(process.env.CONCURRENT_BATCHES || '2', 10)
-// Retries for processing a given file
-const RETRIES = parseInt(process.env.RETRIES || '10', 10)
-const RETRY_DELAY_MS = parseInt(process.env.RETRY_DELAY_MS || '100', 10)
-
-const RETRY_FILESTORE_404 = process.env.RETRY_FILESTORE_404 === 'true'
-const BUFFER_DIR = fs.mkdtempSync(
- process.env.BUFFER_DIR_PREFIX || '/tmp/back_fill_file_hash-'
-)
-// https://nodejs.org/api/stream.html#streamgetdefaulthighwatermarkobjectmode
-const STREAM_HIGH_WATER_MARK = parseInt(
- process.env.STREAM_HIGH_WATER_MARK || (64 * 1024).toString(),
- 10
-)
-const LOGGING_INTERVAL = parseInt(process.env.LOGGING_INTERVAL || '60000', 10)
-const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10)
-
-// Filestore endpoint location
-const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1'
-const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009'
-
-async function fetchFromFilestore(projectId, fileId) {
- const url = `http://${FILESTORE_HOST}:${FILESTORE_PORT}/project/${projectId}/file/${fileId}`
- const response = await fetch(url)
- if (!response.ok) {
- if (response.status === 404) {
- throw new NotFoundError('file not found in filestore', {
- status: response.status,
- })
- }
- const body = await response.text()
- throw new OError('fetchFromFilestore failed', {
- projectId,
- fileId,
- status: response.status,
- body,
- })
- }
- if (!response.body) {
- throw new OError('fetchFromFilestore response has no body', {
- projectId,
- fileId,
- status: response.status,
- })
- }
- return response.body
-}
-
-const projectsCollection = db.collection('projects')
-/** @type {ProjectsCollection} */
-const typedProjectsCollection = db.collection('projects')
-const deletedProjectsCollection = db.collection('deletedProjects')
-/** @type {DeletedProjectsCollection} */
-const typedDeletedProjectsCollection = db.collection('deletedProjects')
-
-const concurrencyLimit = pLimit(CONCURRENCY)
-
-/**
- * @template T
- * @template V
- * @param {Array} array
- * @param {(arg: T) => Promise} fn
- * @return {Promise>>}
- */
-async function processConcurrently(array, fn) {
- return await Promise.all(array.map(x => concurrencyLimit(() => fn(x))))
-}
-
-const STATS = {
- projects: 0,
- blobs: 0,
- filesWithHash: 0,
- filesWithoutHash: 0,
- filesDuplicated: 0,
- filesRetries: 0,
- filesFailed: 0,
- fileTreeUpdated: 0,
- badFileTrees: 0,
- globalBlobsCount: 0,
- globalBlobsEgress: 0,
- projectDeleted: 0,
- projectHardDeleted: 0,
- fileHardDeleted: 0,
- mongoUpdates: 0,
- readFromGCSCount: 0,
- readFromGCSIngress: 0,
- writeToGCSCount: 0,
- writeToGCSEgress: 0,
-}
-
-const processStart = performance.now()
-let lastLogTS = processStart
-let lastLog = Object.assign({}, STATS)
-let lastEventLoopStats = performance.eventLoopUtilization()
-
-/**
- * @param {number} v
- * @param {number} ms
- */
-function toMiBPerSecond(v, ms) {
- const ONE_MiB = 1024 * 1024
- return v / ONE_MiB / (ms / 1000)
-}
-
-/**
- * @param {any} stats
- * @param {number} ms
- * @return {{readFromGCSThroughputMiBPerSecond: number}}
- */
-function bandwidthStats(stats, ms) {
- return {
- readFromGCSThroughputMiBPerSecond: toMiBPerSecond(
- stats.readFromGCSIngress,
- ms
- ),
- }
-}
-
-/**
- * @param {EventLoopUtilization} nextEventLoopStats
- * @param {number} now
- * @return {Object}
- */
-function computeDiff(nextEventLoopStats, now) {
- const ms = now - lastLogTS
- lastLogTS = now
- const diff = {
- eventLoop: performance.eventLoopUtilization(
- nextEventLoopStats,
- lastEventLoopStats
- ),
- }
- for (const [name, v] of Object.entries(STATS)) {
- diff[name] = v - lastLog[name]
- }
- return Object.assign(diff, bandwidthStats(diff, ms))
-}
-
-/**
- * @param {boolean} isLast
- */
-function printStats(isLast = false) {
- const now = performance.now()
- const nextEventLoopStats = performance.eventLoopUtilization()
- const logLine = JSON.stringify({
- time: new Date(),
- LOGGING_IDENTIFIER,
- ...STATS,
- ...bandwidthStats(STATS, now - processStart),
- eventLoop: nextEventLoopStats,
- diff: computeDiff(nextEventLoopStats, now),
- deferredBatches: Array.from(deferredBatches.keys()),
- })
- if (isLast) {
- console.warn(logLine)
- } else {
- console.log(logLine)
- }
- lastEventLoopStats = nextEventLoopStats
- lastLog = Object.assign({}, STATS)
-}
-
-setInterval(printStats, LOGGING_INTERVAL)
-
-let gracefulShutdownInitiated = false
-
-process.on('SIGINT', handleSignal)
-process.on('SIGTERM', handleSignal)
-
-function handleSignal() {
- gracefulShutdownInitiated = true
- console.warn('graceful shutdown initiated, draining queue')
-}
-
-/**
- * @param {QueueEntry} entry
- * @return {Promise}
- */
-async function processFileWithCleanup(entry) {
- const {
- ctx: { projectId },
- cacheKey,
- } = entry
- const filePath = Path.join(BUFFER_DIR, projectId.toString() + cacheKey)
- try {
- return await processFile(entry, filePath)
- } finally {
- await Promise.all([
- fs.promises.rm(filePath, { force: true }),
- fs.promises.rm(filePath + GZ_SUFFIX, { force: true }),
- ])
- }
-}
-
-/**
- * @param {QueueEntry} entry
- * @param {string} filePath
- * @return {Promise}
- */
-async function processFile(entry, filePath) {
- for (let attempt = 0; attempt < RETRIES; attempt++) {
- try {
- return await processFileOnce(entry, filePath)
- } catch (err) {
- if (gracefulShutdownInitiated) throw err
- if (err instanceof NotFoundError) {
- if (!RETRY_FILESTORE_404) {
- throw err // disable retries for not found in filestore bucket case
- }
- }
- STATS.filesRetries++
- const {
- ctx: { projectId },
- fileId,
- hash,
- path,
- } = entry
- logger.warn(
- { err, projectId, fileId, hash, path, attempt },
- 'failed to process file, trying again'
- )
- const jitter = Math.random() * RETRY_DELAY_MS
- await setTimeout(RETRY_DELAY_MS + jitter)
- }
- }
- return await processFileOnce(entry, filePath)
-}
-
-/**
- * @param {QueueEntry} entry
- * @param {string} filePath
- * @return {Promise}
- */
-async function processFileOnce(entry, filePath) {
- const {
- ctx: { projectId, historyId },
- fileId,
- } = entry
- if (entry.hash && entry.ctx.hasCompletedBlob(entry.hash)) {
- // We can enter this case for two identical files in the same project,
- // one with hash, the other without. When the one without hash gets
- // processed first, we can skip downloading the other one we already
- // know the hash of.
- return entry.hash
- }
- const blobStore = new BlobStore(historyId)
- STATS.readFromGCSCount++
- // make a fetch request to filestore itself
- const src = await fetchFromFilestore(projectId, fileId)
- const dst = fs.createWriteStream(filePath, {
- highWaterMark: STREAM_HIGH_WATER_MARK,
- })
- try {
- await Stream.promises.pipeline(src, dst)
- } finally {
- STATS.readFromGCSIngress += dst.bytesWritten
- }
- const blob = await makeBlobForFile(filePath)
- blob.setStringLength(
- await getStringLengthOfFile(blob.getByteLength(), filePath)
- )
- const hash = blob.getHash()
- if (entry.hash && hash !== entry.hash) {
- throw new OError('hash mismatch', { entry, hash })
- }
-
- if (GLOBAL_BLOBS.has(hash)) {
- STATS.globalBlobsCount++
- STATS.globalBlobsEgress += estimateBlobSize(blob)
- return hash
- }
- if (entry.ctx.hasCompletedBlob(hash)) {
- return hash
- }
- entry.ctx.recordPendingBlob(hash)
-
- try {
- await uploadBlobToGCS(blobStore, entry, blob, hash, filePath)
- entry.ctx.recordCompletedBlob(hash) // mark upload as completed
- } catch (err) {
- entry.ctx.recordFailedBlob(hash)
- throw err
- }
- return hash
-}
-
-/**
- * @param {BlobStore} blobStore
- * @param {QueueEntry} entry
- * @param {Blob} blob
- * @param {string} hash
- * @param {string} filePath
- * @return {Promise}
- */
-async function uploadBlobToGCS(blobStore, entry, blob, hash, filePath) {
- if (entry.ctx.getCachedHistoryBlob(hash)) {
- return // fast-path using hint from pre-fetched blobs
- }
- if (!PROCESS_BLOBS) {
- // round trip to postgres/mongo when not pre-fetched
- const blob = await blobStore.getBlob(hash)
- if (blob) {
- entry.ctx.recordHistoryBlob(blob)
- return
- }
- }
- // blob missing in history-v1, create in GCS and persist in postgres/mongo
- STATS.writeToGCSCount++
- STATS.writeToGCSEgress += blob.getByteLength()
- await blobStore.putBlob(filePath, blob)
- entry.ctx.recordHistoryBlob(blob)
-}
-
-const GZ_SUFFIX = '.gz'
-
-/**
- * @param {Array} files
- * @return {Promise}
- */
-async function processFiles(files) {
- await processConcurrently(
- files,
- /**
- * @param {QueueEntry} entry
- * @return {Promise}
- */
- async function (entry) {
- if (gracefulShutdownInitiated) return
- try {
- await entry.ctx.processFile(entry)
- } catch (err) {
- STATS.filesFailed++
- const {
- ctx: { projectId },
- fileId,
- hash,
- path,
- } = entry
- logger.error(
- { err, projectId, fileId, hash, path },
- 'failed to process file'
- )
- }
- }
- )
-}
-
-/** @type {Map} */
-const deferredBatches = new Map()
-
-async function waitForDeferredQueues() {
- // Wait for ALL pending batches to finish, especially wait for their mongo
- // writes to finish to avoid extra work when resuming the batch.
- const all = await Promise.allSettled(deferredBatches.values())
- // Now that all batches finished, we can throw if needed.
- for (const res of all) {
- if (res.status === 'rejected') {
- throw res.reason
- }
- }
-}
-
-/**
- * @param {Array} batch
- * @param {string} prefix
- */
-async function queueNextBatch(batch, prefix = 'rootFolder.0') {
- if (gracefulShutdownInitiated) {
- throw new Error('graceful shutdown: aborting batch processing')
- }
-
- // Read ids now, the batch will get trimmed by processBatch shortly.
- const start = renderObjectId(batch[0]._id)
- const end = renderObjectId(batch[batch.length - 1]._id)
- const deferred = processBatch(batch, prefix)
- .then(() => {
- console.error(`Actually completed batch ending ${end}`)
- })
- .catch(err => {
- logger.error({ err, start, end }, 'fatal error processing batch')
- throw err
- })
- .finally(() => {
- deferredBatches.delete(end)
- })
- deferredBatches.set(end, deferred)
-
- if (deferredBatches.size >= CONCURRENT_BATCHES) {
- // Wait for any of the deferred batches to finish before fetching the next.
- // We should never have more than CONCURRENT_BATCHES batches in memory.
- await Promise.race(deferredBatches.values())
- }
-}
-
-/**
- * @param {Array} batch
- * @param {string} prefix
- * @return {Promise}
- */
-async function processBatch(batch, prefix = 'rootFolder.0') {
- const { nBlobs, blobs } = await collectProjectBlobs(batch)
- const files = Array.from(findFileInBatch(batch, prefix, blobs))
- STATS.projects += batch.length
- STATS.blobs += nBlobs
-
- // GC
- batch.length = 0
- blobs.clear()
-
- // The files are currently ordered by project-id.
- // Order them by file-id ASC then hash ASC to
- // increase the hit rate on the "already processed
- // hash for project" checks.
- files.sort(
- /**
- * @param {QueueEntry} a
- * @param {QueueEntry} b
- * @return {number}
- */
- function (a, b) {
- if (a.fileId && b.fileId) return a.fileId > b.fileId ? 1 : -1
- if (a.hash && b.hash) return a.hash > b.hash ? 1 : -1
- if (a.fileId) return -1
- return 1
- }
- )
- await processFiles(files)
- await processConcurrently(
- files,
- /**
- * @param {QueueEntry} entry
- * @return {Promise}
- */
- async function (entry) {
- await entry.ctx.flushMongoQueues()
- }
- )
-}
-
-/**
- * @param {Array<{project: Project}>} batch
- * @return {Promise}
- */
-async function handleDeletedFileTreeBatch(batch) {
- await queueNextBatch(
- batch.map(d => d.project),
- 'project.rootFolder.0'
- )
-}
-
-/**
- * @param {QueueEntry} entry
- * @return {Promise}
- */
-async function tryUpdateFileRefInMongo(entry) {
- if (entry.path.startsWith('project.')) {
- return await tryUpdateFileRefInMongoInDeletedProject(entry)
- }
-
- STATS.mongoUpdates++
- const result = await projectsCollection.updateOne(
- {
- _id: entry.ctx.projectId,
- [`${entry.path}._id`]: new ObjectId(entry.fileId),
- },
- {
- $set: { [`${entry.path}.hash`]: entry.hash },
- }
- )
- return result.matchedCount === 1
-}
-
-/**
- * @param {QueueEntry} entry
- * @return {Promise}
- */
-async function tryUpdateFileRefInMongoInDeletedProject(entry) {
- STATS.mongoUpdates++
- const result = await deletedProjectsCollection.updateOne(
- {
- 'deleterData.deletedProjectId': entry.ctx.projectId,
- [`${entry.path}._id`]: new ObjectId(entry.fileId),
- },
- {
- $set: { [`${entry.path}.hash`]: entry.hash },
- }
- )
- return result.matchedCount === 1
-}
-
-const RETRY_UPDATE_HASH = 100
-
-/**
- * @param {QueueEntry} entry
- * @return {Promise}
- */
-async function updateFileRefInMongo(entry) {
- if (await tryUpdateFileRefInMongo(entry)) return
-
- const { fileId } = entry
- const { projectId } = entry.ctx
- for (let i = 0; i < RETRY_UPDATE_HASH; i++) {
- let prefix = 'rootFolder.0'
- let p = await projectsCollection.findOne(
- { _id: projectId },
- { projection: { rootFolder: 1 } }
- )
- if (!p) {
- STATS.projectDeleted++
- prefix = 'project.rootFolder.0'
- const deletedProject = await deletedProjectsCollection.findOne(
- {
- 'deleterData.deletedProjectId': projectId,
- project: { $exists: true },
- },
- { projection: { 'project.rootFolder': 1 } }
- )
- p = deletedProject?.project
- if (!p) {
- STATS.projectHardDeleted++
- console.warn(
- 'bug: project hard-deleted while processing',
- projectId,
- fileId
- )
- return
- }
- }
- let found = false
- for (const e of findFiles(entry.ctx, p.rootFolder[0], prefix)) {
- found = e.fileId === fileId
- if (!found) continue
- if (await tryUpdateFileRefInMongo(e)) return
- break
- }
- if (!found) {
- STATS.fileHardDeleted++
- console.warn('bug: file hard-deleted while processing', projectId, fileId)
- return
- }
-
- STATS.fileTreeUpdated++
- }
- throw new OError(
- 'file-tree updated repeatedly while trying to add hash',
- entry
- )
-}
-
-/**
- * @param {ProjectContext} ctx
- * @param {Folder} folder
- * @param {string} path
- * @param {boolean} isInputLoop
- * @return Generator
- */
-function* findFiles(ctx, folder, path, isInputLoop = false) {
- if (!folder || typeof folder !== 'object') {
- ctx.fileTreeBroken = true
- logger.warn({ projectId: ctx.projectId, path }, 'bad file-tree, bad folder')
- return
- }
- if (!Array.isArray(folder.folders)) {
- folder.folders = []
- ctx.fileTreeBroken = true
- logger.warn(
- { projectId: ctx.projectId, path: `${path}.folders` },
- 'bad file-tree, bad folders'
- )
- }
- let i = 0
- for (const child of folder.folders) {
- const idx = i++
- yield* findFiles(ctx, child, `${path}.folders.${idx}`, isInputLoop)
- }
- if (!Array.isArray(folder.fileRefs)) {
- folder.fileRefs = []
- ctx.fileTreeBroken = true
- logger.warn(
- { projectId: ctx.projectId, path: `${path}.fileRefs` },
- 'bad file-tree, bad fileRefs'
- )
- }
- i = 0
- for (const fileRef of folder.fileRefs) {
- const idx = i++
- const fileRefPath = `${path}.fileRefs.${idx}`
- if (!fileRef._id || !(fileRef._id instanceof ObjectId)) {
- ctx.fileTreeBroken = true
- logger.warn(
- { projectId: ctx.projectId, path: fileRefPath },
- 'bad file-tree, bad fileRef id'
- )
- continue
- }
- const fileId = fileRef._id.toString()
- if (PROCESS_HASHED_FILES && fileRef.hash) {
- if (ctx.canSkipProcessingHashedFile(fileRef.hash)) continue
- if (isInputLoop) {
- ctx.remainingQueueEntries++
- STATS.filesWithHash++
- }
- yield {
- ctx,
- cacheKey: fileRef.hash,
- fileId,
- path: MONGO_PATH_SKIP_WRITE_HASH_TO_FILE_TREE,
- hash: fileRef.hash,
- }
- }
- if (!fileRef.hash) {
- if (isInputLoop) {
- ctx.remainingQueueEntries++
- STATS.filesWithoutHash++
- }
- yield {
- ctx,
- cacheKey: fileId,
- fileId,
- path: fileRefPath,
- }
- }
- }
-}
-
-/**
- * @param {Array} projects
- * @param {string} prefix
- * @param {Map>} blobs
- * @return Generator
- */
-function* findFileInBatch(projects, prefix, blobs) {
- for (const project of projects) {
- const projectIdS = project._id.toString()
- const historyIdS = project.overleaf.history.id.toString()
- const projectBlobs = blobs.get(historyIdS) || []
- const ctx = new ProjectContext(project._id, historyIdS, projectBlobs)
- try {
- yield* findFiles(ctx, project.rootFolder?.[0], prefix, true)
- } catch (err) {
- logger.error(
- { err, projectId: projectIdS },
- 'bad file-tree, processing error'
- )
- } finally {
- if (ctx.fileTreeBroken) STATS.badFileTrees++
- }
- }
-}
-
-/**
- * @param {Array} batch
- * @return {Promise<{nBlobs: number, blobs: Map>}>}
- */
-async function collectProjectBlobs(batch) {
- if (!PROCESS_BLOBS) return { nBlobs: 0, blobs: new Map() }
- return await getProjectBlobsBatch(batch.map(p => p.overleaf.history.id))
-}
-
-const BATCH_FILE_UPDATES = 100
-
-const MONGO_PATH_SKIP_WRITE_HASH_TO_FILE_TREE = 'skip-write-to-file-tree'
-
-class ProjectContext {
- /** @type {Map} */
- #historyBlobs
-
- /** @type {number} */
- remainingQueueEntries = 0
-
- /** @type {boolean} */
- fileTreeBroken = false
-
- /**
- * @param {ObjectId} projectId
- * @param {string} historyId
- * @param {Array} blobs
- */
- constructor(projectId, historyId, blobs) {
- this.projectId = projectId
- this.historyId = historyId
- this.#historyBlobs = new Map(blobs.map(b => [b.getHash(), b]))
- }
-
- /**
- * @param {string} hash
- * @return {Blob | undefined}
- */
- getCachedHistoryBlob(hash) {
- return this.#historyBlobs.get(hash)
- }
-
- /**
- * @param {Blob} blob
- */
- recordHistoryBlob(blob) {
- this.#historyBlobs.set(blob.getHash(), blob)
- }
-
- /**
- * @param {string} hash
- * @return {boolean}
- */
- canSkipProcessingHashedFile(hash) {
- if (this.#historyBlobs.has(hash)) return true // This file will be processed as blob.
- if (GLOBAL_BLOBS.has(hash)) return true // global blob
- return false
- }
-
- async flushMongoQueuesIfNeeded() {
- if (this.remainingQueueEntries === 0) {
- await this.flushMongoQueues()
- }
-
- if (this.#pendingFileWrites.length > BATCH_FILE_UPDATES) {
- await this.#storeFileHashes()
- }
- }
-
- async flushMongoQueues() {
- await this.#storeFileHashes()
- }
-
- /** @type {Set} */
- #pendingBlobs = new Set()
- /** @type {Set} */
- #completedBlobs = new Set()
-
- /**
- * @param {string} hash
- */
- recordPendingBlob(hash) {
- this.#pendingBlobs.add(hash)
- }
-
- /**
- * @param {string} hash
- */
- recordFailedBlob(hash) {
- this.#pendingBlobs.delete(hash)
- }
-
- /**
- * @param {string} hash
- */
- recordCompletedBlob(hash) {
- this.#completedBlobs.add(hash)
- this.#pendingBlobs.delete(hash)
- }
-
- /**
- * @param {string} hash
- * @return {boolean}
- */
- hasCompletedBlob(hash) {
- return this.#pendingBlobs.has(hash) || this.#completedBlobs.has(hash)
- }
-
- /** @type {Array} */
- #pendingFileWrites = []
-
- /**
- * @param {QueueEntry} entry
- */
- queueFileForWritingHash(entry) {
- if (entry.path === MONGO_PATH_SKIP_WRITE_HASH_TO_FILE_TREE) return
- this.#pendingFileWrites.push(entry)
- }
-
- /**
- * @param {Collection} collection
- * @param {Array} entries
- * @param {Object} query
- * @return {Promise