diff --git a/package-lock.json b/package-lock.json index 2b3a5868a2..d9d8285618 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35581,6 +35581,7 @@ "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142", + "license": "Apache-2.0", "dependencies": { "aws-sign2": "~0.7.0", "aws4": "^1.8.0", @@ -35638,15 +35639,15 @@ } }, "node_modules/request/node_modules/tough-cookie": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", - "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", + "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", + "license": "BSD-3-Clause", "dependencies": { - "psl": "^1.1.28", - "punycode": "^2.1.1" + "tldts": "^6.1.32" }, "engines": { - "node": ">=0.8" + "node": ">=16" } }, "node_modules/requestretry": { @@ -39612,6 +39613,24 @@ "tlds": "bin.js" } }, + "node_modules/tldts": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", + "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", + "license": "MIT", + "dependencies": { + "tldts-core": "^6.1.86" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", + "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", + "license": "MIT" + }, "node_modules/tmp": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", diff --git a/package.json b/package.json index 388b750c3d..44fffc4664 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,9 @@ "path-to-regexp": "3.3.0", "body-parser": "1.20.3", "multer": "2.0.1" + }, + "request@2.88.2": { + "tough-cookie": "5.1.2" } }, "scripts": { diff --git a/server-ce/test/Makefile b/server-ce/test/Makefile index 6c56b7e8fe..fb7c980293 100644 --- a/server-ce/test/Makefile +++ b/server-ce/test/Makefile @@ -21,9 +21,11 @@ test-e2e-native: test-e2e: docker compose build host-admin + docker compose up -d host-admin docker compose up --no-log-prefix --exit-code-from=e2e e2e test-e2e-open: + docker compose up -d host-admin docker compose up --no-log-prefix --exit-code-from=e2e-open e2e-open clean: diff --git a/server-ce/test/docker-compose.yml b/server-ce/test/docker-compose.yml index 029b73fc62..1652baeae9 100644 --- a/server-ce/test/docker-compose.yml +++ b/server-ce/test/docker-compose.yml @@ -35,7 +35,7 @@ services: MAILTRAP_PASSWORD: 'password-for-mailtrap' mongo: - image: mongo:6.0 + image: mongo:8.0.11 command: '--replSet overleaf' volumes: - ../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/server-ce/test/editor.spec.ts b/server-ce/test/editor.spec.ts index d0060518de..3e57b94f8f 100644 --- a/server-ce/test/editor.spec.ts +++ b/server-ce/test/editor.spec.ts @@ -2,6 +2,7 @@ import { createNewFile, createProject, openProjectById, + testNewFileUpload, } from './helpers/project' import { isExcludedBySharding, startWith } from './helpers/config' import { ensureUserExists, login } from './helpers/login' @@ -119,24 +120,7 @@ describe('editor', () => { cy.get('button').contains('New file').click({ force: true }) }) - it('can upload file', () => { - const name = `${uuid()}.txt` - const content = `Test File Content ${name}` - cy.get('button').contains('Upload').click({ force: true }) - cy.get('input[type=file]') - .first() - .selectFile( - { - contents: Cypress.Buffer.from(content), - fileName: name, - lastModified: Date.now(), - }, - { force: true } - ) - // force: The file-tree pane is too narrow to display the full name. - cy.findByTestId('file-tree').findByText(name).click({ force: true }) - cy.findByText(content) - }) + testNewFileUpload() it('should not display import from URL', () => { cy.findByText('From external URL').should('not.exist') diff --git a/server-ce/test/filestore-migration.spec.ts b/server-ce/test/filestore-migration.spec.ts new file mode 100644 index 0000000000..25875ad374 --- /dev/null +++ b/server-ce/test/filestore-migration.spec.ts @@ -0,0 +1,104 @@ +import { ensureUserExists, login } from './helpers/login' +import { + createProject, + openProjectById, + prepareFileUploadTest, +} from './helpers/project' +import { isExcludedBySharding, startWith } from './helpers/config' +import { prepareWaitForNextCompileSlot } from './helpers/compile' +import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry' +import { v4 as uuid } from 'uuid' +import { purgeFilestoreData, runScript } from './helpers/hostAdminClient' + +describe('filestore migration', function () { + if (isExcludedBySharding('CE_CUSTOM_3')) return + startWith({ withDataDir: true, resetData: true, vars: {} }) + ensureUserExists({ email: 'user@example.com' }) + + let projectName: string + let projectId: string + let waitForCompileRateLimitCoolOff: (fn: () => void) => void + const previousBinaryFiles: (() => void)[] = [] + beforeWithReRunOnTestRetry(function () { + projectName = `project-${uuid()}` + login('user@example.com') + createProject(projectName, { type: 'Example project' }).then( + id => (projectId = id) + ) + let queueReset + ;({ waitForCompileRateLimitCoolOff, queueReset } = + prepareWaitForNextCompileSlot()) + queueReset() + previousBinaryFiles.push(prepareFileUploadTest(true)) + }) + + beforeEach(() => { + login('user@example.com') + waitForCompileRateLimitCoolOff(() => { + openProjectById(projectId) + }) + }) + + function checkFilesAreAccessible() { + it('can upload new binary file and read previous uploads', function () { + previousBinaryFiles.push(prepareFileUploadTest(true)) + for (const check of previousBinaryFiles) { + check() + } + }) + + it('renders frog jpg', () => { + cy.findByTestId('file-tree').findByText('frog.jpg').click() + cy.get('[alt="frog.jpg"]') + .should('be.visible') + .and('have.prop', 'naturalWidth') + .should('be.greaterThan', 0) + }) + } + + describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL not set', function () { + startWith({ withDataDir: true, vars: {} }) + checkFilesAreAccessible() + }) + + describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=0', function () { + startWith({ + withDataDir: true, + vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '0' }, + }) + checkFilesAreAccessible() + + describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=1', function () { + startWith({ + withDataDir: true, + vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '1' }, + }) + checkFilesAreAccessible() + + describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=2', function () { + startWith({ + withDataDir: true, + vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '1' }, + }) + before(async function () { + await runScript({ + cwd: 'services/history-v1', + script: 'storage/scripts/back_fill_file_hash.mjs', + }) + }) + startWith({ + withDataDir: true, + vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '2' }, + }) + checkFilesAreAccessible() + + describe('purge filestore data', function () { + before(async function () { + await purgeFilestoreData() + }) + checkFilesAreAccessible() + }) + }) + }) + }) +}) diff --git a/server-ce/test/helpers/config.ts b/server-ce/test/helpers/config.ts index 030e70ceb5..78e81be1f7 100644 --- a/server-ce/test/helpers/config.ts +++ b/server-ce/test/helpers/config.ts @@ -9,6 +9,7 @@ export function isExcludedBySharding( | 'CE_DEFAULT' | 'CE_CUSTOM_1' | 'CE_CUSTOM_2' + | 'CE_CUSTOM_3' | 'PRO_DEFAULT_1' | 'PRO_DEFAULT_2' | 'PRO_CUSTOM_1' diff --git a/server-ce/test/helpers/hostAdminClient.ts b/server-ce/test/helpers/hostAdminClient.ts index cafeaa2db6..dadfe2b059 100644 --- a/server-ce/test/helpers/hostAdminClient.ts +++ b/server-ce/test/helpers/hostAdminClient.ts @@ -85,6 +85,12 @@ export async function getRedisKeys() { return stdout.split('\n') } +export async function purgeFilestoreData() { + await fetchJSON(`${hostAdminURL}/data/user_files`, { + method: 'DELETE', + }) +} + async function sleep(ms: number) { return new Promise(resolve => { setTimeout(resolve, ms) diff --git a/server-ce/test/helpers/project.ts b/server-ce/test/helpers/project.ts index abcce3f9b2..4b3197afed 100644 --- a/server-ce/test/helpers/project.ts +++ b/server-ce/test/helpers/project.ts @@ -216,3 +216,43 @@ export function createNewFile() { return fileName } + +export function prepareFileUploadTest(binary = false) { + const name = `${uuid()}.txt` + const content = `Test File Content ${name}${binary ? ' \x00' : ''}` + cy.get('button').contains('Upload').click({ force: true }) + cy.get('input[type=file]') + .first() + .selectFile( + { + contents: Cypress.Buffer.from(content), + fileName: name, + lastModified: Date.now(), + }, + { force: true } + ) + + // wait for the upload to finish + cy.findByRole('treeitem', { name }) + + return function check() { + cy.findByRole('treeitem', { name }).click() + if (binary) { + cy.findByText(content).should('not.have.class', 'cm-line') + } else { + cy.findByText(content).should('have.class', 'cm-line') + } + } +} + +export function testNewFileUpload() { + it('can upload text file', () => { + const check = prepareFileUploadTest(false) + check() + }) + + it('can upload binary file', () => { + const check = prepareFileUploadTest(true) + check() + }) +} diff --git a/server-ce/test/host-admin.js b/server-ce/test/host-admin.js index f73209d58f..b3dcd72b1f 100644 --- a/server-ce/test/host-admin.js +++ b/server-ce/test/host-admin.js @@ -29,6 +29,17 @@ const IMAGES = { PRO: process.env.IMAGE_TAG_PRO.replace(/:.+/, ''), } +function defaultDockerComposeOverride() { + return { + services: { + sharelatex: { + environment: {}, + }, + 'git-bridge': {}, + }, + } +} + let previousConfig = '' function readDockerComposeOverride() { @@ -38,14 +49,7 @@ function readDockerComposeOverride() { if (error.code !== 'ENOENT') { throw error } - return { - services: { - sharelatex: { - environment: {}, - }, - 'git-bridge': {}, - }, - } + return defaultDockerComposeOverride } } @@ -77,12 +81,21 @@ app.use(bodyParser.json()) app.use((req, res, next) => { // Basic access logs console.log(req.method, req.url, req.body) + const json = res.json + res.json = body => { + console.log(req.method, req.url, req.body, '->', body) + json.call(res, body) + } + next() +}) +app.use((req, res, next) => { // Add CORS headers const accessControlAllowOrigin = process.env.ACCESS_CONTROL_ALLOW_ORIGIN || 'http://sharelatex' res.setHeader('Access-Control-Allow-Origin', accessControlAllowOrigin) res.setHeader('Access-Control-Allow-Headers', 'Content-Type') res.setHeader('Access-Control-Max-Age', '3600') + res.setHeader('Access-Control-Allow-Methods', 'DELETE, GET, HEAD, POST, PUT') next() }) @@ -133,6 +146,7 @@ const allowedVars = Joi.object( 'V1_HISTORY_URL', 'SANDBOXED_COMPILES', 'ALL_TEX_LIVE_DOCKER_IMAGE_NAMES', + 'OVERLEAF_FILESTORE_MIGRATION_LEVEL', 'OVERLEAF_TEMPLATES_USER_ID', 'OVERLEAF_NEW_PROJECT_TEMPLATE_LINKS', 'OVERLEAF_ALLOW_PUBLIC_ACCESS', @@ -319,8 +333,19 @@ app.get('/redis/keys', (req, res) => { ) }) +app.delete('/data/user_files', (req, res) => { + runDockerCompose( + 'exec', + ['sharelatex', 'rm', '-rf', '/var/lib/overleaf/data/user_files'], + (error, stdout, stderr) => { + res.json({ error, stdout, stderr }) + } + ) +}) + app.use(handleValidationErrors()) purgeDataDir() +writeDockerComposeOverride(defaultDockerComposeOverride()) app.listen(80) diff --git a/services/chat/docker-compose.ci.yml b/services/chat/docker-compose.ci.yml index 24b57ab084..ca3303a079 100644 --- a/services/chat/docker-compose.ci.yml +++ b/services/chat/docker-compose.ci.yml @@ -42,7 +42,7 @@ services: command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/chat/docker-compose.yml b/services/chat/docker-compose.yml index ddc5f9e698..e7b8ce7385 100644 --- a/services/chat/docker-compose.yml +++ b/services/chat/docker-compose.yml @@ -44,7 +44,7 @@ services: command: npm run --silent test:acceptance mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/contacts/docker-compose.ci.yml b/services/contacts/docker-compose.ci.yml index 24b57ab084..ca3303a079 100644 --- a/services/contacts/docker-compose.ci.yml +++ b/services/contacts/docker-compose.ci.yml @@ -42,7 +42,7 @@ services: command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/contacts/docker-compose.yml b/services/contacts/docker-compose.yml index 6c77ef5e31..474ea224f8 100644 --- a/services/contacts/docker-compose.yml +++ b/services/contacts/docker-compose.yml @@ -44,7 +44,7 @@ services: command: npm run --silent test:acceptance mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/docstore/docker-compose.ci.yml b/services/docstore/docker-compose.ci.yml index 40decc4aea..cdb4783c5a 100644 --- a/services/docstore/docker-compose.ci.yml +++ b/services/docstore/docker-compose.ci.yml @@ -47,7 +47,7 @@ services: command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/docstore/docker-compose.yml b/services/docstore/docker-compose.yml index 8c11eb5a91..a9099c7e7b 100644 --- a/services/docstore/docker-compose.yml +++ b/services/docstore/docker-compose.yml @@ -49,7 +49,7 @@ services: command: npm run --silent test:acceptance mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index ca15f35fef..c6ec24a84b 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -55,7 +55,7 @@ services: retries: 20 mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index cf7c9a2eb6..c1b23c11c5 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -57,7 +57,7 @@ services: retries: 20 mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/history-v1/docker-compose.ci.yml b/services/history-v1/docker-compose.ci.yml index da664d6b30..cf6ec3357d 100644 --- a/services/history-v1/docker-compose.ci.yml +++ b/services/history-v1/docker-compose.ci.yml @@ -75,7 +75,7 @@ services: retries: 20 mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/history-v1/docker-compose.yml b/services/history-v1/docker-compose.yml index 22b739abf9..3a33882d28 100644 --- a/services/history-v1/docker-compose.yml +++ b/services/history-v1/docker-compose.yml @@ -83,7 +83,7 @@ services: retries: 20 mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs index 0ccadaf5a9..2e12328e5c 100644 --- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs +++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs @@ -150,10 +150,6 @@ const CONCURRENT_BATCHES = parseInt(process.env.CONCURRENT_BATCHES || '2', 10) const RETRIES = parseInt(process.env.RETRIES || '10', 10) const RETRY_DELAY_MS = parseInt(process.env.RETRY_DELAY_MS || '100', 10) -const USER_FILES_BUCKET_NAME = process.env.USER_FILES_BUCKET_NAME || '' -if (!USER_FILES_BUCKET_NAME) { - throw new Error('env var USER_FILES_BUCKET_NAME is missing') -} const RETRY_FILESTORE_404 = process.env.RETRY_FILESTORE_404 === 'true' const BUFFER_DIR = fs.mkdtempSync( process.env.BUFFER_DIR_PREFIX || '/tmp/back_fill_file_hash-' diff --git a/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs b/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs index 7bab794692..2525ee1d6e 100644 --- a/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs +++ b/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs @@ -9,15 +9,12 @@ import { Blob } from 'overleaf-editor-core' import { BlobStore, getStringLengthOfFile, - GLOBAL_BLOBS, makeBlobForFile, } from '../lib/blob_store/index.js' import { db } from '../lib/mongodb.js' import commandLineArgs from 'command-line-args' import readline from 'node:readline' -import { _blobIsBackedUp, backupBlob } from '../lib/backupBlob.mjs' import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js' -import filestorePersistor from '../lib/persistor.js' import { setTimeout } from 'node:timers/promises' // Silence warning. @@ -52,12 +49,11 @@ ObjectId.cacheHexString = true */ /** - * @return {{FIX_NOT_FOUND: boolean, FIX_HASH_MISMATCH: boolean, FIX_DELETE_PERMISSION: boolean, FIX_MISSING_HASH: boolean, LOGS: string}} + * @return {{FIX_NOT_FOUND: boolean, FIX_HASH_MISMATCH: boolean, FIX_MISSING_HASH: boolean, LOGS: string}} */ function parseArgs() { const args = commandLineArgs([ { name: 'fixNotFound', type: String, defaultValue: 'true' }, - { name: 'fixDeletePermission', type: String, defaultValue: 'true' }, { name: 'fixHashMismatch', type: String, defaultValue: 'true' }, { name: 'fixMissingHash', type: String, defaultValue: 'true' }, { name: 'logs', type: String, defaultValue: '' }, @@ -74,20 +70,13 @@ function parseArgs() { } return { FIX_HASH_MISMATCH: boolVal('fixNotFound'), - FIX_DELETE_PERMISSION: boolVal('fixDeletePermission'), FIX_NOT_FOUND: boolVal('fixHashMismatch'), FIX_MISSING_HASH: boolVal('fixMissingHash'), LOGS: args.logs, } } -const { - FIX_HASH_MISMATCH, - FIX_DELETE_PERMISSION, - FIX_NOT_FOUND, - FIX_MISSING_HASH, - LOGS, -} = parseArgs() +const { FIX_HASH_MISMATCH, FIX_NOT_FOUND, FIX_MISSING_HASH, LOGS } = parseArgs() if (!LOGS) { throw new Error('--logs parameter missing') } @@ -105,6 +94,37 @@ const STREAM_HIGH_WATER_MARK = parseInt( ) const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10) +// Filestore endpoint location +const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1' +const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009' + +async function fetchFromFilestore(projectId, fileId) { + const url = `http://${FILESTORE_HOST}:${FILESTORE_PORT}/project/${projectId}/file/${fileId}` + const response = await fetch(url) + if (!response.ok) { + if (response.status === 404) { + throw new NotFoundError('file not found in filestore', { + status: response.status, + }) + } + const body = await response.text() + throw new OError('fetchFromFilestore failed', { + projectId, + fileId, + status: response.status, + body, + }) + } + if (!response.body) { + throw new OError('fetchFromFilestore response has no body', { + projectId, + fileId, + status: response.status, + }) + } + return response.body +} + /** @type {ProjectsCollection} */ const projectsCollection = db.collection('projects') /** @type {DeletedProjectsCollection} */ @@ -302,19 +322,16 @@ async function setHashInMongo(projectId, fileId, hash) { * @return {Promise} */ async function importRestoredFilestoreFile(projectId, fileId, historyId) { - const filestoreKey = `${projectId}/${fileId}` const path = `${BUFFER_DIR}/${projectId}_${fileId}` try { let s try { - s = await filestorePersistor.getObjectStream( - USER_FILES_BUCKET_NAME, - filestoreKey - ) + s = await fetchFromFilestore(projectId, fileId) } catch (err) { if (err instanceof NotFoundError) { throw new OError('missing blob, need to restore filestore file', { - filestoreKey, + projectId, + fileId, }) } throw err @@ -325,7 +342,6 @@ async function importRestoredFilestoreFile(projectId, fileId, historyId) { ) const blobStore = new BlobStore(historyId) const blob = await blobStore.putFile(path) - await backupBlob(historyId, blob, path) await setHashInMongo(projectId, fileId, blob.getHash()) } finally { await fs.promises.rm(path, { force: true }) @@ -339,13 +355,9 @@ async function importRestoredFilestoreFile(projectId, fileId, historyId) { * @return {Promise} */ async function bufferFilestoreFileToDisk(projectId, fileId, path) { - const filestoreKey = `${projectId}/${fileId}` try { await Stream.promises.pipeline( - await filestorePersistor.getObjectStream( - USER_FILES_BUCKET_NAME, - filestoreKey - ), + await fetchFromFilestore(projectId, fileId), fs.createWriteStream(path, { highWaterMark: STREAM_HIGH_WATER_MARK }) ) const blob = await makeBlobForFile(path) @@ -356,7 +368,8 @@ async function bufferFilestoreFileToDisk(projectId, fileId, path) { } catch (err) { if (err instanceof NotFoundError) { throw new OError('missing blob, need to restore filestore file', { - filestoreKey, + projectId, + fileId, }) } throw err @@ -389,7 +402,7 @@ async function uploadFilestoreFile(projectId, fileId) { const blob = await bufferFilestoreFileToDisk(projectId, fileId, path) const hash = blob.getHash() try { - await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) + await ensureBlobExistsForFile(projectId, fileId, hash) } catch (err) { if (!(err instanceof Blob.NotFoundError)) throw err @@ -397,7 +410,7 @@ async function uploadFilestoreFile(projectId, fileId) { const historyId = project.overleaf.history.id.toString() const blobStore = new BlobStore(historyId) await blobStore.putBlob(path, blob) - await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) + await ensureBlobExistsForFile(projectId, fileId, hash) } } finally { await fs.promises.rm(path, { force: true }) @@ -426,11 +439,7 @@ async function fixHashMismatch(line) { await importRestoredFilestoreFile(projectId, fileId, historyId) return true } - return await ensureBlobExistsForFileAndUploadToAWS( - projectId, - fileId, - computedHash - ) + return await ensureBlobExistsForFile(projectId, fileId, computedHash) } /** @@ -444,30 +453,19 @@ async function hashAlreadyUpdatedInFileTree(projectId, fileId, hash) { return fileRef.hash === hash } -/** - * @param {string} projectId - * @param {string} hash - * @return {Promise} - */ -async function needsBackingUpToAWS(projectId, hash) { - if (GLOBAL_BLOBS.has(hash)) return false - return !(await _blobIsBackedUp(projectId, hash)) -} - /** * @param {string} projectId * @param {string} fileId * @param {string} hash * @return {Promise} */ -async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) { +async function ensureBlobExistsForFile(projectId, fileId, hash) { const { project } = await getProject(projectId) const historyId = project.overleaf.history.id.toString() const blobStore = new BlobStore(historyId) if ( (await hashAlreadyUpdatedInFileTree(projectId, fileId, hash)) && - (await blobStore.getBlob(hash)) && - !(await needsBackingUpToAWS(projectId, hash)) + (await blobStore.getBlob(hash)) ) { return false // already processed } @@ -488,7 +486,7 @@ async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) { ) if (writtenBlob.getHash() !== hash) { // Double check download, better safe than sorry. - throw new OError('blob corrupted', { writtenBlob }) + throw new OError('blob corrupted', { writtenBlob, hash }) } let blob = await blobStore.getBlob(hash) @@ -497,7 +495,6 @@ async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) { // HACK: Skip upload to GCS and finalize putBlob operation directly. await blobStore.backend.insertBlob(historyId, writtenBlob) } - await backupBlob(historyId, writtenBlob, path) } finally { await fs.promises.rm(path, { force: true }) } @@ -505,16 +502,6 @@ async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) { return true } -/** - * @param {string} line - * @return {Promise} - */ -async function fixDeletePermission(line) { - let { projectId, fileId, hash } = JSON.parse(line) - if (!hash) hash = await computeFilestoreFileHash(projectId, fileId) - return await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) -} - /** * @param {string} line * @return {Promise} @@ -526,7 +513,7 @@ async function fixMissingHash(line) { } = await findFile(projectId, fileId) if (hash) { // processed, double check - return await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) + return await ensureBlobExistsForFile(projectId, fileId, hash) } await uploadFilestoreFile(projectId, fileId) return true @@ -543,11 +530,6 @@ const CASES = { flag: FIX_HASH_MISMATCH, action: fixHashMismatch, }, - 'delete permission': { - match: 'storage.objects.delete', - flag: FIX_DELETE_PERMISSION, - action: fixDeletePermission, - }, 'missing file hash': { match: '"bad file hash"', flag: FIX_MISSING_HASH, diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs index 62b0b1de25..b6cdd4b9bf 100644 --- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs @@ -20,7 +20,7 @@ import { makeProjectKey, } from '../../../../storage/lib/blob_store/index.js' -import express from 'express' +import { mockFilestore } from './support/MockFilestore.mjs' chai.use(chaiExclude) const TIMEOUT = 20 * 1_000 @@ -28,59 +28,6 @@ const TIMEOUT = 20 * 1_000 const projectsCollection = db.collection('projects') const deletedProjectsCollection = db.collection('deletedProjects') -class MockFilestore { - constructor() { - this.host = process.env.FILESTORE_HOST || '127.0.0.1' - this.port = process.env.FILESTORE_PORT || 3009 - // create a server listening on this.host and this.port - this.files = {} - - this.app = express() - - this.app.get('/project/:projectId/file/:fileId', (req, res) => { - const { projectId, fileId } = req.params - const content = this.files[projectId]?.[fileId] - if (!content) return res.status(404).end() - res.status(200).end(content) - }) - } - - start() { - // reset stored files - this.files = {} - // start the server - if (this.serverPromise) { - return this.serverPromise - } else { - this.serverPromise = new Promise((resolve, reject) => { - this.server = this.app.listen(this.port, this.host, err => { - if (err) return reject(err) - resolve() - }) - }) - return this.serverPromise - } - } - - addFile(projectId, fileId, fileContent) { - if (!this.files[projectId]) { - this.files[projectId] = {} - } - this.files[projectId][fileId] = fileContent - } - - deleteObject(projectId, fileId) { - if (this.files[projectId]) { - delete this.files[projectId][fileId] - if (Object.keys(this.files[projectId]).length === 0) { - delete this.files[projectId] - } - } - } -} - -const mockFilestore = new MockFilestore() - /** * @param {ObjectId} objectId * @return {string} diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs index ceafa24c3a..3aa00d685a 100644 --- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs +++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs @@ -1,48 +1,24 @@ import fs from 'node:fs' import Crypto from 'node:crypto' -import Stream from 'node:stream' import { promisify } from 'node:util' import { Binary, ObjectId } from 'mongodb' import { Blob } from 'overleaf-editor-core' -import { backedUpBlobs, blobs, db } from '../../../../storage/lib/mongodb.js' +import { db } from '../../../../storage/lib/mongodb.js' import cleanup from './support/cleanup.js' import testProjects from '../api/support/test_projects.js' import { execFile } from 'node:child_process' import chai, { expect } from 'chai' import chaiExclude from 'chai-exclude' -import config from 'config' -import { WritableBuffer } from '@overleaf/stream-utils' -import { - backupPersistor, - projectBlobsBucket, -} from '../../../../storage/lib/backupPersistor.mjs' -import projectKey from '../../../../storage/lib/project_key.js' -import { - BlobStore, - makeProjectKey, -} from '../../../../storage/lib/blob_store/index.js' -import ObjectPersistor from '@overleaf/object-persistor' +import { BlobStore } from '../../../../storage/lib/blob_store/index.js' +import { mockFilestore } from './support/MockFilestore.mjs' chai.use(chaiExclude) const TIMEOUT = 20 * 1_000 -const { deksBucket } = config.get('backupStore') -const { tieringStorageClass } = config.get('backupPersistor') - const projectsCollection = db.collection('projects') const deletedProjectsCollection = db.collection('deletedProjects') -const FILESTORE_PERSISTOR = ObjectPersistor({ - backend: 'gcs', - gcs: { - endpoint: { - apiEndpoint: process.env.GCS_API_ENDPOINT, - projectId: process.env.GCS_PROJECT_ID, - }, - }, -}) - /** * @param {ObjectId} objectId * @return {string} @@ -70,17 +46,6 @@ function binaryForGitBlobHash(gitBlobHash) { return new Binary(Buffer.from(gitBlobHash, 'hex')) } -async function listS3Bucket(bucket, wantStorageClass) { - const client = backupPersistor._getClientForBucket(bucket) - const response = await client.listObjectsV2({ Bucket: bucket }).promise() - - for (const object of response.Contents || []) { - expect(object).to.have.property('StorageClass', wantStorageClass) - } - - return (response.Contents || []).map(item => item.Key || '') -} - function objectIdFromTime(timestamp) { return ObjectId.createFromTime(new Date(timestamp).getTime() / 1000) } @@ -97,7 +62,6 @@ describe('back_fill_file_hash_fix_up script', function () { const historyIdDeleted0 = projectIdDeleted0.toString() const fileIdWithDifferentHashFound = objectIdFromTime('2017-02-01T00:00:00Z') const fileIdInGoodState = objectIdFromTime('2017-02-01T00:01:00Z') - const fileIdBlobExistsInGCS0 = objectIdFromTime('2017-02-01T00:02:00Z') const fileIdWithDifferentHashNotFound0 = objectIdFromTime( '2017-02-01T00:03:00Z' ) @@ -112,9 +76,6 @@ describe('back_fill_file_hash_fix_up script', function () { const fileIdWithDifferentHashRestore = objectIdFromTime( '2017-02-01T00:08:00Z' ) - const fileIdBlobExistsInGCS1 = objectIdFromTime('2017-02-01T00:09:00Z') - const fileIdRestoreFromFilestore0 = objectIdFromTime('2017-02-01T00:10:00Z') - const fileIdRestoreFromFilestore1 = objectIdFromTime('2017-02-01T00:11:00Z') const fileIdMissing2 = objectIdFromTime('2017-02-01T00:12:00Z') const fileIdHashMissing0 = objectIdFromTime('2017-02-01T00:13:00Z') const fileIdHashMissing1 = objectIdFromTime('2017-02-01T00:14:00Z') @@ -125,31 +86,11 @@ describe('back_fill_file_hash_fix_up script', function () { ) const deleteProjectsRecordId0 = new ObjectId() const writtenBlobs = [ - { - projectId: projectId0, - historyId: historyId0, - fileId: fileIdBlobExistsInGCS0, - }, - { - projectId: projectId0, - historyId: historyId0, - fileId: fileIdBlobExistsInGCS1, - }, { projectId: projectId0, historyId: historyId0, fileId: fileIdWithDifferentHashNotFound0, }, - { - projectId: projectId0, - historyId: historyId0, - fileId: fileIdRestoreFromFilestore0, - }, - { - projectId: projectId0, - historyId: historyId0, - fileId: fileIdRestoreFromFilestore1, - }, { projectId: projectId0, historyId: historyId0, @@ -200,17 +141,6 @@ describe('back_fill_file_hash_fix_up script', function () { }, msg: 'failed to process file', }, - { - projectId: projectId0, - fileId: fileIdRestoreFromFilestore0, - err: { message: 'OError: hash mismatch' }, - hash: gitBlobHash(fileIdRestoreFromFilestore0), - entry: { - ctx: { historyId: historyId0.toString() }, - hash: hashDoesNotExistAsBlob, - }, - msg: 'failed to process file', - }, { projectId: projectIdDeleted0, fileId: fileIdWithDifferentHashNotFound1, @@ -236,33 +166,6 @@ describe('back_fill_file_hash_fix_up script', function () { err: { message: 'NotFoundError' }, msg: 'failed to process file', }, - { - projectId: projectId0, - fileId: fileIdBlobExistsInGCS0, - hash: gitBlobHash(fileIdBlobExistsInGCS0), - err: { message: 'storage.objects.delete' }, - msg: 'failed to process file', - }, - { - projectId: projectId0, - fileId: fileIdBlobExistsInGCSCorrupted, - hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted), - err: { message: 'storage.objects.delete' }, - msg: 'failed to process file', - }, - { - projectId: projectId0, - fileId: fileIdBlobExistsInGCS1, - hash: gitBlobHash(fileIdBlobExistsInGCS1), - err: { message: 'storage.objects.delete' }, - msg: 'failed to process file', - }, - { - projectId: projectId0, - fileId: fileIdRestoreFromFilestore1, - err: { message: 'storage.objects.delete' }, - msg: 'failed to process file', - }, { projectId: projectIdDeleted0, fileId: fileIdMissing1, @@ -291,22 +194,23 @@ describe('back_fill_file_hash_fix_up script', function () { reason: 'bad file hash', msg: 'bad file-tree path', }, + { + projectId: projectId0, + _id: fileIdBlobExistsInGCSCorrupted, + reason: 'bad file hash', + msg: 'bad file-tree path', + }, ] if (PRINT_IDS_AND_HASHES_FOR_DEBUGGING) { const fileIds = { fileIdWithDifferentHashFound, fileIdInGoodState, - fileIdBlobExistsInGCS0, - fileIdBlobExistsInGCS1, fileIdWithDifferentHashNotFound0, fileIdWithDifferentHashNotFound1, - fileIdBlobExistsInGCSCorrupted, fileIdMissing0, fileIdMissing1, fileIdMissing2, fileIdWithDifferentHashRestore, - fileIdRestoreFromFilestore0, - fileIdRestoreFromFilestore1, fileIdHashMissing0, fileIdHashMissing1, } @@ -330,38 +234,25 @@ describe('back_fill_file_hash_fix_up script', function () { before(cleanup.everything) before('populate blobs/GCS', async function () { - await FILESTORE_PERSISTOR.sendStream( - USER_FILES_BUCKET_NAME, - `${projectId0}/${fileIdRestoreFromFilestore0}`, - Stream.Readable.from([fileIdRestoreFromFilestore0.toString()]) + await mockFilestore.start() + mockFilestore.addFile( + projectId0, + fileIdHashMissing0, + fileIdHashMissing0.toString() ) - await FILESTORE_PERSISTOR.sendStream( - USER_FILES_BUCKET_NAME, - `${projectId0}/${fileIdRestoreFromFilestore1}`, - Stream.Readable.from([fileIdRestoreFromFilestore1.toString()]) + mockFilestore.addFile( + projectId0, + fileIdHashMissing1, + fileIdHashMissing1.toString() ) - await FILESTORE_PERSISTOR.sendStream( - USER_FILES_BUCKET_NAME, - `${projectId0}/${fileIdHashMissing0}`, - Stream.Readable.from([fileIdHashMissing0.toString()]) - ) - await FILESTORE_PERSISTOR.sendStream( - USER_FILES_BUCKET_NAME, - `${projectId0}/${fileIdHashMissing1}`, - Stream.Readable.from([fileIdHashMissing1.toString()]) + mockFilestore.addFile( + projectId0, + fileIdBlobExistsInGCSCorrupted, + fileIdBlobExistsInGCSCorrupted.toString() ) await new BlobStore(historyId0.toString()).putString( fileIdHashMissing1.toString() // partially processed ) - await new BlobStore(historyId0.toString()).putString( - fileIdBlobExistsInGCS0.toString() - ) - await new BlobStore(historyId0.toString()).putString( - fileIdBlobExistsInGCS1.toString() - ) - await new BlobStore(historyId0.toString()).putString( - fileIdRestoreFromFilestore1.toString() - ) const path = '/tmp/test-blob-corrupted' try { await fs.promises.writeFile(path, contentCorruptedBlob) @@ -426,22 +317,10 @@ describe('back_fill_file_hash_fix_up script', function () { _id: fileIdWithDifferentHashNotFound0, hash: hashDoesNotExistAsBlob, }, - { - _id: fileIdRestoreFromFilestore0, - hash: hashDoesNotExistAsBlob, - }, - { - _id: fileIdRestoreFromFilestore1, - }, - { - _id: fileIdBlobExistsInGCS0, - hash: gitBlobHash(fileIdBlobExistsInGCS0), - }, { _id: fileIdBlobExistsInGCSCorrupted, hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted), }, - { _id: fileIdBlobExistsInGCS1 }, ], folders: [], }, @@ -546,8 +425,8 @@ describe('back_fill_file_hash_fix_up script', function () { }) it('should print stats', function () { expect(stats).to.contain({ - processedLines: 16, - success: 11, + processedLines: 12, + success: 7, alreadyProcessed: 0, fileDeleted: 0, skipped: 0, @@ -558,9 +437,9 @@ describe('back_fill_file_hash_fix_up script', function () { it('should handle re-run on same logs', async function () { ;({ stats } = await runScriptWithLogs()) expect(stats).to.contain({ - processedLines: 16, + processedLines: 12, success: 0, - alreadyProcessed: 8, + alreadyProcessed: 4, fileDeleted: 3, skipped: 0, failed: 3, @@ -663,31 +542,11 @@ describe('back_fill_file_hash_fix_up script', function () { _id: fileIdWithDifferentHashNotFound0, hash: gitBlobHash(fileIdWithDifferentHashNotFound0), }, - // Updated hash - { - _id: fileIdRestoreFromFilestore0, - hash: gitBlobHash(fileIdRestoreFromFilestore0), - }, - // Added hash - { - _id: fileIdRestoreFromFilestore1, - hash: gitBlobHash(fileIdRestoreFromFilestore1), - }, - // No change, blob created - { - _id: fileIdBlobExistsInGCS0, - hash: gitBlobHash(fileIdBlobExistsInGCS0), - }, // No change, flagged { _id: fileIdBlobExistsInGCSCorrupted, hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted), }, - // Added hash - { - _id: fileIdBlobExistsInGCS1, - hash: gitBlobHash(fileIdBlobExistsInGCS1), - }, ], folders: [], }, @@ -696,7 +555,7 @@ describe('back_fill_file_hash_fix_up script', function () { ], overleaf: { history: { id: historyId0 } }, // Incremented when removing file/updating hash - version: 8, + version: 5, }, ]) expect(await deletedProjectsCollection.find({}).toArray()).to.deep.equal([ @@ -745,62 +604,6 @@ describe('back_fill_file_hash_fix_up script', function () { (writtenBlobsByProject.get(projectId) || []).concat([fileId]) ) } - expect( - (await backedUpBlobs.find({}, { sort: { _id: 1 } }).toArray()).map( - entry => { - // blobs are pushed unordered into mongo. Sort the list for consistency. - entry.blobs.sort() - return entry - } - ) - ).to.deep.equal( - Array.from(writtenBlobsByProject.entries()).map( - ([projectId, fileIds]) => { - return { - _id: projectId, - blobs: fileIds - .map(fileId => binaryForGitBlobHash(gitBlobHash(fileId))) - .sort(), - } - } - ) - ) - }) - it('should have backed up all the files', async function () { - expect(tieringStorageClass).to.exist - const objects = await listS3Bucket(projectBlobsBucket, tieringStorageClass) - expect(objects.sort()).to.deep.equal( - writtenBlobs - .map(({ historyId, fileId, hash }) => - makeProjectKey(historyId, hash || gitBlobHash(fileId)) - ) - .sort() - ) - for (let { historyId, fileId } of writtenBlobs) { - const hash = gitBlobHash(fileId.toString()) - const s = await backupPersistor.getObjectStream( - projectBlobsBucket, - makeProjectKey(historyId, hash), - { autoGunzip: true } - ) - const buf = new WritableBuffer() - await Stream.promises.pipeline(s, buf) - expect(gitBlobHashBuffer(buf.getContents())).to.equal(hash) - const id = buf.getContents().toString('utf-8') - expect(id).to.equal(fileId.toString()) - // double check we are not comparing 'undefined' or '[object Object]' above - expect(id).to.match(/^[a-f0-9]{24}$/) - } - const deks = await listS3Bucket(deksBucket, 'STANDARD') - expect(deks.sort()).to.deep.equal( - Array.from( - new Set( - writtenBlobs.map( - ({ historyId }) => projectKey.format(historyId) + '/dek' - ) - ) - ).sort() - ) }) it('should have written the back filled files to history v1', async function () { for (const { historyId, fileId } of writtenBlobs) { diff --git a/services/history-v1/test/acceptance/js/storage/support/MockFilestore.mjs b/services/history-v1/test/acceptance/js/storage/support/MockFilestore.mjs new file mode 100644 index 0000000000..55d0923c34 --- /dev/null +++ b/services/history-v1/test/acceptance/js/storage/support/MockFilestore.mjs @@ -0,0 +1,54 @@ +import express from 'express' + +class MockFilestore { + constructor() { + this.host = process.env.FILESTORE_HOST || '127.0.0.1' + this.port = process.env.FILESTORE_PORT || 3009 + // create a server listening on this.host and this.port + this.files = {} + + this.app = express() + + this.app.get('/project/:projectId/file/:fileId', (req, res) => { + const { projectId, fileId } = req.params + const content = this.files[projectId]?.[fileId] + if (!content) return res.status(404).end() + res.status(200).end(content) + }) + } + + start() { + // reset stored files + this.files = {} + // start the server + if (this.serverPromise) { + return this.serverPromise + } else { + this.serverPromise = new Promise((resolve, reject) => { + this.server = this.app.listen(this.port, this.host, err => { + if (err) return reject(err) + resolve() + }) + }) + return this.serverPromise + } + } + + addFile(projectId, fileId, fileContent) { + if (!this.files[projectId]) { + this.files[projectId] = {} + } + this.files[projectId][fileId] = fileContent + } + + deleteObject(projectId, fileId) { + if (this.files[projectId]) { + delete this.files[projectId][fileId] + if (Object.keys(this.files[projectId]).length === 0) { + delete this.files[projectId] + } + } + } +} + +export const mockFilestore = new MockFilestore() diff --git a/services/notifications/docker-compose.ci.yml b/services/notifications/docker-compose.ci.yml index 24b57ab084..ca3303a079 100644 --- a/services/notifications/docker-compose.ci.yml +++ b/services/notifications/docker-compose.ci.yml @@ -42,7 +42,7 @@ services: command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/notifications/docker-compose.yml b/services/notifications/docker-compose.yml index 081bbfa002..e43e9aeef5 100644 --- a/services/notifications/docker-compose.yml +++ b/services/notifications/docker-compose.yml @@ -44,7 +44,7 @@ services: command: npm run --silent test:acceptance mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/project-history/docker-compose.ci.yml b/services/project-history/docker-compose.ci.yml index ca15f35fef..c6ec24a84b 100644 --- a/services/project-history/docker-compose.ci.yml +++ b/services/project-history/docker-compose.ci.yml @@ -55,7 +55,7 @@ services: retries: 20 mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/project-history/docker-compose.yml b/services/project-history/docker-compose.yml index eeca03de6e..dd3c6468fe 100644 --- a/services/project-history/docker-compose.yml +++ b/services/project-history/docker-compose.yml @@ -57,7 +57,7 @@ services: retries: 20 mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/web/app.mjs b/services/web/app.mjs index b7c723da3d..3f54cc36a8 100644 --- a/services/web/app.mjs +++ b/services/web/app.mjs @@ -56,14 +56,8 @@ if (Settings.catchErrors) { // Create ./data/dumpFolder if needed FileWriter.ensureDumpFolderExists() -if ( - !Features.hasFeature('project-history-blobs') && - !Features.hasFeature('filestore') -) { - throw new Error( - 'invalid config: must enable either project-history-blobs (Settings.enableProjectHistoryBlobs=true) or enable filestore (Settings.disableFilestore=false)' - ) -} +// Validate combination of feature flags. +Features.validateSettings() // handle SIGTERM for graceful shutdown in kubernetes process.on('SIGTERM', function (signal) { diff --git a/services/web/app/src/Features/Authentication/AuthenticationController.js b/services/web/app/src/Features/Authentication/AuthenticationController.js index 7a97d2ac9c..99c418df1b 100644 --- a/services/web/app/src/Features/Authentication/AuthenticationController.js +++ b/services/web/app/src/Features/Authentication/AuthenticationController.js @@ -36,7 +36,22 @@ function send401WithChallenge(res) { function checkCredentials(userDetailsMap, user, password) { const expectedPassword = userDetailsMap.get(user) const userExists = userDetailsMap.has(user) && expectedPassword // user exists with a non-null password - const isValid = userExists && tsscmp(expectedPassword, password) + + let isValid = false + if (userExists) { + if (Array.isArray(expectedPassword)) { + const isValidPrimary = Boolean( + expectedPassword[0] && tsscmp(expectedPassword[0], password) + ) + const isValidFallback = Boolean( + expectedPassword[1] && tsscmp(expectedPassword[1], password) + ) + isValid = isValidPrimary || isValidFallback + } else { + isValid = tsscmp(expectedPassword, password) + } + } + if (!isValid) { logger.err({ user }, 'invalid login details') } diff --git a/services/web/app/src/Features/Documents/DocumentController.mjs b/services/web/app/src/Features/Documents/DocumentController.mjs index 6998c0b36a..9a16811894 100644 --- a/services/web/app/src/Features/Documents/DocumentController.mjs +++ b/services/web/app/src/Features/Documents/DocumentController.mjs @@ -7,6 +7,7 @@ import logger from '@overleaf/logger' import _ from 'lodash' import { plainTextResponse } from '../../infrastructure/Response.js' import { expressify } from '@overleaf/promise-utils' +import Modules from '../../infrastructure/Modules.js' async function getDocument(req, res) { const { Project_id: projectId, doc_id: docId } = req.params @@ -92,6 +93,9 @@ async function setDocument(req, res) { { docId, projectId }, 'finished receiving set document request from api (docupdater)' ) + + await Modules.promises.hooks.fire('docModified', projectId, docId) + res.json(result) } diff --git a/services/web/app/src/Features/History/HistoryURLHelper.js b/services/web/app/src/Features/History/HistoryURLHelper.js index 8b8d8cbdd7..acb43ced68 100644 --- a/services/web/app/src/Features/History/HistoryURLHelper.js +++ b/services/web/app/src/Features/History/HistoryURLHelper.js @@ -8,7 +8,7 @@ function projectHistoryURLWithFilestoreFallback( ) { const filestoreURL = `${Settings.apis.filestore.url}/project/${projectId}/file/${fileRef._id}?from=${origin}` // TODO: When this file is converted to ES modules we will be able to use Features.hasFeature('project-history-blobs'). Currently we can't stub the feature return value in tests. - if (fileRef.hash && Settings.enableProjectHistoryBlobs) { + if (fileRef.hash && Settings.filestoreMigrationLevel >= 1) { return { url: `${Settings.apis.project_history.url}/project/${historyId}/blob/${fileRef.hash}`, fallbackURL: filestoreURL, diff --git a/services/web/app/src/Features/Uploads/ProjectUploadController.mjs b/services/web/app/src/Features/Uploads/ProjectUploadController.mjs index a3bc434ed7..84b8738af3 100644 --- a/services/web/app/src/Features/Uploads/ProjectUploadController.mjs +++ b/services/web/app/src/Features/Uploads/ProjectUploadController.mjs @@ -66,7 +66,7 @@ function uploadProject(req, res, next) { async function uploadFile(req, res, next) { const timer = new metrics.Timer('file-upload') const name = req.body.name - const path = req.file?.path + const { path } = req.file const projectId = req.params.Project_id const userId = SessionManager.getLoggedInUserId(req.session) let { folder_id: folderId } = req.query @@ -162,8 +162,14 @@ function multerMiddleware(req, res, next) { .status(422) .json({ success: false, error: req.i18n.translate('file_too_large') }) } - - return next(err) + if (err) return next(err) + if (!req.file?.path) { + logger.info({ req }, 'missing req.file.path on upload') + return res + .status(400) + .json({ success: false, error: 'invalid_upload_request' }) + } + next() }) } diff --git a/services/web/app/src/infrastructure/Features.js b/services/web/app/src/infrastructure/Features.js index ed231a6caa..786f6ef88d 100644 --- a/services/web/app/src/infrastructure/Features.js +++ b/services/web/app/src/infrastructure/Features.js @@ -19,8 +19,7 @@ const trackChangesModuleAvailable = * @property {boolean | undefined} enableGithubSync * @property {boolean | undefined} enableGitBridge * @property {boolean | undefined} enableHomepage - * @property {boolean | undefined} enableProjectHistoryBlobs - * @property {boolean | undefined} disableFilestore + * @property {number} filestoreMigrationLevel * @property {boolean | undefined} enableSaml * @property {boolean | undefined} ldap * @property {boolean | undefined} oauth @@ -30,6 +29,14 @@ const trackChangesModuleAvailable = */ const Features = { + validateSettings() { + if (![0, 1, 2].includes(Settings.filestoreMigrationLevel)) { + throw new Error( + `invalid OVERLEAF_FILESTORE_MIGRATION_LEVEL=${Settings.filestoreMigrationLevel}, expected 0, 1 or 2` + ) + } + }, + /** * @returns {boolean} */ @@ -89,9 +96,9 @@ const Features = { Settings.enabledLinkedFileTypes.includes('url') ) case 'project-history-blobs': - return Boolean(Settings.enableProjectHistoryBlobs) + return Settings.filestoreMigrationLevel > 0 case 'filestore': - return Boolean(Settings.disableFilestore) === false + return Settings.filestoreMigrationLevel < 2 case 'support': return supportModuleAvailable case 'symbol-palette': diff --git a/services/web/app/src/infrastructure/Modules.js b/services/web/app/src/infrastructure/Modules.js index 20975a3642..aea3aeb087 100644 --- a/services/web/app/src/infrastructure/Modules.js +++ b/services/web/app/src/infrastructure/Modules.js @@ -150,8 +150,7 @@ async function linkedFileAgentsIncludes() { async function attachHooks() { for (const module of await modules()) { const { promises, ...hooks } = module.hooks || {} - for (const hook in promises || {}) { - const method = promises[hook] + for (const [hook, method] of Object.entries(promises || {})) { attachHook(hook, method) } for (const hook in hooks || {}) { diff --git a/services/web/app/views/_cookie_banner.pug b/services/web/app/views/_cookie_banner.pug index 56974326cd..7cbc569bc1 100644 --- a/services/web/app/views/_cookie_banner.pug +++ b/services/web/app/views/_cookie_banner.pug @@ -1,13 +1,13 @@ -section.cookie-banner.hidden-print.hidden(aria-label='Cookie banner') - .cookie-banner-content We only use cookies for essential purposes and to improve your experience on our site. You can find out more in our cookie policy. +section.cookie-banner.hidden-print.hidden(aria-label=translate('cookie_banner')) + .cookie-banner-content !{translate('cookie_banner_info', {}, [{ name: 'a', attrs: { href: '/legal#Cookies' }}])} .cookie-banner-actions button( type='button' class='btn btn-link btn-sm' data-ol-cookie-banner-set-consent='essential' - ) Essential cookies only + ) #{translate('essential_cookies_only')} button( type='button' class='btn btn-primary btn-sm' data-ol-cookie-banner-set-consent='all' - ) Accept all cookies + ) #{translate('accept_all_cookies')} diff --git a/services/web/app/views/general/post-gateway.pug b/services/web/app/views/general/post-gateway.pug index c6bbc92d01..86f379ac1b 100644 --- a/services/web/app/views/general/post-gateway.pug +++ b/services/web/app/views/general/post-gateway.pug @@ -4,7 +4,7 @@ block vars - var suppressNavbar = true - var suppressFooter = true - var suppressSkipToContent = true - - var suppressCookieBanner = true + - var suppressPugCookieBanner = true block content .content.content-alt diff --git a/services/web/app/views/layout-marketing.pug b/services/web/app/views/layout-marketing.pug index b54c30f033..26e4eb539d 100644 --- a/services/web/app/views/layout-marketing.pug +++ b/services/web/app/views/layout-marketing.pug @@ -24,7 +24,7 @@ block body else include layout/fat-footer - if typeof suppressCookieBanner == 'undefined' + if typeof suppressPugCookieBanner == 'undefined' include _cookie_banner if bootstrapVersion === 5 diff --git a/services/web/app/views/layout-react.pug b/services/web/app/views/layout-react.pug index 94ff3ba247..e9c4c932c4 100644 --- a/services/web/app/views/layout-react.pug +++ b/services/web/app/views/layout-react.pug @@ -69,5 +69,5 @@ block body else include layout/fat-footer-react-bootstrap-5 - if typeof suppressCookieBanner === 'undefined' + if typeof suppressPugCookieBanner === 'undefined' include _cookie_banner diff --git a/services/web/app/views/layout-website-redesign.pug b/services/web/app/views/layout-website-redesign.pug index 61ed83043b..aa7fea9f07 100644 --- a/services/web/app/views/layout-website-redesign.pug +++ b/services/web/app/views/layout-website-redesign.pug @@ -27,7 +27,7 @@ block body else include layout/fat-footer-website-redesign - if typeof suppressCookieBanner == 'undefined' + if typeof suppressPugCookieBanner == 'undefined' include _cookie_banner block contactModal diff --git a/services/web/app/views/project/editor/new_from_template.pug b/services/web/app/views/project/editor/new_from_template.pug index 7e4e01967e..6d35913141 100644 --- a/services/web/app/views/project/editor/new_from_template.pug +++ b/services/web/app/views/project/editor/new_from_template.pug @@ -2,7 +2,7 @@ extends ../../layout-marketing block vars - var suppressFooter = true - - var suppressCookieBanner = true + - var suppressPugCookieBanner = true - var suppressSkipToContent = true block content diff --git a/services/web/app/views/project/ide-react-detached.pug b/services/web/app/views/project/ide-react-detached.pug index ca1a178bbf..fa695b1af5 100644 --- a/services/web/app/views/project/ide-react-detached.pug +++ b/services/web/app/views/project/ide-react-detached.pug @@ -7,7 +7,7 @@ block vars - var suppressNavbar = true - var suppressFooter = true - var suppressSkipToContent = true - - var suppressCookieBanner = true + - var suppressPugCookieBanner = true - metadata.robotsNoindexNofollow = true block content diff --git a/services/web/app/views/project/list-react.pug b/services/web/app/views/project/list-react.pug index 78103e75a6..47bff344b6 100644 --- a/services/web/app/views/project/list-react.pug +++ b/services/web/app/views/project/list-react.pug @@ -7,6 +7,7 @@ block vars - const suppressNavContentLinks = true - const suppressNavbar = true - const suppressFooter = true + - const suppressPugCookieBanner = true block append meta meta( diff --git a/services/web/app/views/project/token/access-react.pug b/services/web/app/views/project/token/access-react.pug index 80b91f1a99..6c01ad15b1 100644 --- a/services/web/app/views/project/token/access-react.pug +++ b/services/web/app/views/project/token/access-react.pug @@ -5,7 +5,7 @@ block entrypointVar block vars - var suppressFooter = true - - var suppressCookieBanner = true + - var suppressPugCookieBanner = true - var suppressSkipToContent = true block append meta diff --git a/services/web/app/views/project/token/sharing-updates.pug b/services/web/app/views/project/token/sharing-updates.pug index d1818be0af..2f67e5a3c1 100644 --- a/services/web/app/views/project/token/sharing-updates.pug +++ b/services/web/app/views/project/token/sharing-updates.pug @@ -5,7 +5,7 @@ block entrypointVar block vars - var suppressFooter = true - - var suppressCookieBanner = true + - var suppressPugCookieBanner = true - var suppressSkipToContent = true block append meta diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 72a74f56a8..4311eccd96 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -440,6 +440,9 @@ module.exports = { ',' ), + filestoreMigrationLevel: + parseInt(process.env.OVERLEAF_FILESTORE_MIGRATION_LEVEL, 10) || 0, + // i18n // ------ // diff --git a/services/web/docker-compose.ci.yml b/services/web/docker-compose.ci.yml index 33b5a3ca2e..8376103315 100644 --- a/services/web/docker-compose.ci.yml +++ b/services/web/docker-compose.ci.yml @@ -95,7 +95,7 @@ services: image: redis:7.4.3 mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 logging: driver: none command: --replSet overleaf diff --git a/services/web/docker-compose.yml b/services/web/docker-compose.yml index 069c1e77de..e0a4a064c5 100644 --- a/services/web/docker-compose.yml +++ b/services/web/docker-compose.yml @@ -91,7 +91,7 @@ services: image: redis:7.4.3 mongo: - image: mongo:7.0.20 + image: mongo:8.0.11 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/web/frontend/extracted-translations.json b/services/web/frontend/extracted-translations.json index 604879f26d..4bf3a15725 100644 --- a/services/web/frontend/extracted-translations.json +++ b/services/web/frontend/extracted-translations.json @@ -36,6 +36,7 @@ "about_to_remove_user_preamble": "", "about_to_trash_projects": "", "abstract": "", + "accept_all_cookies": "", "accept_and_continue": "", "accept_change": "", "accept_change_error_description": "", @@ -337,6 +338,8 @@ "continue_to": "", "continue_using_free_features": "", "continue_with_free_plan": "", + "cookie_banner": "", + "cookie_banner_info": "", "copied": "", "copy": "", "copy_code": "", @@ -552,6 +555,7 @@ "error_opening_document_detail": "", "error_performing_request": "", "error_processing_file": "", + "essential_cookies_only": "", "example_project": "", "existing_plan_active_until_term_end": "", "expand": "", @@ -871,6 +875,7 @@ "invalid_password_too_similar": "", "invalid_regular_expression": "", "invalid_request": "", + "invalid_upload_request": "", "invite": "", "invite_expired": "", "invite_more_collabs": "", diff --git a/services/web/frontend/js/features/cookie-banner/index.js b/services/web/frontend/js/features/cookie-banner/index.js deleted file mode 100644 index 3d9b2b8d6c..0000000000 --- a/services/web/frontend/js/features/cookie-banner/index.js +++ /dev/null @@ -1,53 +0,0 @@ -import getMeta from '@/utils/meta' - -function loadGA() { - if (window.olLoadGA) { - window.olLoadGA() - } -} - -function setConsent(value) { - document.querySelector('.cookie-banner').classList.add('hidden') - const cookieDomain = getMeta('ol-ExposedSettings').cookieDomain - const oneYearInSeconds = 60 * 60 * 24 * 365 - const cookieAttributes = - '; path=/' + - '; domain=' + - cookieDomain + - '; max-age=' + - oneYearInSeconds + - '; SameSite=Lax; Secure' - if (value === 'all') { - document.cookie = 'oa=1' + cookieAttributes - loadGA() - window.dispatchEvent(new CustomEvent('cookie-consent', { detail: true })) - } else { - document.cookie = 'oa=0' + cookieAttributes - window.dispatchEvent(new CustomEvent('cookie-consent', { detail: false })) - } -} - -if ( - getMeta('ol-ExposedSettings').gaToken || - getMeta('ol-ExposedSettings').gaTokenV4 || - getMeta('ol-ExposedSettings').propensityId || - getMeta('ol-ExposedSettings').hotjarId -) { - document - .querySelectorAll('[data-ol-cookie-banner-set-consent]') - .forEach(el => { - el.addEventListener('click', function (e) { - e.preventDefault() - const consentType = el.getAttribute('data-ol-cookie-banner-set-consent') - setConsent(consentType) - }) - }) - - const oaCookie = document.cookie.split('; ').find(c => c.startsWith('oa=')) - if (!oaCookie) { - const cookieBannerEl = document.querySelector('.cookie-banner') - if (cookieBannerEl) { - cookieBannerEl.classList.remove('hidden') - } - } -} diff --git a/services/web/frontend/js/features/cookie-banner/index.ts b/services/web/frontend/js/features/cookie-banner/index.ts new file mode 100644 index 0000000000..2ea97e875a --- /dev/null +++ b/services/web/frontend/js/features/cookie-banner/index.ts @@ -0,0 +1,32 @@ +import { + CookieConsentValue, + cookieBannerRequired, + hasMadeCookieChoice, + setConsent, +} from '@/features/cookie-banner/utils' + +function toggleCookieBanner(hidden: boolean) { + const cookieBannerEl = document.querySelector('.cookie-banner') + if (cookieBannerEl) { + cookieBannerEl.classList.toggle('hidden', hidden) + } +} + +if (cookieBannerRequired()) { + document + .querySelectorAll('[data-ol-cookie-banner-set-consent]') + .forEach(el => { + el.addEventListener('click', function (e) { + e.preventDefault() + toggleCookieBanner(true) + const consentType = el.getAttribute( + 'data-ol-cookie-banner-set-consent' + ) as CookieConsentValue | null + setConsent(consentType) + }) + }) + + if (!hasMadeCookieChoice()) { + toggleCookieBanner(false) + } +} diff --git a/services/web/frontend/js/features/cookie-banner/utils.ts b/services/web/frontend/js/features/cookie-banner/utils.ts new file mode 100644 index 0000000000..5c045d4e71 --- /dev/null +++ b/services/web/frontend/js/features/cookie-banner/utils.ts @@ -0,0 +1,43 @@ +import getMeta from '@/utils/meta' + +export type CookieConsentValue = 'all' | 'essential' + +function loadGA() { + if (window.olLoadGA) { + window.olLoadGA() + } +} + +export function setConsent(value: CookieConsentValue | null) { + const cookieDomain = getMeta('ol-ExposedSettings').cookieDomain + const oneYearInSeconds = 60 * 60 * 24 * 365 + const cookieAttributes = + '; path=/' + + '; domain=' + + cookieDomain + + '; max-age=' + + oneYearInSeconds + + '; SameSite=Lax; Secure' + if (value === 'all') { + document.cookie = 'oa=1' + cookieAttributes + loadGA() + window.dispatchEvent(new CustomEvent('cookie-consent', { detail: true })) + } else { + document.cookie = 'oa=0' + cookieAttributes + window.dispatchEvent(new CustomEvent('cookie-consent', { detail: false })) + } +} + +export function cookieBannerRequired() { + const exposedSettings = getMeta('ol-ExposedSettings') + return Boolean( + exposedSettings.gaToken || + exposedSettings.gaTokenV4 || + exposedSettings.propensityId || + exposedSettings.hotjarId + ) +} + +export function hasMadeCookieChoice() { + return document.cookie.split('; ').some(c => c.startsWith('oa=')) +} diff --git a/services/web/frontend/js/features/file-tree/components/file-tree-create/error-message.tsx b/services/web/frontend/js/features/file-tree/components/file-tree-create/error-message.tsx index 02cc083928..244ef1a76b 100644 --- a/services/web/frontend/js/features/file-tree/components/file-tree-create/error-message.tsx +++ b/services/web/frontend/js/features/file-tree/components/file-tree-create/error-message.tsx @@ -1,4 +1,4 @@ -import { useTranslation } from 'react-i18next' +import { useTranslation, Trans } from 'react-i18next' import { FetchError } from '../../../../infrastructure/fetch-json' import RedirectToLogin from './redirect-to-login' import { @@ -7,6 +7,7 @@ import { InvalidFilenameError, } from '../../errors' import DangerMessage from './danger-message' +import getMeta from '@/utils/meta' // TODO: Update the error type when we properly type FileTreeActionableContext export default function ErrorMessage({ @@ -15,6 +16,7 @@ export default function ErrorMessage({ error: string | Record }) { const { t } = useTranslation() + const { isOverleaf } = getMeta('ol-ExposedSettings') const fileNameLimit = 150 // the error is a string @@ -46,6 +48,22 @@ export default function ErrorMessage({ ) + case 'invalid_upload_request': + if (!isOverleaf) { + return ( + {t('generic_something_went_wrong')} + ) + } + return ( + + ]} + /> + + ) + case 'duplicate_file_name': return ( diff --git a/services/web/frontend/js/features/ide-redesign/components/chat/message.tsx b/services/web/frontend/js/features/ide-redesign/components/chat/message.tsx index 9a4ffe3a1b..6822db39da 100644 --- a/services/web/frontend/js/features/ide-redesign/components/chat/message.tsx +++ b/services/web/frontend/js/features/ide-redesign/components/chat/message.tsx @@ -1,15 +1,14 @@ import { MessageProps } from '@/features/chat/components/message' import { User } from '../../../../../../types/user' -import { getHueForUserId } from '@/shared/utils/colors' +import { + getBackgroundColorForUserId, + hslStringToLuminance, +} from '@/shared/utils/colors' import MessageContent from '@/features/chat/components/message-content' import classNames from 'classnames' import MaterialIcon from '@/shared/components/material-icon' import { t } from 'i18next' -function hue(user?: User) { - return user ? getHueForUserId(user.id) : 0 -} - function getAvatarStyle(user?: User) { if (!user?.id) { // Deleted user @@ -20,9 +19,15 @@ function getAvatarStyle(user?: User) { } } + const backgroundColor = getBackgroundColorForUserId(user.id) + return { - borderColor: `hsl(${hue(user)}, 85%, 40%)`, - backgroundColor: `hsl(${hue(user)}, 85%, 40%`, + borderColor: backgroundColor, + backgroundColor, + color: + hslStringToLuminance(backgroundColor) < 0.5 + ? 'var(--content-primary-dark)' + : 'var(--content-primary)', } } diff --git a/services/web/frontend/js/features/ide-redesign/components/online-users/online-users-widget.tsx b/services/web/frontend/js/features/ide-redesign/components/online-users/online-users-widget.tsx index 07aaa647a9..2d30297e51 100644 --- a/services/web/frontend/js/features/ide-redesign/components/online-users/online-users-widget.tsx +++ b/services/web/frontend/js/features/ide-redesign/components/online-users/online-users-widget.tsx @@ -7,7 +7,11 @@ import { DropdownToggle, } from '@/features/ui/components/bootstrap-5/dropdown-menu' import OLTooltip from '@/features/ui/components/ol/ol-tooltip' -import { getBackgroundColorForUserId } from '@/shared/utils/colors' +import { + getBackgroundColorForUserId, + hslStringToLuminance, +} from '@/shared/utils/colors' +import classNames from 'classnames' import { useCallback, useMemo } from 'react' import { useTranslation } from 'react-i18next' @@ -86,9 +90,16 @@ const OnlineUserWidget = ({ const OnlineUserCircle = ({ user }: { user: OnlineUser }) => { const backgroundColor = getBackgroundColorForUserId(user.user_id) + const luminance = hslStringToLuminance(backgroundColor) const [character] = [...user.name] return ( - + = 0.5, + })} + style={{ backgroundColor }} + > {character} ) diff --git a/services/web/frontend/js/features/project-list/components/project-list-ds-nav.tsx b/services/web/frontend/js/features/project-list/components/project-list-ds-nav.tsx index 3d24f9845c..07319ffaf1 100644 --- a/services/web/frontend/js/features/project-list/components/project-list-ds-nav.tsx +++ b/services/web/frontend/js/features/project-list/components/project-list-ds-nav.tsx @@ -20,6 +20,7 @@ import Footer from '@/features/ui/components/bootstrap-5/footer/footer' import SidebarDsNav from '@/features/project-list/components/sidebar/sidebar-ds-nav' import SystemMessages from '@/shared/components/system-messages' import overleafLogo from '@/shared/svgs/overleaf-a-ds-solution-mallard.svg' +import CookieBanner from '@/shared/components/cookie-banner' export function ProjectListDsNav() { const navbarProps = getMeta('ol-navbar') @@ -125,6 +126,7 @@ export function ProjectListDsNav() {