diff --git a/package-lock.json b/package-lock.json index d9d8285618..2b3a5868a2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35581,7 +35581,6 @@ "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", "deprecated": "request has been deprecated, see https://github.com/request/request/issues/3142", - "license": "Apache-2.0", "dependencies": { "aws-sign2": "~0.7.0", "aws4": "^1.8.0", @@ -35639,15 +35638,15 @@ } }, "node_modules/request/node_modules/tough-cookie": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", - "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", - "license": "BSD-3-Clause", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", "dependencies": { - "tldts": "^6.1.32" + "psl": "^1.1.28", + "punycode": "^2.1.1" }, "engines": { - "node": ">=16" + "node": ">=0.8" } }, "node_modules/requestretry": { @@ -39613,24 +39612,6 @@ "tlds": "bin.js" } }, - "node_modules/tldts": { - "version": "6.1.86", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", - "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", - "license": "MIT", - "dependencies": { - "tldts-core": "^6.1.86" - }, - "bin": { - "tldts": "bin/cli.js" - } - }, - "node_modules/tldts-core": { - "version": "6.1.86", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", - "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", - "license": "MIT" - }, "node_modules/tmp": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", diff --git a/package.json b/package.json index 44fffc4664..388b750c3d 100644 --- a/package.json +++ b/package.json @@ -33,9 +33,6 @@ "path-to-regexp": "3.3.0", "body-parser": "1.20.3", "multer": "2.0.1" - }, - "request@2.88.2": { - "tough-cookie": "5.1.2" } }, "scripts": { diff --git a/server-ce/test/Makefile b/server-ce/test/Makefile index fb7c980293..6c56b7e8fe 100644 --- a/server-ce/test/Makefile +++ b/server-ce/test/Makefile @@ -21,11 +21,9 @@ test-e2e-native: test-e2e: docker compose build host-admin - docker compose up -d host-admin docker compose up --no-log-prefix --exit-code-from=e2e e2e test-e2e-open: - docker compose up -d host-admin docker compose up --no-log-prefix --exit-code-from=e2e-open e2e-open clean: diff --git a/server-ce/test/docker-compose.yml b/server-ce/test/docker-compose.yml index 1652baeae9..029b73fc62 100644 --- a/server-ce/test/docker-compose.yml +++ b/server-ce/test/docker-compose.yml @@ -35,7 +35,7 @@ services: MAILTRAP_PASSWORD: 'password-for-mailtrap' mongo: - image: mongo:8.0.11 + image: mongo:6.0 command: '--replSet overleaf' volumes: - ../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/server-ce/test/editor.spec.ts b/server-ce/test/editor.spec.ts index 3e57b94f8f..d0060518de 100644 --- a/server-ce/test/editor.spec.ts +++ b/server-ce/test/editor.spec.ts @@ -2,7 +2,6 @@ import { createNewFile, createProject, openProjectById, - testNewFileUpload, } from './helpers/project' import { isExcludedBySharding, startWith } from './helpers/config' import { ensureUserExists, login } from './helpers/login' @@ -120,7 +119,24 @@ describe('editor', () => { cy.get('button').contains('New file').click({ force: true }) }) - testNewFileUpload() + it('can upload file', () => { + const name = `${uuid()}.txt` + const content = `Test File Content ${name}` + cy.get('button').contains('Upload').click({ force: true }) + cy.get('input[type=file]') + .first() + .selectFile( + { + contents: Cypress.Buffer.from(content), + fileName: name, + lastModified: Date.now(), + }, + { force: true } + ) + // force: The file-tree pane is too narrow to display the full name. + cy.findByTestId('file-tree').findByText(name).click({ force: true }) + cy.findByText(content) + }) it('should not display import from URL', () => { cy.findByText('From external URL').should('not.exist') diff --git a/server-ce/test/filestore-migration.spec.ts b/server-ce/test/filestore-migration.spec.ts deleted file mode 100644 index 25875ad374..0000000000 --- a/server-ce/test/filestore-migration.spec.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { ensureUserExists, login } from './helpers/login' -import { - createProject, - openProjectById, - prepareFileUploadTest, -} from './helpers/project' -import { isExcludedBySharding, startWith } from './helpers/config' -import { prepareWaitForNextCompileSlot } from './helpers/compile' -import { beforeWithReRunOnTestRetry } from './helpers/beforeWithReRunOnTestRetry' -import { v4 as uuid } from 'uuid' -import { purgeFilestoreData, runScript } from './helpers/hostAdminClient' - -describe('filestore migration', function () { - if (isExcludedBySharding('CE_CUSTOM_3')) return - startWith({ withDataDir: true, resetData: true, vars: {} }) - ensureUserExists({ email: 'user@example.com' }) - - let projectName: string - let projectId: string - let waitForCompileRateLimitCoolOff: (fn: () => void) => void - const previousBinaryFiles: (() => void)[] = [] - beforeWithReRunOnTestRetry(function () { - projectName = `project-${uuid()}` - login('user@example.com') - createProject(projectName, { type: 'Example project' }).then( - id => (projectId = id) - ) - let queueReset - ;({ waitForCompileRateLimitCoolOff, queueReset } = - prepareWaitForNextCompileSlot()) - queueReset() - previousBinaryFiles.push(prepareFileUploadTest(true)) - }) - - beforeEach(() => { - login('user@example.com') - waitForCompileRateLimitCoolOff(() => { - openProjectById(projectId) - }) - }) - - function checkFilesAreAccessible() { - it('can upload new binary file and read previous uploads', function () { - previousBinaryFiles.push(prepareFileUploadTest(true)) - for (const check of previousBinaryFiles) { - check() - } - }) - - it('renders frog jpg', () => { - cy.findByTestId('file-tree').findByText('frog.jpg').click() - cy.get('[alt="frog.jpg"]') - .should('be.visible') - .and('have.prop', 'naturalWidth') - .should('be.greaterThan', 0) - }) - } - - describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL not set', function () { - startWith({ withDataDir: true, vars: {} }) - checkFilesAreAccessible() - }) - - describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=0', function () { - startWith({ - withDataDir: true, - vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '0' }, - }) - checkFilesAreAccessible() - - describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=1', function () { - startWith({ - withDataDir: true, - vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '1' }, - }) - checkFilesAreAccessible() - - describe('OVERLEAF_FILESTORE_MIGRATION_LEVEL=2', function () { - startWith({ - withDataDir: true, - vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '1' }, - }) - before(async function () { - await runScript({ - cwd: 'services/history-v1', - script: 'storage/scripts/back_fill_file_hash.mjs', - }) - }) - startWith({ - withDataDir: true, - vars: { OVERLEAF_FILESTORE_MIGRATION_LEVEL: '2' }, - }) - checkFilesAreAccessible() - - describe('purge filestore data', function () { - before(async function () { - await purgeFilestoreData() - }) - checkFilesAreAccessible() - }) - }) - }) - }) -}) diff --git a/server-ce/test/helpers/config.ts b/server-ce/test/helpers/config.ts index 78e81be1f7..030e70ceb5 100644 --- a/server-ce/test/helpers/config.ts +++ b/server-ce/test/helpers/config.ts @@ -9,7 +9,6 @@ export function isExcludedBySharding( | 'CE_DEFAULT' | 'CE_CUSTOM_1' | 'CE_CUSTOM_2' - | 'CE_CUSTOM_3' | 'PRO_DEFAULT_1' | 'PRO_DEFAULT_2' | 'PRO_CUSTOM_1' diff --git a/server-ce/test/helpers/hostAdminClient.ts b/server-ce/test/helpers/hostAdminClient.ts index dadfe2b059..cafeaa2db6 100644 --- a/server-ce/test/helpers/hostAdminClient.ts +++ b/server-ce/test/helpers/hostAdminClient.ts @@ -85,12 +85,6 @@ export async function getRedisKeys() { return stdout.split('\n') } -export async function purgeFilestoreData() { - await fetchJSON(`${hostAdminURL}/data/user_files`, { - method: 'DELETE', - }) -} - async function sleep(ms: number) { return new Promise(resolve => { setTimeout(resolve, ms) diff --git a/server-ce/test/helpers/project.ts b/server-ce/test/helpers/project.ts index 4b3197afed..abcce3f9b2 100644 --- a/server-ce/test/helpers/project.ts +++ b/server-ce/test/helpers/project.ts @@ -216,43 +216,3 @@ export function createNewFile() { return fileName } - -export function prepareFileUploadTest(binary = false) { - const name = `${uuid()}.txt` - const content = `Test File Content ${name}${binary ? ' \x00' : ''}` - cy.get('button').contains('Upload').click({ force: true }) - cy.get('input[type=file]') - .first() - .selectFile( - { - contents: Cypress.Buffer.from(content), - fileName: name, - lastModified: Date.now(), - }, - { force: true } - ) - - // wait for the upload to finish - cy.findByRole('treeitem', { name }) - - return function check() { - cy.findByRole('treeitem', { name }).click() - if (binary) { - cy.findByText(content).should('not.have.class', 'cm-line') - } else { - cy.findByText(content).should('have.class', 'cm-line') - } - } -} - -export function testNewFileUpload() { - it('can upload text file', () => { - const check = prepareFileUploadTest(false) - check() - }) - - it('can upload binary file', () => { - const check = prepareFileUploadTest(true) - check() - }) -} diff --git a/server-ce/test/host-admin.js b/server-ce/test/host-admin.js index b3dcd72b1f..f73209d58f 100644 --- a/server-ce/test/host-admin.js +++ b/server-ce/test/host-admin.js @@ -29,17 +29,6 @@ const IMAGES = { PRO: process.env.IMAGE_TAG_PRO.replace(/:.+/, ''), } -function defaultDockerComposeOverride() { - return { - services: { - sharelatex: { - environment: {}, - }, - 'git-bridge': {}, - }, - } -} - let previousConfig = '' function readDockerComposeOverride() { @@ -49,7 +38,14 @@ function readDockerComposeOverride() { if (error.code !== 'ENOENT') { throw error } - return defaultDockerComposeOverride + return { + services: { + sharelatex: { + environment: {}, + }, + 'git-bridge': {}, + }, + } } } @@ -81,21 +77,12 @@ app.use(bodyParser.json()) app.use((req, res, next) => { // Basic access logs console.log(req.method, req.url, req.body) - const json = res.json - res.json = body => { - console.log(req.method, req.url, req.body, '->', body) - json.call(res, body) - } - next() -}) -app.use((req, res, next) => { // Add CORS headers const accessControlAllowOrigin = process.env.ACCESS_CONTROL_ALLOW_ORIGIN || 'http://sharelatex' res.setHeader('Access-Control-Allow-Origin', accessControlAllowOrigin) res.setHeader('Access-Control-Allow-Headers', 'Content-Type') res.setHeader('Access-Control-Max-Age', '3600') - res.setHeader('Access-Control-Allow-Methods', 'DELETE, GET, HEAD, POST, PUT') next() }) @@ -146,7 +133,6 @@ const allowedVars = Joi.object( 'V1_HISTORY_URL', 'SANDBOXED_COMPILES', 'ALL_TEX_LIVE_DOCKER_IMAGE_NAMES', - 'OVERLEAF_FILESTORE_MIGRATION_LEVEL', 'OVERLEAF_TEMPLATES_USER_ID', 'OVERLEAF_NEW_PROJECT_TEMPLATE_LINKS', 'OVERLEAF_ALLOW_PUBLIC_ACCESS', @@ -333,19 +319,8 @@ app.get('/redis/keys', (req, res) => { ) }) -app.delete('/data/user_files', (req, res) => { - runDockerCompose( - 'exec', - ['sharelatex', 'rm', '-rf', '/var/lib/overleaf/data/user_files'], - (error, stdout, stderr) => { - res.json({ error, stdout, stderr }) - } - ) -}) - app.use(handleValidationErrors()) purgeDataDir() -writeDockerComposeOverride(defaultDockerComposeOverride()) app.listen(80) diff --git a/services/chat/docker-compose.ci.yml b/services/chat/docker-compose.ci.yml index ca3303a079..24b57ab084 100644 --- a/services/chat/docker-compose.ci.yml +++ b/services/chat/docker-compose.ci.yml @@ -42,7 +42,7 @@ services: command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/chat/docker-compose.yml b/services/chat/docker-compose.yml index e7b8ce7385..ddc5f9e698 100644 --- a/services/chat/docker-compose.yml +++ b/services/chat/docker-compose.yml @@ -44,7 +44,7 @@ services: command: npm run --silent test:acceptance mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/contacts/docker-compose.ci.yml b/services/contacts/docker-compose.ci.yml index ca3303a079..24b57ab084 100644 --- a/services/contacts/docker-compose.ci.yml +++ b/services/contacts/docker-compose.ci.yml @@ -42,7 +42,7 @@ services: command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/contacts/docker-compose.yml b/services/contacts/docker-compose.yml index 474ea224f8..6c77ef5e31 100644 --- a/services/contacts/docker-compose.yml +++ b/services/contacts/docker-compose.yml @@ -44,7 +44,7 @@ services: command: npm run --silent test:acceptance mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/docstore/docker-compose.ci.yml b/services/docstore/docker-compose.ci.yml index cdb4783c5a..40decc4aea 100644 --- a/services/docstore/docker-compose.ci.yml +++ b/services/docstore/docker-compose.ci.yml @@ -47,7 +47,7 @@ services: command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/docstore/docker-compose.yml b/services/docstore/docker-compose.yml index a9099c7e7b..8c11eb5a91 100644 --- a/services/docstore/docker-compose.yml +++ b/services/docstore/docker-compose.yml @@ -49,7 +49,7 @@ services: command: npm run --silent test:acceptance mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/document-updater/docker-compose.ci.yml b/services/document-updater/docker-compose.ci.yml index c6ec24a84b..ca15f35fef 100644 --- a/services/document-updater/docker-compose.ci.yml +++ b/services/document-updater/docker-compose.ci.yml @@ -55,7 +55,7 @@ services: retries: 20 mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/document-updater/docker-compose.yml b/services/document-updater/docker-compose.yml index c1b23c11c5..cf7c9a2eb6 100644 --- a/services/document-updater/docker-compose.yml +++ b/services/document-updater/docker-compose.yml @@ -57,7 +57,7 @@ services: retries: 20 mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/history-v1/docker-compose.ci.yml b/services/history-v1/docker-compose.ci.yml index cf6ec3357d..da664d6b30 100644 --- a/services/history-v1/docker-compose.ci.yml +++ b/services/history-v1/docker-compose.ci.yml @@ -75,7 +75,7 @@ services: retries: 20 mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/history-v1/docker-compose.yml b/services/history-v1/docker-compose.yml index 3a33882d28..22b739abf9 100644 --- a/services/history-v1/docker-compose.yml +++ b/services/history-v1/docker-compose.yml @@ -83,7 +83,7 @@ services: retries: 20 mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/history-v1/storage/scripts/back_fill_file_hash.mjs b/services/history-v1/storage/scripts/back_fill_file_hash.mjs index 2e12328e5c..0ccadaf5a9 100644 --- a/services/history-v1/storage/scripts/back_fill_file_hash.mjs +++ b/services/history-v1/storage/scripts/back_fill_file_hash.mjs @@ -150,6 +150,10 @@ const CONCURRENT_BATCHES = parseInt(process.env.CONCURRENT_BATCHES || '2', 10) const RETRIES = parseInt(process.env.RETRIES || '10', 10) const RETRY_DELAY_MS = parseInt(process.env.RETRY_DELAY_MS || '100', 10) +const USER_FILES_BUCKET_NAME = process.env.USER_FILES_BUCKET_NAME || '' +if (!USER_FILES_BUCKET_NAME) { + throw new Error('env var USER_FILES_BUCKET_NAME is missing') +} const RETRY_FILESTORE_404 = process.env.RETRY_FILESTORE_404 === 'true' const BUFFER_DIR = fs.mkdtempSync( process.env.BUFFER_DIR_PREFIX || '/tmp/back_fill_file_hash-' diff --git a/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs b/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs index 2525ee1d6e..7bab794692 100644 --- a/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs +++ b/services/history-v1/storage/scripts/back_fill_file_hash_fix_up.mjs @@ -9,12 +9,15 @@ import { Blob } from 'overleaf-editor-core' import { BlobStore, getStringLengthOfFile, + GLOBAL_BLOBS, makeBlobForFile, } from '../lib/blob_store/index.js' import { db } from '../lib/mongodb.js' import commandLineArgs from 'command-line-args' import readline from 'node:readline' +import { _blobIsBackedUp, backupBlob } from '../lib/backupBlob.mjs' import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js' +import filestorePersistor from '../lib/persistor.js' import { setTimeout } from 'node:timers/promises' // Silence warning. @@ -49,11 +52,12 @@ ObjectId.cacheHexString = true */ /** - * @return {{FIX_NOT_FOUND: boolean, FIX_HASH_MISMATCH: boolean, FIX_MISSING_HASH: boolean, LOGS: string}} + * @return {{FIX_NOT_FOUND: boolean, FIX_HASH_MISMATCH: boolean, FIX_DELETE_PERMISSION: boolean, FIX_MISSING_HASH: boolean, LOGS: string}} */ function parseArgs() { const args = commandLineArgs([ { name: 'fixNotFound', type: String, defaultValue: 'true' }, + { name: 'fixDeletePermission', type: String, defaultValue: 'true' }, { name: 'fixHashMismatch', type: String, defaultValue: 'true' }, { name: 'fixMissingHash', type: String, defaultValue: 'true' }, { name: 'logs', type: String, defaultValue: '' }, @@ -70,13 +74,20 @@ function parseArgs() { } return { FIX_HASH_MISMATCH: boolVal('fixNotFound'), + FIX_DELETE_PERMISSION: boolVal('fixDeletePermission'), FIX_NOT_FOUND: boolVal('fixHashMismatch'), FIX_MISSING_HASH: boolVal('fixMissingHash'), LOGS: args.logs, } } -const { FIX_HASH_MISMATCH, FIX_NOT_FOUND, FIX_MISSING_HASH, LOGS } = parseArgs() +const { + FIX_HASH_MISMATCH, + FIX_DELETE_PERMISSION, + FIX_NOT_FOUND, + FIX_MISSING_HASH, + LOGS, +} = parseArgs() if (!LOGS) { throw new Error('--logs parameter missing') } @@ -94,37 +105,6 @@ const STREAM_HIGH_WATER_MARK = parseInt( ) const SLEEP_BEFORE_EXIT = parseInt(process.env.SLEEP_BEFORE_EXIT || '1000', 10) -// Filestore endpoint location -const FILESTORE_HOST = process.env.FILESTORE_HOST || '127.0.0.1' -const FILESTORE_PORT = process.env.FILESTORE_PORT || '3009' - -async function fetchFromFilestore(projectId, fileId) { - const url = `http://${FILESTORE_HOST}:${FILESTORE_PORT}/project/${projectId}/file/${fileId}` - const response = await fetch(url) - if (!response.ok) { - if (response.status === 404) { - throw new NotFoundError('file not found in filestore', { - status: response.status, - }) - } - const body = await response.text() - throw new OError('fetchFromFilestore failed', { - projectId, - fileId, - status: response.status, - body, - }) - } - if (!response.body) { - throw new OError('fetchFromFilestore response has no body', { - projectId, - fileId, - status: response.status, - }) - } - return response.body -} - /** @type {ProjectsCollection} */ const projectsCollection = db.collection('projects') /** @type {DeletedProjectsCollection} */ @@ -322,16 +302,19 @@ async function setHashInMongo(projectId, fileId, hash) { * @return {Promise} */ async function importRestoredFilestoreFile(projectId, fileId, historyId) { + const filestoreKey = `${projectId}/${fileId}` const path = `${BUFFER_DIR}/${projectId}_${fileId}` try { let s try { - s = await fetchFromFilestore(projectId, fileId) + s = await filestorePersistor.getObjectStream( + USER_FILES_BUCKET_NAME, + filestoreKey + ) } catch (err) { if (err instanceof NotFoundError) { throw new OError('missing blob, need to restore filestore file', { - projectId, - fileId, + filestoreKey, }) } throw err @@ -342,6 +325,7 @@ async function importRestoredFilestoreFile(projectId, fileId, historyId) { ) const blobStore = new BlobStore(historyId) const blob = await blobStore.putFile(path) + await backupBlob(historyId, blob, path) await setHashInMongo(projectId, fileId, blob.getHash()) } finally { await fs.promises.rm(path, { force: true }) @@ -355,9 +339,13 @@ async function importRestoredFilestoreFile(projectId, fileId, historyId) { * @return {Promise} */ async function bufferFilestoreFileToDisk(projectId, fileId, path) { + const filestoreKey = `${projectId}/${fileId}` try { await Stream.promises.pipeline( - await fetchFromFilestore(projectId, fileId), + await filestorePersistor.getObjectStream( + USER_FILES_BUCKET_NAME, + filestoreKey + ), fs.createWriteStream(path, { highWaterMark: STREAM_HIGH_WATER_MARK }) ) const blob = await makeBlobForFile(path) @@ -368,8 +356,7 @@ async function bufferFilestoreFileToDisk(projectId, fileId, path) { } catch (err) { if (err instanceof NotFoundError) { throw new OError('missing blob, need to restore filestore file', { - projectId, - fileId, + filestoreKey, }) } throw err @@ -402,7 +389,7 @@ async function uploadFilestoreFile(projectId, fileId) { const blob = await bufferFilestoreFileToDisk(projectId, fileId, path) const hash = blob.getHash() try { - await ensureBlobExistsForFile(projectId, fileId, hash) + await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) } catch (err) { if (!(err instanceof Blob.NotFoundError)) throw err @@ -410,7 +397,7 @@ async function uploadFilestoreFile(projectId, fileId) { const historyId = project.overleaf.history.id.toString() const blobStore = new BlobStore(historyId) await blobStore.putBlob(path, blob) - await ensureBlobExistsForFile(projectId, fileId, hash) + await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) } } finally { await fs.promises.rm(path, { force: true }) @@ -439,7 +426,11 @@ async function fixHashMismatch(line) { await importRestoredFilestoreFile(projectId, fileId, historyId) return true } - return await ensureBlobExistsForFile(projectId, fileId, computedHash) + return await ensureBlobExistsForFileAndUploadToAWS( + projectId, + fileId, + computedHash + ) } /** @@ -453,19 +444,30 @@ async function hashAlreadyUpdatedInFileTree(projectId, fileId, hash) { return fileRef.hash === hash } +/** + * @param {string} projectId + * @param {string} hash + * @return {Promise} + */ +async function needsBackingUpToAWS(projectId, hash) { + if (GLOBAL_BLOBS.has(hash)) return false + return !(await _blobIsBackedUp(projectId, hash)) +} + /** * @param {string} projectId * @param {string} fileId * @param {string} hash * @return {Promise} */ -async function ensureBlobExistsForFile(projectId, fileId, hash) { +async function ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) { const { project } = await getProject(projectId) const historyId = project.overleaf.history.id.toString() const blobStore = new BlobStore(historyId) if ( (await hashAlreadyUpdatedInFileTree(projectId, fileId, hash)) && - (await blobStore.getBlob(hash)) + (await blobStore.getBlob(hash)) && + !(await needsBackingUpToAWS(projectId, hash)) ) { return false // already processed } @@ -486,7 +488,7 @@ async function ensureBlobExistsForFile(projectId, fileId, hash) { ) if (writtenBlob.getHash() !== hash) { // Double check download, better safe than sorry. - throw new OError('blob corrupted', { writtenBlob, hash }) + throw new OError('blob corrupted', { writtenBlob }) } let blob = await blobStore.getBlob(hash) @@ -495,6 +497,7 @@ async function ensureBlobExistsForFile(projectId, fileId, hash) { // HACK: Skip upload to GCS and finalize putBlob operation directly. await blobStore.backend.insertBlob(historyId, writtenBlob) } + await backupBlob(historyId, writtenBlob, path) } finally { await fs.promises.rm(path, { force: true }) } @@ -502,6 +505,16 @@ async function ensureBlobExistsForFile(projectId, fileId, hash) { return true } +/** + * @param {string} line + * @return {Promise} + */ +async function fixDeletePermission(line) { + let { projectId, fileId, hash } = JSON.parse(line) + if (!hash) hash = await computeFilestoreFileHash(projectId, fileId) + return await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) +} + /** * @param {string} line * @return {Promise} @@ -513,7 +526,7 @@ async function fixMissingHash(line) { } = await findFile(projectId, fileId) if (hash) { // processed, double check - return await ensureBlobExistsForFile(projectId, fileId, hash) + return await ensureBlobExistsForFileAndUploadToAWS(projectId, fileId, hash) } await uploadFilestoreFile(projectId, fileId) return true @@ -530,6 +543,11 @@ const CASES = { flag: FIX_HASH_MISMATCH, action: fixHashMismatch, }, + 'delete permission': { + match: 'storage.objects.delete', + flag: FIX_DELETE_PERMISSION, + action: fixDeletePermission, + }, 'missing file hash': { match: '"bad file hash"', flag: FIX_MISSING_HASH, diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs index b6cdd4b9bf..62b0b1de25 100644 --- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs +++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash.test.mjs @@ -20,7 +20,7 @@ import { makeProjectKey, } from '../../../../storage/lib/blob_store/index.js' -import { mockFilestore } from './support/MockFilestore.mjs' +import express from 'express' chai.use(chaiExclude) const TIMEOUT = 20 * 1_000 @@ -28,6 +28,59 @@ const TIMEOUT = 20 * 1_000 const projectsCollection = db.collection('projects') const deletedProjectsCollection = db.collection('deletedProjects') +class MockFilestore { + constructor() { + this.host = process.env.FILESTORE_HOST || '127.0.0.1' + this.port = process.env.FILESTORE_PORT || 3009 + // create a server listening on this.host and this.port + this.files = {} + + this.app = express() + + this.app.get('/project/:projectId/file/:fileId', (req, res) => { + const { projectId, fileId } = req.params + const content = this.files[projectId]?.[fileId] + if (!content) return res.status(404).end() + res.status(200).end(content) + }) + } + + start() { + // reset stored files + this.files = {} + // start the server + if (this.serverPromise) { + return this.serverPromise + } else { + this.serverPromise = new Promise((resolve, reject) => { + this.server = this.app.listen(this.port, this.host, err => { + if (err) return reject(err) + resolve() + }) + }) + return this.serverPromise + } + } + + addFile(projectId, fileId, fileContent) { + if (!this.files[projectId]) { + this.files[projectId] = {} + } + this.files[projectId][fileId] = fileContent + } + + deleteObject(projectId, fileId) { + if (this.files[projectId]) { + delete this.files[projectId][fileId] + if (Object.keys(this.files[projectId]).length === 0) { + delete this.files[projectId] + } + } + } +} + +const mockFilestore = new MockFilestore() + /** * @param {ObjectId} objectId * @return {string} diff --git a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs index 3aa00d685a..ceafa24c3a 100644 --- a/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs +++ b/services/history-v1/test/acceptance/js/storage/back_fill_file_hash_fix_up.test.mjs @@ -1,24 +1,48 @@ import fs from 'node:fs' import Crypto from 'node:crypto' +import Stream from 'node:stream' import { promisify } from 'node:util' import { Binary, ObjectId } from 'mongodb' import { Blob } from 'overleaf-editor-core' -import { db } from '../../../../storage/lib/mongodb.js' +import { backedUpBlobs, blobs, db } from '../../../../storage/lib/mongodb.js' import cleanup from './support/cleanup.js' import testProjects from '../api/support/test_projects.js' import { execFile } from 'node:child_process' import chai, { expect } from 'chai' import chaiExclude from 'chai-exclude' -import { BlobStore } from '../../../../storage/lib/blob_store/index.js' -import { mockFilestore } from './support/MockFilestore.mjs' +import config from 'config' +import { WritableBuffer } from '@overleaf/stream-utils' +import { + backupPersistor, + projectBlobsBucket, +} from '../../../../storage/lib/backupPersistor.mjs' +import projectKey from '../../../../storage/lib/project_key.js' +import { + BlobStore, + makeProjectKey, +} from '../../../../storage/lib/blob_store/index.js' +import ObjectPersistor from '@overleaf/object-persistor' chai.use(chaiExclude) const TIMEOUT = 20 * 1_000 +const { deksBucket } = config.get('backupStore') +const { tieringStorageClass } = config.get('backupPersistor') + const projectsCollection = db.collection('projects') const deletedProjectsCollection = db.collection('deletedProjects') +const FILESTORE_PERSISTOR = ObjectPersistor({ + backend: 'gcs', + gcs: { + endpoint: { + apiEndpoint: process.env.GCS_API_ENDPOINT, + projectId: process.env.GCS_PROJECT_ID, + }, + }, +}) + /** * @param {ObjectId} objectId * @return {string} @@ -46,6 +70,17 @@ function binaryForGitBlobHash(gitBlobHash) { return new Binary(Buffer.from(gitBlobHash, 'hex')) } +async function listS3Bucket(bucket, wantStorageClass) { + const client = backupPersistor._getClientForBucket(bucket) + const response = await client.listObjectsV2({ Bucket: bucket }).promise() + + for (const object of response.Contents || []) { + expect(object).to.have.property('StorageClass', wantStorageClass) + } + + return (response.Contents || []).map(item => item.Key || '') +} + function objectIdFromTime(timestamp) { return ObjectId.createFromTime(new Date(timestamp).getTime() / 1000) } @@ -62,6 +97,7 @@ describe('back_fill_file_hash_fix_up script', function () { const historyIdDeleted0 = projectIdDeleted0.toString() const fileIdWithDifferentHashFound = objectIdFromTime('2017-02-01T00:00:00Z') const fileIdInGoodState = objectIdFromTime('2017-02-01T00:01:00Z') + const fileIdBlobExistsInGCS0 = objectIdFromTime('2017-02-01T00:02:00Z') const fileIdWithDifferentHashNotFound0 = objectIdFromTime( '2017-02-01T00:03:00Z' ) @@ -76,6 +112,9 @@ describe('back_fill_file_hash_fix_up script', function () { const fileIdWithDifferentHashRestore = objectIdFromTime( '2017-02-01T00:08:00Z' ) + const fileIdBlobExistsInGCS1 = objectIdFromTime('2017-02-01T00:09:00Z') + const fileIdRestoreFromFilestore0 = objectIdFromTime('2017-02-01T00:10:00Z') + const fileIdRestoreFromFilestore1 = objectIdFromTime('2017-02-01T00:11:00Z') const fileIdMissing2 = objectIdFromTime('2017-02-01T00:12:00Z') const fileIdHashMissing0 = objectIdFromTime('2017-02-01T00:13:00Z') const fileIdHashMissing1 = objectIdFromTime('2017-02-01T00:14:00Z') @@ -86,11 +125,31 @@ describe('back_fill_file_hash_fix_up script', function () { ) const deleteProjectsRecordId0 = new ObjectId() const writtenBlobs = [ + { + projectId: projectId0, + historyId: historyId0, + fileId: fileIdBlobExistsInGCS0, + }, + { + projectId: projectId0, + historyId: historyId0, + fileId: fileIdBlobExistsInGCS1, + }, { projectId: projectId0, historyId: historyId0, fileId: fileIdWithDifferentHashNotFound0, }, + { + projectId: projectId0, + historyId: historyId0, + fileId: fileIdRestoreFromFilestore0, + }, + { + projectId: projectId0, + historyId: historyId0, + fileId: fileIdRestoreFromFilestore1, + }, { projectId: projectId0, historyId: historyId0, @@ -141,6 +200,17 @@ describe('back_fill_file_hash_fix_up script', function () { }, msg: 'failed to process file', }, + { + projectId: projectId0, + fileId: fileIdRestoreFromFilestore0, + err: { message: 'OError: hash mismatch' }, + hash: gitBlobHash(fileIdRestoreFromFilestore0), + entry: { + ctx: { historyId: historyId0.toString() }, + hash: hashDoesNotExistAsBlob, + }, + msg: 'failed to process file', + }, { projectId: projectIdDeleted0, fileId: fileIdWithDifferentHashNotFound1, @@ -166,6 +236,33 @@ describe('back_fill_file_hash_fix_up script', function () { err: { message: 'NotFoundError' }, msg: 'failed to process file', }, + { + projectId: projectId0, + fileId: fileIdBlobExistsInGCS0, + hash: gitBlobHash(fileIdBlobExistsInGCS0), + err: { message: 'storage.objects.delete' }, + msg: 'failed to process file', + }, + { + projectId: projectId0, + fileId: fileIdBlobExistsInGCSCorrupted, + hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted), + err: { message: 'storage.objects.delete' }, + msg: 'failed to process file', + }, + { + projectId: projectId0, + fileId: fileIdBlobExistsInGCS1, + hash: gitBlobHash(fileIdBlobExistsInGCS1), + err: { message: 'storage.objects.delete' }, + msg: 'failed to process file', + }, + { + projectId: projectId0, + fileId: fileIdRestoreFromFilestore1, + err: { message: 'storage.objects.delete' }, + msg: 'failed to process file', + }, { projectId: projectIdDeleted0, fileId: fileIdMissing1, @@ -194,23 +291,22 @@ describe('back_fill_file_hash_fix_up script', function () { reason: 'bad file hash', msg: 'bad file-tree path', }, - { - projectId: projectId0, - _id: fileIdBlobExistsInGCSCorrupted, - reason: 'bad file hash', - msg: 'bad file-tree path', - }, ] if (PRINT_IDS_AND_HASHES_FOR_DEBUGGING) { const fileIds = { fileIdWithDifferentHashFound, fileIdInGoodState, + fileIdBlobExistsInGCS0, + fileIdBlobExistsInGCS1, fileIdWithDifferentHashNotFound0, fileIdWithDifferentHashNotFound1, + fileIdBlobExistsInGCSCorrupted, fileIdMissing0, fileIdMissing1, fileIdMissing2, fileIdWithDifferentHashRestore, + fileIdRestoreFromFilestore0, + fileIdRestoreFromFilestore1, fileIdHashMissing0, fileIdHashMissing1, } @@ -234,25 +330,38 @@ describe('back_fill_file_hash_fix_up script', function () { before(cleanup.everything) before('populate blobs/GCS', async function () { - await mockFilestore.start() - mockFilestore.addFile( - projectId0, - fileIdHashMissing0, - fileIdHashMissing0.toString() + await FILESTORE_PERSISTOR.sendStream( + USER_FILES_BUCKET_NAME, + `${projectId0}/${fileIdRestoreFromFilestore0}`, + Stream.Readable.from([fileIdRestoreFromFilestore0.toString()]) ) - mockFilestore.addFile( - projectId0, - fileIdHashMissing1, - fileIdHashMissing1.toString() + await FILESTORE_PERSISTOR.sendStream( + USER_FILES_BUCKET_NAME, + `${projectId0}/${fileIdRestoreFromFilestore1}`, + Stream.Readable.from([fileIdRestoreFromFilestore1.toString()]) ) - mockFilestore.addFile( - projectId0, - fileIdBlobExistsInGCSCorrupted, - fileIdBlobExistsInGCSCorrupted.toString() + await FILESTORE_PERSISTOR.sendStream( + USER_FILES_BUCKET_NAME, + `${projectId0}/${fileIdHashMissing0}`, + Stream.Readable.from([fileIdHashMissing0.toString()]) + ) + await FILESTORE_PERSISTOR.sendStream( + USER_FILES_BUCKET_NAME, + `${projectId0}/${fileIdHashMissing1}`, + Stream.Readable.from([fileIdHashMissing1.toString()]) ) await new BlobStore(historyId0.toString()).putString( fileIdHashMissing1.toString() // partially processed ) + await new BlobStore(historyId0.toString()).putString( + fileIdBlobExistsInGCS0.toString() + ) + await new BlobStore(historyId0.toString()).putString( + fileIdBlobExistsInGCS1.toString() + ) + await new BlobStore(historyId0.toString()).putString( + fileIdRestoreFromFilestore1.toString() + ) const path = '/tmp/test-blob-corrupted' try { await fs.promises.writeFile(path, contentCorruptedBlob) @@ -317,10 +426,22 @@ describe('back_fill_file_hash_fix_up script', function () { _id: fileIdWithDifferentHashNotFound0, hash: hashDoesNotExistAsBlob, }, + { + _id: fileIdRestoreFromFilestore0, + hash: hashDoesNotExistAsBlob, + }, + { + _id: fileIdRestoreFromFilestore1, + }, + { + _id: fileIdBlobExistsInGCS0, + hash: gitBlobHash(fileIdBlobExistsInGCS0), + }, { _id: fileIdBlobExistsInGCSCorrupted, hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted), }, + { _id: fileIdBlobExistsInGCS1 }, ], folders: [], }, @@ -425,8 +546,8 @@ describe('back_fill_file_hash_fix_up script', function () { }) it('should print stats', function () { expect(stats).to.contain({ - processedLines: 12, - success: 7, + processedLines: 16, + success: 11, alreadyProcessed: 0, fileDeleted: 0, skipped: 0, @@ -437,9 +558,9 @@ describe('back_fill_file_hash_fix_up script', function () { it('should handle re-run on same logs', async function () { ;({ stats } = await runScriptWithLogs()) expect(stats).to.contain({ - processedLines: 12, + processedLines: 16, success: 0, - alreadyProcessed: 4, + alreadyProcessed: 8, fileDeleted: 3, skipped: 0, failed: 3, @@ -542,11 +663,31 @@ describe('back_fill_file_hash_fix_up script', function () { _id: fileIdWithDifferentHashNotFound0, hash: gitBlobHash(fileIdWithDifferentHashNotFound0), }, + // Updated hash + { + _id: fileIdRestoreFromFilestore0, + hash: gitBlobHash(fileIdRestoreFromFilestore0), + }, + // Added hash + { + _id: fileIdRestoreFromFilestore1, + hash: gitBlobHash(fileIdRestoreFromFilestore1), + }, + // No change, blob created + { + _id: fileIdBlobExistsInGCS0, + hash: gitBlobHash(fileIdBlobExistsInGCS0), + }, // No change, flagged { _id: fileIdBlobExistsInGCSCorrupted, hash: gitBlobHash(fileIdBlobExistsInGCSCorrupted), }, + // Added hash + { + _id: fileIdBlobExistsInGCS1, + hash: gitBlobHash(fileIdBlobExistsInGCS1), + }, ], folders: [], }, @@ -555,7 +696,7 @@ describe('back_fill_file_hash_fix_up script', function () { ], overleaf: { history: { id: historyId0 } }, // Incremented when removing file/updating hash - version: 5, + version: 8, }, ]) expect(await deletedProjectsCollection.find({}).toArray()).to.deep.equal([ @@ -604,6 +745,62 @@ describe('back_fill_file_hash_fix_up script', function () { (writtenBlobsByProject.get(projectId) || []).concat([fileId]) ) } + expect( + (await backedUpBlobs.find({}, { sort: { _id: 1 } }).toArray()).map( + entry => { + // blobs are pushed unordered into mongo. Sort the list for consistency. + entry.blobs.sort() + return entry + } + ) + ).to.deep.equal( + Array.from(writtenBlobsByProject.entries()).map( + ([projectId, fileIds]) => { + return { + _id: projectId, + blobs: fileIds + .map(fileId => binaryForGitBlobHash(gitBlobHash(fileId))) + .sort(), + } + } + ) + ) + }) + it('should have backed up all the files', async function () { + expect(tieringStorageClass).to.exist + const objects = await listS3Bucket(projectBlobsBucket, tieringStorageClass) + expect(objects.sort()).to.deep.equal( + writtenBlobs + .map(({ historyId, fileId, hash }) => + makeProjectKey(historyId, hash || gitBlobHash(fileId)) + ) + .sort() + ) + for (let { historyId, fileId } of writtenBlobs) { + const hash = gitBlobHash(fileId.toString()) + const s = await backupPersistor.getObjectStream( + projectBlobsBucket, + makeProjectKey(historyId, hash), + { autoGunzip: true } + ) + const buf = new WritableBuffer() + await Stream.promises.pipeline(s, buf) + expect(gitBlobHashBuffer(buf.getContents())).to.equal(hash) + const id = buf.getContents().toString('utf-8') + expect(id).to.equal(fileId.toString()) + // double check we are not comparing 'undefined' or '[object Object]' above + expect(id).to.match(/^[a-f0-9]{24}$/) + } + const deks = await listS3Bucket(deksBucket, 'STANDARD') + expect(deks.sort()).to.deep.equal( + Array.from( + new Set( + writtenBlobs.map( + ({ historyId }) => projectKey.format(historyId) + '/dek' + ) + ) + ).sort() + ) }) it('should have written the back filled files to history v1', async function () { for (const { historyId, fileId } of writtenBlobs) { diff --git a/services/history-v1/test/acceptance/js/storage/support/MockFilestore.mjs b/services/history-v1/test/acceptance/js/storage/support/MockFilestore.mjs deleted file mode 100644 index 55d0923c34..0000000000 --- a/services/history-v1/test/acceptance/js/storage/support/MockFilestore.mjs +++ /dev/null @@ -1,54 +0,0 @@ -import express from 'express' - -class MockFilestore { - constructor() { - this.host = process.env.FILESTORE_HOST || '127.0.0.1' - this.port = process.env.FILESTORE_PORT || 3009 - // create a server listening on this.host and this.port - this.files = {} - - this.app = express() - - this.app.get('/project/:projectId/file/:fileId', (req, res) => { - const { projectId, fileId } = req.params - const content = this.files[projectId]?.[fileId] - if (!content) return res.status(404).end() - res.status(200).end(content) - }) - } - - start() { - // reset stored files - this.files = {} - // start the server - if (this.serverPromise) { - return this.serverPromise - } else { - this.serverPromise = new Promise((resolve, reject) => { - this.server = this.app.listen(this.port, this.host, err => { - if (err) return reject(err) - resolve() - }) - }) - return this.serverPromise - } - } - - addFile(projectId, fileId, fileContent) { - if (!this.files[projectId]) { - this.files[projectId] = {} - } - this.files[projectId][fileId] = fileContent - } - - deleteObject(projectId, fileId) { - if (this.files[projectId]) { - delete this.files[projectId][fileId] - if (Object.keys(this.files[projectId]).length === 0) { - delete this.files[projectId] - } - } - } -} - -export const mockFilestore = new MockFilestore() diff --git a/services/notifications/docker-compose.ci.yml b/services/notifications/docker-compose.ci.yml index ca3303a079..24b57ab084 100644 --- a/services/notifications/docker-compose.ci.yml +++ b/services/notifications/docker-compose.ci.yml @@ -42,7 +42,7 @@ services: command: tar -czf /tmp/build/build.tar.gz --exclude=build.tar.gz --exclude-vcs . user: root mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/notifications/docker-compose.yml b/services/notifications/docker-compose.yml index e43e9aeef5..081bbfa002 100644 --- a/services/notifications/docker-compose.yml +++ b/services/notifications/docker-compose.yml @@ -44,7 +44,7 @@ services: command: npm run --silent test:acceptance mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/project-history/docker-compose.ci.yml b/services/project-history/docker-compose.ci.yml index c6ec24a84b..ca15f35fef 100644 --- a/services/project-history/docker-compose.ci.yml +++ b/services/project-history/docker-compose.ci.yml @@ -55,7 +55,7 @@ services: retries: 20 mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/project-history/docker-compose.yml b/services/project-history/docker-compose.yml index dd3c6468fe..eeca03de6e 100644 --- a/services/project-history/docker-compose.yml +++ b/services/project-history/docker-compose.yml @@ -57,7 +57,7 @@ services: retries: 20 mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/web/app.mjs b/services/web/app.mjs index 3f54cc36a8..b7c723da3d 100644 --- a/services/web/app.mjs +++ b/services/web/app.mjs @@ -56,8 +56,14 @@ if (Settings.catchErrors) { // Create ./data/dumpFolder if needed FileWriter.ensureDumpFolderExists() -// Validate combination of feature flags. -Features.validateSettings() +if ( + !Features.hasFeature('project-history-blobs') && + !Features.hasFeature('filestore') +) { + throw new Error( + 'invalid config: must enable either project-history-blobs (Settings.enableProjectHistoryBlobs=true) or enable filestore (Settings.disableFilestore=false)' + ) +} // handle SIGTERM for graceful shutdown in kubernetes process.on('SIGTERM', function (signal) { diff --git a/services/web/app/src/Features/Authentication/AuthenticationController.js b/services/web/app/src/Features/Authentication/AuthenticationController.js index 99c418df1b..7a97d2ac9c 100644 --- a/services/web/app/src/Features/Authentication/AuthenticationController.js +++ b/services/web/app/src/Features/Authentication/AuthenticationController.js @@ -36,22 +36,7 @@ function send401WithChallenge(res) { function checkCredentials(userDetailsMap, user, password) { const expectedPassword = userDetailsMap.get(user) const userExists = userDetailsMap.has(user) && expectedPassword // user exists with a non-null password - - let isValid = false - if (userExists) { - if (Array.isArray(expectedPassword)) { - const isValidPrimary = Boolean( - expectedPassword[0] && tsscmp(expectedPassword[0], password) - ) - const isValidFallback = Boolean( - expectedPassword[1] && tsscmp(expectedPassword[1], password) - ) - isValid = isValidPrimary || isValidFallback - } else { - isValid = tsscmp(expectedPassword, password) - } - } - + const isValid = userExists && tsscmp(expectedPassword, password) if (!isValid) { logger.err({ user }, 'invalid login details') } diff --git a/services/web/app/src/Features/Documents/DocumentController.mjs b/services/web/app/src/Features/Documents/DocumentController.mjs index 9a16811894..6998c0b36a 100644 --- a/services/web/app/src/Features/Documents/DocumentController.mjs +++ b/services/web/app/src/Features/Documents/DocumentController.mjs @@ -7,7 +7,6 @@ import logger from '@overleaf/logger' import _ from 'lodash' import { plainTextResponse } from '../../infrastructure/Response.js' import { expressify } from '@overleaf/promise-utils' -import Modules from '../../infrastructure/Modules.js' async function getDocument(req, res) { const { Project_id: projectId, doc_id: docId } = req.params @@ -93,9 +92,6 @@ async function setDocument(req, res) { { docId, projectId }, 'finished receiving set document request from api (docupdater)' ) - - await Modules.promises.hooks.fire('docModified', projectId, docId) - res.json(result) } diff --git a/services/web/app/src/Features/History/HistoryURLHelper.js b/services/web/app/src/Features/History/HistoryURLHelper.js index acb43ced68..8b8d8cbdd7 100644 --- a/services/web/app/src/Features/History/HistoryURLHelper.js +++ b/services/web/app/src/Features/History/HistoryURLHelper.js @@ -8,7 +8,7 @@ function projectHistoryURLWithFilestoreFallback( ) { const filestoreURL = `${Settings.apis.filestore.url}/project/${projectId}/file/${fileRef._id}?from=${origin}` // TODO: When this file is converted to ES modules we will be able to use Features.hasFeature('project-history-blobs'). Currently we can't stub the feature return value in tests. - if (fileRef.hash && Settings.filestoreMigrationLevel >= 1) { + if (fileRef.hash && Settings.enableProjectHistoryBlobs) { return { url: `${Settings.apis.project_history.url}/project/${historyId}/blob/${fileRef.hash}`, fallbackURL: filestoreURL, diff --git a/services/web/app/src/Features/Uploads/ProjectUploadController.mjs b/services/web/app/src/Features/Uploads/ProjectUploadController.mjs index 84b8738af3..a3bc434ed7 100644 --- a/services/web/app/src/Features/Uploads/ProjectUploadController.mjs +++ b/services/web/app/src/Features/Uploads/ProjectUploadController.mjs @@ -66,7 +66,7 @@ function uploadProject(req, res, next) { async function uploadFile(req, res, next) { const timer = new metrics.Timer('file-upload') const name = req.body.name - const { path } = req.file + const path = req.file?.path const projectId = req.params.Project_id const userId = SessionManager.getLoggedInUserId(req.session) let { folder_id: folderId } = req.query @@ -162,14 +162,8 @@ function multerMiddleware(req, res, next) { .status(422) .json({ success: false, error: req.i18n.translate('file_too_large') }) } - if (err) return next(err) - if (!req.file?.path) { - logger.info({ req }, 'missing req.file.path on upload') - return res - .status(400) - .json({ success: false, error: 'invalid_upload_request' }) - } - next() + + return next(err) }) } diff --git a/services/web/app/src/infrastructure/Features.js b/services/web/app/src/infrastructure/Features.js index 6147e70e0f..aaf51103b9 100644 --- a/services/web/app/src/infrastructure/Features.js +++ b/services/web/app/src/infrastructure/Features.js @@ -19,7 +19,8 @@ const trackChangesModuleAvailable = * @property {boolean | undefined} enableGithubSync * @property {boolean | undefined} enableGitBridge * @property {boolean | undefined} enableHomepage - * @property {number} filestoreMigrationLevel + * @property {boolean | undefined} enableProjectHistoryBlobs + * @property {boolean | undefined} disableFilestore * @property {boolean | undefined} enableSaml * @property {boolean | undefined} ldap * @property {boolean | undefined} oauth @@ -29,14 +30,6 @@ const trackChangesModuleAvailable = */ const Features = { - validateSettings() { - if (![0, 1, 2].includes(Settings.filestoreMigrationLevel)) { - throw new Error( - `invalid OVERLEAF_FILESTORE_MIGRATION_LEVEL=${Settings.filestoreMigrationLevel}, expected 0, 1 or 2` - ) - } - }, - /** * @returns {boolean} */ @@ -96,9 +89,9 @@ const Features = { Settings.enabledLinkedFileTypes.includes('url') ) case 'project-history-blobs': - return Settings.filestoreMigrationLevel > 0 + return Boolean(Settings.enableProjectHistoryBlobs) case 'filestore': - return Settings.filestoreMigrationLevel < 2 + return Boolean(Settings.disableFilestore) === false case 'support': return supportModuleAvailable case 'symbol-palette': diff --git a/services/web/app/src/infrastructure/Modules.js b/services/web/app/src/infrastructure/Modules.js index aea3aeb087..20975a3642 100644 --- a/services/web/app/src/infrastructure/Modules.js +++ b/services/web/app/src/infrastructure/Modules.js @@ -150,7 +150,8 @@ async function linkedFileAgentsIncludes() { async function attachHooks() { for (const module of await modules()) { const { promises, ...hooks } = module.hooks || {} - for (const [hook, method] of Object.entries(promises || {})) { + for (const hook in promises || {}) { + const method = promises[hook] attachHook(hook, method) } for (const hook in hooks || {}) { diff --git a/services/web/app/views/_cookie_banner.pug b/services/web/app/views/_cookie_banner.pug index 7cbc569bc1..56974326cd 100644 --- a/services/web/app/views/_cookie_banner.pug +++ b/services/web/app/views/_cookie_banner.pug @@ -1,13 +1,13 @@ -section.cookie-banner.hidden-print.hidden(aria-label=translate('cookie_banner')) - .cookie-banner-content !{translate('cookie_banner_info', {}, [{ name: 'a', attrs: { href: '/legal#Cookies' }}])} +section.cookie-banner.hidden-print.hidden(aria-label='Cookie banner') + .cookie-banner-content We only use cookies for essential purposes and to improve your experience on our site. You can find out more in our cookie policy. .cookie-banner-actions button( type='button' class='btn btn-link btn-sm' data-ol-cookie-banner-set-consent='essential' - ) #{translate('essential_cookies_only')} + ) Essential cookies only button( type='button' class='btn btn-primary btn-sm' data-ol-cookie-banner-set-consent='all' - ) #{translate('accept_all_cookies')} + ) Accept all cookies diff --git a/services/web/app/views/general/post-gateway.pug b/services/web/app/views/general/post-gateway.pug index 86f379ac1b..c6bbc92d01 100644 --- a/services/web/app/views/general/post-gateway.pug +++ b/services/web/app/views/general/post-gateway.pug @@ -4,7 +4,7 @@ block vars - var suppressNavbar = true - var suppressFooter = true - var suppressSkipToContent = true - - var suppressPugCookieBanner = true + - var suppressCookieBanner = true block content .content.content-alt diff --git a/services/web/app/views/layout-marketing.pug b/services/web/app/views/layout-marketing.pug index 26e4eb539d..b54c30f033 100644 --- a/services/web/app/views/layout-marketing.pug +++ b/services/web/app/views/layout-marketing.pug @@ -24,7 +24,7 @@ block body else include layout/fat-footer - if typeof suppressPugCookieBanner == 'undefined' + if typeof suppressCookieBanner == 'undefined' include _cookie_banner if bootstrapVersion === 5 diff --git a/services/web/app/views/layout-react.pug b/services/web/app/views/layout-react.pug index e9c4c932c4..94ff3ba247 100644 --- a/services/web/app/views/layout-react.pug +++ b/services/web/app/views/layout-react.pug @@ -69,5 +69,5 @@ block body else include layout/fat-footer-react-bootstrap-5 - if typeof suppressPugCookieBanner === 'undefined' + if typeof suppressCookieBanner === 'undefined' include _cookie_banner diff --git a/services/web/app/views/layout-website-redesign.pug b/services/web/app/views/layout-website-redesign.pug index aa7fea9f07..61ed83043b 100644 --- a/services/web/app/views/layout-website-redesign.pug +++ b/services/web/app/views/layout-website-redesign.pug @@ -27,7 +27,7 @@ block body else include layout/fat-footer-website-redesign - if typeof suppressPugCookieBanner == 'undefined' + if typeof suppressCookieBanner == 'undefined' include _cookie_banner block contactModal diff --git a/services/web/app/views/project/editor/new_from_template.pug b/services/web/app/views/project/editor/new_from_template.pug index a5dc3ff33c..c84288a21a 100644 --- a/services/web/app/views/project/editor/new_from_template.pug +++ b/services/web/app/views/project/editor/new_from_template.pug @@ -2,7 +2,7 @@ extends ../../layout-marketing block vars - var suppressFooter = true - - var suppressPugCookieBanner = true + - var suppressCookieBanner = true - var suppressSkipToContent = true block content diff --git a/services/web/app/views/project/ide-react-detached.pug b/services/web/app/views/project/ide-react-detached.pug index fa695b1af5..ca1a178bbf 100644 --- a/services/web/app/views/project/ide-react-detached.pug +++ b/services/web/app/views/project/ide-react-detached.pug @@ -7,7 +7,7 @@ block vars - var suppressNavbar = true - var suppressFooter = true - var suppressSkipToContent = true - - var suppressPugCookieBanner = true + - var suppressCookieBanner = true - metadata.robotsNoindexNofollow = true block content diff --git a/services/web/app/views/project/list-react.pug b/services/web/app/views/project/list-react.pug index 47bff344b6..78103e75a6 100644 --- a/services/web/app/views/project/list-react.pug +++ b/services/web/app/views/project/list-react.pug @@ -7,7 +7,6 @@ block vars - const suppressNavContentLinks = true - const suppressNavbar = true - const suppressFooter = true - - const suppressPugCookieBanner = true block append meta meta( diff --git a/services/web/app/views/project/token/access-react.pug b/services/web/app/views/project/token/access-react.pug index 6c01ad15b1..80b91f1a99 100644 --- a/services/web/app/views/project/token/access-react.pug +++ b/services/web/app/views/project/token/access-react.pug @@ -5,7 +5,7 @@ block entrypointVar block vars - var suppressFooter = true - - var suppressPugCookieBanner = true + - var suppressCookieBanner = true - var suppressSkipToContent = true block append meta diff --git a/services/web/app/views/project/token/sharing-updates.pug b/services/web/app/views/project/token/sharing-updates.pug index 2f67e5a3c1..d1818be0af 100644 --- a/services/web/app/views/project/token/sharing-updates.pug +++ b/services/web/app/views/project/token/sharing-updates.pug @@ -5,7 +5,7 @@ block entrypointVar block vars - var suppressFooter = true - - var suppressPugCookieBanner = true + - var suppressCookieBanner = true - var suppressSkipToContent = true block append meta diff --git a/services/web/config/settings.defaults.js b/services/web/config/settings.defaults.js index 27f68c1c87..4d5e2a5d78 100644 --- a/services/web/config/settings.defaults.js +++ b/services/web/config/settings.defaults.js @@ -443,9 +443,6 @@ module.exports = { ',' ), - filestoreMigrationLevel: - parseInt(process.env.OVERLEAF_FILESTORE_MIGRATION_LEVEL, 10) || 0, - // i18n // ------ // diff --git a/services/web/docker-compose.ci.yml b/services/web/docker-compose.ci.yml index 8376103315..33b5a3ca2e 100644 --- a/services/web/docker-compose.ci.yml +++ b/services/web/docker-compose.ci.yml @@ -95,7 +95,7 @@ services: image: redis:7.4.3 mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 logging: driver: none command: --replSet overleaf diff --git a/services/web/docker-compose.yml b/services/web/docker-compose.yml index e0a4a064c5..069c1e77de 100644 --- a/services/web/docker-compose.yml +++ b/services/web/docker-compose.yml @@ -91,7 +91,7 @@ services: image: redis:7.4.3 mongo: - image: mongo:8.0.11 + image: mongo:7.0.20 command: --replSet overleaf volumes: - ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js diff --git a/services/web/frontend/extracted-translations.json b/services/web/frontend/extracted-translations.json index 639c9fcdfc..ef2a9c6a2c 100644 --- a/services/web/frontend/extracted-translations.json +++ b/services/web/frontend/extracted-translations.json @@ -35,7 +35,6 @@ "about_to_remove_user_preamble": "", "about_to_trash_projects": "", "abstract": "", - "accept_all_cookies": "", "accept_and_continue": "", "accept_change": "", "accept_change_error_description": "", @@ -333,8 +332,6 @@ "continue_to": "", "continue_using_free_features": "", "continue_with_free_plan": "", - "cookie_banner": "", - "cookie_banner_info": "", "copied": "", "copy": "", "copy_code": "", @@ -547,7 +544,6 @@ "error_opening_document_detail": "", "error_performing_request": "", "error_processing_file": "", - "essential_cookies_only": "", "example_project": "", "existing_plan_active_until_term_end": "", "expand": "", @@ -867,7 +863,6 @@ "invalid_password_too_similar": "", "invalid_regular_expression": "", "invalid_request": "", - "invalid_upload_request": "", "invite": "", "invite_expired": "", "invite_more_collabs": "", diff --git a/services/web/frontend/js/features/cookie-banner/index.js b/services/web/frontend/js/features/cookie-banner/index.js new file mode 100644 index 0000000000..3d9b2b8d6c --- /dev/null +++ b/services/web/frontend/js/features/cookie-banner/index.js @@ -0,0 +1,53 @@ +import getMeta from '@/utils/meta' + +function loadGA() { + if (window.olLoadGA) { + window.olLoadGA() + } +} + +function setConsent(value) { + document.querySelector('.cookie-banner').classList.add('hidden') + const cookieDomain = getMeta('ol-ExposedSettings').cookieDomain + const oneYearInSeconds = 60 * 60 * 24 * 365 + const cookieAttributes = + '; path=/' + + '; domain=' + + cookieDomain + + '; max-age=' + + oneYearInSeconds + + '; SameSite=Lax; Secure' + if (value === 'all') { + document.cookie = 'oa=1' + cookieAttributes + loadGA() + window.dispatchEvent(new CustomEvent('cookie-consent', { detail: true })) + } else { + document.cookie = 'oa=0' + cookieAttributes + window.dispatchEvent(new CustomEvent('cookie-consent', { detail: false })) + } +} + +if ( + getMeta('ol-ExposedSettings').gaToken || + getMeta('ol-ExposedSettings').gaTokenV4 || + getMeta('ol-ExposedSettings').propensityId || + getMeta('ol-ExposedSettings').hotjarId +) { + document + .querySelectorAll('[data-ol-cookie-banner-set-consent]') + .forEach(el => { + el.addEventListener('click', function (e) { + e.preventDefault() + const consentType = el.getAttribute('data-ol-cookie-banner-set-consent') + setConsent(consentType) + }) + }) + + const oaCookie = document.cookie.split('; ').find(c => c.startsWith('oa=')) + if (!oaCookie) { + const cookieBannerEl = document.querySelector('.cookie-banner') + if (cookieBannerEl) { + cookieBannerEl.classList.remove('hidden') + } + } +} diff --git a/services/web/frontend/js/features/cookie-banner/index.ts b/services/web/frontend/js/features/cookie-banner/index.ts deleted file mode 100644 index 2ea97e875a..0000000000 --- a/services/web/frontend/js/features/cookie-banner/index.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { - CookieConsentValue, - cookieBannerRequired, - hasMadeCookieChoice, - setConsent, -} from '@/features/cookie-banner/utils' - -function toggleCookieBanner(hidden: boolean) { - const cookieBannerEl = document.querySelector('.cookie-banner') - if (cookieBannerEl) { - cookieBannerEl.classList.toggle('hidden', hidden) - } -} - -if (cookieBannerRequired()) { - document - .querySelectorAll('[data-ol-cookie-banner-set-consent]') - .forEach(el => { - el.addEventListener('click', function (e) { - e.preventDefault() - toggleCookieBanner(true) - const consentType = el.getAttribute( - 'data-ol-cookie-banner-set-consent' - ) as CookieConsentValue | null - setConsent(consentType) - }) - }) - - if (!hasMadeCookieChoice()) { - toggleCookieBanner(false) - } -} diff --git a/services/web/frontend/js/features/cookie-banner/utils.ts b/services/web/frontend/js/features/cookie-banner/utils.ts deleted file mode 100644 index 5c045d4e71..0000000000 --- a/services/web/frontend/js/features/cookie-banner/utils.ts +++ /dev/null @@ -1,43 +0,0 @@ -import getMeta from '@/utils/meta' - -export type CookieConsentValue = 'all' | 'essential' - -function loadGA() { - if (window.olLoadGA) { - window.olLoadGA() - } -} - -export function setConsent(value: CookieConsentValue | null) { - const cookieDomain = getMeta('ol-ExposedSettings').cookieDomain - const oneYearInSeconds = 60 * 60 * 24 * 365 - const cookieAttributes = - '; path=/' + - '; domain=' + - cookieDomain + - '; max-age=' + - oneYearInSeconds + - '; SameSite=Lax; Secure' - if (value === 'all') { - document.cookie = 'oa=1' + cookieAttributes - loadGA() - window.dispatchEvent(new CustomEvent('cookie-consent', { detail: true })) - } else { - document.cookie = 'oa=0' + cookieAttributes - window.dispatchEvent(new CustomEvent('cookie-consent', { detail: false })) - } -} - -export function cookieBannerRequired() { - const exposedSettings = getMeta('ol-ExposedSettings') - return Boolean( - exposedSettings.gaToken || - exposedSettings.gaTokenV4 || - exposedSettings.propensityId || - exposedSettings.hotjarId - ) -} - -export function hasMadeCookieChoice() { - return document.cookie.split('; ').some(c => c.startsWith('oa=')) -} diff --git a/services/web/frontend/js/features/file-tree/components/file-tree-create/error-message.tsx b/services/web/frontend/js/features/file-tree/components/file-tree-create/error-message.tsx index 244ef1a76b..02cc083928 100644 --- a/services/web/frontend/js/features/file-tree/components/file-tree-create/error-message.tsx +++ b/services/web/frontend/js/features/file-tree/components/file-tree-create/error-message.tsx @@ -1,4 +1,4 @@ -import { useTranslation, Trans } from 'react-i18next' +import { useTranslation } from 'react-i18next' import { FetchError } from '../../../../infrastructure/fetch-json' import RedirectToLogin from './redirect-to-login' import { @@ -7,7 +7,6 @@ import { InvalidFilenameError, } from '../../errors' import DangerMessage from './danger-message' -import getMeta from '@/utils/meta' // TODO: Update the error type when we properly type FileTreeActionableContext export default function ErrorMessage({ @@ -16,7 +15,6 @@ export default function ErrorMessage({ error: string | Record }) { const { t } = useTranslation() - const { isOverleaf } = getMeta('ol-ExposedSettings') const fileNameLimit = 150 // the error is a string @@ -48,22 +46,6 @@ export default function ErrorMessage({ ) - case 'invalid_upload_request': - if (!isOverleaf) { - return ( - {t('generic_something_went_wrong')} - ) - } - return ( - - ]} - /> - - ) - case 'duplicate_file_name': return ( diff --git a/services/web/frontend/js/features/ide-redesign/components/chat/message.tsx b/services/web/frontend/js/features/ide-redesign/components/chat/message.tsx index 6822db39da..9a4ffe3a1b 100644 --- a/services/web/frontend/js/features/ide-redesign/components/chat/message.tsx +++ b/services/web/frontend/js/features/ide-redesign/components/chat/message.tsx @@ -1,14 +1,15 @@ import { MessageProps } from '@/features/chat/components/message' import { User } from '../../../../../../types/user' -import { - getBackgroundColorForUserId, - hslStringToLuminance, -} from '@/shared/utils/colors' +import { getHueForUserId } from '@/shared/utils/colors' import MessageContent from '@/features/chat/components/message-content' import classNames from 'classnames' import MaterialIcon from '@/shared/components/material-icon' import { t } from 'i18next' +function hue(user?: User) { + return user ? getHueForUserId(user.id) : 0 +} + function getAvatarStyle(user?: User) { if (!user?.id) { // Deleted user @@ -19,15 +20,9 @@ function getAvatarStyle(user?: User) { } } - const backgroundColor = getBackgroundColorForUserId(user.id) - return { - borderColor: backgroundColor, - backgroundColor, - color: - hslStringToLuminance(backgroundColor) < 0.5 - ? 'var(--content-primary-dark)' - : 'var(--content-primary)', + borderColor: `hsl(${hue(user)}, 85%, 40%)`, + backgroundColor: `hsl(${hue(user)}, 85%, 40%`, } } diff --git a/services/web/frontend/js/features/ide-redesign/components/online-users/online-users-widget.tsx b/services/web/frontend/js/features/ide-redesign/components/online-users/online-users-widget.tsx index 2d30297e51..07aaa647a9 100644 --- a/services/web/frontend/js/features/ide-redesign/components/online-users/online-users-widget.tsx +++ b/services/web/frontend/js/features/ide-redesign/components/online-users/online-users-widget.tsx @@ -7,11 +7,7 @@ import { DropdownToggle, } from '@/features/ui/components/bootstrap-5/dropdown-menu' import OLTooltip from '@/features/ui/components/ol/ol-tooltip' -import { - getBackgroundColorForUserId, - hslStringToLuminance, -} from '@/shared/utils/colors' -import classNames from 'classnames' +import { getBackgroundColorForUserId } from '@/shared/utils/colors' import { useCallback, useMemo } from 'react' import { useTranslation } from 'react-i18next' @@ -90,16 +86,9 @@ const OnlineUserWidget = ({ const OnlineUserCircle = ({ user }: { user: OnlineUser }) => { const backgroundColor = getBackgroundColorForUserId(user.user_id) - const luminance = hslStringToLuminance(backgroundColor) const [character] = [...user.name] return ( - = 0.5, - })} - style={{ backgroundColor }} - > + {character} ) diff --git a/services/web/frontend/js/features/project-list/components/project-list-ds-nav.tsx b/services/web/frontend/js/features/project-list/components/project-list-ds-nav.tsx index 07319ffaf1..3d24f9845c 100644 --- a/services/web/frontend/js/features/project-list/components/project-list-ds-nav.tsx +++ b/services/web/frontend/js/features/project-list/components/project-list-ds-nav.tsx @@ -20,7 +20,6 @@ import Footer from '@/features/ui/components/bootstrap-5/footer/footer' import SidebarDsNav from '@/features/project-list/components/sidebar/sidebar-ds-nav' import SystemMessages from '@/shared/components/system-messages' import overleafLogo from '@/shared/svgs/overleaf-a-ds-solution-mallard.svg' -import CookieBanner from '@/shared/components/cookie-banner' export function ProjectListDsNav() { const navbarProps = getMeta('ol-navbar') @@ -126,7 +125,6 @@ export function ProjectListDsNav() {