mirror of
https://github.com/yu-i-i/overleaf-cep.git
synced 2025-08-07 05:00:05 +02:00
Compare commits
239 commits
c4e6dfbbbd
...
1741e48d59
Author | SHA1 | Date | |
---|---|---|---|
![]() |
1741e48d59 | ||
![]() |
94a067d7c8 | ||
![]() |
3f10b29869 | ||
![]() |
ca111771c2 | ||
![]() |
dadf6f0ed4 | ||
![]() |
fc56d1690d | ||
![]() |
aa723a70c2 | ||
![]() |
73e141a4a3 | ||
![]() |
620edfa347 | ||
![]() |
4e192f760d | ||
![]() |
44926d3519 | ||
![]() |
b41f8164b8 | ||
![]() |
bfd9ab6b8f | ||
![]() |
9ea0f2ec29 | ||
![]() |
4cee376878 | ||
![]() |
4e8f982ca2 | ||
![]() |
eb276c7403 | ||
![]() |
a68e96400b | ||
![]() |
0f3f78cde7 | ||
![]() |
d08fa01110 | ||
![]() |
cdf04d695c | ||
![]() |
71bc4c45bc | ||
![]() |
55a13ca1de | ||
![]() |
92dd62975e | ||
![]() |
040f70471c | ||
![]() |
f08532dfb0 | ||
![]() |
7eecfe9e27 | ||
![]() |
670ed44963 | ||
![]() |
f2030789d1 | ||
![]() |
ee11889431 | ||
![]() |
1fb18b092d | ||
![]() |
6169a5d3df | ||
![]() |
51250ca45f | ||
![]() |
bfe5871e9e | ||
![]() |
0cc244c516 | ||
![]() |
767ac1632e | ||
![]() |
1d1bad23e3 | ||
![]() |
73b4584575 | ||
![]() |
bbf85ae6d2 | ||
![]() |
90fac6b206 | ||
![]() |
9f527f10e1 | ||
![]() |
fc6df69e41 | ||
![]() |
3b5a148cdc | ||
![]() |
4e27add5b7 | ||
![]() |
c3e34f8850 | ||
![]() |
f36c87b301 | ||
![]() |
830d0daa38 | ||
![]() |
fd62142b21 | ||
![]() |
6501314616 | ||
![]() |
fef5ab7255 | ||
![]() |
0c6c61b654 | ||
![]() |
774292d8ba | ||
![]() |
6969a12fae | ||
![]() |
c52b23af57 | ||
![]() |
e08e58485e | ||
![]() |
b9bc2b01fd | ||
![]() |
b223bb8da8 | ||
![]() |
61ce012fb5 | ||
![]() |
f0fe0db10c | ||
![]() |
93793fe723 | ||
![]() |
46f3b595a9 | ||
![]() |
639690bb50 | ||
![]() |
90b6dbdf55 | ||
![]() |
640c699042 | ||
![]() |
ad677afa81 | ||
![]() |
f46fd6f2d5 | ||
![]() |
373b222929 | ||
![]() |
99ab41fd76 | ||
![]() |
c921c8f586 | ||
![]() |
4707842642 | ||
![]() |
dabf610764 | ||
![]() |
b44dd4d3c5 | ||
![]() |
2d4b5e51f1 | ||
![]() |
c55df0e803 | ||
![]() |
b14490d6aa | ||
![]() |
d67f3d3181 | ||
![]() |
2f0254a2c8 | ||
![]() |
fa058a5ca8 | ||
![]() |
25d1b2a24d | ||
![]() |
363e426e9f | ||
![]() |
d04cc1d8ac | ||
![]() |
dda94cdfbc | ||
![]() |
e754ee9cb4 | ||
![]() |
7eb5c8a38e | ||
![]() |
1ab8302254 | ||
![]() |
29238e54e3 | ||
![]() |
20c7f14b3c | ||
![]() |
b480903426 | ||
![]() |
235f1a5a59 | ||
![]() |
d59afb21be | ||
![]() |
5ba43eb56c | ||
![]() |
9eb84d6ad5 | ||
![]() |
374acf8119 | ||
![]() |
ab19677a6c | ||
![]() |
e8462f4250 | ||
![]() |
1fb94dee18 | ||
![]() |
e4dae982d2 | ||
![]() |
f5c92cb627 | ||
![]() |
bdcf1d3a83 | ||
![]() |
e827540a6d | ||
![]() |
7f019d3880 | ||
![]() |
716fe07e84 | ||
![]() |
0e9c310d1d | ||
![]() |
7308ac0e1f | ||
![]() |
a853a92765 | ||
![]() |
562ef81389 | ||
![]() |
8fe07b196b | ||
![]() |
7f67df2468 | ||
![]() |
47fb3a644c | ||
![]() |
e44f892cb0 | ||
![]() |
6932b3deb7 | ||
![]() |
800b151024 | ||
![]() |
47d8e59938 | ||
![]() |
56f8993bd7 | ||
![]() |
432a92173a | ||
![]() |
2779691cd9 | ||
![]() |
d2d556ddf6 | ||
![]() |
0895b5c6ee | ||
![]() |
7168572e74 | ||
![]() |
c2da12939e | ||
![]() |
b8d74c6ae0 | ||
![]() |
e56c4304a1 | ||
![]() |
9fd4e4ab87 | ||
![]() |
d7cddd14fa | ||
![]() |
67a6857ca6 | ||
![]() |
57f389646c | ||
![]() |
f3a19f48d8 | ||
![]() |
19852ed180 | ||
![]() |
5a33a51076 | ||
![]() |
48e0bc28f8 | ||
![]() |
e98ec386cb | ||
![]() |
9680fd115b | ||
![]() |
54f5c3115c | ||
![]() |
1117ea1b3e | ||
![]() |
ff78f687d8 | ||
![]() |
31b57e2991 | ||
![]() |
8ada51158f | ||
![]() |
6cb5360c88 | ||
![]() |
70601db76f | ||
![]() |
7b69d61540 | ||
![]() |
13bf214a3c | ||
![]() |
47ea64c30a | ||
![]() |
e94473a1ce | ||
![]() |
a7818e9b11 | ||
![]() |
f94adbf039 | ||
![]() |
ec13227fc6 | ||
![]() |
a530cca2c5 | ||
![]() |
0292bc418d | ||
![]() |
e99cd74cca | ||
![]() |
dcabf55882 | ||
![]() |
35dc7faab6 | ||
![]() |
c5b584e3d8 | ||
![]() |
09b68de041 | ||
![]() |
7c7cc0fce0 | ||
![]() |
3f7c88108c | ||
![]() |
f134746c9c | ||
![]() |
b5d6484991 | ||
![]() |
78481e010e | ||
![]() |
9d72eeeeac | ||
![]() |
f85fdd3a97 | ||
![]() |
f6bd485863 | ||
![]() |
5cdaa424ee | ||
![]() |
0c3a62297a | ||
![]() |
a097577e29 | ||
![]() |
f7e716c826 | ||
![]() |
84996ea88c | ||
![]() |
e6371ec197 | ||
![]() |
971c63a636 | ||
![]() |
5a67353dc3 | ||
![]() |
eaefecb91d | ||
![]() |
8d569815e6 | ||
![]() |
2d48c86e61 | ||
![]() |
a178c0f400 | ||
![]() |
cf105cf01d | ||
![]() |
3b93efdf5c | ||
![]() |
28ff69b51b | ||
![]() |
4ddd3ee772 | ||
![]() |
92499f6260 | ||
![]() |
3d9bc77fce | ||
![]() |
c3ade4dce1 | ||
![]() |
9470c3a44b | ||
![]() |
89b4eaf391 | ||
![]() |
79dcab4ef5 | ||
![]() |
542a52c510 | ||
![]() |
10b0d6333f | ||
![]() |
1fcf046c81 | ||
![]() |
cc72d8b11b | ||
![]() |
c4493ebc90 | ||
![]() |
3d9b1bb177 | ||
![]() |
ea33c7d896 | ||
![]() |
768180c456 | ||
![]() |
dad6f97cce | ||
![]() |
273ae4aecd | ||
![]() |
b5f8bfa28e | ||
![]() |
959562661f | ||
![]() |
80abd0ac2c | ||
![]() |
19eefebe95 | ||
![]() |
087a9daf34 | ||
![]() |
a7be1f3430 | ||
![]() |
c373db4f86 | ||
![]() |
3bc21faeaf | ||
![]() |
f9515f10cd | ||
![]() |
0b387c5116 | ||
![]() |
149b590413 | ||
![]() |
282f5f92ff | ||
![]() |
afedce1b0e | ||
![]() |
061d67ee4b | ||
![]() |
36056e75d7 | ||
![]() |
d04bfbcdea | ||
![]() |
ecc2f1f544 | ||
![]() |
a11266471c | ||
![]() |
302362c70d | ||
![]() |
efd53e567c | ||
![]() |
0002e008bb | ||
![]() |
96af83a4ed | ||
![]() |
00aa26cd1a | ||
![]() |
3cad54b215 | ||
![]() |
a04d3198a8 | ||
![]() |
aaa15a2733 | ||
![]() |
cae698b705 | ||
![]() |
c233243948 | ||
![]() |
f045361b49 | ||
![]() |
441c7a89a7 | ||
![]() |
7ec4cbd841 | ||
![]() |
1ea577ef12 | ||
![]() |
eedf5367fc | ||
![]() |
fe4f41501f | ||
![]() |
f11ad91249 | ||
![]() |
1c4a761478 | ||
![]() |
c3c14ccfbc | ||
![]() |
9824151e62 | ||
![]() |
d7ad742ba3 | ||
![]() |
087c41190e | ||
![]() |
69bc8a135b | ||
![]() |
6a344c7a52 | ||
![]() |
b5031fdee5 | ||
![]() |
15e5501ddd | ||
![]() |
5974eed4aa | ||
![]() |
8a4c84e7dd |
726 changed files with 15163 additions and 15356 deletions
|
@ -73,4 +73,4 @@ Please see the [CONTRIBUTING](CONTRIBUTING.md) file for information on contribut
|
||||||
|
|
||||||
The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the [`LICENSE`](LICENSE) file.
|
The code in this repository is released under the GNU AFFERO GENERAL PUBLIC LICENSE, version 3. A copy can be found in the [`LICENSE`](LICENSE) file.
|
||||||
|
|
||||||
Copyright (c) Overleaf, 2014-2024.
|
Copyright (c) Overleaf, 2014-2025.
|
||||||
|
|
|
@ -11,12 +11,6 @@ bin/build
|
||||||
> [!NOTE]
|
> [!NOTE]
|
||||||
> If Docker is running out of RAM while building the services in parallel, create a `.env` file in this directory containing `COMPOSE_PARALLEL_LIMIT=1`.
|
> If Docker is running out of RAM while building the services in parallel, create a `.env` file in this directory containing `COMPOSE_PARALLEL_LIMIT=1`.
|
||||||
|
|
||||||
Next, initialize the database:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
bin/init
|
|
||||||
```
|
|
||||||
|
|
||||||
Then start the services:
|
Then start the services:
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
|
|
|
@ -1,6 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
docker compose up --detach mongo
|
|
||||||
curl --max-time 10 --retry 5 --retry-delay 5 --retry-all-errors --silent --output /dev/null localhost:27017
|
|
||||||
docker compose exec mongo mongosh --eval "rs.initiate({ _id: 'overleaf', members: [{ _id: 0, host: 'mongo:27017' }] })"
|
|
||||||
docker compose down mongo
|
|
|
@ -94,6 +94,14 @@ services:
|
||||||
- "127.0.0.1:27017:27017" # for debugging
|
- "127.0.0.1:27017:27017" # for debugging
|
||||||
volumes:
|
volumes:
|
||||||
- mongo-data:/data/db
|
- mongo-data:/data/db
|
||||||
|
- ../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
|
environment:
|
||||||
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
|
extra_hosts:
|
||||||
|
# Required when using the automatic database setup for initializing the
|
||||||
|
# replica set. This override is not needed when running the setup after
|
||||||
|
# starting up mongo.
|
||||||
|
- mongo:127.0.0.1
|
||||||
|
|
||||||
notifications:
|
notifications:
|
||||||
build:
|
build:
|
||||||
|
|
|
@ -103,7 +103,7 @@ services:
|
||||||
command: '--replSet overleaf'
|
command: '--replSet overleaf'
|
||||||
volumes:
|
volumes:
|
||||||
- ~/mongo_data:/data/db
|
- ~/mongo_data:/data/db
|
||||||
- ./server-ce/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
- ./bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
environment:
|
environment:
|
||||||
MONGO_INITDB_DATABASE: sharelatex
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
extra_hosts:
|
extra_hosts:
|
||||||
|
|
|
@ -95,6 +95,19 @@ async function fetchNothing(url, opts = {}) {
|
||||||
* @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
|
* @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
|
||||||
*/
|
*/
|
||||||
async function fetchRedirect(url, opts = {}) {
|
async function fetchRedirect(url, opts = {}) {
|
||||||
|
const { location } = await fetchRedirectWithResponse(url, opts)
|
||||||
|
return location
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Make a request and extract the redirect from the response.
|
||||||
|
*
|
||||||
|
* @param {string | URL} url - request URL
|
||||||
|
* @param {object} opts - fetch options
|
||||||
|
* @return {Promise<{location: string, response: Response}>}
|
||||||
|
* @throws {RequestFailedError} if the response has a non redirect status code or missing Location header
|
||||||
|
*/
|
||||||
|
async function fetchRedirectWithResponse(url, opts = {}) {
|
||||||
const { fetchOpts } = parseOpts(opts)
|
const { fetchOpts } = parseOpts(opts)
|
||||||
fetchOpts.redirect = 'manual'
|
fetchOpts.redirect = 'manual'
|
||||||
const response = await performRequest(url, fetchOpts)
|
const response = await performRequest(url, fetchOpts)
|
||||||
|
@ -112,7 +125,7 @@ async function fetchRedirect(url, opts = {}) {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
await discardResponseBody(response)
|
await discardResponseBody(response)
|
||||||
return location
|
return { location, response }
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -297,6 +310,7 @@ module.exports = {
|
||||||
fetchStreamWithResponse,
|
fetchStreamWithResponse,
|
||||||
fetchNothing,
|
fetchNothing,
|
||||||
fetchRedirect,
|
fetchRedirect,
|
||||||
|
fetchRedirectWithResponse,
|
||||||
fetchString,
|
fetchString,
|
||||||
fetchStringWithResponse,
|
fetchStringWithResponse,
|
||||||
RequestFailedError,
|
RequestFailedError,
|
||||||
|
|
|
@ -16,6 +16,7 @@ let VERBOSE_LOGGING
|
||||||
let BATCH_RANGE_START
|
let BATCH_RANGE_START
|
||||||
let BATCH_RANGE_END
|
let BATCH_RANGE_END
|
||||||
let BATCH_MAX_TIME_SPAN_IN_MS
|
let BATCH_MAX_TIME_SPAN_IN_MS
|
||||||
|
let BATCHED_UPDATE_RUNNING = false
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @typedef {import("mongodb").Collection} Collection
|
* @typedef {import("mongodb").Collection} Collection
|
||||||
|
@ -211,57 +212,66 @@ async function batchedUpdate(
|
||||||
findOptions,
|
findOptions,
|
||||||
batchedUpdateOptions
|
batchedUpdateOptions
|
||||||
) {
|
) {
|
||||||
ID_EDGE_PAST = await getIdEdgePast(collection)
|
// only a single batchedUpdate can run at a time due to global variables
|
||||||
if (!ID_EDGE_PAST) {
|
if (BATCHED_UPDATE_RUNNING) {
|
||||||
console.warn(
|
throw new Error('batchedUpdate is already running')
|
||||||
`The collection ${collection.collectionName} appears to be empty.`
|
|
||||||
)
|
|
||||||
return 0
|
|
||||||
}
|
}
|
||||||
refreshGlobalOptionsForBatchedUpdate(batchedUpdateOptions)
|
try {
|
||||||
|
BATCHED_UPDATE_RUNNING = true
|
||||||
findOptions = findOptions || {}
|
ID_EDGE_PAST = await getIdEdgePast(collection)
|
||||||
findOptions.readPreference = READ_PREFERENCE_SECONDARY
|
if (!ID_EDGE_PAST) {
|
||||||
|
console.warn(
|
||||||
projection = projection || { _id: 1 }
|
`The collection ${collection.collectionName} appears to be empty.`
|
||||||
let nextBatch
|
)
|
||||||
let updated = 0
|
return 0
|
||||||
let start = BATCH_RANGE_START
|
|
||||||
|
|
||||||
while (start !== BATCH_RANGE_END) {
|
|
||||||
let end = getNextEnd(start)
|
|
||||||
nextBatch = await getNextBatch(
|
|
||||||
collection,
|
|
||||||
query,
|
|
||||||
start,
|
|
||||||
end,
|
|
||||||
projection,
|
|
||||||
findOptions
|
|
||||||
)
|
|
||||||
if (nextBatch.length > 0) {
|
|
||||||
end = nextBatch[nextBatch.length - 1]._id
|
|
||||||
updated += nextBatch.length
|
|
||||||
|
|
||||||
if (VERBOSE_LOGGING) {
|
|
||||||
console.log(
|
|
||||||
`Running update on batch with ids ${JSON.stringify(
|
|
||||||
nextBatch.map(entry => entry._id)
|
|
||||||
)}`
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
console.error(`Running update on batch ending ${renderObjectId(end)}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (typeof update === 'function') {
|
|
||||||
await update(nextBatch)
|
|
||||||
} else {
|
|
||||||
await performUpdate(collection, nextBatch, update)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
console.error(`Completed batch ending ${renderObjectId(end)}`)
|
refreshGlobalOptionsForBatchedUpdate(batchedUpdateOptions)
|
||||||
start = end
|
|
||||||
|
findOptions = findOptions || {}
|
||||||
|
findOptions.readPreference = READ_PREFERENCE_SECONDARY
|
||||||
|
|
||||||
|
projection = projection || { _id: 1 }
|
||||||
|
let nextBatch
|
||||||
|
let updated = 0
|
||||||
|
let start = BATCH_RANGE_START
|
||||||
|
|
||||||
|
while (start !== BATCH_RANGE_END) {
|
||||||
|
let end = getNextEnd(start)
|
||||||
|
nextBatch = await getNextBatch(
|
||||||
|
collection,
|
||||||
|
query,
|
||||||
|
start,
|
||||||
|
end,
|
||||||
|
projection,
|
||||||
|
findOptions
|
||||||
|
)
|
||||||
|
if (nextBatch.length > 0) {
|
||||||
|
end = nextBatch[nextBatch.length - 1]._id
|
||||||
|
updated += nextBatch.length
|
||||||
|
|
||||||
|
if (VERBOSE_LOGGING) {
|
||||||
|
console.log(
|
||||||
|
`Running update on batch with ids ${JSON.stringify(
|
||||||
|
nextBatch.map(entry => entry._id)
|
||||||
|
)}`
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
console.error(`Running update on batch ending ${renderObjectId(end)}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof update === 'function') {
|
||||||
|
await update(nextBatch)
|
||||||
|
} else {
|
||||||
|
await performUpdate(collection, nextBatch, update)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.error(`Completed batch ending ${renderObjectId(end)}`)
|
||||||
|
start = end
|
||||||
|
}
|
||||||
|
return updated
|
||||||
|
} finally {
|
||||||
|
BATCHED_UPDATE_RUNNING = false
|
||||||
}
|
}
|
||||||
return updated
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -414,6 +414,16 @@ class CachedPerProjectEncryptedS3Persistor {
|
||||||
return await this.sendStream(bucketName, path, fs.createReadStream(fsPath))
|
return await this.sendStream(bucketName, path, fs.createReadStream(fsPath))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {string} bucketName
|
||||||
|
* @param {string} path
|
||||||
|
* @return {Promise<number>}
|
||||||
|
*/
|
||||||
|
async getObjectSize(bucketName, path) {
|
||||||
|
return await this.#parent.getObjectSize(bucketName, path)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param {string} bucketName
|
* @param {string} bucketName
|
||||||
* @param {string} path
|
* @param {string} path
|
||||||
|
|
|
@ -13,6 +13,7 @@ module.exports = {
|
||||||
expressify,
|
expressify,
|
||||||
expressifyErrorHandler,
|
expressifyErrorHandler,
|
||||||
promiseMapWithLimit,
|
promiseMapWithLimit,
|
||||||
|
promiseMapSettledWithLimit,
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -264,3 +265,19 @@ async function promiseMapWithLimit(concurrency, array, fn) {
|
||||||
const limit = pLimit(concurrency)
|
const limit = pLimit(concurrency)
|
||||||
return await Promise.all(array.map(x => limit(() => fn(x))))
|
return await Promise.all(array.map(x => limit(() => fn(x))))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Map values in `array` with the async function `fn`
|
||||||
|
*
|
||||||
|
* Limit the number of unresolved promises to `concurrency`.
|
||||||
|
*
|
||||||
|
* @template T, U
|
||||||
|
* @param {number} concurrency
|
||||||
|
* @param {Array<T>} array
|
||||||
|
* @param {(T) => Promise<U>} fn
|
||||||
|
* @return {Promise<Array<PromiseSettledResult<U>>>}
|
||||||
|
*/
|
||||||
|
function promiseMapSettledWithLimit(concurrency, array, fn) {
|
||||||
|
const limit = pLimit(concurrency)
|
||||||
|
return Promise.allSettled(array.map(x => limit(() => fn(x))))
|
||||||
|
}
|
||||||
|
|
2468
package-lock.json
generated
2468
package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
@ -73,6 +73,7 @@
|
||||||
"services/third-party-datastore",
|
"services/third-party-datastore",
|
||||||
"services/third-party-references",
|
"services/third-party-references",
|
||||||
"services/tpdsworker",
|
"services/tpdsworker",
|
||||||
"services/web"
|
"services/web",
|
||||||
|
"tools/saas-e2e"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,7 +10,7 @@ ENV TEXMFVAR=/var/lib/overleaf/tmp/texmf-var
|
||||||
|
|
||||||
# Update to ensure dependencies are updated
|
# Update to ensure dependencies are updated
|
||||||
# ------------------------------------------
|
# ------------------------------------------
|
||||||
ENV REBUILT_AFTER="2024-15-10"
|
ENV REBUILT_AFTER="2025-03-27"
|
||||||
|
|
||||||
# Install dependencies
|
# Install dependencies
|
||||||
# --------------------
|
# --------------------
|
||||||
|
|
1
server-ce/bin/shared
Symbolic link
1
server-ce/bin/shared
Symbolic link
|
@ -0,0 +1 @@
|
||||||
|
../../bin/shared/
|
1
server-ce/hotfix/4.2.9/Dockerfile
Normal file
1
server-ce/hotfix/4.2.9/Dockerfile
Normal file
|
@ -0,0 +1 @@
|
||||||
|
FROM sharelatex/sharelatex:4.2.8
|
1
server-ce/hotfix/5.3.3/Dockerfile
Normal file
1
server-ce/hotfix/5.3.3/Dockerfile
Normal file
|
@ -0,0 +1 @@
|
||||||
|
FROM sharelatex/sharelatex:5.3.2
|
|
@ -1 +0,0 @@
|
||||||
rs.initiate({ _id: "overleaf", members: [ { _id: 0, host: "mongo:27017" } ] })
|
|
|
@ -30,7 +30,7 @@ server {
|
||||||
application/pdf pdf;
|
application/pdf pdf;
|
||||||
}
|
}
|
||||||
# handle output files for specific users
|
# handle output files for specific users
|
||||||
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
|
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
|
||||||
alias /var/lib/overleaf/data/output/$1-$2/generated-files/$3/output.$4;
|
alias /var/lib/overleaf/data/output/$1-$2/generated-files/$3/output.$4;
|
||||||
}
|
}
|
||||||
# handle .blg files for specific users
|
# handle .blg files for specific users
|
||||||
|
@ -38,7 +38,7 @@ server {
|
||||||
alias /var/lib/overleaf/data/output/$1-$2/generated-files/$3/$4.blg;
|
alias /var/lib/overleaf/data/output/$1-$2/generated-files/$3/$4.blg;
|
||||||
}
|
}
|
||||||
# handle output files for anonymous users
|
# handle output files for anonymous users
|
||||||
location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
|
location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
|
||||||
alias /var/lib/overleaf/data/output/$1/generated-files/$2/output.$3;
|
alias /var/lib/overleaf/data/output/$1/generated-files/$2/output.$3;
|
||||||
}
|
}
|
||||||
# handle .blg files for anonymous users
|
# handle .blg files for anonymous users
|
||||||
|
|
|
@ -47,12 +47,12 @@ server {
|
||||||
}
|
}
|
||||||
|
|
||||||
# handle output files for specific users
|
# handle output files for specific users
|
||||||
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
|
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
|
||||||
proxy_pass http://127.0.0.1:8080; # clsi-nginx.conf
|
proxy_pass http://127.0.0.1:8080; # clsi-nginx.conf
|
||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
}
|
}
|
||||||
# handle output files for anonymous users
|
# handle output files for anonymous users
|
||||||
location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
|
location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
|
||||||
proxy_pass http://127.0.0.1:8080; # clsi-nginx.conf
|
proxy_pass http://127.0.0.1:8080; # clsi-nginx.conf
|
||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
.
|
|
|
@ -95,7 +95,9 @@ describe('Project creation and compilation', function () {
|
||||||
|
|
||||||
cy.findByText('Share').click()
|
cy.findByText('Share').click()
|
||||||
cy.findByRole('dialog').within(() => {
|
cy.findByRole('dialog').within(() => {
|
||||||
cy.get('input').type('collaborator@example.com,')
|
cy.findByTestId('collaborator-email-input').type(
|
||||||
|
'collaborator@example.com,'
|
||||||
|
)
|
||||||
cy.findByText('Invite').click({ force: true })
|
cy.findByText('Invite').click({ force: true })
|
||||||
cy.findByText('Invite not yet accepted.')
|
cy.findByText('Invite not yet accepted.')
|
||||||
})
|
})
|
||||||
|
|
|
@ -38,7 +38,7 @@ services:
|
||||||
image: mongo:6.0
|
image: mongo:6.0
|
||||||
command: '--replSet overleaf'
|
command: '--replSet overleaf'
|
||||||
volumes:
|
volumes:
|
||||||
- ../mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
- ../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
environment:
|
environment:
|
||||||
MONGO_INITDB_DATABASE: sharelatex
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
extra_hosts:
|
extra_hosts:
|
||||||
|
|
|
@ -149,10 +149,10 @@ describe('editor', () => {
|
||||||
openFile(fileName, 'static')
|
openFile(fileName, 'static')
|
||||||
|
|
||||||
cy.log('reject changes')
|
cy.log('reject changes')
|
||||||
cy.findByText('Review').click()
|
cy.contains('.toolbar-item', 'Review').click()
|
||||||
cy.get('.cm-content').should('not.contain.text', oldContent)
|
cy.get('.cm-content').should('not.contain.text', oldContent)
|
||||||
cy.findByText('Reject').click({ force: true })
|
cy.findByText('Reject change').click({ force: true })
|
||||||
cy.findByText('Review').click()
|
cy.contains('.toolbar-item', 'Review').click()
|
||||||
|
|
||||||
cy.log('recompile to force flush')
|
cy.log('recompile to force flush')
|
||||||
recompile()
|
recompile()
|
||||||
|
@ -205,10 +205,10 @@ describe('editor', () => {
|
||||||
openFile(fileName, 'static')
|
openFile(fileName, 'static')
|
||||||
|
|
||||||
cy.log('reject changes')
|
cy.log('reject changes')
|
||||||
cy.findByText('Review').click()
|
cy.contains('.toolbar-item', 'Review').click()
|
||||||
cy.get('.cm-content').should('not.contain.text', oldContent)
|
cy.get('.cm-content').should('not.contain.text', oldContent)
|
||||||
cy.findAllByText('Reject').first().click({ force: true })
|
cy.findAllByText('Reject change').first().click({ force: true })
|
||||||
cy.findByText('Review').click()
|
cy.contains('.toolbar-item', 'Review').click()
|
||||||
|
|
||||||
cy.log('recompile to force flush')
|
cy.log('recompile to force flush')
|
||||||
recompile()
|
recompile()
|
||||||
|
|
|
@ -59,7 +59,7 @@ describe('LDAP', () => {
|
||||||
|
|
||||||
it('login', () => {
|
it('login', () => {
|
||||||
cy.visit('/')
|
cy.visit('/')
|
||||||
cy.findByText('Login LDAP')
|
cy.findByText('Log in LDAP')
|
||||||
|
|
||||||
cy.get('input[name="login"]').type('fry')
|
cy.get('input[name="login"]').type('fry')
|
||||||
cy.get('input[name="password"]').type('fry')
|
cy.get('input[name="password"]').type('fry')
|
||||||
|
|
|
@ -136,7 +136,7 @@ describe('git-bridge', function () {
|
||||||
shareProjectByEmailAndAcceptInviteViaDash(
|
shareProjectByEmailAndAcceptInviteViaDash(
|
||||||
projectName,
|
projectName,
|
||||||
'collaborator-rw@example.com',
|
'collaborator-rw@example.com',
|
||||||
'Can edit'
|
'Editor'
|
||||||
)
|
)
|
||||||
maybeClearAllTokens()
|
maybeClearAllTokens()
|
||||||
openProjectByName(projectName)
|
openProjectByName(projectName)
|
||||||
|
@ -147,7 +147,7 @@ describe('git-bridge', function () {
|
||||||
shareProjectByEmailAndAcceptInviteViaDash(
|
shareProjectByEmailAndAcceptInviteViaDash(
|
||||||
projectName,
|
projectName,
|
||||||
'collaborator-ro@example.com',
|
'collaborator-ro@example.com',
|
||||||
'Can view'
|
'Viewer'
|
||||||
)
|
)
|
||||||
maybeClearAllTokens()
|
maybeClearAllTokens()
|
||||||
openProjectByName(projectName)
|
openProjectByName(projectName)
|
||||||
|
|
|
@ -24,7 +24,7 @@ export function prepareWaitForNextCompileSlot() {
|
||||||
queueReset()
|
queueReset()
|
||||||
triggerCompile()
|
triggerCompile()
|
||||||
cy.log('Wait for compile to finish')
|
cy.log('Wait for compile to finish')
|
||||||
cy.findByText('Recompile')
|
cy.findByText('Recompile').should('be.visible')
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
function recompile() {
|
function recompile() {
|
||||||
|
|
|
@ -100,7 +100,7 @@ export function openProjectViaInviteNotification(projectName: string) {
|
||||||
function shareProjectByEmail(
|
function shareProjectByEmail(
|
||||||
projectName: string,
|
projectName: string,
|
||||||
email: string,
|
email: string,
|
||||||
level: 'Can view' | 'Can edit'
|
level: 'Viewer' | 'Editor'
|
||||||
) {
|
) {
|
||||||
openProjectByName(projectName)
|
openProjectByName(projectName)
|
||||||
cy.findByText('Share').click()
|
cy.findByText('Share').click()
|
||||||
|
@ -108,7 +108,13 @@ function shareProjectByEmail(
|
||||||
cy.findByLabelText('Add people', { selector: 'input' }).type(`${email},`)
|
cy.findByLabelText('Add people', { selector: 'input' }).type(`${email},`)
|
||||||
cy.findByLabelText('Add people', { selector: 'input' })
|
cy.findByLabelText('Add people', { selector: 'input' })
|
||||||
.parents('form')
|
.parents('form')
|
||||||
.within(() => cy.findByText('Can edit').parent().select(level))
|
.within(() => {
|
||||||
|
cy.findByTestId('add-collaborator-select')
|
||||||
|
.click()
|
||||||
|
.then(() => {
|
||||||
|
cy.findByText(level).click()
|
||||||
|
})
|
||||||
|
})
|
||||||
cy.findByText('Invite').click({ force: true })
|
cy.findByText('Invite').click({ force: true })
|
||||||
cy.findByText('Invite not yet accepted.')
|
cy.findByText('Invite not yet accepted.')
|
||||||
})
|
})
|
||||||
|
@ -117,7 +123,7 @@ function shareProjectByEmail(
|
||||||
export function shareProjectByEmailAndAcceptInviteViaDash(
|
export function shareProjectByEmailAndAcceptInviteViaDash(
|
||||||
projectName: string,
|
projectName: string,
|
||||||
email: string,
|
email: string,
|
||||||
level: 'Can view' | 'Can edit'
|
level: 'Viewer' | 'Editor'
|
||||||
) {
|
) {
|
||||||
shareProjectByEmail(projectName, email, level)
|
shareProjectByEmail(projectName, email, level)
|
||||||
|
|
||||||
|
@ -128,7 +134,7 @@ export function shareProjectByEmailAndAcceptInviteViaDash(
|
||||||
export function shareProjectByEmailAndAcceptInviteViaEmail(
|
export function shareProjectByEmailAndAcceptInviteViaEmail(
|
||||||
projectName: string,
|
projectName: string,
|
||||||
email: string,
|
email: string,
|
||||||
level: 'Can view' | 'Can edit'
|
level: 'Viewer' | 'Editor'
|
||||||
) {
|
) {
|
||||||
shareProjectByEmail(projectName, email, level)
|
shareProjectByEmail(projectName, email, level)
|
||||||
|
|
||||||
|
@ -212,11 +218,11 @@ export function createNewFile() {
|
||||||
|
|
||||||
export function toggleTrackChanges(state: boolean) {
|
export function toggleTrackChanges(state: boolean) {
|
||||||
cy.findByText('Review').click()
|
cy.findByText('Review').click()
|
||||||
cy.get('.rp-tc-state-collapse').then(el => {
|
cy.get('.track-changes-menu-button').then(el => {
|
||||||
// TODO: simplify this in the frontend?
|
// when the menu is expanded renders the `expand_more` icon,
|
||||||
if (el.hasClass('rp-tc-state-collapse-on')) {
|
// and the `chevron_right` icon when it's collapsed
|
||||||
// make track-changes switches visible
|
if (!el.text().includes('expand_more')) {
|
||||||
cy.get('.rp-tc-state-collapse').click()
|
el.click()
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -241,5 +247,5 @@ export function toggleTrackChanges(state: boolean) {
|
||||||
cy.wait(alias)
|
cy.wait(alias)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
cy.findByText('Review').click()
|
cy.contains('.toolbar-item', 'Review').click()
|
||||||
}
|
}
|
||||||
|
|
|
@ -154,7 +154,7 @@ describe('Project Sharing', function () {
|
||||||
|
|
||||||
beforeEach(function () {
|
beforeEach(function () {
|
||||||
login('user@example.com')
|
login('user@example.com')
|
||||||
shareProjectByEmailAndAcceptInviteViaEmail(projectName, email, 'Can view')
|
shareProjectByEmailAndAcceptInviteViaEmail(projectName, email, 'Viewer')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should grant the collaborator read access', () => {
|
it('should grant the collaborator read access', () => {
|
||||||
|
@ -169,7 +169,7 @@ describe('Project Sharing', function () {
|
||||||
|
|
||||||
beforeWithReRunOnTestRetry(function () {
|
beforeWithReRunOnTestRetry(function () {
|
||||||
login('user@example.com')
|
login('user@example.com')
|
||||||
shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Can view')
|
shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Viewer')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should grant the collaborator read access', () => {
|
it('should grant the collaborator read access', () => {
|
||||||
|
@ -186,7 +186,7 @@ describe('Project Sharing', function () {
|
||||||
|
|
||||||
beforeWithReRunOnTestRetry(function () {
|
beforeWithReRunOnTestRetry(function () {
|
||||||
login('user@example.com')
|
login('user@example.com')
|
||||||
shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Can edit')
|
shareProjectByEmailAndAcceptInviteViaDash(projectName, email, 'Editor')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should grant the collaborator write access', () => {
|
it('should grant the collaborator write access', () => {
|
||||||
|
|
|
@ -204,9 +204,9 @@ describe('SandboxedCompiles', function () {
|
||||||
cy.log('wait for compile')
|
cy.log('wait for compile')
|
||||||
cy.get('.pdf-viewer').should('contain.text', 'sandboxed')
|
cy.get('.pdf-viewer').should('contain.text', 'sandboxed')
|
||||||
|
|
||||||
cy.log('Check which compiler version was used, expect 2024')
|
cy.log('Check which compiler version was used, expect 2025')
|
||||||
cy.get('[aria-label="View logs"]').click()
|
cy.get('[aria-label="View logs"]').click()
|
||||||
cy.findByText(/This is pdfTeX, Version .+ \(TeX Live 2024\) /)
|
cy.findByText(/This is pdfTeX, Version .+ \(TeX Live 2025\) /)
|
||||||
|
|
||||||
cy.log('Check that there is no TeX Live version toggle')
|
cy.log('Check that there is no TeX Live version toggle')
|
||||||
cy.get('header').findByText('Menu').click()
|
cy.get('header').findByText('Menu').click()
|
||||||
|
|
|
@ -116,13 +116,6 @@ test_acceptance_clean:
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
||||||
|
|
||||||
test_acceptance_pre_run:
|
test_acceptance_pre_run:
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
|
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
|
|
||||||
while ! mongosh --eval "db.version()" > /dev/null; do \
|
|
||||||
echo "Waiting for Mongo..."; \
|
|
||||||
sleep 1; \
|
|
||||||
done; \
|
|
||||||
mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
|
|
||||||
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
|
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
|
||||||
endif
|
endif
|
||||||
|
|
|
@ -26,7 +26,7 @@ services:
|
||||||
NODE_OPTIONS: "--unhandled-rejections=strict"
|
NODE_OPTIONS: "--unhandled-rejections=strict"
|
||||||
depends_on:
|
depends_on:
|
||||||
mongo:
|
mongo:
|
||||||
condition: service_healthy
|
condition: service_started
|
||||||
user: node
|
user: node
|
||||||
command: npm run test:acceptance
|
command: npm run test:acceptance
|
||||||
|
|
||||||
|
@ -41,7 +41,12 @@ services:
|
||||||
mongo:
|
mongo:
|
||||||
image: mongo:6.0.13
|
image: mongo:6.0.13
|
||||||
command: --replSet overleaf
|
command: --replSet overleaf
|
||||||
healthcheck:
|
volumes:
|
||||||
test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
|
- ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
interval: 1s
|
environment:
|
||||||
retries: 20
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
|
extra_hosts:
|
||||||
|
# Required when using the automatic database setup for initializing the
|
||||||
|
# replica set. This override is not needed when running the setup after
|
||||||
|
# starting up mongo.
|
||||||
|
- mongo:127.0.0.1
|
||||||
|
|
|
@ -38,14 +38,19 @@ services:
|
||||||
user: node
|
user: node
|
||||||
depends_on:
|
depends_on:
|
||||||
mongo:
|
mongo:
|
||||||
condition: service_healthy
|
condition: service_started
|
||||||
command: npm run --silent test:acceptance
|
command: npm run --silent test:acceptance
|
||||||
|
|
||||||
mongo:
|
mongo:
|
||||||
image: mongo:6.0.13
|
image: mongo:6.0.13
|
||||||
command: --replSet overleaf
|
command: --replSet overleaf
|
||||||
healthcheck:
|
volumes:
|
||||||
test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
|
- ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
interval: 1s
|
environment:
|
||||||
retries: 20
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
|
extra_hosts:
|
||||||
|
# Required when using the automatic database setup for initializing the
|
||||||
|
# replica set. This override is not needed when running the setup after
|
||||||
|
# starting up mongo.
|
||||||
|
- mongo:127.0.0.1
|
||||||
|
|
||||||
|
|
|
@ -309,6 +309,10 @@ const loadTcpServer = net.createServer(function (socket) {
|
||||||
} else {
|
} else {
|
||||||
// Ready will cancel the maint state.
|
// Ready will cancel the maint state.
|
||||||
socket.write(`up, ready, ${Math.max(freeLoadPercentage, 1)}%\n`, 'ASCII')
|
socket.write(`up, ready, ${Math.max(freeLoadPercentage, 1)}%\n`, 'ASCII')
|
||||||
|
if (freeLoadPercentage <= 0) {
|
||||||
|
// This metric records how often we would have gone into maintenance mode.
|
||||||
|
Metrics.inc('clsi-prevented-maint')
|
||||||
|
}
|
||||||
}
|
}
|
||||||
socket.end()
|
socket.end()
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -98,12 +98,11 @@ module.exports = OutputCacheManager = {
|
||||||
CONTENT_SUBDIR: 'content',
|
CONTENT_SUBDIR: 'content',
|
||||||
CACHE_SUBDIR: 'generated-files',
|
CACHE_SUBDIR: 'generated-files',
|
||||||
ARCHIVE_SUBDIR: 'archived-logs',
|
ARCHIVE_SUBDIR: 'archived-logs',
|
||||||
// build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
|
// build id is HEXDATE-HEXRANDOM from Date.now() and RandomBytes
|
||||||
// for backwards compatibility, make the randombytes part optional
|
BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/,
|
||||||
BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
|
CONTENT_REGEX: /^[0-9a-f]+-[0-9a-f]+$/,
|
||||||
CONTENT_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/,
|
|
||||||
CACHE_LIMIT: 2, // maximum number of cache directories
|
CACHE_LIMIT: 2, // maximum number of cache directories
|
||||||
CACHE_AGE: 60 * 60 * 1000, // up to one hour old
|
CACHE_AGE: 90 * 60 * 1000, // up to 90 minutes old
|
||||||
|
|
||||||
init,
|
init,
|
||||||
queueDirOperation: callbackify(queueDirOperation),
|
queueDirOperation: callbackify(queueDirOperation),
|
||||||
|
@ -137,7 +136,11 @@ module.exports = OutputCacheManager = {
|
||||||
outputDir,
|
outputDir,
|
||||||
callback
|
callback
|
||||||
) {
|
) {
|
||||||
OutputCacheManager.generateBuildId(function (err, buildId) {
|
const getBuildId = cb => {
|
||||||
|
if (request.buildId) return cb(null, request.buildId)
|
||||||
|
OutputCacheManager.generateBuildId(cb)
|
||||||
|
}
|
||||||
|
getBuildId(function (err, buildId) {
|
||||||
if (err) {
|
if (err) {
|
||||||
return callback(err)
|
return callback(err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,7 @@ const { NotFoundError } = require('./Errors')
|
||||||
const logger = require('@overleaf/logger')
|
const logger = require('@overleaf/logger')
|
||||||
|
|
||||||
// NOTE: Updating this list requires a corresponding change in
|
// NOTE: Updating this list requires a corresponding change in
|
||||||
// * services/web/frontend/js/features/pdf-preview/util/file-list.js
|
// * services/web/frontend/js/features/pdf-preview/util/file-list.ts
|
||||||
const ignoreFiles = ['output.fls', 'output.fdb_latexmk']
|
const ignoreFiles = ['output.fls', 'output.fdb_latexmk']
|
||||||
|
|
||||||
function getContentDir(projectId, userId) {
|
function getContentDir(projectId, userId) {
|
||||||
|
|
|
@ -13,6 +13,7 @@ const CompileManager = require('./CompileManager')
|
||||||
const async = require('async')
|
const async = require('async')
|
||||||
const logger = require('@overleaf/logger')
|
const logger = require('@overleaf/logger')
|
||||||
const oneDay = 24 * 60 * 60 * 1000
|
const oneDay = 24 * 60 * 60 * 1000
|
||||||
|
const Metrics = require('@overleaf/metrics')
|
||||||
const Settings = require('@overleaf/settings')
|
const Settings = require('@overleaf/settings')
|
||||||
const diskusage = require('diskusage')
|
const diskusage = require('diskusage')
|
||||||
const { callbackify } = require('node:util')
|
const { callbackify } = require('node:util')
|
||||||
|
@ -22,33 +23,48 @@ const fs = require('node:fs')
|
||||||
// projectId -> timestamp mapping.
|
// projectId -> timestamp mapping.
|
||||||
const LAST_ACCESS = new Map()
|
const LAST_ACCESS = new Map()
|
||||||
|
|
||||||
async function refreshExpiryTimeout() {
|
async function collectDiskStats() {
|
||||||
const paths = [
|
const paths = [
|
||||||
Settings.path.compilesDir,
|
Settings.path.compilesDir,
|
||||||
Settings.path.outputDir,
|
Settings.path.outputDir,
|
||||||
Settings.path.clsiCacheDir,
|
Settings.path.clsiCacheDir,
|
||||||
]
|
]
|
||||||
|
|
||||||
|
const diskStats = {}
|
||||||
for (const path of paths) {
|
for (const path of paths) {
|
||||||
try {
|
try {
|
||||||
const stats = await diskusage.check(path)
|
const stats = await diskusage.check(path)
|
||||||
const lowDisk = stats.available / stats.total < 0.1
|
const diskAvailablePercent = (stats.available / stats.total) * 100
|
||||||
|
Metrics.gauge('disk_available_percent', diskAvailablePercent, 1, {
|
||||||
const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
|
path,
|
||||||
if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
|
})
|
||||||
logger.warn(
|
const lowDisk = diskAvailablePercent < 10
|
||||||
{
|
diskStats[path] = { stats, lowDisk }
|
||||||
stats,
|
|
||||||
newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2),
|
|
||||||
},
|
|
||||||
'disk running low on space, modifying EXPIRY_TIMEOUT'
|
|
||||||
)
|
|
||||||
ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
|
|
||||||
break
|
|
||||||
}
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
logger.err({ err, path }, 'error getting disk usage')
|
logger.err({ err, path }, 'error getting disk usage')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return diskStats
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshExpiryTimeout() {
|
||||||
|
for (const [path, { stats, lowDisk }] of Object.entries(
|
||||||
|
await collectDiskStats()
|
||||||
|
)) {
|
||||||
|
const lowerExpiry = ProjectPersistenceManager.EXPIRY_TIMEOUT * 0.9
|
||||||
|
if (lowDisk && Settings.project_cache_length_ms / 2 < lowerExpiry) {
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
path,
|
||||||
|
stats,
|
||||||
|
newExpiryTimeoutInDays: (lowerExpiry / oneDay).toFixed(2),
|
||||||
|
},
|
||||||
|
'disk running low on space, modifying EXPIRY_TIMEOUT'
|
||||||
|
)
|
||||||
|
ProjectPersistenceManager.EXPIRY_TIMEOUT = lowerExpiry
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = ProjectPersistenceManager = {
|
module.exports = ProjectPersistenceManager = {
|
||||||
|
@ -103,6 +119,13 @@ module.exports = ProjectPersistenceManager = {
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Collect disk stats frequently to have them ready the next time /metrics is scraped (60s +- jitter).
|
||||||
|
setInterval(() => {
|
||||||
|
collectDiskStats().catch(err => {
|
||||||
|
logger.err({ err }, 'low level error collecting disk stats')
|
||||||
|
})
|
||||||
|
}, 50_000)
|
||||||
},
|
},
|
||||||
|
|
||||||
markProjectAsJustAccessed(projectId, callback) {
|
markProjectAsJustAccessed(projectId, callback) {
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
const settings = require('@overleaf/settings')
|
const settings = require('@overleaf/settings')
|
||||||
|
const OutputCacheManager = require('./OutputCacheManager')
|
||||||
|
|
||||||
const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex']
|
const VALID_COMPILERS = ['pdflatex', 'latex', 'xelatex', 'lualatex']
|
||||||
const MAX_TIMEOUT = 600
|
const MAX_TIMEOUT = 600
|
||||||
|
@ -135,6 +136,11 @@ function parse(body, callback) {
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
response.rootResourcePath = _checkPath(rootResourcePath)
|
response.rootResourcePath = _checkPath(rootResourcePath)
|
||||||
|
|
||||||
|
response.buildId = _parseAttribute('buildId', compile.options.buildId, {
|
||||||
|
type: 'string',
|
||||||
|
regex: OutputCacheManager.BUILD_REGEX,
|
||||||
|
})
|
||||||
} catch (error1) {
|
} catch (error1) {
|
||||||
const error = error1
|
const error = error1
|
||||||
return callback(error)
|
return callback(error)
|
||||||
|
@ -199,6 +205,13 @@ function _parseAttribute(name, attribute, options) {
|
||||||
throw new Error(`${name} attribute should be a ${options.type}`)
|
throw new Error(`${name} attribute should be a ${options.type}`)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (options.type === 'string' && options.regex instanceof RegExp) {
|
||||||
|
if (!options.regex.test(attribute)) {
|
||||||
|
throw new Error(
|
||||||
|
`${name} attribute does not match regex ${options.regex}`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
if (options.default != null) {
|
if (options.default != null) {
|
||||||
return options.default
|
return options.default
|
||||||
|
|
|
@ -200,73 +200,22 @@ module.exports = ResourceWriter = {
|
||||||
return OutputFileFinder.findOutputFiles(
|
return OutputFileFinder.findOutputFiles(
|
||||||
resources,
|
resources,
|
||||||
basePath,
|
basePath,
|
||||||
function (error, outputFiles, allFiles) {
|
(error, outputFiles, allFiles) => {
|
||||||
if (error != null) {
|
if (error != null) {
|
||||||
return callback(error)
|
return callback(error)
|
||||||
}
|
}
|
||||||
|
|
||||||
const jobs = []
|
const jobs = []
|
||||||
for (const file of Array.from(outputFiles || [])) {
|
for (const { path } of outputFiles || []) {
|
||||||
;(function (file) {
|
const shouldDelete = ResourceWriter.isExtraneousFile(path)
|
||||||
const { path } = file
|
if (shouldDelete) {
|
||||||
let shouldDelete = true
|
jobs.push(callback =>
|
||||||
if (
|
ResourceWriter._deleteFileIfNotDirectory(
|
||||||
path.match(/^output\./) ||
|
Path.join(basePath, path),
|
||||||
path.match(/\.aux$/) ||
|
callback
|
||||||
path.match(/^cache\//)
|
|
||||||
) {
|
|
||||||
// knitr cache
|
|
||||||
shouldDelete = false
|
|
||||||
}
|
|
||||||
if (path.match(/^output-.*/)) {
|
|
||||||
// Tikz cached figures (default case)
|
|
||||||
shouldDelete = false
|
|
||||||
}
|
|
||||||
if (path.match(/\.(pdf|dpth|md5)$/)) {
|
|
||||||
// Tikz cached figures (by extension)
|
|
||||||
shouldDelete = false
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
path.match(/\.(pygtex|pygstyle)$/) ||
|
|
||||||
path.match(/(^|\/)_minted-[^\/]+\//)
|
|
||||||
) {
|
|
||||||
// minted files/directory
|
|
||||||
shouldDelete = false
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
path.match(/\.md\.tex$/) ||
|
|
||||||
path.match(/(^|\/)_markdown_[^\/]+\//)
|
|
||||||
) {
|
|
||||||
// markdown files/directory
|
|
||||||
shouldDelete = false
|
|
||||||
}
|
|
||||||
if (path.match(/-eps-converted-to\.pdf$/)) {
|
|
||||||
// Epstopdf generated files
|
|
||||||
shouldDelete = false
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
path === 'output.pdf' ||
|
|
||||||
path === 'output.dvi' ||
|
|
||||||
path === 'output.log' ||
|
|
||||||
path === 'output.xdv' ||
|
|
||||||
path === 'output.stdout' ||
|
|
||||||
path === 'output.stderr'
|
|
||||||
) {
|
|
||||||
shouldDelete = true
|
|
||||||
}
|
|
||||||
if (path === 'output.tex') {
|
|
||||||
// created by TikzManager if present in output files
|
|
||||||
shouldDelete = true
|
|
||||||
}
|
|
||||||
if (shouldDelete) {
|
|
||||||
return jobs.push(callback =>
|
|
||||||
ResourceWriter._deleteFileIfNotDirectory(
|
|
||||||
Path.join(basePath, path),
|
|
||||||
callback
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
}
|
)
|
||||||
})(file)
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return async.series(jobs, function (error) {
|
return async.series(jobs, function (error) {
|
||||||
|
@ -279,6 +228,58 @@ module.exports = ResourceWriter = {
|
||||||
)
|
)
|
||||||
},
|
},
|
||||||
|
|
||||||
|
isExtraneousFile(path) {
|
||||||
|
let shouldDelete = true
|
||||||
|
if (
|
||||||
|
path.match(/^output\./) ||
|
||||||
|
path.match(/\.aux$/) ||
|
||||||
|
path.match(/^cache\//)
|
||||||
|
) {
|
||||||
|
// knitr cache
|
||||||
|
shouldDelete = false
|
||||||
|
}
|
||||||
|
if (path.match(/^output-.*/)) {
|
||||||
|
// Tikz cached figures (default case)
|
||||||
|
shouldDelete = false
|
||||||
|
}
|
||||||
|
if (path.match(/\.(pdf|dpth|md5)$/)) {
|
||||||
|
// Tikz cached figures (by extension)
|
||||||
|
shouldDelete = false
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
path.match(/\.(pygtex|pygstyle)$/) ||
|
||||||
|
path.match(/(^|\/)_minted-[^\/]+\//)
|
||||||
|
) {
|
||||||
|
// minted files/directory
|
||||||
|
shouldDelete = false
|
||||||
|
}
|
||||||
|
if (path.match(/\.md\.tex$/) || path.match(/(^|\/)_markdown_[^\/]+\//)) {
|
||||||
|
// markdown files/directory
|
||||||
|
shouldDelete = false
|
||||||
|
}
|
||||||
|
if (path.match(/-eps-converted-to\.pdf$/)) {
|
||||||
|
// Epstopdf generated files
|
||||||
|
shouldDelete = false
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
path === 'output.synctex.gz' ||
|
||||||
|
path === 'output.pdfxref' ||
|
||||||
|
path === 'output.pdf' ||
|
||||||
|
path === 'output.dvi' ||
|
||||||
|
path === 'output.log' ||
|
||||||
|
path === 'output.xdv' ||
|
||||||
|
path === 'output.stdout' ||
|
||||||
|
path === 'output.stderr'
|
||||||
|
) {
|
||||||
|
shouldDelete = true
|
||||||
|
}
|
||||||
|
if (path === 'output.tex') {
|
||||||
|
// created by TikzManager if present in output files
|
||||||
|
shouldDelete = true
|
||||||
|
}
|
||||||
|
return shouldDelete
|
||||||
|
},
|
||||||
|
|
||||||
_deleteFileIfNotDirectory(path, callback) {
|
_deleteFileIfNotDirectory(path, callback) {
|
||||||
if (callback == null) {
|
if (callback == null) {
|
||||||
callback = function () {}
|
callback = function () {}
|
||||||
|
|
|
@ -46,7 +46,7 @@ server {
|
||||||
}
|
}
|
||||||
|
|
||||||
# handle output files for specific users
|
# handle output files for specific users
|
||||||
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
|
location ~ ^/project/([0-9a-f]+)/user/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
|
||||||
if ($request_method = 'OPTIONS') {
|
if ($request_method = 'OPTIONS') {
|
||||||
# handle OPTIONS method for CORS requests
|
# handle OPTIONS method for CORS requests
|
||||||
add_header 'Allow' 'GET,HEAD';
|
add_header 'Allow' 'GET,HEAD';
|
||||||
|
@ -64,7 +64,7 @@ server {
|
||||||
alias /output/$1-$2/generated-files/$3/$4.blg;
|
alias /output/$1-$2/generated-files/$3/$4.blg;
|
||||||
}
|
}
|
||||||
# handle output files for anonymous users
|
# handle output files for anonymous users
|
||||||
location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z]+)$ {
|
location ~ ^/project/([0-9a-f]+)/build/([0-9a-f-]+)/output/output\.([a-z.]+)$ {
|
||||||
if ($request_method = 'OPTIONS') {
|
if ($request_method = 'OPTIONS') {
|
||||||
# handle OPTIONS method for CORS requests
|
# handle OPTIONS method for CORS requests
|
||||||
add_header 'Allow' 'GET,HEAD';
|
add_header 'Allow' 'GET,HEAD';
|
||||||
|
|
|
@ -28,7 +28,7 @@
|
||||||
"body-parser": "^1.20.3",
|
"body-parser": "^1.20.3",
|
||||||
"bunyan": "^1.8.15",
|
"bunyan": "^1.8.15",
|
||||||
"diskusage": "^1.1.3",
|
"diskusage": "^1.1.3",
|
||||||
"dockerode": "^3.1.0",
|
"dockerode": "^4.0.5",
|
||||||
"express": "^4.21.2",
|
"express": "^4.21.2",
|
||||||
"lodash": "^4.17.21",
|
"lodash": "^4.17.21",
|
||||||
"p-limit": "^3.1.0",
|
"p-limit": "^3.1.0",
|
||||||
|
|
|
@ -107,7 +107,6 @@ Hello world
|
||||||
'output.fdb_latexmk',
|
'output.fdb_latexmk',
|
||||||
'output.fls',
|
'output.fls',
|
||||||
'output.log',
|
'output.log',
|
||||||
'output.pdfxref',
|
|
||||||
'output.stderr',
|
'output.stderr',
|
||||||
'output.stdout',
|
'output.stdout',
|
||||||
])
|
])
|
||||||
|
|
|
@ -16,7 +16,7 @@ const modulePath = require('node:path').join(
|
||||||
'../../../app/js/DockerLockManager'
|
'../../../app/js/DockerLockManager'
|
||||||
)
|
)
|
||||||
|
|
||||||
describe('LockManager', function () {
|
describe('DockerLockManager', function () {
|
||||||
beforeEach(function () {
|
beforeEach(function () {
|
||||||
return (this.LockManager = SandboxedModule.require(modulePath, {
|
return (this.LockManager = SandboxedModule.require(modulePath, {
|
||||||
requires: {
|
requires: {
|
||||||
|
|
|
@ -21,6 +21,7 @@ describe('LockManager', function () {
|
||||||
compileConcurrencyLimit: 5,
|
compileConcurrencyLimit: 5,
|
||||||
}),
|
}),
|
||||||
'./Errors': (this.Erros = Errors),
|
'./Errors': (this.Erros = Errors),
|
||||||
|
'./RequestParser': { MAX_TIMEOUT: 600 },
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
|
@ -23,6 +23,7 @@ describe('ProjectPersistenceManager', function () {
|
||||||
beforeEach(function () {
|
beforeEach(function () {
|
||||||
this.ProjectPersistenceManager = SandboxedModule.require(modulePath, {
|
this.ProjectPersistenceManager = SandboxedModule.require(modulePath, {
|
||||||
requires: {
|
requires: {
|
||||||
|
'@overleaf/metrics': (this.Metrics = { gauge: sinon.stub() }),
|
||||||
'./UrlCache': (this.UrlCache = {}),
|
'./UrlCache': (this.UrlCache = {}),
|
||||||
'./CompileManager': (this.CompileManager = {}),
|
'./CompileManager': (this.CompileManager = {}),
|
||||||
diskusage: (this.diskusage = { check: sinon.stub() }),
|
diskusage: (this.diskusage = { check: sinon.stub() }),
|
||||||
|
@ -49,6 +50,10 @@ describe('ProjectPersistenceManager', function () {
|
||||||
})
|
})
|
||||||
|
|
||||||
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
|
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
|
||||||
|
this.Metrics.gauge.should.have.been.calledWith(
|
||||||
|
'disk_available_percent',
|
||||||
|
40
|
||||||
|
)
|
||||||
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(
|
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(
|
||||||
this.settings.project_cache_length_ms
|
this.settings.project_cache_length_ms
|
||||||
)
|
)
|
||||||
|
@ -63,6 +68,10 @@ describe('ProjectPersistenceManager', function () {
|
||||||
})
|
})
|
||||||
|
|
||||||
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
|
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
|
||||||
|
this.Metrics.gauge.should.have.been.calledWith(
|
||||||
|
'disk_available_percent',
|
||||||
|
5
|
||||||
|
)
|
||||||
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(900)
|
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(900)
|
||||||
done()
|
done()
|
||||||
})
|
})
|
||||||
|
@ -75,6 +84,10 @@ describe('ProjectPersistenceManager', function () {
|
||||||
})
|
})
|
||||||
this.ProjectPersistenceManager.EXPIRY_TIMEOUT = 500
|
this.ProjectPersistenceManager.EXPIRY_TIMEOUT = 500
|
||||||
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
|
this.ProjectPersistenceManager.refreshExpiryTimeout(() => {
|
||||||
|
this.Metrics.gauge.should.have.been.calledWith(
|
||||||
|
'disk_available_percent',
|
||||||
|
5
|
||||||
|
)
|
||||||
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(500)
|
this.ProjectPersistenceManager.EXPIRY_TIMEOUT.should.equal(500)
|
||||||
done()
|
done()
|
||||||
})
|
})
|
||||||
|
|
|
@ -30,6 +30,7 @@ describe('RequestParser', function () {
|
||||||
this.RequestParser = SandboxedModule.require(modulePath, {
|
this.RequestParser = SandboxedModule.require(modulePath, {
|
||||||
requires: {
|
requires: {
|
||||||
'@overleaf/settings': (this.settings = {}),
|
'@overleaf/settings': (this.settings = {}),
|
||||||
|
'./OutputCacheManager': { BUILD_REGEX: /^[0-9a-f]+-[0-9a-f]+$/ },
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -274,6 +275,37 @@ describe('RequestParser', function () {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('with a valid buildId', function () {
|
||||||
|
beforeEach(function (done) {
|
||||||
|
this.validRequest.compile.options.buildId = '195a4869176-a4ad60bee7bf35e4'
|
||||||
|
this.RequestParser.parse(this.validRequest, (error, data) => {
|
||||||
|
if (error) return done(error)
|
||||||
|
this.data = data
|
||||||
|
done()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return an error', function () {
|
||||||
|
this.data.buildId.should.equal('195a4869176-a4ad60bee7bf35e4')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('with a bad buildId', function () {
|
||||||
|
beforeEach(function () {
|
||||||
|
this.validRequest.compile.options.buildId = 'foo/bar'
|
||||||
|
this.RequestParser.parse(this.validRequest, this.callback)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should return an error', function () {
|
||||||
|
this.callback
|
||||||
|
.calledWithMatch({
|
||||||
|
message:
|
||||||
|
'buildId attribute does not match regex /^[0-9a-f]+-[0-9a-f]+$/',
|
||||||
|
})
|
||||||
|
.should.equal(true)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
describe('with a resource with a valid date', function () {
|
describe('with a resource with a valid date', function () {
|
||||||
beforeEach(function () {
|
beforeEach(function () {
|
||||||
this.date = '12:00 01/02/03'
|
this.date = '12:00 01/02/03'
|
||||||
|
|
|
@ -116,13 +116,6 @@ test_acceptance_clean:
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
||||||
|
|
||||||
test_acceptance_pre_run:
|
test_acceptance_pre_run:
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
|
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
|
|
||||||
while ! mongosh --eval "db.version()" > /dev/null; do \
|
|
||||||
echo "Waiting for Mongo..."; \
|
|
||||||
sleep 1; \
|
|
||||||
done; \
|
|
||||||
mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
|
|
||||||
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
|
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
|
||||||
endif
|
endif
|
||||||
|
|
|
@ -26,7 +26,7 @@ services:
|
||||||
NODE_OPTIONS: "--unhandled-rejections=strict"
|
NODE_OPTIONS: "--unhandled-rejections=strict"
|
||||||
depends_on:
|
depends_on:
|
||||||
mongo:
|
mongo:
|
||||||
condition: service_healthy
|
condition: service_started
|
||||||
user: node
|
user: node
|
||||||
command: npm run test:acceptance
|
command: npm run test:acceptance
|
||||||
|
|
||||||
|
@ -41,7 +41,12 @@ services:
|
||||||
mongo:
|
mongo:
|
||||||
image: mongo:6.0.13
|
image: mongo:6.0.13
|
||||||
command: --replSet overleaf
|
command: --replSet overleaf
|
||||||
healthcheck:
|
volumes:
|
||||||
test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
|
- ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
interval: 1s
|
environment:
|
||||||
retries: 20
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
|
extra_hosts:
|
||||||
|
# Required when using the automatic database setup for initializing the
|
||||||
|
# replica set. This override is not needed when running the setup after
|
||||||
|
# starting up mongo.
|
||||||
|
- mongo:127.0.0.1
|
||||||
|
|
|
@ -38,14 +38,19 @@ services:
|
||||||
user: node
|
user: node
|
||||||
depends_on:
|
depends_on:
|
||||||
mongo:
|
mongo:
|
||||||
condition: service_healthy
|
condition: service_started
|
||||||
command: npm run --silent test:acceptance
|
command: npm run --silent test:acceptance
|
||||||
|
|
||||||
mongo:
|
mongo:
|
||||||
image: mongo:6.0.13
|
image: mongo:6.0.13
|
||||||
command: --replSet overleaf
|
command: --replSet overleaf
|
||||||
healthcheck:
|
volumes:
|
||||||
test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
|
- ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
interval: 1s
|
environment:
|
||||||
retries: 20
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
|
extra_hosts:
|
||||||
|
# Required when using the automatic database setup for initializing the
|
||||||
|
# replica set. This override is not needed when running the setup after
|
||||||
|
# starting up mongo.
|
||||||
|
- mongo:127.0.0.1
|
||||||
|
|
||||||
|
|
|
@ -116,13 +116,6 @@ test_acceptance_clean:
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
||||||
|
|
||||||
test_acceptance_pre_run:
|
test_acceptance_pre_run:
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
|
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
|
|
||||||
while ! mongosh --eval "db.version()" > /dev/null; do \
|
|
||||||
echo "Waiting for Mongo..."; \
|
|
||||||
sleep 1; \
|
|
||||||
done; \
|
|
||||||
mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
|
|
||||||
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
|
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
|
||||||
endif
|
endif
|
||||||
|
|
|
@ -88,14 +88,17 @@ app.get('/status', (req, res) => res.send('docstore is alive'))
|
||||||
|
|
||||||
app.use(handleValidationErrors())
|
app.use(handleValidationErrors())
|
||||||
app.use(function (error, req, res, next) {
|
app.use(function (error, req, res, next) {
|
||||||
logger.error({ err: error, req }, 'request errored')
|
|
||||||
if (error instanceof Errors.NotFoundError) {
|
if (error instanceof Errors.NotFoundError) {
|
||||||
|
logger.warn({ req }, 'not found')
|
||||||
res.sendStatus(404)
|
res.sendStatus(404)
|
||||||
} else if (error instanceof Errors.DocModifiedError) {
|
} else if (error instanceof Errors.DocModifiedError) {
|
||||||
|
logger.warn({ req }, 'conflict: doc modified')
|
||||||
res.sendStatus(409)
|
res.sendStatus(409)
|
||||||
} else if (error instanceof Errors.DocVersionDecrementedError) {
|
} else if (error instanceof Errors.DocVersionDecrementedError) {
|
||||||
|
logger.warn({ req }, 'conflict: doc version decremented')
|
||||||
res.sendStatus(409)
|
res.sendStatus(409)
|
||||||
} else {
|
} else {
|
||||||
|
logger.error({ err: error, req }, 'request errored')
|
||||||
res.status(500).send('Oops, something went wrong')
|
res.status(500).send('Oops, something went wrong')
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
|
@ -29,7 +29,7 @@ services:
|
||||||
NODE_OPTIONS: "--unhandled-rejections=strict"
|
NODE_OPTIONS: "--unhandled-rejections=strict"
|
||||||
depends_on:
|
depends_on:
|
||||||
mongo:
|
mongo:
|
||||||
condition: service_healthy
|
condition: service_started
|
||||||
gcs:
|
gcs:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
user: node
|
user: node
|
||||||
|
@ -46,10 +46,15 @@ services:
|
||||||
mongo:
|
mongo:
|
||||||
image: mongo:6.0.13
|
image: mongo:6.0.13
|
||||||
command: --replSet overleaf
|
command: --replSet overleaf
|
||||||
healthcheck:
|
volumes:
|
||||||
test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
|
- ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
interval: 1s
|
environment:
|
||||||
retries: 20
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
|
extra_hosts:
|
||||||
|
# Required when using the automatic database setup for initializing the
|
||||||
|
# replica set. This override is not needed when running the setup after
|
||||||
|
# starting up mongo.
|
||||||
|
- mongo:127.0.0.1
|
||||||
gcs:
|
gcs:
|
||||||
image: fsouza/fake-gcs-server:1.45.2
|
image: fsouza/fake-gcs-server:1.45.2
|
||||||
command: ["--port=9090", "--scheme=http"]
|
command: ["--port=9090", "--scheme=http"]
|
||||||
|
|
|
@ -41,7 +41,7 @@ services:
|
||||||
user: node
|
user: node
|
||||||
depends_on:
|
depends_on:
|
||||||
mongo:
|
mongo:
|
||||||
condition: service_healthy
|
condition: service_started
|
||||||
gcs:
|
gcs:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
command: npm run --silent test:acceptance
|
command: npm run --silent test:acceptance
|
||||||
|
@ -49,10 +49,15 @@ services:
|
||||||
mongo:
|
mongo:
|
||||||
image: mongo:6.0.13
|
image: mongo:6.0.13
|
||||||
command: --replSet overleaf
|
command: --replSet overleaf
|
||||||
healthcheck:
|
volumes:
|
||||||
test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
|
- ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
interval: 1s
|
environment:
|
||||||
retries: 20
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
|
extra_hosts:
|
||||||
|
# Required when using the automatic database setup for initializing the
|
||||||
|
# replica set. This override is not needed when running the setup after
|
||||||
|
# starting up mongo.
|
||||||
|
- mongo:127.0.0.1
|
||||||
|
|
||||||
gcs:
|
gcs:
|
||||||
image: fsouza/fake-gcs-server:1.45.2
|
image: fsouza/fake-gcs-server:1.45.2
|
||||||
|
|
|
@ -116,13 +116,6 @@ test_acceptance_clean:
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
||||||
|
|
||||||
test_acceptance_pre_run:
|
test_acceptance_pre_run:
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
|
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
|
|
||||||
while ! mongosh --eval "db.version()" > /dev/null; do \
|
|
||||||
echo "Waiting for Mongo..."; \
|
|
||||||
sleep 1; \
|
|
||||||
done; \
|
|
||||||
mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
|
|
||||||
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
|
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
|
||||||
endif
|
endif
|
||||||
|
|
|
@ -147,6 +147,10 @@ app.post(
|
||||||
'/project/:project_id/get_and_flush_if_old',
|
'/project/:project_id/get_and_flush_if_old',
|
||||||
HttpController.getProjectDocsAndFlushIfOld
|
HttpController.getProjectDocsAndFlushIfOld
|
||||||
)
|
)
|
||||||
|
app.get(
|
||||||
|
'/project/:project_id/last_updated_at',
|
||||||
|
HttpController.getProjectLastUpdatedAt
|
||||||
|
)
|
||||||
app.post('/project/:project_id/clearState', HttpController.clearProjectState)
|
app.post('/project/:project_id/clearState', HttpController.clearProjectState)
|
||||||
app.post('/project/:project_id/doc/:doc_id', HttpController.setDoc)
|
app.post('/project/:project_id/doc/:doc_id', HttpController.setDoc)
|
||||||
app.post('/project/:project_id/doc/:doc_id/append', HttpController.appendToDoc)
|
app.post('/project/:project_id/doc/:doc_id/append', HttpController.appendToDoc)
|
||||||
|
|
|
@ -129,6 +129,22 @@ function getProjectDocsAndFlushIfOld(req, res, next) {
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getProjectLastUpdatedAt(req, res, next) {
|
||||||
|
const projectId = req.params.project_id
|
||||||
|
ProjectManager.getProjectDocsTimestamps(projectId, (err, timestamps) => {
|
||||||
|
if (err) return next(err)
|
||||||
|
|
||||||
|
// Filter out nulls. This can happen when
|
||||||
|
// - docs get flushed between the listing and getting the individual docs ts
|
||||||
|
// - a doc flush failed half way (doc keys removed, project tracking not updated)
|
||||||
|
timestamps = timestamps.filter(ts => !!ts)
|
||||||
|
|
||||||
|
timestamps = timestamps.map(ts => parseInt(ts, 10))
|
||||||
|
timestamps.sort((a, b) => (a > b ? 1 : -1))
|
||||||
|
res.json({ lastUpdatedAt: timestamps.pop() })
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
function clearProjectState(req, res, next) {
|
function clearProjectState(req, res, next) {
|
||||||
const projectId = req.params.project_id
|
const projectId = req.params.project_id
|
||||||
const timer = new Metrics.Timer('http.clearProjectState')
|
const timer = new Metrics.Timer('http.clearProjectState')
|
||||||
|
@ -521,6 +537,7 @@ module.exports = {
|
||||||
getDoc,
|
getDoc,
|
||||||
peekDoc,
|
peekDoc,
|
||||||
getProjectDocsAndFlushIfOld,
|
getProjectDocsAndFlushIfOld,
|
||||||
|
getProjectLastUpdatedAt,
|
||||||
clearProjectState,
|
clearProjectState,
|
||||||
appendToDoc,
|
appendToDoc,
|
||||||
setDoc,
|
setDoc,
|
||||||
|
|
|
@ -29,7 +29,7 @@ services:
|
||||||
NODE_OPTIONS: "--unhandled-rejections=strict"
|
NODE_OPTIONS: "--unhandled-rejections=strict"
|
||||||
depends_on:
|
depends_on:
|
||||||
mongo:
|
mongo:
|
||||||
condition: service_healthy
|
condition: service_started
|
||||||
redis:
|
redis:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
user: node
|
user: node
|
||||||
|
@ -53,7 +53,12 @@ services:
|
||||||
mongo:
|
mongo:
|
||||||
image: mongo:6.0.13
|
image: mongo:6.0.13
|
||||||
command: --replSet overleaf
|
command: --replSet overleaf
|
||||||
healthcheck:
|
volumes:
|
||||||
test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
|
- ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
interval: 1s
|
environment:
|
||||||
retries: 20
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
|
extra_hosts:
|
||||||
|
# Required when using the automatic database setup for initializing the
|
||||||
|
# replica set. This override is not needed when running the setup after
|
||||||
|
# starting up mongo.
|
||||||
|
- mongo:127.0.0.1
|
||||||
|
|
|
@ -41,7 +41,7 @@ services:
|
||||||
user: node
|
user: node
|
||||||
depends_on:
|
depends_on:
|
||||||
mongo:
|
mongo:
|
||||||
condition: service_healthy
|
condition: service_started
|
||||||
redis:
|
redis:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
command: npm run --silent test:acceptance
|
command: npm run --silent test:acceptance
|
||||||
|
@ -56,8 +56,13 @@ services:
|
||||||
mongo:
|
mongo:
|
||||||
image: mongo:6.0.13
|
image: mongo:6.0.13
|
||||||
command: --replSet overleaf
|
command: --replSet overleaf
|
||||||
healthcheck:
|
volumes:
|
||||||
test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
|
- ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
interval: 1s
|
environment:
|
||||||
retries: 20
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
|
extra_hosts:
|
||||||
|
# Required when using the automatic database setup for initializing the
|
||||||
|
# replica set. This override is not needed when running the setup after
|
||||||
|
# starting up mongo.
|
||||||
|
- mongo:127.0.0.1
|
||||||
|
|
||||||
|
|
|
@ -109,11 +109,40 @@ describe('Applying updates to a doc', function () {
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('should yield last updated time', function (done) {
|
||||||
|
DocUpdaterClient.getProjectLastUpdatedAt(
|
||||||
|
this.project_id,
|
||||||
|
(error, res, body) => {
|
||||||
|
if (error != null) {
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
res.statusCode.should.equal(200)
|
||||||
|
body.lastUpdatedAt.should.be.within(this.startTime, Date.now())
|
||||||
|
done()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('should yield no last updated time for another project', function (done) {
|
||||||
|
DocUpdaterClient.getProjectLastUpdatedAt(
|
||||||
|
DocUpdaterClient.randomId(),
|
||||||
|
(error, res, body) => {
|
||||||
|
if (error != null) {
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
res.statusCode.should.equal(200)
|
||||||
|
body.should.deep.equal({})
|
||||||
|
done()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
describe('when sending another update', function () {
|
describe('when sending another update', function () {
|
||||||
before(function (done) {
|
before(function (done) {
|
||||||
this.timeout = 10000
|
this.timeout(10000)
|
||||||
this.second_update = Object.create(this.update)
|
this.second_update = Object.assign({}, this.update)
|
||||||
this.second_update.v = this.version + 1
|
this.second_update.v = this.version + 1
|
||||||
|
this.secondStartTime = Date.now()
|
||||||
DocUpdaterClient.sendUpdate(
|
DocUpdaterClient.sendUpdate(
|
||||||
this.project_id,
|
this.project_id,
|
||||||
this.doc_id,
|
this.doc_id,
|
||||||
|
@ -127,6 +156,24 @@ describe('Applying updates to a doc', function () {
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('should update the doc', function (done) {
|
||||||
|
DocUpdaterClient.getDoc(
|
||||||
|
this.project_id,
|
||||||
|
this.doc_id,
|
||||||
|
(error, res, doc) => {
|
||||||
|
if (error) done(error)
|
||||||
|
doc.lines.should.deep.equal([
|
||||||
|
'one',
|
||||||
|
'one and a half',
|
||||||
|
'one and a half',
|
||||||
|
'two',
|
||||||
|
'three',
|
||||||
|
])
|
||||||
|
done()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
it('should not change the first op timestamp', function (done) {
|
it('should not change the first op timestamp', function (done) {
|
||||||
rclientProjectHistory.get(
|
rclientProjectHistory.get(
|
||||||
ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
|
ProjectHistoryKeys.projectHistoryFirstOpTimestamp({
|
||||||
|
@ -142,6 +189,23 @@ describe('Applying updates to a doc', function () {
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('should yield last updated time', function (done) {
|
||||||
|
DocUpdaterClient.getProjectLastUpdatedAt(
|
||||||
|
this.project_id,
|
||||||
|
(error, res, body) => {
|
||||||
|
if (error != null) {
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
res.statusCode.should.equal(200)
|
||||||
|
body.lastUpdatedAt.should.be.within(
|
||||||
|
this.secondStartTime,
|
||||||
|
Date.now()
|
||||||
|
)
|
||||||
|
done()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
@ -119,6 +119,18 @@ module.exports = DocUpdaterClient = {
|
||||||
)
|
)
|
||||||
},
|
},
|
||||||
|
|
||||||
|
getProjectLastUpdatedAt(projectId, callback) {
|
||||||
|
request.get(
|
||||||
|
`http://127.0.0.1:3003/project/${projectId}/last_updated_at`,
|
||||||
|
(error, res, body) => {
|
||||||
|
if (body != null && res.statusCode >= 200 && res.statusCode < 300) {
|
||||||
|
body = JSON.parse(body)
|
||||||
|
}
|
||||||
|
callback(error, res, body)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
},
|
||||||
|
|
||||||
preloadDoc(projectId, docId, callback) {
|
preloadDoc(projectId, docId, callback) {
|
||||||
DocUpdaterClient.getDoc(projectId, docId, callback)
|
DocUpdaterClient.getDoc(projectId, docId, callback)
|
||||||
},
|
},
|
||||||
|
|
|
@ -76,12 +76,10 @@ The configuration file is in `.json` format.
|
||||||
"postbackBaseUrl" (string): the postback url,
|
"postbackBaseUrl" (string): the postback url,
|
||||||
"serviceName" (string): current name of writeLaTeX
|
"serviceName" (string): current name of writeLaTeX
|
||||||
in case it ever changes,
|
in case it ever changes,
|
||||||
"oauth2" (object): { null or missing if oauth2 shouldn't be used
|
"oauth2Server" (string): oauth2 server,
|
||||||
"oauth2ClientID" (string): oauth2 client ID,
|
with protocol and
|
||||||
"oauth2ClientSecret" (string): oauth2 client secret,
|
without trailing slash,
|
||||||
"oauth2Server" (string): oauth2 server,
|
null or missing if oauth2 shouldn't be used
|
||||||
with protocol and
|
|
||||||
without trailing slash
|
|
||||||
},
|
},
|
||||||
"repoStore" (object, optional): { configure the repo store
|
"repoStore" (object, optional): { configure the repo store
|
||||||
"maxFileSize" (long, optional): maximum size of a file, inclusive
|
"maxFileSize" (long, optional): maximum size of a file, inclusive
|
||||||
|
|
|
@ -7,11 +7,7 @@
|
||||||
"apiBaseUrl": "${GIT_BRIDGE_API_BASE_URL:-https://localhost/api/v0}",
|
"apiBaseUrl": "${GIT_BRIDGE_API_BASE_URL:-https://localhost/api/v0}",
|
||||||
"postbackBaseUrl": "${GIT_BRIDGE_POSTBACK_BASE_URL:-https://localhost}",
|
"postbackBaseUrl": "${GIT_BRIDGE_POSTBACK_BASE_URL:-https://localhost}",
|
||||||
"serviceName": "${GIT_BRIDGE_SERVICE_NAME:-Overleaf}",
|
"serviceName": "${GIT_BRIDGE_SERVICE_NAME:-Overleaf}",
|
||||||
"oauth2": {
|
"oauth2Server": "${GIT_BRIDGE_OAUTH2_SERVER:-https://localhost}",
|
||||||
"oauth2ClientID": "${GIT_BRIDGE_OAUTH2_CLIENT_ID}",
|
|
||||||
"oauth2ClientSecret": "${GIT_BRIDGE_OAUTH2_CLIENT_SECRET}",
|
|
||||||
"oauth2Server": "${GIT_BRIDGE_OAUTH2_SERVER:-https://localhost}"
|
|
||||||
},
|
|
||||||
"userPasswordEnabled": ${GIT_BRIDGE_USER_PASSWORD_ENABLED:-false},
|
"userPasswordEnabled": ${GIT_BRIDGE_USER_PASSWORD_ENABLED:-false},
|
||||||
"repoStore": {
|
"repoStore": {
|
||||||
"maxFileNum": ${GIT_BRIDGE_REPOSTORE_MAX_FILE_NUM:-2000},
|
"maxFileNum": ${GIT_BRIDGE_REPOSTORE_MAX_FILE_NUM:-2000},
|
||||||
|
|
|
@ -7,11 +7,7 @@
|
||||||
"apiBaseUrl": "https://localhost/api/v0",
|
"apiBaseUrl": "https://localhost/api/v0",
|
||||||
"postbackBaseUrl": "https://localhost",
|
"postbackBaseUrl": "https://localhost",
|
||||||
"serviceName": "Overleaf",
|
"serviceName": "Overleaf",
|
||||||
"oauth2": {
|
"oauth2Server": "https://localhost",
|
||||||
"oauth2ClientID": "asdf",
|
|
||||||
"oauth2ClientSecret": "asdf",
|
|
||||||
"oauth2Server": "https://localhost"
|
|
||||||
},
|
|
||||||
"repoStore": {
|
"repoStore": {
|
||||||
"maxFileNum": 2000,
|
"maxFileNum": 2000,
|
||||||
"maxFileSize": 52428800
|
"maxFileSize": 52428800
|
||||||
|
|
|
@ -7,11 +7,7 @@
|
||||||
"apiBaseUrl": "http://v2.overleaf.test:3000/api/v0",
|
"apiBaseUrl": "http://v2.overleaf.test:3000/api/v0",
|
||||||
"postbackBaseUrl": "http://git-bridge:8000",
|
"postbackBaseUrl": "http://git-bridge:8000",
|
||||||
"serviceName": "Overleaf",
|
"serviceName": "Overleaf",
|
||||||
"oauth2": {
|
"oauth2Server": "http://v2.overleaf.test:3000",
|
||||||
"oauth2ClientID": "264c723c925c13590880751f861f13084934030c13b4452901e73bdfab226edc",
|
|
||||||
"oauth2ClientSecret": "e6b2e9eee7ae2bb653823250bb69594a91db0547fe3790a7135acb497108e62d",
|
|
||||||
"oauth2Server": "http://v2.overleaf.test:3000"
|
|
||||||
},
|
|
||||||
"repoStore": {
|
"repoStore": {
|
||||||
"maxFileNum": 2000,
|
"maxFileNum": 2000,
|
||||||
"maxFileSize": 52428800
|
"maxFileSize": 52428800
|
||||||
|
|
|
@ -30,7 +30,7 @@ public class Config implements JSONSource {
|
||||||
config.apiBaseURL,
|
config.apiBaseURL,
|
||||||
config.postbackURL,
|
config.postbackURL,
|
||||||
config.serviceName,
|
config.serviceName,
|
||||||
Oauth2.asSanitised(config.oauth2),
|
config.oauth2Server,
|
||||||
config.userPasswordEnabled,
|
config.userPasswordEnabled,
|
||||||
config.repoStore,
|
config.repoStore,
|
||||||
SwapStoreConfig.sanitisedCopy(config.swapStore),
|
SwapStoreConfig.sanitisedCopy(config.swapStore),
|
||||||
|
@ -46,7 +46,7 @@ public class Config implements JSONSource {
|
||||||
private String apiBaseURL;
|
private String apiBaseURL;
|
||||||
private String postbackURL;
|
private String postbackURL;
|
||||||
private String serviceName;
|
private String serviceName;
|
||||||
@Nullable private Oauth2 oauth2;
|
@Nullable private String oauth2Server;
|
||||||
private boolean userPasswordEnabled;
|
private boolean userPasswordEnabled;
|
||||||
@Nullable private RepoStoreConfig repoStore;
|
@Nullable private RepoStoreConfig repoStore;
|
||||||
@Nullable private SwapStoreConfig swapStore;
|
@Nullable private SwapStoreConfig swapStore;
|
||||||
|
@ -70,7 +70,7 @@ public class Config implements JSONSource {
|
||||||
String apiBaseURL,
|
String apiBaseURL,
|
||||||
String postbackURL,
|
String postbackURL,
|
||||||
String serviceName,
|
String serviceName,
|
||||||
Oauth2 oauth2,
|
String oauth2Server,
|
||||||
boolean userPasswordEnabled,
|
boolean userPasswordEnabled,
|
||||||
RepoStoreConfig repoStore,
|
RepoStoreConfig repoStore,
|
||||||
SwapStoreConfig swapStore,
|
SwapStoreConfig swapStore,
|
||||||
|
@ -84,7 +84,7 @@ public class Config implements JSONSource {
|
||||||
this.apiBaseURL = apiBaseURL;
|
this.apiBaseURL = apiBaseURL;
|
||||||
this.postbackURL = postbackURL;
|
this.postbackURL = postbackURL;
|
||||||
this.serviceName = serviceName;
|
this.serviceName = serviceName;
|
||||||
this.oauth2 = oauth2;
|
this.oauth2Server = oauth2Server;
|
||||||
this.userPasswordEnabled = userPasswordEnabled;
|
this.userPasswordEnabled = userPasswordEnabled;
|
||||||
this.repoStore = repoStore;
|
this.repoStore = repoStore;
|
||||||
this.swapStore = swapStore;
|
this.swapStore = swapStore;
|
||||||
|
@ -116,7 +116,7 @@ public class Config implements JSONSource {
|
||||||
if (!postbackURL.endsWith("/")) {
|
if (!postbackURL.endsWith("/")) {
|
||||||
postbackURL += "/";
|
postbackURL += "/";
|
||||||
}
|
}
|
||||||
oauth2 = new Gson().fromJson(configObject.get("oauth2"), Oauth2.class);
|
oauth2Server = getOptionalString(configObject, "oauth2Server");
|
||||||
userPasswordEnabled = getOptionalString(configObject, "userPasswordEnabled").equals("true");
|
userPasswordEnabled = getOptionalString(configObject, "userPasswordEnabled").equals("true");
|
||||||
repoStore = new Gson().fromJson(configObject.get("repoStore"), RepoStoreConfig.class);
|
repoStore = new Gson().fromJson(configObject.get("repoStore"), RepoStoreConfig.class);
|
||||||
swapStore = new Gson().fromJson(configObject.get("swapStore"), SwapStoreConfig.class);
|
swapStore = new Gson().fromJson(configObject.get("swapStore"), SwapStoreConfig.class);
|
||||||
|
@ -166,19 +166,12 @@ public class Config implements JSONSource {
|
||||||
return postbackURL;
|
return postbackURL;
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isUsingOauth2() {
|
|
||||||
return oauth2 != null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isUserPasswordEnabled() {
|
public boolean isUserPasswordEnabled() {
|
||||||
return userPasswordEnabled;
|
return userPasswordEnabled;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Oauth2 getOauth2() {
|
public String getOauth2Server() {
|
||||||
if (!isUsingOauth2()) {
|
return oauth2Server;
|
||||||
throw new AssertionError("Getting oauth2 when not using it");
|
|
||||||
}
|
|
||||||
return oauth2;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Optional<RepoStoreConfig> getRepoStore() {
|
public Optional<RepoStoreConfig> getRepoStore() {
|
||||||
|
|
|
@ -1,33 +0,0 @@
|
||||||
package uk.ac.ic.wlgitbridge.application.config;
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Created by winston on 25/10/15.
|
|
||||||
*/
|
|
||||||
public class Oauth2 {
|
|
||||||
|
|
||||||
private final String oauth2ClientID;
|
|
||||||
private final String oauth2ClientSecret;
|
|
||||||
private final String oauth2Server;
|
|
||||||
|
|
||||||
public Oauth2(String oauth2ClientID, String oauth2ClientSecret, String oauth2Server) {
|
|
||||||
this.oauth2ClientID = oauth2ClientID;
|
|
||||||
this.oauth2ClientSecret = oauth2ClientSecret;
|
|
||||||
this.oauth2Server = oauth2Server;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getOauth2ClientID() {
|
|
||||||
return oauth2ClientID;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getOauth2ClientSecret() {
|
|
||||||
return oauth2ClientSecret;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getOauth2Server() {
|
|
||||||
return oauth2Server;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Oauth2 asSanitised(Oauth2 oauth2) {
|
|
||||||
return new Oauth2("<oauth2ClientID>", "<oauth2ClientSecret>", oauth2.oauth2Server);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -151,9 +151,9 @@ public class GitBridgeServer {
|
||||||
throws ServletException {
|
throws ServletException {
|
||||||
final ServletContextHandler servletContextHandler =
|
final ServletContextHandler servletContextHandler =
|
||||||
new ServletContextHandler(ServletContextHandler.SESSIONS);
|
new ServletContextHandler(ServletContextHandler.SESSIONS);
|
||||||
if (config.isUsingOauth2()) {
|
if (config.getOauth2Server() != null) {
|
||||||
Filter filter =
|
Filter filter =
|
||||||
new Oauth2Filter(snapshotApi, config.getOauth2(), config.isUserPasswordEnabled());
|
new Oauth2Filter(snapshotApi, config.getOauth2Server(), config.isUserPasswordEnabled());
|
||||||
servletContextHandler.addFilter(
|
servletContextHandler.addFilter(
|
||||||
new FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST));
|
new FilterHolder(filter), "/*", EnumSet.of(DispatcherType.REQUEST));
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,7 +13,6 @@ import javax.servlet.*;
|
||||||
import javax.servlet.http.HttpServletRequest;
|
import javax.servlet.http.HttpServletRequest;
|
||||||
import javax.servlet.http.HttpServletResponse;
|
import javax.servlet.http.HttpServletResponse;
|
||||||
import org.apache.commons.codec.binary.Base64;
|
import org.apache.commons.codec.binary.Base64;
|
||||||
import uk.ac.ic.wlgitbridge.application.config.Oauth2;
|
|
||||||
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
|
import uk.ac.ic.wlgitbridge.bridge.snapshot.SnapshotApi;
|
||||||
import uk.ac.ic.wlgitbridge.util.Instance;
|
import uk.ac.ic.wlgitbridge.util.Instance;
|
||||||
import uk.ac.ic.wlgitbridge.util.Log;
|
import uk.ac.ic.wlgitbridge.util.Log;
|
||||||
|
@ -28,13 +27,13 @@ public class Oauth2Filter implements Filter {
|
||||||
|
|
||||||
private final SnapshotApi snapshotApi;
|
private final SnapshotApi snapshotApi;
|
||||||
|
|
||||||
private final Oauth2 oauth2;
|
private final String oauth2Server;
|
||||||
|
|
||||||
private final boolean isUserPasswordEnabled;
|
private final boolean isUserPasswordEnabled;
|
||||||
|
|
||||||
public Oauth2Filter(SnapshotApi snapshotApi, Oauth2 oauth2, boolean isUserPasswordEnabled) {
|
public Oauth2Filter(SnapshotApi snapshotApi, String oauth2Server, boolean isUserPasswordEnabled) {
|
||||||
this.snapshotApi = snapshotApi;
|
this.snapshotApi = snapshotApi;
|
||||||
this.oauth2 = oauth2;
|
this.oauth2Server = oauth2Server;
|
||||||
this.isUserPasswordEnabled = isUserPasswordEnabled;
|
this.isUserPasswordEnabled = isUserPasswordEnabled;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -108,7 +107,7 @@ public class Oauth2Filter implements Filter {
|
||||||
// fail later (for example, in the unlikely event that the token
|
// fail later (for example, in the unlikely event that the token
|
||||||
// expired between the two requests). In that case, JGit will
|
// expired between the two requests). In that case, JGit will
|
||||||
// return a 401 without a custom error message.
|
// return a 401 without a custom error message.
|
||||||
int statusCode = checkAccessToken(oauth2, password, getClientIp(request));
|
int statusCode = checkAccessToken(this.oauth2Server, password, getClientIp(request));
|
||||||
if (statusCode == 429) {
|
if (statusCode == 429) {
|
||||||
handleRateLimit(projectId, username, request, response);
|
handleRateLimit(projectId, username, request, response);
|
||||||
return;
|
return;
|
||||||
|
@ -238,10 +237,9 @@ public class Oauth2Filter implements Filter {
|
||||||
"your Overleaf Account Settings."));
|
"your Overleaf Account Settings."));
|
||||||
}
|
}
|
||||||
|
|
||||||
private int checkAccessToken(Oauth2 oauth2, String accessToken, String clientIp)
|
private int checkAccessToken(String oauth2Server, String accessToken, String clientIp)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
GenericUrl url =
|
GenericUrl url = new GenericUrl(oauth2Server + "/oauth/token/info?client_ip=" + clientIp);
|
||||||
new GenericUrl(oauth2.getOauth2Server() + "/oauth/token/info?client_ip=" + clientIp);
|
|
||||||
HttpRequest request = Instance.httpRequestFactory.buildGetRequest(url);
|
HttpRequest request = Instance.httpRequestFactory.buildGetRequest(url);
|
||||||
HttpHeaders headers = new HttpHeaders();
|
HttpHeaders headers = new HttpHeaders();
|
||||||
headers.setAuthorization("Bearer " + accessToken);
|
headers.setAuthorization("Bearer " + accessToken);
|
||||||
|
|
|
@ -1495,13 +1495,9 @@ public class WLGitBridgeIntegrationTest {
|
||||||
+ port
|
+ port
|
||||||
+ "\",\n"
|
+ "\",\n"
|
||||||
+ " \"serviceName\": \"Overleaf\",\n"
|
+ " \"serviceName\": \"Overleaf\",\n"
|
||||||
+ " \"oauth2\": {\n"
|
+ " \"oauth2Server\": \"http://127.0.0.1:"
|
||||||
+ " \"oauth2ClientID\": \"clientID\",\n"
|
|
||||||
+ " \"oauth2ClientSecret\": \"oauth2 client secret\",\n"
|
|
||||||
+ " \"oauth2Server\": \"http://127.0.0.1:"
|
|
||||||
+ apiPort
|
+ apiPort
|
||||||
+ "\"\n"
|
+ "\"";
|
||||||
+ " }";
|
|
||||||
if (swapCfg != null) {
|
if (swapCfg != null) {
|
||||||
cfgStr +=
|
cfgStr +=
|
||||||
",\n"
|
",\n"
|
||||||
|
@ -1524,7 +1520,6 @@ public class WLGitBridgeIntegrationTest {
|
||||||
+ ",\n"
|
+ ",\n"
|
||||||
+ " \"intervalMillis\": "
|
+ " \"intervalMillis\": "
|
||||||
+ swapCfg.getIntervalMillis()
|
+ swapCfg.getIntervalMillis()
|
||||||
+ "\n"
|
|
||||||
+ " }\n";
|
+ " }\n";
|
||||||
}
|
}
|
||||||
cfgStr += "}\n";
|
cfgStr += "}\n";
|
||||||
|
|
|
@ -23,11 +23,7 @@ public class ConfigTest {
|
||||||
+ " \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
|
+ " \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
|
||||||
+ " \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
|
+ " \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
|
||||||
+ " \"serviceName\": \"Overleaf\",\n"
|
+ " \"serviceName\": \"Overleaf\",\n"
|
||||||
+ " \"oauth2\": {\n"
|
+ " \"oauth2Server\": \"https://www.overleaf.com\"\n"
|
||||||
+ " \"oauth2ClientID\": \"clientID\",\n"
|
|
||||||
+ " \"oauth2ClientSecret\": \"oauth2 client secret\",\n"
|
|
||||||
+ " \"oauth2Server\": \"https://www.overleaf.com\"\n"
|
|
||||||
+ " }\n"
|
|
||||||
+ "}\n");
|
+ "}\n");
|
||||||
Config config = new Config(reader);
|
Config config = new Config(reader);
|
||||||
assertEquals(80, config.getPort());
|
assertEquals(80, config.getPort());
|
||||||
|
@ -35,10 +31,7 @@ public class ConfigTest {
|
||||||
assertEquals("http://127.0.0.1:60000/api/v0/", config.getAPIBaseURL());
|
assertEquals("http://127.0.0.1:60000/api/v0/", config.getAPIBaseURL());
|
||||||
assertEquals("http://127.0.0.1/", config.getPostbackURL());
|
assertEquals("http://127.0.0.1/", config.getPostbackURL());
|
||||||
assertEquals("Overleaf", config.getServiceName());
|
assertEquals("Overleaf", config.getServiceName());
|
||||||
assertTrue(config.isUsingOauth2());
|
assertEquals("https://www.overleaf.com", config.getOauth2Server());
|
||||||
assertEquals("clientID", config.getOauth2().getOauth2ClientID());
|
|
||||||
assertEquals("oauth2 client secret", config.getOauth2().getOauth2ClientSecret());
|
|
||||||
assertEquals("https://www.overleaf.com", config.getOauth2().getOauth2Server());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test(expected = AssertionError.class)
|
@Test(expected = AssertionError.class)
|
||||||
|
@ -53,7 +46,7 @@ public class ConfigTest {
|
||||||
+ " \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
|
+ " \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
|
||||||
+ " \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
|
+ " \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
|
||||||
+ " \"serviceName\": \"Overleaf\",\n"
|
+ " \"serviceName\": \"Overleaf\",\n"
|
||||||
+ " \"oauth2\": null\n"
|
+ " \"oauth2Server\": null\n"
|
||||||
+ "}\n");
|
+ "}\n");
|
||||||
Config config = new Config(reader);
|
Config config = new Config(reader);
|
||||||
assertEquals(80, config.getPort());
|
assertEquals(80, config.getPort());
|
||||||
|
@ -61,8 +54,7 @@ public class ConfigTest {
|
||||||
assertEquals("http://127.0.0.1:60000/api/v0/", config.getAPIBaseURL());
|
assertEquals("http://127.0.0.1:60000/api/v0/", config.getAPIBaseURL());
|
||||||
assertEquals("http://127.0.0.1/", config.getPostbackURL());
|
assertEquals("http://127.0.0.1/", config.getPostbackURL());
|
||||||
assertEquals("Overleaf", config.getServiceName());
|
assertEquals("Overleaf", config.getServiceName());
|
||||||
assertFalse(config.isUsingOauth2());
|
assertNull(config.getOauth2Server());
|
||||||
config.getOauth2();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -77,11 +69,7 @@ public class ConfigTest {
|
||||||
+ " \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
|
+ " \"apiBaseUrl\": \"http://127.0.0.1:60000/api/v0\",\n"
|
||||||
+ " \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
|
+ " \"postbackBaseUrl\": \"http://127.0.0.1\",\n"
|
||||||
+ " \"serviceName\": \"Overleaf\",\n"
|
+ " \"serviceName\": \"Overleaf\",\n"
|
||||||
+ " \"oauth2\": {\n"
|
+ " \"oauth2Server\": \"https://www.overleaf.com\"\n"
|
||||||
+ " \"oauth2ClientID\": \"my oauth2 client id\",\n"
|
|
||||||
+ " \"oauth2ClientSecret\": \"my oauth2 client secret\",\n"
|
|
||||||
+ " \"oauth2Server\": \"https://www.overleaf.com\"\n"
|
|
||||||
+ " }\n"
|
|
||||||
+ "}\n");
|
+ "}\n");
|
||||||
Config config = new Config(reader);
|
Config config = new Config(reader);
|
||||||
String expected =
|
String expected =
|
||||||
|
@ -94,11 +82,7 @@ public class ConfigTest {
|
||||||
+ " \"apiBaseURL\": \"http://127.0.0.1:60000/api/v0/\",\n"
|
+ " \"apiBaseURL\": \"http://127.0.0.1:60000/api/v0/\",\n"
|
||||||
+ " \"postbackURL\": \"http://127.0.0.1/\",\n"
|
+ " \"postbackURL\": \"http://127.0.0.1/\",\n"
|
||||||
+ " \"serviceName\": \"Overleaf\",\n"
|
+ " \"serviceName\": \"Overleaf\",\n"
|
||||||
+ " \"oauth2\": {\n"
|
+ " \"oauth2Server\": \"https://www.overleaf.com\",\n"
|
||||||
+ " \"oauth2ClientID\": \"<oauth2ClientID>\",\n"
|
|
||||||
+ " \"oauth2ClientSecret\": \"<oauth2ClientSecret>\",\n"
|
|
||||||
+ " \"oauth2Server\": \"https://www.overleaf.com\"\n"
|
|
||||||
+ " },\n"
|
|
||||||
+ " \"userPasswordEnabled\": false,\n"
|
+ " \"userPasswordEnabled\": false,\n"
|
||||||
+ " \"repoStore\": null,\n"
|
+ " \"repoStore\": null,\n"
|
||||||
+ " \"swapStore\": null,\n"
|
+ " \"swapStore\": null,\n"
|
||||||
|
|
|
@ -116,13 +116,6 @@ test_acceptance_clean:
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) down -v -t 0
|
||||||
|
|
||||||
test_acceptance_pre_run:
|
test_acceptance_pre_run:
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) up -d mongo
|
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) exec -T mongo sh -c ' \
|
|
||||||
while ! mongosh --eval "db.version()" > /dev/null; do \
|
|
||||||
echo "Waiting for Mongo..."; \
|
|
||||||
sleep 1; \
|
|
||||||
done; \
|
|
||||||
mongosh --eval "rs.initiate({ _id: \"overleaf\", members: [ { _id: 0, host: \"mongo:27017\" } ] })"'
|
|
||||||
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
ifneq (,$(wildcard test/acceptance/js/scripts/pre-run))
|
||||||
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
|
$(DOCKER_COMPOSE_TEST_ACCEPTANCE) run --rm test_acceptance test/acceptance/js/scripts/pre-run
|
||||||
endif
|
endif
|
||||||
|
|
|
@ -22,6 +22,7 @@ const BlobStore = storage.BlobStore
|
||||||
const chunkStore = storage.chunkStore
|
const chunkStore = storage.chunkStore
|
||||||
const HashCheckBlobStore = storage.HashCheckBlobStore
|
const HashCheckBlobStore = storage.HashCheckBlobStore
|
||||||
const persistChanges = storage.persistChanges
|
const persistChanges = storage.persistChanges
|
||||||
|
const InvalidChangeError = storage.InvalidChangeError
|
||||||
|
|
||||||
const render = require('./render')
|
const render = require('./render')
|
||||||
|
|
||||||
|
@ -113,7 +114,8 @@ async function importChanges(req, res, next) {
|
||||||
err instanceof File.NotEditableError ||
|
err instanceof File.NotEditableError ||
|
||||||
err instanceof FileMap.PathnameError ||
|
err instanceof FileMap.PathnameError ||
|
||||||
err instanceof Snapshot.EditMissingFileError ||
|
err instanceof Snapshot.EditMissingFileError ||
|
||||||
err instanceof chunkStore.ChunkVersionConflictError
|
err instanceof chunkStore.ChunkVersionConflictError ||
|
||||||
|
err instanceof InvalidChangeError
|
||||||
) {
|
) {
|
||||||
// If we failed to apply operations, that's probably because they were
|
// If we failed to apply operations, that's probably because they were
|
||||||
// invalid.
|
// invalid.
|
||||||
|
|
|
@ -4,17 +4,24 @@ import '@overleaf/metrics/initialize.js'
|
||||||
import http from 'node:http'
|
import http from 'node:http'
|
||||||
import { fileURLToPath } from 'node:url'
|
import { fileURLToPath } from 'node:url'
|
||||||
import { promisify } from 'node:util'
|
import { promisify } from 'node:util'
|
||||||
|
import { setTimeout } from 'node:timers/promises'
|
||||||
import express from 'express'
|
import express from 'express'
|
||||||
import logger from '@overleaf/logger'
|
import logger from '@overleaf/logger'
|
||||||
import Metrics from '@overleaf/metrics'
|
import Metrics from '@overleaf/metrics'
|
||||||
|
import { healthCheck } from './backupVerifier/healthCheck.mjs'
|
||||||
import {
|
import {
|
||||||
BackupCorruptedError,
|
BackupCorruptedError,
|
||||||
healthCheck,
|
|
||||||
verifyBlob,
|
verifyBlob,
|
||||||
} from './storage/lib/backupVerifier.mjs'
|
} from './storage/lib/backupVerifier.mjs'
|
||||||
import { mongodb } from './storage/index.js'
|
import { mongodb } from './storage/index.js'
|
||||||
import { expressify } from '@overleaf/promise-utils'
|
import { expressify } from '@overleaf/promise-utils'
|
||||||
import { Blob } from 'overleaf-editor-core'
|
import { Blob } from 'overleaf-editor-core'
|
||||||
|
import { loadGlobalBlobs } from './storage/lib/blob_store/index.js'
|
||||||
|
import { EventEmitter } from 'node:events'
|
||||||
|
import {
|
||||||
|
loopRandomProjects,
|
||||||
|
setWriteMetrics,
|
||||||
|
} from './backupVerifier/ProjectVerifier.mjs'
|
||||||
|
|
||||||
const app = express()
|
const app = express()
|
||||||
|
|
||||||
|
@ -64,20 +71,46 @@ app.use((err, req, res, next) => {
|
||||||
next(err)
|
next(err)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const shutdownEmitter = new EventEmitter()
|
||||||
|
|
||||||
|
shutdownEmitter.once('shutdown', async code => {
|
||||||
|
logger.info({ code }, 'shutting down')
|
||||||
|
await mongodb.client.close()
|
||||||
|
await setTimeout(100)
|
||||||
|
process.exit(code)
|
||||||
|
})
|
||||||
|
|
||||||
|
process.on('SIGTERM', () => {
|
||||||
|
shutdownEmitter.emit('shutdown', 0)
|
||||||
|
})
|
||||||
|
|
||||||
|
process.on('SIGINT', () => {
|
||||||
|
shutdownEmitter.emit('shutdown', 0)
|
||||||
|
})
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param {number} port
|
* @param {number} port
|
||||||
* @return {Promise<http.Server>}
|
* @return {Promise<http.Server>}
|
||||||
*/
|
*/
|
||||||
export async function startApp(port) {
|
export async function startApp(port) {
|
||||||
await mongodb.client.connect()
|
await mongodb.client.connect()
|
||||||
|
await loadGlobalBlobs()
|
||||||
await healthCheck()
|
await healthCheck()
|
||||||
const server = http.createServer(app)
|
const server = http.createServer(app)
|
||||||
await promisify(server.listen.bind(server, port))()
|
await promisify(server.listen.bind(server, port))()
|
||||||
|
loopRandomProjects(shutdownEmitter)
|
||||||
return server
|
return server
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setWriteMetrics(true)
|
||||||
|
|
||||||
// Run this if we're called directly
|
// Run this if we're called directly
|
||||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||||
const PORT = parseInt(process.env.PORT || '3102', 10)
|
const PORT = parseInt(process.env.PORT || '3102', 10)
|
||||||
await startApp(PORT)
|
try {
|
||||||
|
await startApp(PORT)
|
||||||
|
} catch (error) {
|
||||||
|
shutdownEmitter.emit('shutdown', 1)
|
||||||
|
logger.error({ error }, 'error starting app')
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -38,10 +38,10 @@ app.use((err, req, res, next) => {
|
||||||
})
|
})
|
||||||
|
|
||||||
async function triggerGracefulShutdown(server, signal) {
|
async function triggerGracefulShutdown(server, signal) {
|
||||||
logger.warn({ signal }, 'graceful shutdown: started shutdown sequence')
|
logger.info({ signal }, 'graceful shutdown: started shutdown sequence')
|
||||||
await drainQueue()
|
await drainQueue()
|
||||||
server.close(function () {
|
server.close(function () {
|
||||||
logger.warn({ signal }, 'graceful shutdown: closed server')
|
logger.info({ signal }, 'graceful shutdown: closed server')
|
||||||
setTimeout(() => {
|
setTimeout(() => {
|
||||||
process.exit(0)
|
process.exit(0)
|
||||||
}, 1000)
|
}, 1000)
|
||||||
|
|
33
services/history-v1/backupVerifier/ProjectMetrics.mjs
Normal file
33
services/history-v1/backupVerifier/ProjectMetrics.mjs
Normal file
|
@ -0,0 +1,33 @@
|
||||||
|
import Metrics from '@overleaf/metrics'
|
||||||
|
import { objectIdFromDate } from './utils.mjs'
|
||||||
|
import { db } from '../storage/lib/mongodb.js'
|
||||||
|
|
||||||
|
const projectsCollection = db.collection('projects')
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {Date} beforeTime
|
||||||
|
* @return {Promise<void>}
|
||||||
|
*/
|
||||||
|
export async function measurePendingChangesBeforeTime(beforeTime) {
|
||||||
|
const pendingChangeCount = await projectsCollection.countDocuments({
|
||||||
|
'overleaf.backup.pendingChangeAt': {
|
||||||
|
$lt: beforeTime,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
Metrics.gauge('backup_verification_pending_changes', pendingChangeCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {Date} graceTime
|
||||||
|
* @return {Promise<void>}
|
||||||
|
*/
|
||||||
|
export async function measureNeverBackedUpProjects(graceTime) {
|
||||||
|
const neverBackedUpCount = await projectsCollection.countDocuments({
|
||||||
|
'overleaf.backup.lastBackedUpVersion': null,
|
||||||
|
_id: { $lt: objectIdFromDate(graceTime) },
|
||||||
|
})
|
||||||
|
Metrics.gauge('backup_verification_never_backed_up', neverBackedUpCount)
|
||||||
|
}
|
79
services/history-v1/backupVerifier/ProjectSampler.mjs
Normal file
79
services/history-v1/backupVerifier/ProjectSampler.mjs
Normal file
|
@ -0,0 +1,79 @@
|
||||||
|
// @ts-check
|
||||||
|
import { objectIdFromDate } from './utils.mjs'
|
||||||
|
import { db } from '../storage/lib/mongodb.js'
|
||||||
|
import config from 'config'
|
||||||
|
|
||||||
|
const projectsCollection = db.collection('projects')
|
||||||
|
|
||||||
|
const HAS_PROJECTS_WITHOUT_HISTORY =
|
||||||
|
config.get('hasProjectsWithoutHistory') === 'true'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {Date} start
|
||||||
|
* @param {Date} end
|
||||||
|
* @param {number} N
|
||||||
|
* @yields {string}
|
||||||
|
*/
|
||||||
|
export async function* getProjectsCreatedInDateRangeCursor(start, end, N) {
|
||||||
|
yield* getSampleProjectsCursor(N, [
|
||||||
|
{
|
||||||
|
$match: {
|
||||||
|
_id: {
|
||||||
|
$gt: objectIdFromDate(start),
|
||||||
|
$lte: objectIdFromDate(end),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function* getProjectsUpdatedInDateRangeCursor(start, end, N) {
|
||||||
|
yield* getSampleProjectsCursor(N, [
|
||||||
|
{
|
||||||
|
$match: {
|
||||||
|
'overleaf.history.updatedAt': {
|
||||||
|
$gt: start,
|
||||||
|
$lte: end,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {import('mongodb').Document} Document
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @generator
|
||||||
|
* @param {number} N
|
||||||
|
* @param {Array<Document>} preSampleAggregationStages
|
||||||
|
* @yields {string}
|
||||||
|
*/
|
||||||
|
export async function* getSampleProjectsCursor(
|
||||||
|
N,
|
||||||
|
preSampleAggregationStages = []
|
||||||
|
) {
|
||||||
|
const cursor = projectsCollection.aggregate([
|
||||||
|
...preSampleAggregationStages,
|
||||||
|
{ $sample: { size: N } },
|
||||||
|
{ $project: { 'overleaf.history.id': 1 } },
|
||||||
|
])
|
||||||
|
|
||||||
|
let validProjects = 0
|
||||||
|
let hasInvalidProject = false
|
||||||
|
|
||||||
|
for await (const project of cursor) {
|
||||||
|
if (HAS_PROJECTS_WITHOUT_HISTORY && !project.overleaf?.history?.id) {
|
||||||
|
hasInvalidProject = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
validProjects++
|
||||||
|
yield project.overleaf.history.id.toString()
|
||||||
|
}
|
||||||
|
|
||||||
|
if (validProjects === 0 && hasInvalidProject) {
|
||||||
|
yield* getSampleProjectsCursor(N, preSampleAggregationStages)
|
||||||
|
}
|
||||||
|
}
|
320
services/history-v1/backupVerifier/ProjectVerifier.mjs
Normal file
320
services/history-v1/backupVerifier/ProjectVerifier.mjs
Normal file
|
@ -0,0 +1,320 @@
|
||||||
|
// @ts-check
|
||||||
|
import { verifyProjectWithErrorContext } from '../storage/lib/backupVerifier.mjs'
|
||||||
|
import { promiseMapSettledWithLimit } from '@overleaf/promise-utils'
|
||||||
|
import logger from '@overleaf/logger'
|
||||||
|
import metrics from '@overleaf/metrics'
|
||||||
|
import {
|
||||||
|
getSampleProjectsCursor,
|
||||||
|
getProjectsCreatedInDateRangeCursor,
|
||||||
|
getProjectsUpdatedInDateRangeCursor,
|
||||||
|
} from './ProjectSampler.mjs'
|
||||||
|
import OError from '@overleaf/o-error'
|
||||||
|
import { setTimeout } from 'node:timers/promises'
|
||||||
|
|
||||||
|
const MS_PER_30_DAYS = 30 * 24 * 60 * 60 * 1000
|
||||||
|
|
||||||
|
const failureCounter = new metrics.prom.Counter({
|
||||||
|
name: 'backup_project_verification_failed',
|
||||||
|
help: 'Number of projects that failed verification',
|
||||||
|
labelNames: ['name'],
|
||||||
|
})
|
||||||
|
|
||||||
|
const successCounter = new metrics.prom.Counter({
|
||||||
|
name: 'backup_project_verification_succeeded',
|
||||||
|
help: 'Number of projects that succeeded verification',
|
||||||
|
})
|
||||||
|
|
||||||
|
let WRITE_METRICS = false
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {import('node:events').EventEmitter} EventEmitter
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Allows writing metrics to be enabled or disabled.
|
||||||
|
* @param {Boolean} writeMetrics
|
||||||
|
*/
|
||||||
|
export function setWriteMetrics(writeMetrics) {
|
||||||
|
WRITE_METRICS = writeMetrics
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {Error|unknown} error
|
||||||
|
* @param {string} historyId
|
||||||
|
*/
|
||||||
|
function handleVerificationError(error, historyId) {
|
||||||
|
const name = error instanceof Error ? error.name : 'UnknownError'
|
||||||
|
logger.error({ historyId, error, name }, 'error verifying project backup')
|
||||||
|
|
||||||
|
WRITE_METRICS && failureCounter.inc({ name })
|
||||||
|
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {Date} startDate
|
||||||
|
* @param {Date} endDate
|
||||||
|
* @param {number} interval
|
||||||
|
* @returns {Array<VerificationJobSpecification>}
|
||||||
|
*/
|
||||||
|
function splitJobs(startDate, endDate, interval) {
|
||||||
|
/** @type {Array<VerificationJobSpecification>} */
|
||||||
|
const jobs = []
|
||||||
|
while (startDate < endDate) {
|
||||||
|
const nextStart = new Date(
|
||||||
|
Math.min(startDate.getTime() + interval, endDate.getTime())
|
||||||
|
)
|
||||||
|
jobs.push({ startDate, endDate: nextStart })
|
||||||
|
startDate = nextStart
|
||||||
|
}
|
||||||
|
return jobs
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {AsyncGenerator<string>} historyIdCursor
|
||||||
|
* @param {EventEmitter} [eventEmitter]
|
||||||
|
* @param {number} [delay] - Allows a delay between each verification
|
||||||
|
* @return {Promise<{verified: number, total: number, errorTypes: *[], hasFailure: boolean}>}
|
||||||
|
*/
|
||||||
|
async function verifyProjectsFromCursor(
|
||||||
|
historyIdCursor,
|
||||||
|
eventEmitter,
|
||||||
|
delay = 0
|
||||||
|
) {
|
||||||
|
const errorTypes = []
|
||||||
|
let verified = 0
|
||||||
|
let total = 0
|
||||||
|
let receivedShutdownSignal = false
|
||||||
|
if (eventEmitter) {
|
||||||
|
eventEmitter.once('shutdown', () => {
|
||||||
|
receivedShutdownSignal = true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for await (const historyId of historyIdCursor) {
|
||||||
|
if (receivedShutdownSignal) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
total++
|
||||||
|
try {
|
||||||
|
await verifyProjectWithErrorContext(historyId)
|
||||||
|
logger.debug({ historyId }, 'verified project backup successfully')
|
||||||
|
WRITE_METRICS && successCounter.inc()
|
||||||
|
verified++
|
||||||
|
} catch (error) {
|
||||||
|
const errorType = handleVerificationError(error, historyId)
|
||||||
|
errorTypes.push(errorType)
|
||||||
|
}
|
||||||
|
if (delay > 0) {
|
||||||
|
await setTimeout(delay)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
verified,
|
||||||
|
total,
|
||||||
|
errorTypes,
|
||||||
|
hasFailure: errorTypes.length > 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {number} nProjectsToSample
|
||||||
|
* @param {EventEmitter} [signal]
|
||||||
|
* @param {number} [delay]
|
||||||
|
* @return {Promise<VerificationJobStatus>}
|
||||||
|
*/
|
||||||
|
export async function verifyRandomProjectSample(
|
||||||
|
nProjectsToSample,
|
||||||
|
signal,
|
||||||
|
delay = 0
|
||||||
|
) {
|
||||||
|
const historyIds = await getSampleProjectsCursor(nProjectsToSample)
|
||||||
|
return await verifyProjectsFromCursor(historyIds, signal, delay)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Samples projects with history IDs between the specified dates and verifies them.
|
||||||
|
*
|
||||||
|
* @param {Date} startDate
|
||||||
|
* @param {Date} endDate
|
||||||
|
* @param {number} projectsPerRange
|
||||||
|
* @param {EventEmitter} [signal]
|
||||||
|
* @return {Promise<VerificationJobStatus>}
|
||||||
|
*/
|
||||||
|
async function verifyRange(startDate, endDate, projectsPerRange, signal) {
|
||||||
|
logger.info({ startDate, endDate }, 'verifying range')
|
||||||
|
|
||||||
|
const results = await verifyProjectsFromCursor(
|
||||||
|
getProjectsCreatedInDateRangeCursor(startDate, endDate, projectsPerRange),
|
||||||
|
signal
|
||||||
|
)
|
||||||
|
|
||||||
|
if (results.total === 0) {
|
||||||
|
logger.debug(
|
||||||
|
{ start: startDate, end: endDate },
|
||||||
|
'No projects found in range'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const jobStatus = {
|
||||||
|
...results,
|
||||||
|
startDate,
|
||||||
|
endDate,
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
{ ...jobStatus, errorTypes: Array.from(new Set(jobStatus.errorTypes)) },
|
||||||
|
'Verified range'
|
||||||
|
)
|
||||||
|
return jobStatus
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} VerificationJobSpecification
|
||||||
|
* @property {Date} startDate
|
||||||
|
* @property {Date} endDate
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {import('./types.d.ts').VerificationJobStatus} VerificationJobStatus
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} VerifyDateRangeOptions
|
||||||
|
* @property {Date} startDate
|
||||||
|
* @property {Date} endDate
|
||||||
|
* @property {number} [interval]
|
||||||
|
* @property {number} [projectsPerRange]
|
||||||
|
* @property {number} [concurrency]
|
||||||
|
* @property {EventEmitter} [signal]
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {VerifyDateRangeOptions} options
|
||||||
|
* @return {Promise<VerificationJobStatus>}
|
||||||
|
*/
|
||||||
|
export async function verifyProjectsCreatedInDateRange({
|
||||||
|
concurrency = 0,
|
||||||
|
projectsPerRange = 10,
|
||||||
|
startDate,
|
||||||
|
endDate,
|
||||||
|
interval = MS_PER_30_DAYS,
|
||||||
|
signal,
|
||||||
|
}) {
|
||||||
|
const jobs = splitJobs(startDate, endDate, interval)
|
||||||
|
if (jobs.length === 0) {
|
||||||
|
throw new OError('Time range could not be split into jobs', {
|
||||||
|
start: startDate,
|
||||||
|
end: endDate,
|
||||||
|
interval,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
const settlements = await promiseMapSettledWithLimit(
|
||||||
|
concurrency,
|
||||||
|
jobs,
|
||||||
|
({ startDate, endDate }) =>
|
||||||
|
verifyRange(startDate, endDate, projectsPerRange, signal)
|
||||||
|
)
|
||||||
|
return settlements.reduce(
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {VerificationJobStatus} acc
|
||||||
|
* @param settlement
|
||||||
|
* @return {VerificationJobStatus}
|
||||||
|
*/
|
||||||
|
(acc, settlement) => {
|
||||||
|
if (settlement.status !== 'rejected') {
|
||||||
|
if (settlement.value.hasFailure) {
|
||||||
|
acc.hasFailure = true
|
||||||
|
}
|
||||||
|
acc.total += settlement.value.total
|
||||||
|
acc.verified += settlement.value.verified
|
||||||
|
acc.errorTypes = acc.errorTypes.concat(settlement.value.errorTypes)
|
||||||
|
} else {
|
||||||
|
logger.error({ ...settlement.reason }, 'Error processing range')
|
||||||
|
}
|
||||||
|
return acc
|
||||||
|
},
|
||||||
|
/** @type {VerificationJobStatus} */
|
||||||
|
{
|
||||||
|
startDate,
|
||||||
|
endDate,
|
||||||
|
verified: 0,
|
||||||
|
total: 0,
|
||||||
|
hasFailure: false,
|
||||||
|
errorTypes: [],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verifies that projects that have recently gone out of RPO have been updated.
|
||||||
|
*
|
||||||
|
* @param {Date} startDate
|
||||||
|
* @param {Date} endDate
|
||||||
|
* @param {number} nProjects
|
||||||
|
* @param {EventEmitter} [signal]
|
||||||
|
* @return {Promise<VerificationJobStatus>}
|
||||||
|
*/
|
||||||
|
export async function verifyProjectsUpdatedInDateRange(
|
||||||
|
startDate,
|
||||||
|
endDate,
|
||||||
|
nProjects,
|
||||||
|
signal
|
||||||
|
) {
|
||||||
|
logger.debug(
|
||||||
|
{ startDate, endDate, nProjects },
|
||||||
|
'Sampling projects updated in date range'
|
||||||
|
)
|
||||||
|
const results = await verifyProjectsFromCursor(
|
||||||
|
getProjectsUpdatedInDateRangeCursor(startDate, endDate, nProjects),
|
||||||
|
signal
|
||||||
|
)
|
||||||
|
|
||||||
|
if (results.total === 0) {
|
||||||
|
logger.debug(
|
||||||
|
{ start: startDate, end: endDate },
|
||||||
|
'No projects updated recently'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const jobStatus = {
|
||||||
|
...results,
|
||||||
|
startDate,
|
||||||
|
endDate,
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug(
|
||||||
|
{ ...jobStatus, errorTypes: Array.from(new Set(jobStatus.errorTypes)) },
|
||||||
|
'Verified recently updated projects'
|
||||||
|
)
|
||||||
|
return jobStatus
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {EventEmitter} signal
|
||||||
|
* @return {void}
|
||||||
|
*/
|
||||||
|
export function loopRandomProjects(signal) {
|
||||||
|
let shutdown = false
|
||||||
|
signal.on('shutdown', function () {
|
||||||
|
shutdown = true
|
||||||
|
})
|
||||||
|
async function loop() {
|
||||||
|
do {
|
||||||
|
try {
|
||||||
|
const result = await verifyRandomProjectSample(100, signal, 2_000)
|
||||||
|
logger.debug({ result }, 'verified random project sample')
|
||||||
|
} catch (error) {
|
||||||
|
logger.error({ error }, 'error verifying random project sample')
|
||||||
|
}
|
||||||
|
// eslint-disable-next-line no-unmodified-loop-condition
|
||||||
|
} while (!shutdown)
|
||||||
|
}
|
||||||
|
loop()
|
||||||
|
}
|
32
services/history-v1/backupVerifier/healthCheck.mjs
Normal file
32
services/history-v1/backupVerifier/healthCheck.mjs
Normal file
|
@ -0,0 +1,32 @@
|
||||||
|
import config from 'config'
|
||||||
|
import { verifyProjectWithErrorContext } from '../storage/lib/backupVerifier.mjs'
|
||||||
|
import {
|
||||||
|
measureNeverBackedUpProjects,
|
||||||
|
measurePendingChangesBeforeTime,
|
||||||
|
} from './ProjectMetrics.mjs'
|
||||||
|
import { getEndDateForRPO, RPO } from './utils.mjs'
|
||||||
|
|
||||||
|
/** @type {Array<string>} */
|
||||||
|
const HEALTH_CHECK_PROJECTS = JSON.parse(config.get('healthCheckProjects'))
|
||||||
|
|
||||||
|
export async function healthCheck() {
|
||||||
|
if (!Array.isArray(HEALTH_CHECK_PROJECTS)) {
|
||||||
|
throw new Error('expected healthCheckProjects to be an array')
|
||||||
|
}
|
||||||
|
if (HEALTH_CHECK_PROJECTS.length !== 2) {
|
||||||
|
throw new Error('expected 2 healthCheckProjects')
|
||||||
|
}
|
||||||
|
if (!HEALTH_CHECK_PROJECTS.some(id => id.length === 24)) {
|
||||||
|
throw new Error('expected mongo id in healthCheckProjects')
|
||||||
|
}
|
||||||
|
if (!HEALTH_CHECK_PROJECTS.some(id => id.length < 24)) {
|
||||||
|
throw new Error('expected postgres id in healthCheckProjects')
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const historyId of HEALTH_CHECK_PROJECTS) {
|
||||||
|
await verifyProjectWithErrorContext(historyId)
|
||||||
|
}
|
||||||
|
|
||||||
|
await measurePendingChangesBeforeTime(getEndDateForRPO(2))
|
||||||
|
await measureNeverBackedUpProjects(getEndDateForRPO(2))
|
||||||
|
}
|
8
services/history-v1/backupVerifier/types.d.ts
vendored
Normal file
8
services/history-v1/backupVerifier/types.d.ts
vendored
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
export type VerificationJobStatus = {
|
||||||
|
verified: number
|
||||||
|
total: number
|
||||||
|
startDate?: Date
|
||||||
|
endDate?: Date
|
||||||
|
hasFailure: boolean
|
||||||
|
errorTypes: Array<string>
|
||||||
|
}
|
35
services/history-v1/backupVerifier/utils.mjs
Normal file
35
services/history-v1/backupVerifier/utils.mjs
Normal file
|
@ -0,0 +1,35 @@
|
||||||
|
import { ObjectId } from 'mongodb'
|
||||||
|
import config from 'config'
|
||||||
|
|
||||||
|
export const RPO = parseInt(config.get('backupRPOInMS'), 10)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {Date} time
|
||||||
|
* @return {ObjectId}
|
||||||
|
*/
|
||||||
|
export function objectIdFromDate(time) {
|
||||||
|
return ObjectId.createFromTime(time.getTime() / 1000)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {number} [factor] - Multiply RPO by this factor, default is 1
|
||||||
|
* @return {Date}
|
||||||
|
*/
|
||||||
|
export function getEndDateForRPO(factor = 1) {
|
||||||
|
return new Date(Date.now() - RPO * factor)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a startDate, endDate pair that checks a period of time before the RPO horizon
|
||||||
|
*
|
||||||
|
* @param {number} offset - How many seconds we should check
|
||||||
|
* @return {{endDate: Date, startDate: Date}}
|
||||||
|
*/
|
||||||
|
export function getDatesBeforeRPO(offset) {
|
||||||
|
const now = new Date()
|
||||||
|
const endDate = new Date(now.getTime() - RPO)
|
||||||
|
return {
|
||||||
|
endDate,
|
||||||
|
startDate: new Date(endDate.getTime() - offset * 1000),
|
||||||
|
}
|
||||||
|
}
|
|
@ -7,4 +7,4 @@ history-v1
|
||||||
--node-version=20.18.2
|
--node-version=20.18.2
|
||||||
--public-repo=False
|
--public-repo=False
|
||||||
--script-version=4.5.0
|
--script-version=4.5.0
|
||||||
--tsconfig-extra-includes=backup-deletion-app.mjs,backup-verifier-app.mjs,api/**/*,migrations/**/*,storage/**/*
|
--tsconfig-extra-includes=backup-deletion-app.mjs,backup-verifier-app.mjs,backup-worker-app.mjs,api/**/*,migrations/**/*,storage/**/*
|
||||||
|
|
|
@ -66,6 +66,7 @@
|
||||||
},
|
},
|
||||||
"healthCheckBlobs": "HEALTH_CHECK_BLOBS",
|
"healthCheckBlobs": "HEALTH_CHECK_BLOBS",
|
||||||
"healthCheckProjects": "HEALTH_CHECK_PROJECTS",
|
"healthCheckProjects": "HEALTH_CHECK_PROJECTS",
|
||||||
|
"backupRPOInMS": "BACKUP_RPO_IN_MS",
|
||||||
"minSoftDeletionPeriodDays": "MIN_SOFT_DELETION_PERIOD_DAYS",
|
"minSoftDeletionPeriodDays": "MIN_SOFT_DELETION_PERIOD_DAYS",
|
||||||
"mongo": {
|
"mongo": {
|
||||||
"uri": "MONGO_CONNECTION_STRING"
|
"uri": "MONGO_CONNECTION_STRING"
|
||||||
|
|
|
@ -23,12 +23,14 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"backupRPOInMS": "3600000",
|
||||||
"chunkStore": {
|
"chunkStore": {
|
||||||
"historyStoreConcurrency": "4"
|
"historyStoreConcurrency": "4"
|
||||||
},
|
},
|
||||||
"zipStore": {
|
"zipStore": {
|
||||||
"zipTimeoutMs": "360000"
|
"zipTimeoutMs": "360000"
|
||||||
},
|
},
|
||||||
|
"hasProjectsWithoutHistory": false,
|
||||||
"minSoftDeletionPeriodDays": "90",
|
"minSoftDeletionPeriodDays": "90",
|
||||||
"maxDeleteKeys": "1000",
|
"maxDeleteKeys": "1000",
|
||||||
"useDeleteObjects": "true",
|
"useDeleteObjects": "true",
|
||||||
|
|
|
@ -36,6 +36,7 @@
|
||||||
},
|
},
|
||||||
"healthCheckBlobs": "[\"42/f70d7bba4ae1f07682e0358bd7a2068094fc023b\",\"000000000000000000000042/98d5521fe746bc2d11761edab5d0829bee286009\"]",
|
"healthCheckBlobs": "[\"42/f70d7bba4ae1f07682e0358bd7a2068094fc023b\",\"000000000000000000000042/98d5521fe746bc2d11761edab5d0829bee286009\"]",
|
||||||
"healthCheckProjects": "[\"42\",\"000000000000000000000042\"]",
|
"healthCheckProjects": "[\"42\",\"000000000000000000000042\"]",
|
||||||
|
"backupRPOInMS": "360000",
|
||||||
"maxDeleteKeys": "3",
|
"maxDeleteKeys": "3",
|
||||||
"useDeleteObjects": "false",
|
"useDeleteObjects": "false",
|
||||||
"mongo": {
|
"mongo": {
|
||||||
|
|
|
@ -40,7 +40,7 @@ services:
|
||||||
- ./test/acceptance/certs:/certs
|
- ./test/acceptance/certs:/certs
|
||||||
depends_on:
|
depends_on:
|
||||||
mongo:
|
mongo:
|
||||||
condition: service_healthy
|
condition: service_started
|
||||||
redis:
|
redis:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
postgres:
|
postgres:
|
||||||
|
@ -74,10 +74,15 @@ services:
|
||||||
mongo:
|
mongo:
|
||||||
image: mongo:6.0.13
|
image: mongo:6.0.13
|
||||||
command: --replSet overleaf
|
command: --replSet overleaf
|
||||||
healthcheck:
|
volumes:
|
||||||
test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
|
- ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
interval: 1s
|
environment:
|
||||||
retries: 20
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
|
extra_hosts:
|
||||||
|
# Required when using the automatic database setup for initializing the
|
||||||
|
# replica set. This override is not needed when running the setup after
|
||||||
|
# starting up mongo.
|
||||||
|
- mongo:127.0.0.1
|
||||||
postgres:
|
postgres:
|
||||||
image: postgres:10
|
image: postgres:10
|
||||||
environment:
|
environment:
|
||||||
|
|
|
@ -57,7 +57,7 @@ services:
|
||||||
user: node
|
user: node
|
||||||
depends_on:
|
depends_on:
|
||||||
mongo:
|
mongo:
|
||||||
condition: service_healthy
|
condition: service_started
|
||||||
redis:
|
redis:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
postgres:
|
postgres:
|
||||||
|
@ -82,10 +82,15 @@ services:
|
||||||
mongo:
|
mongo:
|
||||||
image: mongo:6.0.13
|
image: mongo:6.0.13
|
||||||
command: --replSet overleaf
|
command: --replSet overleaf
|
||||||
healthcheck:
|
volumes:
|
||||||
test: "mongosh --quiet localhost/test --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 1)'"
|
- ../../bin/shared/mongodb-init-replica-set.js:/docker-entrypoint-initdb.d/mongodb-init-replica-set.js
|
||||||
interval: 1s
|
environment:
|
||||||
retries: 20
|
MONGO_INITDB_DATABASE: sharelatex
|
||||||
|
extra_hosts:
|
||||||
|
# Required when using the automatic database setup for initializing the
|
||||||
|
# replica set. This override is not needed when running the setup after
|
||||||
|
# starting up mongo.
|
||||||
|
- mongo:127.0.0.1
|
||||||
|
|
||||||
postgres:
|
postgres:
|
||||||
image: postgres:10
|
image: postgres:10
|
||||||
|
|
|
@ -15,3 +15,6 @@ exports.zipStore = require('./lib/zip_store')
|
||||||
const { BlobStore, loadGlobalBlobs } = require('./lib/blob_store')
|
const { BlobStore, loadGlobalBlobs } = require('./lib/blob_store')
|
||||||
exports.BlobStore = BlobStore
|
exports.BlobStore = BlobStore
|
||||||
exports.loadGlobalBlobs = loadGlobalBlobs
|
exports.loadGlobalBlobs = loadGlobalBlobs
|
||||||
|
|
||||||
|
const { InvalidChangeError } = require('./lib/errors')
|
||||||
|
exports.InvalidChangeError = InvalidChangeError
|
||||||
|
|
|
@ -1,14 +1,24 @@
|
||||||
// @ts-check
|
// @ts-check
|
||||||
import config from 'config'
|
|
||||||
import OError from '@overleaf/o-error'
|
import OError from '@overleaf/o-error'
|
||||||
import { backupPersistor, projectBlobsBucket } from './backupPersistor.mjs'
|
import chunkStore from '../lib/chunk_store/index.js'
|
||||||
import { Blob } from 'overleaf-editor-core'
|
import {
|
||||||
import { BlobStore, makeProjectKey } from './blob_store/index.js'
|
backupPersistor,
|
||||||
|
chunksBucket,
|
||||||
|
projectBlobsBucket,
|
||||||
|
} from './backupPersistor.mjs'
|
||||||
|
import { Blob, Chunk, History } from 'overleaf-editor-core'
|
||||||
|
import { BlobStore, GLOBAL_BLOBS, makeProjectKey } from './blob_store/index.js'
|
||||||
import blobHash from './blob_hash.js'
|
import blobHash from './blob_hash.js'
|
||||||
import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
|
import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
|
||||||
|
import logger from '@overleaf/logger'
|
||||||
|
import path from 'node:path'
|
||||||
|
import projectKey from './project_key.js'
|
||||||
|
import streams from './streams.js'
|
||||||
|
import objectPersistor from '@overleaf/object-persistor'
|
||||||
|
import { getEndDateForRPO } from '../../backupVerifier/utils.mjs'
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @typedef {import("@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor").CachedPerProjectEncryptedS3Persistor} CachedPerProjectEncryptedS3Persistor
|
* @typedef {import("@overleaf/object-persistor/src/PerProjectEncryptedS3Persistor.js").CachedPerProjectEncryptedS3Persistor} CachedPerProjectEncryptedS3Persistor
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -20,13 +30,13 @@ export async function verifyBlob(historyId, hash) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
*
|
||||||
* @param {string} historyId
|
* @param {string} historyId
|
||||||
* @param {Array<string>} hashes
|
* @return {Promise<CachedPerProjectEncryptedS3Persistor>}
|
||||||
*/
|
*/
|
||||||
export async function verifyBlobs(historyId, hashes) {
|
async function getProjectPersistor(historyId) {
|
||||||
let projectCache
|
|
||||||
try {
|
try {
|
||||||
projectCache = await backupPersistor.forProjectRO(
|
return await backupPersistor.forProjectRO(
|
||||||
projectBlobsBucket,
|
projectBlobsBucket,
|
||||||
makeProjectKey(historyId, '')
|
makeProjectKey(historyId, '')
|
||||||
)
|
)
|
||||||
|
@ -36,16 +46,19 @@ export async function verifyBlobs(historyId, hashes) {
|
||||||
}
|
}
|
||||||
throw err
|
throw err
|
||||||
}
|
}
|
||||||
await verifyBlobsWithCache(historyId, projectCache, hashes)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param {string} historyId
|
* @param {string} historyId
|
||||||
* @param {CachedPerProjectEncryptedS3Persistor} projectCache
|
|
||||||
* @param {Array<string>} hashes
|
* @param {Array<string>} hashes
|
||||||
|
* @param {CachedPerProjectEncryptedS3Persistor} [projectCache]
|
||||||
*/
|
*/
|
||||||
export async function verifyBlobsWithCache(historyId, projectCache, hashes) {
|
export async function verifyBlobs(historyId, hashes, projectCache) {
|
||||||
if (hashes.length === 0) throw new Error('bug: empty hashes')
|
if (hashes.length === 0) throw new Error('bug: empty hashes')
|
||||||
|
|
||||||
|
if (!projectCache) {
|
||||||
|
projectCache = await getProjectPersistor(historyId)
|
||||||
|
}
|
||||||
const blobStore = new BlobStore(historyId)
|
const blobStore = new BlobStore(historyId)
|
||||||
for (const hash of hashes) {
|
for (const hash of hashes) {
|
||||||
const path = makeProjectKey(historyId, hash)
|
const path = makeProjectKey(historyId, hash)
|
||||||
|
@ -58,41 +71,146 @@ export async function verifyBlobsWithCache(historyId, projectCache, hashes) {
|
||||||
})
|
})
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err instanceof NotFoundError) {
|
if (err instanceof NotFoundError) {
|
||||||
throw new BackupCorruptedError('missing blob')
|
throw new BackupCorruptedMissingBlobError('missing blob', {
|
||||||
|
path,
|
||||||
|
hash,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
throw err
|
throw err
|
||||||
}
|
}
|
||||||
const backupHash = await blobHash.fromStream(blob.getByteLength(), stream)
|
const backupHash = await blobHash.fromStream(blob.getByteLength(), stream)
|
||||||
if (backupHash !== hash) {
|
if (backupHash !== hash) {
|
||||||
throw new BackupCorruptedError('hash mismatch for backed up blob', {
|
throw new BackupCorruptedInvalidBlobError(
|
||||||
path,
|
'hash mismatch for backed up blob',
|
||||||
hash,
|
{
|
||||||
backupHash,
|
path,
|
||||||
})
|
hash,
|
||||||
|
backupHash,
|
||||||
|
}
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export class BackupCorruptedError extends OError {}
|
/**
|
||||||
|
* @param {string} historyId
|
||||||
export async function healthCheck() {
|
* @param {Date} [endTimestamp]
|
||||||
/** @type {Array<string>} */
|
*/
|
||||||
const HEALTH_CHECK_BLOBS = JSON.parse(config.get('healthCheckBlobs'))
|
export async function verifyProjectWithErrorContext(
|
||||||
if (HEALTH_CHECK_BLOBS.length !== 2) {
|
historyId,
|
||||||
throw new Error('expected 2 healthCheckBlobs')
|
endTimestamp = getEndDateForRPO()
|
||||||
}
|
) {
|
||||||
if (!HEALTH_CHECK_BLOBS.some(path => path.split('/')[0].length === 24)) {
|
try {
|
||||||
throw new Error('expected mongo id in healthCheckBlobs')
|
await verifyProject(historyId, endTimestamp)
|
||||||
}
|
} catch (err) {
|
||||||
if (!HEALTH_CHECK_BLOBS.some(path => path.split('/')[0].length < 24)) {
|
// @ts-ignore err is Error instance
|
||||||
throw new Error('expected postgres id in healthCheckBlobs')
|
throw OError.tag(err, 'verifyProject', { historyId, endTimestamp })
|
||||||
}
|
|
||||||
if (HEALTH_CHECK_BLOBS.some(path => path.split('/')[1]?.length !== 40)) {
|
|
||||||
throw new Error('expected hash in healthCheckBlobs')
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const path of HEALTH_CHECK_BLOBS) {
|
|
||||||
const [historyId, hash] = path.split('/')
|
|
||||||
await verifyBlob(historyId, hash)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {string} historyId
|
||||||
|
* @param {number} startVersion
|
||||||
|
* @param {CachedPerProjectEncryptedS3Persistor} backupPersistorForProject
|
||||||
|
* @return {Promise<any>}
|
||||||
|
*/
|
||||||
|
async function loadChunk(historyId, startVersion, backupPersistorForProject) {
|
||||||
|
const key = path.join(
|
||||||
|
projectKey.format(historyId),
|
||||||
|
projectKey.pad(startVersion)
|
||||||
|
)
|
||||||
|
try {
|
||||||
|
const buf = await streams.gunzipStreamToBuffer(
|
||||||
|
await backupPersistorForProject.getObjectStream(chunksBucket, key)
|
||||||
|
)
|
||||||
|
return JSON.parse(buf.toString('utf-8'))
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof objectPersistor.Errors.NotFoundError) {
|
||||||
|
throw new Chunk.NotPersistedError(historyId)
|
||||||
|
}
|
||||||
|
if (err instanceof Error) {
|
||||||
|
throw OError.tag(err, 'Failed to load chunk', { historyId, startVersion })
|
||||||
|
}
|
||||||
|
throw err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {string} historyId
|
||||||
|
* @param {Date} endTimestamp
|
||||||
|
*/
|
||||||
|
export async function verifyProject(historyId, endTimestamp) {
|
||||||
|
const backend = chunkStore.getBackend(historyId)
|
||||||
|
const [first, last] = await Promise.all([
|
||||||
|
backend.getFirstChunkBeforeTimestamp(historyId, endTimestamp),
|
||||||
|
backend.getLastActiveChunkBeforeTimestamp(historyId, endTimestamp),
|
||||||
|
])
|
||||||
|
|
||||||
|
const chunksRecordsToVerify = [
|
||||||
|
{
|
||||||
|
chunkId: first.id,
|
||||||
|
chunkLabel: 'first',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
if (first.startVersion !== last.startVersion) {
|
||||||
|
chunksRecordsToVerify.push({
|
||||||
|
chunkId: last.id,
|
||||||
|
chunkLabel: 'last before RPO',
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
const projectCache = await getProjectPersistor(historyId)
|
||||||
|
|
||||||
|
const chunks = await Promise.all(
|
||||||
|
chunksRecordsToVerify.map(async chunk => {
|
||||||
|
try {
|
||||||
|
return History.fromRaw(
|
||||||
|
await loadChunk(historyId, chunk.startVersion, projectCache)
|
||||||
|
)
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof Chunk.NotPersistedError) {
|
||||||
|
throw new BackupRPOViolationChunkNotBackedUpError(
|
||||||
|
'BackupRPOviolation: chunk not backed up',
|
||||||
|
chunk
|
||||||
|
)
|
||||||
|
}
|
||||||
|
throw err
|
||||||
|
}
|
||||||
|
})
|
||||||
|
)
|
||||||
|
const seenBlobs = new Set()
|
||||||
|
const blobsToVerify = []
|
||||||
|
for (const chunk of chunks) {
|
||||||
|
/** @type {Set<string>} */
|
||||||
|
const chunkBlobs = new Set()
|
||||||
|
chunk.findBlobHashes(chunkBlobs)
|
||||||
|
let hasAddedBlobFromThisChunk = false
|
||||||
|
for (const blobHash of chunkBlobs) {
|
||||||
|
if (seenBlobs.has(blobHash)) continue // old blob
|
||||||
|
if (GLOBAL_BLOBS.has(blobHash)) continue // global blob
|
||||||
|
seenBlobs.add(blobHash)
|
||||||
|
if (!hasAddedBlobFromThisChunk) {
|
||||||
|
blobsToVerify.push(blobHash)
|
||||||
|
hasAddedBlobFromThisChunk = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (blobsToVerify.length === 0) {
|
||||||
|
logger.debug(
|
||||||
|
{
|
||||||
|
historyId,
|
||||||
|
chunksRecordsToVerify: chunksRecordsToVerify.map(c => c.chunkId),
|
||||||
|
},
|
||||||
|
'chunks contain no blobs to verify'
|
||||||
|
)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
await verifyBlobs(historyId, blobsToVerify, projectCache)
|
||||||
|
}
|
||||||
|
|
||||||
|
export class BackupCorruptedError extends OError {}
|
||||||
|
export class BackupRPOViolationError extends OError {}
|
||||||
|
export class BackupCorruptedMissingBlobError extends BackupCorruptedError {}
|
||||||
|
export class BackupCorruptedInvalidBlobError extends BackupCorruptedError {}
|
||||||
|
export class BackupRPOViolationChunkNotBackedUpError extends OError {}
|
||||||
|
|
|
@ -3,8 +3,18 @@ const { projects, backedUpBlobs } = require('../mongodb')
|
||||||
const OError = require('@overleaf/o-error')
|
const OError = require('@overleaf/o-error')
|
||||||
|
|
||||||
// List projects with pending backups older than the specified interval
|
// List projects with pending backups older than the specified interval
|
||||||
function listPendingBackups(timeIntervalMs = 0) {
|
function listPendingBackups(timeIntervalMs = 0, limit = null) {
|
||||||
const cutoffTime = new Date(Date.now() - timeIntervalMs)
|
const cutoffTime = new Date(Date.now() - timeIntervalMs)
|
||||||
|
const options = {
|
||||||
|
projection: { 'overleaf.backup.pendingChangeAt': 1 },
|
||||||
|
sort: { 'overleaf.backup.pendingChangeAt': 1 },
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply limit if provided
|
||||||
|
if (limit) {
|
||||||
|
options.limit = limit
|
||||||
|
}
|
||||||
|
|
||||||
const cursor = projects.find(
|
const cursor = projects.find(
|
||||||
{
|
{
|
||||||
'overleaf.backup.pendingChangeAt': {
|
'overleaf.backup.pendingChangeAt': {
|
||||||
|
@ -12,10 +22,30 @@ function listPendingBackups(timeIntervalMs = 0) {
|
||||||
$lt: cutoffTime,
|
$lt: cutoffTime,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
options
|
||||||
|
)
|
||||||
|
return cursor
|
||||||
|
}
|
||||||
|
|
||||||
|
// List projects that have never been backed up and are older than the specified interval
|
||||||
|
function listUninitializedBackups(timeIntervalMs = 0, limit = null) {
|
||||||
|
const cutoffTimeInSeconds = (Date.now() - timeIntervalMs) / 1000
|
||||||
|
const options = {
|
||||||
|
projection: { _id: 1 },
|
||||||
|
sort: { _id: 1 },
|
||||||
|
}
|
||||||
|
// Apply limit if provided
|
||||||
|
if (limit) {
|
||||||
|
options.limit = limit
|
||||||
|
}
|
||||||
|
const cursor = projects.find(
|
||||||
{
|
{
|
||||||
projection: { 'overleaf.backup': 1, 'overleaf.history': 1 },
|
'overleaf.backup.lastBackedUpVersion': null,
|
||||||
sort: { 'overleaf.backup.pendingChangeAt': 1 },
|
_id: {
|
||||||
}
|
$lt: ObjectId.createFromTime(cutoffTimeInSeconds),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
options
|
||||||
)
|
)
|
||||||
return cursor
|
return cursor
|
||||||
}
|
}
|
||||||
|
@ -176,6 +206,7 @@ module.exports = {
|
||||||
updateCurrentMetadataIfNotSet,
|
updateCurrentMetadataIfNotSet,
|
||||||
updatePendingChangeTimestamp,
|
updatePendingChangeTimestamp,
|
||||||
listPendingBackups,
|
listPendingBackups,
|
||||||
|
listUninitializedBackups,
|
||||||
getBackedUpBlobHashes,
|
getBackedUpBlobHashes,
|
||||||
unsetBackedUpBlobHashes,
|
unsetBackedUpBlobHashes,
|
||||||
}
|
}
|
||||||
|
|
|
@ -155,15 +155,22 @@ async function loadAtTimestamp(projectId, timestamp) {
|
||||||
*
|
*
|
||||||
* @param {string} projectId
|
* @param {string} projectId
|
||||||
* @param {Chunk} chunk
|
* @param {Chunk} chunk
|
||||||
|
* @param {Date} [earliestChangeTimestamp]
|
||||||
* @return {Promise.<number>} for the chunkId of the inserted chunk
|
* @return {Promise.<number>} for the chunkId of the inserted chunk
|
||||||
*/
|
*/
|
||||||
async function create(projectId, chunk) {
|
async function create(projectId, chunk, earliestChangeTimestamp) {
|
||||||
assert.projectId(projectId, 'bad projectId')
|
assert.projectId(projectId, 'bad projectId')
|
||||||
assert.instance(chunk, Chunk, 'bad chunk')
|
assert.instance(chunk, Chunk, 'bad chunk')
|
||||||
|
assert.maybe.date(earliestChangeTimestamp, 'bad timestamp')
|
||||||
|
|
||||||
const backend = getBackend(projectId)
|
const backend = getBackend(projectId)
|
||||||
const chunkId = await uploadChunk(projectId, chunk)
|
const chunkId = await uploadChunk(projectId, chunk)
|
||||||
await backend.confirmCreate(projectId, chunk, chunkId)
|
await backend.confirmCreate(
|
||||||
|
projectId,
|
||||||
|
chunk,
|
||||||
|
chunkId,
|
||||||
|
earliestChangeTimestamp
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -195,18 +202,31 @@ async function uploadChunk(projectId, chunk) {
|
||||||
* @param {string} projectId
|
* @param {string} projectId
|
||||||
* @param {number} oldEndVersion
|
* @param {number} oldEndVersion
|
||||||
* @param {Chunk} newChunk
|
* @param {Chunk} newChunk
|
||||||
|
* @param {Date} [earliestChangeTimestamp]
|
||||||
* @return {Promise}
|
* @return {Promise}
|
||||||
*/
|
*/
|
||||||
async function update(projectId, oldEndVersion, newChunk) {
|
async function update(
|
||||||
|
projectId,
|
||||||
|
oldEndVersion,
|
||||||
|
newChunk,
|
||||||
|
earliestChangeTimestamp
|
||||||
|
) {
|
||||||
assert.projectId(projectId, 'bad projectId')
|
assert.projectId(projectId, 'bad projectId')
|
||||||
assert.integer(oldEndVersion, 'bad oldEndVersion')
|
assert.integer(oldEndVersion, 'bad oldEndVersion')
|
||||||
assert.instance(newChunk, Chunk, 'bad newChunk')
|
assert.instance(newChunk, Chunk, 'bad newChunk')
|
||||||
|
assert.maybe.date(earliestChangeTimestamp, 'bad timestamp')
|
||||||
|
|
||||||
const backend = getBackend(projectId)
|
const backend = getBackend(projectId)
|
||||||
const oldChunkId = await getChunkIdForVersion(projectId, oldEndVersion)
|
const oldChunkId = await getChunkIdForVersion(projectId, oldEndVersion)
|
||||||
const newChunkId = await uploadChunk(projectId, newChunk)
|
const newChunkId = await uploadChunk(projectId, newChunk)
|
||||||
|
|
||||||
await backend.confirmUpdate(projectId, oldChunkId, newChunk, newChunkId)
|
await backend.confirmUpdate(
|
||||||
|
projectId,
|
||||||
|
oldChunkId,
|
||||||
|
newChunk,
|
||||||
|
newChunkId,
|
||||||
|
earliestChangeTimestamp
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -54,6 +54,35 @@ async function getChunkForVersion(projectId, version) {
|
||||||
return chunkFromRecord(record)
|
return chunkFromRecord(record)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the metadata for the chunk that contains the given version before the endTime.
|
||||||
|
*/
|
||||||
|
async function getFirstChunkBeforeTimestamp(projectId, timestamp) {
|
||||||
|
assert.mongoId(projectId, 'bad projectId')
|
||||||
|
assert.date(timestamp, 'bad timestamp')
|
||||||
|
|
||||||
|
const recordActive = await getChunkForVersion(projectId, 0)
|
||||||
|
if (recordActive && recordActive.endTimestamp <= timestamp) {
|
||||||
|
return recordActive
|
||||||
|
}
|
||||||
|
|
||||||
|
// fallback to deleted chunk
|
||||||
|
const recordDeleted = await mongodb.chunks.findOne(
|
||||||
|
{
|
||||||
|
projectId: new ObjectId(projectId),
|
||||||
|
state: 'deleted',
|
||||||
|
startVersion: 0,
|
||||||
|
updatedAt: { $lte: timestamp }, // indexed for state=deleted
|
||||||
|
endTimestamp: { $lte: timestamp },
|
||||||
|
},
|
||||||
|
{ sort: { updatedAt: -1 } }
|
||||||
|
)
|
||||||
|
if (recordDeleted) {
|
||||||
|
return chunkFromRecord(recordDeleted)
|
||||||
|
}
|
||||||
|
throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the metadata for the chunk that contains the version that was current at
|
* Get the metadata for the chunk that contains the version that was current at
|
||||||
* the given timestamp.
|
* the given timestamp.
|
||||||
|
@ -86,6 +115,39 @@ async function getChunkForTimestamp(projectId, timestamp) {
|
||||||
return chunkFromRecord(record)
|
return chunkFromRecord(record)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the metadata for the chunk that contains the version that was current before
|
||||||
|
* the given timestamp.
|
||||||
|
*/
|
||||||
|
async function getLastActiveChunkBeforeTimestamp(projectId, timestamp) {
|
||||||
|
assert.mongoId(projectId, 'bad projectId')
|
||||||
|
assert.date(timestamp, 'bad timestamp')
|
||||||
|
|
||||||
|
const record = await mongodb.chunks.findOne(
|
||||||
|
{
|
||||||
|
projectId: new ObjectId(projectId),
|
||||||
|
state: 'active',
|
||||||
|
$or: [
|
||||||
|
{
|
||||||
|
endTimestamp: {
|
||||||
|
$lte: timestamp,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
endTimestamp: null,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
// We use the index on the startVersion for sorting records. This assumes
|
||||||
|
// that timestamps go up with each version.
|
||||||
|
{ sort: { startVersion: -1 } }
|
||||||
|
)
|
||||||
|
if (record == null) {
|
||||||
|
throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
|
||||||
|
}
|
||||||
|
return chunkFromRecord(record)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get all of a project's chunk ids
|
* Get all of a project's chunk ids
|
||||||
*/
|
*/
|
||||||
|
@ -137,7 +199,13 @@ async function insertPendingChunk(projectId, chunk) {
|
||||||
/**
|
/**
|
||||||
* Record that a new chunk was created.
|
* Record that a new chunk was created.
|
||||||
*/
|
*/
|
||||||
async function confirmCreate(projectId, chunk, chunkId, mongoOpts = {}) {
|
async function confirmCreate(
|
||||||
|
projectId,
|
||||||
|
chunk,
|
||||||
|
chunkId,
|
||||||
|
earliestChangeTimestamp,
|
||||||
|
mongoOpts = {}
|
||||||
|
) {
|
||||||
assert.mongoId(projectId, 'bad projectId')
|
assert.mongoId(projectId, 'bad projectId')
|
||||||
assert.instance(chunk, Chunk, 'bad chunk')
|
assert.instance(chunk, Chunk, 'bad chunk')
|
||||||
assert.mongoId(chunkId, 'bad chunkId')
|
assert.mongoId(chunkId, 'bad chunkId')
|
||||||
|
@ -166,13 +234,23 @@ async function confirmCreate(projectId, chunk, chunkId, mongoOpts = {}) {
|
||||||
if (result.matchedCount === 0) {
|
if (result.matchedCount === 0) {
|
||||||
throw new OError('pending chunk not found', { projectId, chunkId })
|
throw new OError('pending chunk not found', { projectId, chunkId })
|
||||||
}
|
}
|
||||||
await updateProjectRecord(projectId, chunk, mongoOpts)
|
await updateProjectRecord(
|
||||||
|
projectId,
|
||||||
|
chunk,
|
||||||
|
earliestChangeTimestamp,
|
||||||
|
mongoOpts
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Write the metadata to the project record
|
* Write the metadata to the project record
|
||||||
*/
|
*/
|
||||||
async function updateProjectRecord(projectId, chunk, mongoOpts = {}) {
|
async function updateProjectRecord(
|
||||||
|
projectId,
|
||||||
|
chunk,
|
||||||
|
earliestChangeTimestamp,
|
||||||
|
mongoOpts = {}
|
||||||
|
) {
|
||||||
// record the end version against the project
|
// record the end version against the project
|
||||||
await mongodb.projects.updateOne(
|
await mongodb.projects.updateOne(
|
||||||
{
|
{
|
||||||
|
@ -189,7 +267,7 @@ async function updateProjectRecord(projectId, chunk, mongoOpts = {}) {
|
||||||
// be cleared every time a backup is completed.
|
// be cleared every time a backup is completed.
|
||||||
$min: {
|
$min: {
|
||||||
'overleaf.backup.pendingChangeAt':
|
'overleaf.backup.pendingChangeAt':
|
||||||
chunk.getEndTimestamp() || new Date(),
|
earliestChangeTimestamp || chunk.getEndTimestamp() || new Date(),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
mongoOpts
|
mongoOpts
|
||||||
|
@ -199,7 +277,13 @@ async function updateProjectRecord(projectId, chunk, mongoOpts = {}) {
|
||||||
/**
|
/**
|
||||||
* Record that a chunk was replaced by a new one.
|
* Record that a chunk was replaced by a new one.
|
||||||
*/
|
*/
|
||||||
async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
|
async function confirmUpdate(
|
||||||
|
projectId,
|
||||||
|
oldChunkId,
|
||||||
|
newChunk,
|
||||||
|
newChunkId,
|
||||||
|
earliestChangeTimestamp
|
||||||
|
) {
|
||||||
assert.mongoId(projectId, 'bad projectId')
|
assert.mongoId(projectId, 'bad projectId')
|
||||||
assert.mongoId(oldChunkId, 'bad oldChunkId')
|
assert.mongoId(oldChunkId, 'bad oldChunkId')
|
||||||
assert.instance(newChunk, Chunk, 'bad newChunk')
|
assert.instance(newChunk, Chunk, 'bad newChunk')
|
||||||
|
@ -209,7 +293,13 @@ async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
|
||||||
try {
|
try {
|
||||||
await session.withTransaction(async () => {
|
await session.withTransaction(async () => {
|
||||||
await deleteChunk(projectId, oldChunkId, { session })
|
await deleteChunk(projectId, oldChunkId, { session })
|
||||||
await confirmCreate(projectId, newChunk, newChunkId, { session })
|
await confirmCreate(
|
||||||
|
projectId,
|
||||||
|
newChunk,
|
||||||
|
newChunkId,
|
||||||
|
earliestChangeTimestamp,
|
||||||
|
{ session }
|
||||||
|
)
|
||||||
})
|
})
|
||||||
} finally {
|
} finally {
|
||||||
await session.endSession()
|
await session.endSession()
|
||||||
|
@ -310,6 +400,8 @@ function chunkFromRecord(record) {
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
getLatestChunk,
|
getLatestChunk,
|
||||||
|
getFirstChunkBeforeTimestamp,
|
||||||
|
getLastActiveChunkBeforeTimestamp,
|
||||||
getChunkForVersion,
|
getChunkForVersion,
|
||||||
getChunkForTimestamp,
|
getChunkForTimestamp,
|
||||||
getProjectChunkIds,
|
getProjectChunkIds,
|
||||||
|
|
|
@ -46,6 +46,59 @@ async function getChunkForVersion(projectId, version) {
|
||||||
return chunkFromRecord(record)
|
return chunkFromRecord(record)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the metadata for the chunk that contains the given version.
|
||||||
|
*/
|
||||||
|
async function getFirstChunkBeforeTimestamp(projectId, timestamp) {
|
||||||
|
assert.date(timestamp, 'bad timestamp')
|
||||||
|
|
||||||
|
const recordActive = await getChunkForVersion(projectId, 0)
|
||||||
|
// projectId must be valid if getChunkForVersion did not throw
|
||||||
|
projectId = parseInt(projectId, 10)
|
||||||
|
if (recordActive && recordActive.endTimestamp <= timestamp) {
|
||||||
|
return recordActive
|
||||||
|
}
|
||||||
|
|
||||||
|
// fallback to deleted chunk
|
||||||
|
const recordDeleted = await knex('old_chunks')
|
||||||
|
.where('doc_id', projectId)
|
||||||
|
.where('start_version', '=', 0)
|
||||||
|
.where('end_timestamp', '<=', timestamp)
|
||||||
|
.orderBy('end_version', 'desc')
|
||||||
|
.first()
|
||||||
|
if (recordDeleted) {
|
||||||
|
return chunkFromRecord(recordDeleted)
|
||||||
|
}
|
||||||
|
throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the metadata for the chunk that contains the version that was current at
|
||||||
|
* the given timestamp.
|
||||||
|
*/
|
||||||
|
async function getLastActiveChunkBeforeTimestamp(projectId, timestamp) {
|
||||||
|
assert.date(timestamp, 'bad timestamp')
|
||||||
|
assert.postgresId(projectId, 'bad projectId')
|
||||||
|
projectId = parseInt(projectId, 10)
|
||||||
|
|
||||||
|
const query = knex('chunks')
|
||||||
|
.where('doc_id', projectId)
|
||||||
|
.where(function () {
|
||||||
|
this.where('end_timestamp', '<=', timestamp).orWhere(
|
||||||
|
'end_timestamp',
|
||||||
|
null
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.orderBy('end_version', 'desc', 'last')
|
||||||
|
|
||||||
|
const record = await query.first()
|
||||||
|
|
||||||
|
if (!record) {
|
||||||
|
throw new Chunk.BeforeTimestampNotFoundError(projectId, timestamp)
|
||||||
|
}
|
||||||
|
return chunkFromRecord(record)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the metadata for the chunk that contains the version that was current at
|
* Get the metadata for the chunk that contains the version that was current at
|
||||||
* the given timestamp.
|
* the given timestamp.
|
||||||
|
@ -140,7 +193,12 @@ async function insertPendingChunk(projectId, chunk) {
|
||||||
/**
|
/**
|
||||||
* Record that a new chunk was created.
|
* Record that a new chunk was created.
|
||||||
*/
|
*/
|
||||||
async function confirmCreate(projectId, chunk, chunkId) {
|
async function confirmCreate(
|
||||||
|
projectId,
|
||||||
|
chunk,
|
||||||
|
chunkId,
|
||||||
|
earliestChangeTimestamp
|
||||||
|
) {
|
||||||
assert.postgresId(projectId, `bad projectId ${projectId}`)
|
assert.postgresId(projectId, `bad projectId ${projectId}`)
|
||||||
projectId = parseInt(projectId, 10)
|
projectId = parseInt(projectId, 10)
|
||||||
|
|
||||||
|
@ -149,14 +207,20 @@ async function confirmCreate(projectId, chunk, chunkId) {
|
||||||
_deletePendingChunk(tx, projectId, chunkId),
|
_deletePendingChunk(tx, projectId, chunkId),
|
||||||
_insertChunk(tx, projectId, chunk, chunkId),
|
_insertChunk(tx, projectId, chunk, chunkId),
|
||||||
])
|
])
|
||||||
await updateProjectRecord(projectId, chunk)
|
await updateProjectRecord(projectId, chunk, earliestChangeTimestamp)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Record that a chunk was replaced by a new one.
|
* Record that a chunk was replaced by a new one.
|
||||||
*/
|
*/
|
||||||
async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
|
async function confirmUpdate(
|
||||||
|
projectId,
|
||||||
|
oldChunkId,
|
||||||
|
newChunk,
|
||||||
|
newChunkId,
|
||||||
|
earliestChangeTimestamp
|
||||||
|
) {
|
||||||
assert.postgresId(projectId, `bad projectId ${projectId}`)
|
assert.postgresId(projectId, `bad projectId ${projectId}`)
|
||||||
projectId = parseInt(projectId, 10)
|
projectId = parseInt(projectId, 10)
|
||||||
|
|
||||||
|
@ -166,7 +230,7 @@ async function confirmUpdate(projectId, oldChunkId, newChunk, newChunkId) {
|
||||||
_deletePendingChunk(tx, projectId, newChunkId),
|
_deletePendingChunk(tx, projectId, newChunkId),
|
||||||
_insertChunk(tx, projectId, newChunk, newChunkId),
|
_insertChunk(tx, projectId, newChunk, newChunkId),
|
||||||
])
|
])
|
||||||
await updateProjectRecord(projectId, newChunk)
|
await updateProjectRecord(projectId, newChunk, earliestChangeTimestamp)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -280,6 +344,8 @@ async function generateProjectId() {
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
getLatestChunk,
|
getLatestChunk,
|
||||||
|
getFirstChunkBeforeTimestamp,
|
||||||
|
getLastActiveChunkBeforeTimestamp,
|
||||||
getChunkForVersion,
|
getChunkForVersion,
|
||||||
getChunkForTimestamp,
|
getChunkForTimestamp,
|
||||||
getProjectChunkIds,
|
getProjectChunkIds,
|
||||||
|
|
|
@ -65,6 +65,9 @@ async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
|
||||||
|
|
||||||
const blobStore = new BlobStore(projectId)
|
const blobStore = new BlobStore(projectId)
|
||||||
|
|
||||||
|
const earliestChangeTimestamp =
|
||||||
|
allChanges.length > 0 ? allChanges[0].getTimestamp() : null
|
||||||
|
|
||||||
let currentChunk
|
let currentChunk
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -78,12 +81,6 @@ async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
|
||||||
let originalEndVersion
|
let originalEndVersion
|
||||||
let changesToPersist
|
let changesToPersist
|
||||||
|
|
||||||
/**
|
|
||||||
* It's only useful to log validation errors once per flush. When we enforce
|
|
||||||
* content hash validation, it will stop the flush right away anyway.
|
|
||||||
*/
|
|
||||||
let validationErrorLogged = false
|
|
||||||
|
|
||||||
limits = limits || {}
|
limits = limits || {}
|
||||||
_.defaults(limits, {
|
_.defaults(limits, {
|
||||||
changeBucketMinutes: 60,
|
changeBucketMinutes: 60,
|
||||||
|
@ -128,22 +125,7 @@ async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
|
||||||
for (const operation of change.iterativelyApplyTo(currentSnapshot, {
|
for (const operation of change.iterativelyApplyTo(currentSnapshot, {
|
||||||
strict: true,
|
strict: true,
|
||||||
})) {
|
})) {
|
||||||
try {
|
await validateContentHash(operation)
|
||||||
await validateContentHash(operation)
|
|
||||||
} catch (err) {
|
|
||||||
// Temporary: skip validation errors
|
|
||||||
if (err instanceof InvalidChangeError) {
|
|
||||||
if (!validationErrorLogged) {
|
|
||||||
logger.warn(
|
|
||||||
{ err, projectId },
|
|
||||||
'content snapshot mismatch (ignored)'
|
|
||||||
)
|
|
||||||
validationErrorLogged = true
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
throw err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
chunk.pushChanges([change])
|
chunk.pushChanges([change])
|
||||||
|
@ -220,7 +202,12 @@ async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
|
||||||
|
|
||||||
checkElapsedTime(timer)
|
checkElapsedTime(timer)
|
||||||
|
|
||||||
await chunkStore.update(projectId, originalEndVersion, currentChunk)
|
await chunkStore.update(
|
||||||
|
projectId,
|
||||||
|
originalEndVersion,
|
||||||
|
currentChunk,
|
||||||
|
earliestChangeTimestamp
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function createNewChunksAsNeeded() {
|
async function createNewChunksAsNeeded() {
|
||||||
|
@ -234,7 +221,7 @@ async function persistChanges(projectId, allChanges, limits, clientEndVersion) {
|
||||||
if (changesPushed) {
|
if (changesPushed) {
|
||||||
checkElapsedTime(timer)
|
checkElapsedTime(timer)
|
||||||
currentChunk = chunk
|
currentChunk = chunk
|
||||||
await chunkStore.create(projectId, chunk)
|
await chunkStore.create(projectId, chunk, earliestChangeTimestamp)
|
||||||
} else {
|
} else {
|
||||||
throw new Error('failed to fill empty chunk')
|
throw new Error('failed to fill empty chunk')
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,8 +2,12 @@
|
||||||
|
|
||||||
import logger from '@overleaf/logger'
|
import logger from '@overleaf/logger'
|
||||||
import commandLineArgs from 'command-line-args'
|
import commandLineArgs from 'command-line-args'
|
||||||
import { History } from 'overleaf-editor-core'
|
import { Chunk, History, Snapshot } from 'overleaf-editor-core'
|
||||||
import { getProjectChunks, loadLatestRaw } from '../lib/chunk_store/index.js'
|
import {
|
||||||
|
getProjectChunks,
|
||||||
|
loadLatestRaw,
|
||||||
|
create,
|
||||||
|
} from '../lib/chunk_store/index.js'
|
||||||
import { client } from '../lib/mongodb.js'
|
import { client } from '../lib/mongodb.js'
|
||||||
import knex from '../lib/knex.js'
|
import knex from '../lib/knex.js'
|
||||||
import { historyStore } from '../lib/history_store.js'
|
import { historyStore } from '../lib/history_store.js'
|
||||||
|
@ -30,7 +34,7 @@ import {
|
||||||
projectBlobsBucket,
|
projectBlobsBucket,
|
||||||
} from '../lib/backupPersistor.mjs'
|
} from '../lib/backupPersistor.mjs'
|
||||||
import { backupGenerator } from '../lib/backupGenerator.mjs'
|
import { backupGenerator } from '../lib/backupGenerator.mjs'
|
||||||
import { promises as fs } from 'node:fs'
|
import { promises as fs, createWriteStream } from 'node:fs'
|
||||||
import os from 'node:os'
|
import os from 'node:os'
|
||||||
import path from 'node:path'
|
import path from 'node:path'
|
||||||
import projectKey from '../lib/project_key.js'
|
import projectKey from '../lib/project_key.js'
|
||||||
|
@ -89,7 +93,7 @@ process.on('SIGTERM', handleSignal)
|
||||||
|
|
||||||
function handleSignal() {
|
function handleSignal() {
|
||||||
gracefulShutdownInitiated = true
|
gracefulShutdownInitiated = true
|
||||||
console.warn('graceful shutdown initiated, draining queue')
|
logger.info({}, 'graceful shutdown initiated, draining queue')
|
||||||
}
|
}
|
||||||
|
|
||||||
async function retry(fn, times, delayMs) {
|
async function retry(fn, times, delayMs) {
|
||||||
|
@ -321,12 +325,18 @@ const optionDefinitions = [
|
||||||
description: 'Time interval in seconds for pending backups (default: 3600)',
|
description: 'Time interval in seconds for pending backups (default: 3600)',
|
||||||
defaultValue: 3600,
|
defaultValue: 3600,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: 'fix',
|
||||||
|
type: Number,
|
||||||
|
description: 'Fix projects without chunks',
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: 'init',
|
name: 'init',
|
||||||
alias: 'I',
|
alias: 'I',
|
||||||
type: Boolean,
|
type: Boolean,
|
||||||
description: 'Initialize backups for all projects.',
|
description: 'Initialize backups for all projects.',
|
||||||
},
|
},
|
||||||
|
{ name: 'output', alias: 'o', type: String, description: 'Output file' },
|
||||||
{
|
{
|
||||||
name: 'start-date',
|
name: 'start-date',
|
||||||
type: String,
|
type: String,
|
||||||
|
@ -366,6 +376,7 @@ function handleOptions() {
|
||||||
!options.list &&
|
!options.list &&
|
||||||
!options.pending &&
|
!options.pending &&
|
||||||
!options.init &&
|
!options.init &&
|
||||||
|
!(options.fix >= 0) &&
|
||||||
!(options.compare && options['start-date'] && options['end-date'])
|
!(options.compare && options['start-date'] && options['end-date'])
|
||||||
|
|
||||||
if (projectIdRequired && !options.projectId) {
|
if (projectIdRequired && !options.projectId) {
|
||||||
|
@ -680,19 +691,68 @@ function convertToISODate(dateStr) {
|
||||||
return new Date(dateStr + 'T00:00:00.000Z').toISOString()
|
return new Date(dateStr + 'T00:00:00.000Z').toISOString()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function fixProjectsWithoutChunks(options) {
|
||||||
|
const limit = options.fix || 1
|
||||||
|
const query = {
|
||||||
|
'overleaf.history.id': { $exists: true },
|
||||||
|
'overleaf.backup.lastBackedUpVersion': { $in: [null] },
|
||||||
|
}
|
||||||
|
const cursor = client
|
||||||
|
.db()
|
||||||
|
.collection('projects')
|
||||||
|
.find(query, {
|
||||||
|
projection: { _id: 1, 'overleaf.history.id': 1 },
|
||||||
|
readPreference: READ_PREFERENCE_SECONDARY,
|
||||||
|
})
|
||||||
|
.limit(limit)
|
||||||
|
for await (const project of cursor) {
|
||||||
|
const historyId = project.overleaf.history.id.toString()
|
||||||
|
const chunks = await getProjectChunks(historyId)
|
||||||
|
if (chunks.length > 0) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (DRY_RUN) {
|
||||||
|
console.log(
|
||||||
|
'Would create new chunk for Project ID:',
|
||||||
|
project._id.toHexString(),
|
||||||
|
'History ID:',
|
||||||
|
historyId,
|
||||||
|
'Chunks:',
|
||||||
|
chunks
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
console.log(
|
||||||
|
'Creating new chunk for Project ID:',
|
||||||
|
project._id.toHexString(),
|
||||||
|
'History ID:',
|
||||||
|
historyId,
|
||||||
|
'Chunks:',
|
||||||
|
chunks
|
||||||
|
)
|
||||||
|
const snapshot = new Snapshot()
|
||||||
|
const history = new History(snapshot, [])
|
||||||
|
const chunk = new Chunk(history, 0)
|
||||||
|
await create(historyId, chunk)
|
||||||
|
const newChunks = await getProjectChunks(historyId)
|
||||||
|
console.log('New chunk:', newChunks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function initializeProjects(options) {
|
export async function initializeProjects(options) {
|
||||||
await ensureGlobalBlobsLoaded()
|
await ensureGlobalBlobsLoaded()
|
||||||
let totalErrors = 0
|
let totalErrors = 0
|
||||||
let totalProjects = 0
|
let totalProjects = 0
|
||||||
|
|
||||||
const query = {
|
const query = {
|
||||||
'overleaf.history.id': { $exists: true },
|
'overleaf.backup.lastBackedUpVersion': { $in: [null] },
|
||||||
'overleaf.backup.lastBackedUpVersion': { $exists: false },
|
}
|
||||||
'overleaf.backup.pendingChangeAt': { $exists: false },
|
|
||||||
_id: {
|
if (options['start-date'] && options['end-date']) {
|
||||||
|
query._id = {
|
||||||
$gte: objectIdFromInput(convertToISODate(options['start-date'])),
|
$gte: objectIdFromInput(convertToISODate(options['start-date'])),
|
||||||
$lt: objectIdFromInput(convertToISODate(options['end-date'])),
|
$lt: objectIdFromInput(convertToISODate(options['end-date'])),
|
||||||
},
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const cursor = client
|
const cursor = client
|
||||||
|
@ -703,6 +763,18 @@ export async function initializeProjects(options) {
|
||||||
readPreference: READ_PREFERENCE_SECONDARY,
|
readPreference: READ_PREFERENCE_SECONDARY,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
if (options.output) {
|
||||||
|
console.log("Writing project IDs to file: '" + options.output + "'")
|
||||||
|
const output = createWriteStream(options.output)
|
||||||
|
for await (const project of cursor) {
|
||||||
|
output.write(project._id.toHexString() + '\n')
|
||||||
|
totalProjects++
|
||||||
|
}
|
||||||
|
output.end()
|
||||||
|
console.log('Wrote ' + totalProjects + ' project IDs to file')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
for await (const project of cursor) {
|
for await (const project of cursor) {
|
||||||
if (gracefulShutdownInitiated) {
|
if (gracefulShutdownInitiated) {
|
||||||
console.warn('graceful shutdown: stopping project initialization')
|
console.warn('graceful shutdown: stopping project initialization')
|
||||||
|
@ -969,11 +1041,12 @@ async function main() {
|
||||||
const options = handleOptions()
|
const options = handleOptions()
|
||||||
await ensureGlobalBlobsLoaded()
|
await ensureGlobalBlobsLoaded()
|
||||||
const projectId = options.projectId
|
const projectId = options.projectId
|
||||||
|
|
||||||
if (options.status) {
|
if (options.status) {
|
||||||
await displayBackupStatus(projectId)
|
await displayBackupStatus(projectId)
|
||||||
} else if (options.list) {
|
} else if (options.list) {
|
||||||
await displayPendingBackups(options)
|
await displayPendingBackups(options)
|
||||||
|
} else if (options.fix !== undefined) {
|
||||||
|
await fixProjectsWithoutChunks(options)
|
||||||
} else if (options.pending) {
|
} else if (options.pending) {
|
||||||
await backupPendingProjects(options)
|
await backupPendingProjects(options)
|
||||||
} else if (options.init) {
|
} else if (options.init) {
|
||||||
|
|
171
services/history-v1/storage/scripts/backup_blob.mjs
Normal file
171
services/history-v1/storage/scripts/backup_blob.mjs
Normal file
|
@ -0,0 +1,171 @@
|
||||||
|
// @ts-check
|
||||||
|
import commandLineArgs from 'command-line-args'
|
||||||
|
import { backupBlob, downloadBlobToDir } from '../lib/backupBlob.mjs'
|
||||||
|
import withTmpDir from '../../api/controllers/with_tmp_dir.js'
|
||||||
|
import {
|
||||||
|
BlobStore,
|
||||||
|
GLOBAL_BLOBS,
|
||||||
|
loadGlobalBlobs,
|
||||||
|
} from '../lib/blob_store/index.js'
|
||||||
|
import assert from '../lib/assert.js'
|
||||||
|
import knex from '../lib/knex.js'
|
||||||
|
import { client } from '../lib/mongodb.js'
|
||||||
|
import { setTimeout } from 'node:timers/promises'
|
||||||
|
import fs from 'node:fs'
|
||||||
|
|
||||||
|
await loadGlobalBlobs()
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gracefully shutdown the process
|
||||||
|
* @return {Promise<void>}
|
||||||
|
*/
|
||||||
|
async function gracefulShutdown() {
|
||||||
|
console.log('Gracefully shutting down')
|
||||||
|
await knex.destroy()
|
||||||
|
await client.close()
|
||||||
|
await setTimeout(100)
|
||||||
|
process.exit()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {string} row
|
||||||
|
* @return {BackupBlobJob}
|
||||||
|
*/
|
||||||
|
function parseCSVRow(row) {
|
||||||
|
const [historyId, hash] = row.split(',')
|
||||||
|
validateBackedUpBlobJob({ historyId, hash })
|
||||||
|
return { historyId, hash }
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {BackupBlobJob} job
|
||||||
|
*/
|
||||||
|
function validateBackedUpBlobJob(job) {
|
||||||
|
assert.projectId(job.historyId)
|
||||||
|
assert.blobHash(job.hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {string} path
|
||||||
|
* @return {Promise<Array<BackupBlobJob>>}
|
||||||
|
*/
|
||||||
|
async function readCSV(path) {
|
||||||
|
let fh
|
||||||
|
/** @type {Array<BackupBlobJob>} */
|
||||||
|
const rows = []
|
||||||
|
try {
|
||||||
|
fh = await fs.promises.open(path, 'r')
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Could not open file: ${error}`)
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
for await (const line of fh.readLines()) {
|
||||||
|
try {
|
||||||
|
const row = parseCSVRow(line)
|
||||||
|
if (GLOBAL_BLOBS.has(row.hash)) {
|
||||||
|
console.log(`Skipping global blob: ${line}`)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rows.push(row)
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error instanceof Error ? error.message : error)
|
||||||
|
console.log(`Skipping invalid row: ${line}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} BackupBlobJob
|
||||||
|
* @property {string} hash
|
||||||
|
* @property {string} historyId
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {Object} options
|
||||||
|
* @property {string} [options.historyId]
|
||||||
|
* @property {string} [options.hash]
|
||||||
|
* @property {string} [options.input]
|
||||||
|
* @return {Promise<Array<BackupBlobJob>>}
|
||||||
|
*/
|
||||||
|
async function initialiseJobs({ historyId, hash, input }) {
|
||||||
|
if (input) {
|
||||||
|
return await readCSV(input)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!historyId) {
|
||||||
|
console.error('historyId is required')
|
||||||
|
process.exitCode = 1
|
||||||
|
await gracefulShutdown()
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!hash) {
|
||||||
|
console.error('hash is required')
|
||||||
|
process.exitCode = 1
|
||||||
|
await gracefulShutdown()
|
||||||
|
}
|
||||||
|
|
||||||
|
validateBackedUpBlobJob({ historyId, hash })
|
||||||
|
|
||||||
|
if (GLOBAL_BLOBS.has(hash)) {
|
||||||
|
console.error(`Blob ${hash} is a global blob; not backing up`)
|
||||||
|
process.exitCode = 1
|
||||||
|
await gracefulShutdown()
|
||||||
|
}
|
||||||
|
return [{ hash, historyId }]
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {string} historyId
|
||||||
|
* @param {string} hash
|
||||||
|
* @return {Promise<void>}
|
||||||
|
*/
|
||||||
|
export async function downloadAndBackupBlob(historyId, hash) {
|
||||||
|
const blobStore = new BlobStore(historyId)
|
||||||
|
const blob = await blobStore.getBlob(hash)
|
||||||
|
if (!blob) {
|
||||||
|
throw new Error(`Blob ${hash} could not be loaded`)
|
||||||
|
}
|
||||||
|
await withTmpDir(`blob-${hash}`, async tmpDir => {
|
||||||
|
const filePath = await downloadBlobToDir(historyId, blob, tmpDir)
|
||||||
|
console.log(`Downloaded blob ${hash} to ${filePath}`)
|
||||||
|
await backupBlob(historyId, blob, filePath)
|
||||||
|
console.log('Backed up blob')
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
let jobs
|
||||||
|
|
||||||
|
const options = commandLineArgs([
|
||||||
|
{ name: 'historyId', type: String },
|
||||||
|
{ name: 'hash', type: String },
|
||||||
|
{ name: 'input', type: String },
|
||||||
|
])
|
||||||
|
|
||||||
|
try {
|
||||||
|
jobs = await initialiseJobs(options)
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error)
|
||||||
|
await gracefulShutdown()
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!Array.isArray(jobs)) {
|
||||||
|
// This is mostly to satisfy typescript
|
||||||
|
process.exitCode = 1
|
||||||
|
await gracefulShutdown()
|
||||||
|
process.exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const { historyId, hash } of jobs) {
|
||||||
|
try {
|
||||||
|
await downloadAndBackupBlob(historyId, hash)
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error)
|
||||||
|
process.exitCode = 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await gracefulShutdown()
|
|
@ -32,34 +32,18 @@ async function takeSample(sampleSize) {
|
||||||
[
|
[
|
||||||
{ $sample: { size: sampleSize } },
|
{ $sample: { size: sampleSize } },
|
||||||
{
|
{
|
||||||
$project: {
|
$match: { 'overleaf.backup.lastBackedUpVersion': { $exists: true } },
|
||||||
_id: 0,
|
|
||||||
hasBackup: {
|
|
||||||
$ifNull: ['$overleaf.backup.lastBackedUpVersion', false],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
$group: {
|
$count: 'total',
|
||||||
_id: null,
|
|
||||||
totalSampled: { $sum: 1 },
|
|
||||||
backedUp: {
|
|
||||||
$sum: {
|
|
||||||
$cond: ['$hasBackup', 1, 0],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
{ readPreference: READ_PREFERENCE_SECONDARY }
|
{ readPreference: READ_PREFERENCE_SECONDARY }
|
||||||
)
|
)
|
||||||
.toArray()
|
.toArray()
|
||||||
|
|
||||||
if (results.length === 0) {
|
const count = results[0]?.total || 0
|
||||||
return { totalSampled: 0, backedUp: 0 }
|
return { totalSampled: sampleSize, backedUp: count }
|
||||||
}
|
|
||||||
|
|
||||||
return results[0]
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function calculateStatistics(
|
function calculateStatistics(
|
||||||
|
@ -67,7 +51,7 @@ function calculateStatistics(
|
||||||
cumulativeBackedUp,
|
cumulativeBackedUp,
|
||||||
totalPopulation
|
totalPopulation
|
||||||
) {
|
) {
|
||||||
const proportion = cumulativeBackedUp / cumulativeSampled
|
const proportion = Math.max(1, cumulativeBackedUp) / cumulativeSampled
|
||||||
|
|
||||||
// Standard error with finite population correction
|
// Standard error with finite population correction
|
||||||
const fpc = Math.sqrt(
|
const fpc = Math.sqrt(
|
||||||
|
|
|
@ -2,6 +2,11 @@ import Queue from 'bull'
|
||||||
import config from 'config'
|
import config from 'config'
|
||||||
import commandLineArgs from 'command-line-args'
|
import commandLineArgs from 'command-line-args'
|
||||||
import logger from '@overleaf/logger'
|
import logger from '@overleaf/logger'
|
||||||
|
import {
|
||||||
|
listPendingBackups,
|
||||||
|
listUninitializedBackups,
|
||||||
|
getBackupStatus,
|
||||||
|
} from '../lib/backup_store/index.js'
|
||||||
|
|
||||||
logger.initialize('backup-queue')
|
logger.initialize('backup-queue')
|
||||||
|
|
||||||
|
@ -28,16 +33,100 @@ const optionDefinitions = [
|
||||||
description: 'Project IDs or date range in YYYY-MM-DD:YYYY-MM-DD format',
|
description: 'Project IDs or date range in YYYY-MM-DD:YYYY-MM-DD format',
|
||||||
},
|
},
|
||||||
{ name: 'monitor', type: Boolean },
|
{ name: 'monitor', type: Boolean },
|
||||||
|
{
|
||||||
|
name: 'queue-pending',
|
||||||
|
type: Number,
|
||||||
|
description:
|
||||||
|
'Find projects with pending changes older than N seconds and add them to the queue',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'show-pending',
|
||||||
|
type: Number,
|
||||||
|
description:
|
||||||
|
'Show count of pending projects older than N seconds without adding to queue',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'limit',
|
||||||
|
type: Number,
|
||||||
|
description: 'Limit the number of jobs to be added',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'interval',
|
||||||
|
type: Number,
|
||||||
|
description: 'Time in seconds to spread jobs over (default: 300)',
|
||||||
|
defaultValue: 300,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'backoff-delay',
|
||||||
|
type: Number,
|
||||||
|
description:
|
||||||
|
'Backoff delay in milliseconds for failed jobs (default: 1000)',
|
||||||
|
defaultValue: 1000,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'attempts',
|
||||||
|
type: Number,
|
||||||
|
description: 'Number of retry attempts for failed jobs (default: 3)',
|
||||||
|
defaultValue: 3,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'warn-threshold',
|
||||||
|
type: Number,
|
||||||
|
description: 'Warn about any project exceeding this pending age',
|
||||||
|
defaultValue: 2 * 3600, // 2 hours
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'verbose',
|
||||||
|
alias: 'v',
|
||||||
|
type: Boolean,
|
||||||
|
description: 'Show detailed information when used with --show-pending',
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
// Parse command line arguments
|
// Parse command line arguments
|
||||||
const options = commandLineArgs(optionDefinitions)
|
const options = commandLineArgs(optionDefinitions)
|
||||||
|
const WARN_THRESHOLD = options['warn-threshold']
|
||||||
|
|
||||||
// Helper to validate date format
|
// Helper to validate date format
|
||||||
function isValidDateFormat(dateStr) {
|
function isValidDateFormat(dateStr) {
|
||||||
return /^\d{4}-\d{2}-\d{2}$/.test(dateStr)
|
return /^\d{4}-\d{2}-\d{2}$/.test(dateStr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Helper to validate the pending time parameter
|
||||||
|
function validatePendingTime(option, value) {
|
||||||
|
if (typeof value !== 'number' || value <= 0) {
|
||||||
|
console.error(
|
||||||
|
`Error: --${option} requires a positive numeric TIME argument in seconds`
|
||||||
|
)
|
||||||
|
console.error(`Example: --${option} 3600`)
|
||||||
|
process.exit(1)
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper to format the pending time display
|
||||||
|
function formatPendingTime(timestamp) {
|
||||||
|
const now = new Date()
|
||||||
|
const diffMs = now - timestamp
|
||||||
|
const seconds = Math.floor(diffMs / 1000)
|
||||||
|
return `${timestamp.toISOString()} (${seconds} seconds ago)`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper to add a job to the queue, checking for duplicates
|
||||||
|
async function addJobWithCheck(queue, data, options) {
|
||||||
|
const jobId = options.jobId
|
||||||
|
|
||||||
|
// Check if the job already exists
|
||||||
|
const existingJob = await queue.getJob(jobId)
|
||||||
|
|
||||||
|
if (existingJob) {
|
||||||
|
return { job: existingJob, added: false }
|
||||||
|
} else {
|
||||||
|
const job = await queue.add(data, options)
|
||||||
|
return { job, added: true }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Setup queue event listeners
|
// Setup queue event listeners
|
||||||
function setupMonitoring() {
|
function setupMonitoring() {
|
||||||
console.log('Starting queue monitoring. Press Ctrl+C to exit.')
|
console.log('Starting queue monitoring. Press Ctrl+C to exit.')
|
||||||
|
@ -99,15 +188,125 @@ async function addDateRangeJob(input) {
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
const job = await backupQueue.add(
|
|
||||||
|
const jobId = `backup-${startDate}-to-${endDate}`
|
||||||
|
const { job, added } = await addJobWithCheck(
|
||||||
|
backupQueue,
|
||||||
{ startDate, endDate },
|
{ startDate, endDate },
|
||||||
{ jobId: `backup-${startDate}-to-${endDate}` }
|
{ jobId }
|
||||||
)
|
)
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
`Added date range backup job: ${startDate} to ${endDate}, job ID: ${job.id}`
|
`${added ? 'Added' : 'Already exists'}: date range backup job: ${startDate} to ${endDate}, job ID: ${job.id}`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Helper to list pending and uninitialized backups
|
||||||
|
// This function combines the two cursors into a single generator
|
||||||
|
// to yield projects from both lists
|
||||||
|
async function* pendingCursor(timeIntervalMs, limit) {
|
||||||
|
for await (const project of listPendingBackups(timeIntervalMs, limit)) {
|
||||||
|
yield project
|
||||||
|
}
|
||||||
|
for await (const project of listUninitializedBackups(timeIntervalMs, limit)) {
|
||||||
|
yield project
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process pending projects with changes older than the specified seconds
|
||||||
|
async function processPendingProjects(
|
||||||
|
age,
|
||||||
|
showOnly,
|
||||||
|
limit,
|
||||||
|
verbose,
|
||||||
|
jobInterval,
|
||||||
|
jobOpts = {}
|
||||||
|
) {
|
||||||
|
const timeIntervalMs = age * 1000
|
||||||
|
console.log(
|
||||||
|
`Finding projects with pending changes older than ${age} seconds${showOnly ? ' (count only)' : ''}`
|
||||||
|
)
|
||||||
|
|
||||||
|
let count = 0
|
||||||
|
let addedCount = 0
|
||||||
|
let existingCount = 0
|
||||||
|
// Pass the limit directly to MongoDB query for better performance
|
||||||
|
const changeTimes = []
|
||||||
|
for await (const project of pendingCursor(timeIntervalMs, limit)) {
|
||||||
|
const projectId = project._id.toHexString()
|
||||||
|
const pendingAt =
|
||||||
|
project.overleaf?.backup?.pendingChangeAt || project._id.getTimestamp()
|
||||||
|
if (pendingAt) {
|
||||||
|
changeTimes.push(pendingAt)
|
||||||
|
const pendingAge = Math.floor((Date.now() - pendingAt.getTime()) / 1000)
|
||||||
|
if (pendingAge > WARN_THRESHOLD) {
|
||||||
|
const backupStatus = await getBackupStatus(projectId)
|
||||||
|
logger.warn(
|
||||||
|
{
|
||||||
|
projectId,
|
||||||
|
pendingAt,
|
||||||
|
pendingAge,
|
||||||
|
backupStatus,
|
||||||
|
warnThreshold: WARN_THRESHOLD,
|
||||||
|
},
|
||||||
|
`pending change exceeds rpo warning threshold`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (showOnly && verbose) {
|
||||||
|
console.log(
|
||||||
|
`Project: ${projectId} (pending since: ${formatPendingTime(pendingAt)})`
|
||||||
|
)
|
||||||
|
} else if (!showOnly) {
|
||||||
|
const delay = Math.floor(Math.random() * jobInterval * 1000) // add random delay to avoid all jobs running simultaneously
|
||||||
|
const { job, added } = await addJobWithCheck(
|
||||||
|
backupQueue,
|
||||||
|
{ projectId, pendingChangeAt: pendingAt.getTime() },
|
||||||
|
{ ...jobOpts, delay, jobId: projectId }
|
||||||
|
)
|
||||||
|
|
||||||
|
if (added) {
|
||||||
|
if (verbose) {
|
||||||
|
console.log(
|
||||||
|
`Added job for project: ${projectId}, job ID: ${job.id} (pending since: ${formatPendingTime(pendingAt)})`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
addedCount++
|
||||||
|
} else {
|
||||||
|
if (verbose) {
|
||||||
|
console.log(
|
||||||
|
`Job already exists for project: ${projectId}, job ID: ${job.id} (pending since: ${formatPendingTime(pendingAt)})`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
existingCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
count++
|
||||||
|
if (count % 1000 === 0) {
|
||||||
|
console.log(
|
||||||
|
`Processed ${count} projects`,
|
||||||
|
showOnly ? '' : `(${addedCount} added, ${existingCount} existing)`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const oldestChange = changeTimes.reduce((min, time) =>
|
||||||
|
time < min ? time : min
|
||||||
|
)
|
||||||
|
|
||||||
|
if (showOnly) {
|
||||||
|
console.log(
|
||||||
|
`Found ${count} projects with pending changes (not added to queue)`
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
console.log(`Found ${count} projects with pending changes:`)
|
||||||
|
console.log(` ${addedCount} jobs added to queue`)
|
||||||
|
console.log(` ${existingCount} jobs already existed in queue`)
|
||||||
|
console.log(` Oldest pending change: ${formatPendingTime(oldestChange)}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Main execution block
|
// Main execution block
|
||||||
async function run() {
|
async function run() {
|
||||||
const optionCount = [
|
const optionCount = [
|
||||||
|
@ -115,6 +314,8 @@ async function run() {
|
||||||
options.status,
|
options.status,
|
||||||
options.add,
|
options.add,
|
||||||
options.monitor,
|
options.monitor,
|
||||||
|
options['queue-pending'] !== undefined,
|
||||||
|
options['show-pending'] !== undefined,
|
||||||
].filter(Boolean).length
|
].filter(Boolean).length
|
||||||
if (optionCount > 1) {
|
if (optionCount > 1) {
|
||||||
console.error('Only one option can be specified')
|
console.error('Only one option can be specified')
|
||||||
|
@ -141,24 +342,65 @@ async function run() {
|
||||||
await addDateRangeJob(input)
|
await addDateRangeJob(input)
|
||||||
} else {
|
} else {
|
||||||
// Handle project ID format
|
// Handle project ID format
|
||||||
const job = await backupQueue.add(
|
const { job, added } = await addJobWithCheck(
|
||||||
|
backupQueue,
|
||||||
{ projectId: input },
|
{ projectId: input },
|
||||||
{ jobId: input }
|
{ jobId: input }
|
||||||
)
|
)
|
||||||
console.log(`Added job for project: ${input}, job ID: ${job.id}`)
|
console.log(
|
||||||
|
`${added ? 'Added' : 'Already exists'}: job for project: ${input}, job ID: ${job.id}`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if (options.monitor) {
|
} else if (options.monitor) {
|
||||||
setupMonitoring()
|
setupMonitoring()
|
||||||
|
} else if (options['queue-pending'] !== undefined) {
|
||||||
|
const age = validatePendingTime('queue-pending', options['queue-pending'])
|
||||||
|
await processPendingProjects(
|
||||||
|
age,
|
||||||
|
false,
|
||||||
|
options.limit,
|
||||||
|
options.verbose,
|
||||||
|
options.interval,
|
||||||
|
{
|
||||||
|
attempts: options.attempts,
|
||||||
|
backoff: {
|
||||||
|
type: 'exponential',
|
||||||
|
delay: options['backoff-delay'],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
} else if (options['show-pending'] !== undefined) {
|
||||||
|
const age = validatePendingTime('show-pending', options['show-pending'])
|
||||||
|
await processPendingProjects(age, true, options.limit, options.verbose)
|
||||||
} else {
|
} else {
|
||||||
console.log('Usage:')
|
console.log('Usage:')
|
||||||
console.log(' --clean Clean up completed and failed jobs')
|
console.log(' --clean Clean up completed and failed jobs')
|
||||||
console.log(' --status Show current job counts')
|
console.log(' --status Show current job counts')
|
||||||
console.log(' --add [projectId] Add a job for the specified projectId')
|
console.log(' --add [projectId] Add a job for the specified projectId')
|
||||||
console.log(
|
console.log(
|
||||||
' --add [YYYY-MM-DD:YYYY-MM-DD] Add a job for the specified date range'
|
' --add [YYYY-MM-DD:YYYY-MM-DD] Add a job for the specified date range'
|
||||||
)
|
)
|
||||||
console.log(' --monitor Monitor queue events')
|
console.log(' --monitor Monitor queue events')
|
||||||
|
console.log(
|
||||||
|
' --queue-pending TIME Find projects with changes older than TIME seconds and add them to the queue'
|
||||||
|
)
|
||||||
|
console.log(
|
||||||
|
' --show-pending TIME Show count of pending projects older than TIME seconds'
|
||||||
|
)
|
||||||
|
console.log(' --limit N Limit the number of jobs to be added')
|
||||||
|
console.log(
|
||||||
|
' --interval TIME Time interval in seconds to spread jobs over'
|
||||||
|
)
|
||||||
|
console.log(
|
||||||
|
' --backoff-delay TIME Backoff delay in milliseconds for failed jobs (default: 1000)'
|
||||||
|
)
|
||||||
|
console.log(
|
||||||
|
' --attempts N Number of retry attempts for failed jobs (default: 3)'
|
||||||
|
)
|
||||||
|
console.log(
|
||||||
|
' --verbose, -v Show detailed information when used with --show-pending'
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -9,8 +9,12 @@ import {
|
||||||
} from './backup.mjs'
|
} from './backup.mjs'
|
||||||
|
|
||||||
const CONCURRENCY = 15
|
const CONCURRENCY = 15
|
||||||
|
const WARN_THRESHOLD = 2 * 60 * 60 * 1000 // warn if projects are older than this
|
||||||
const redisOptions = config.get('redis.queue')
|
const redisOptions = config.get('redis.queue')
|
||||||
const TIME_BUCKETS = [10, 100, 500, 1000, 5000, 10000, 30000, 60000]
|
const JOB_TIME_BUCKETS = [10, 100, 500, 1000, 5000, 10000, 30000, 60000] // milliseconds
|
||||||
|
const LAG_TIME_BUCKETS_HRS = [
|
||||||
|
0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.75, 2, 3, 4, 5, 6,
|
||||||
|
] // hours
|
||||||
|
|
||||||
// Configure backup settings to match worker concurrency
|
// Configure backup settings to match worker concurrency
|
||||||
configureBackup({ concurrency: 50, useSecondary: true })
|
configureBackup({ concurrency: 50, useSecondary: true })
|
||||||
|
@ -27,12 +31,12 @@ const backupQueue = new Queue('backup', {
|
||||||
|
|
||||||
// Log queue events
|
// Log queue events
|
||||||
backupQueue.on('active', job => {
|
backupQueue.on('active', job => {
|
||||||
logger.info({ job }, 'job is now active')
|
logger.debug({ job }, 'job is now active')
|
||||||
})
|
})
|
||||||
|
|
||||||
backupQueue.on('completed', (job, result) => {
|
backupQueue.on('completed', (job, result) => {
|
||||||
metrics.inc('backup_worker_job', 1, { status: 'completed' })
|
metrics.inc('backup_worker_job', 1, { status: 'completed' })
|
||||||
logger.info({ job, result }, 'job completed')
|
logger.debug({ job, result }, 'job completed')
|
||||||
})
|
})
|
||||||
|
|
||||||
backupQueue.on('failed', (job, err) => {
|
backupQueue.on('failed', (job, err) => {
|
||||||
|
@ -41,7 +45,7 @@ backupQueue.on('failed', (job, err) => {
|
||||||
})
|
})
|
||||||
|
|
||||||
backupQueue.on('waiting', jobId => {
|
backupQueue.on('waiting', jobId => {
|
||||||
logger.info({ jobId }, 'job is waiting')
|
logger.debug({ jobId }, 'job is waiting')
|
||||||
})
|
})
|
||||||
|
|
||||||
backupQueue.on('error', error => {
|
backupQueue.on('error', error => {
|
||||||
|
@ -69,7 +73,7 @@ backupQueue.process(CONCURRENCY, async job => {
|
||||||
const { projectId, startDate, endDate } = job.data
|
const { projectId, startDate, endDate } = job.data
|
||||||
|
|
||||||
if (projectId) {
|
if (projectId) {
|
||||||
return await runBackup(projectId)
|
return await runBackup(projectId, job.data, job)
|
||||||
} else if (startDate && endDate) {
|
} else if (startDate && endDate) {
|
||||||
return await runInit(startDate, endDate)
|
return await runInit(startDate, endDate)
|
||||||
} else {
|
} else {
|
||||||
|
@ -77,23 +81,40 @@ backupQueue.process(CONCURRENCY, async job => {
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
async function runBackup(projectId) {
|
async function runBackup(projectId, data, job) {
|
||||||
|
const { pendingChangeAt } = data
|
||||||
|
// record the time it takes to run the backup job
|
||||||
const timer = new metrics.Timer(
|
const timer = new metrics.Timer(
|
||||||
'backup_worker_job_duration',
|
'backup_worker_job_duration',
|
||||||
1,
|
1,
|
||||||
{},
|
{},
|
||||||
TIME_BUCKETS
|
JOB_TIME_BUCKETS
|
||||||
)
|
)
|
||||||
|
const pendingAge = Date.now() - pendingChangeAt
|
||||||
|
if (pendingAge > WARN_THRESHOLD) {
|
||||||
|
logger.warn(
|
||||||
|
{ projectId, pendingAge, job },
|
||||||
|
'project has been pending for a long time'
|
||||||
|
)
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
logger.info({ projectId }, 'processing backup for project')
|
logger.debug({ projectId }, 'processing backup for project')
|
||||||
const { errors, completed } = await backupProject(projectId, {})
|
await backupProject(projectId, {})
|
||||||
metrics.inc('backup_worker_project', completed - errors, {
|
metrics.inc('backup_worker_project', 1, {
|
||||||
status: 'success',
|
status: 'success',
|
||||||
})
|
})
|
||||||
metrics.inc('backup_worker_project', errors, { status: 'failed' })
|
|
||||||
timer.done()
|
timer.done()
|
||||||
return `backup completed ${projectId} (${errors} failed in ${completed} projects)`
|
// record the replication lag (time from change to backup)
|
||||||
|
if (pendingChangeAt) {
|
||||||
|
metrics.histogram(
|
||||||
|
'backup_worker_replication_lag_in_hours',
|
||||||
|
(Date.now() - pendingChangeAt) / (3600 * 1000),
|
||||||
|
LAG_TIME_BUCKETS_HRS
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return `backup completed ${projectId}`
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
|
metrics.inc('backup_worker_project', 1, { status: 'failed' })
|
||||||
logger.error({ projectId, err }, 'backup failed')
|
logger.error({ projectId, err }, 'backup failed')
|
||||||
throw err // Re-throw to mark job as failed
|
throw err // Re-throw to mark job as failed
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,177 @@
|
||||||
|
import fs from 'node:fs'
|
||||||
|
import { makeProjectKey } from '../lib/blob_store/index.js'
|
||||||
|
import { backupPersistor, projectBlobsBucket } from '../lib/backupPersistor.mjs'
|
||||||
|
import { NotFoundError } from '@overleaf/object-persistor/src/Errors.js'
|
||||||
|
import commandLineArgs from 'command-line-args'
|
||||||
|
import OError from '@overleaf/o-error'
|
||||||
|
import assert from '../lib/assert.js'
|
||||||
|
import { client, projects } from '../lib/mongodb.js'
|
||||||
|
import { ObjectId } from 'mongodb'
|
||||||
|
import { setTimeout } from 'node:timers/promises'
|
||||||
|
|
||||||
|
const { input, verbose } = commandLineArgs([
|
||||||
|
{ name: 'input', type: String },
|
||||||
|
{ name: 'verbose', type: Boolean, defaultValue: false },
|
||||||
|
])
|
||||||
|
|
||||||
|
function parseCSVRow(row) {
|
||||||
|
const [path] = row.split(',')
|
||||||
|
const pathSegments = path.split('/')
|
||||||
|
const historyId = `${pathSegments[0]}${pathSegments[1]}${pathSegments[2]}`
|
||||||
|
.split('')
|
||||||
|
.reverse()
|
||||||
|
.join('')
|
||||||
|
|
||||||
|
return { historyId, path, hash: `${pathSegments[3]}${pathSegments[4]}` }
|
||||||
|
}
|
||||||
|
|
||||||
|
async function* readCSV(path) {
|
||||||
|
let fh
|
||||||
|
try {
|
||||||
|
fh = await fs.promises.open(path, 'r')
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Could not open file: ${error}`)
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
for await (const line of fh.readLines()) {
|
||||||
|
try {
|
||||||
|
const row = parseCSVRow(line)
|
||||||
|
yield row
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error instanceof Error ? error.message : error)
|
||||||
|
console.log(`Skipping invalid row: ${line}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class MissingDEKError extends OError {}
|
||||||
|
class InvalidHistoryIdError extends OError {}
|
||||||
|
class MissingProjectError extends OError {}
|
||||||
|
class MissingBlobError extends OError {}
|
||||||
|
|
||||||
|
async function getProjectPersistor(historyId) {
|
||||||
|
try {
|
||||||
|
return await backupPersistor.forProjectRO(
|
||||||
|
projectBlobsBucket,
|
||||||
|
makeProjectKey(historyId, '')
|
||||||
|
)
|
||||||
|
} catch (err) {
|
||||||
|
if (err instanceof NotFoundError) {
|
||||||
|
throw new MissingDEKError('dek does not exist', { historyId }, err)
|
||||||
|
}
|
||||||
|
throw err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function checkBlobExists(path, historyId) {
|
||||||
|
const persistor = await getProjectPersistor(historyId)
|
||||||
|
return await persistor.getObjectSize(projectBlobsBucket, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
let total = 0
|
||||||
|
const errors = {
|
||||||
|
invalidProjectId: 0,
|
||||||
|
notBackedUpProjectId: 0,
|
||||||
|
missingBlob: 0,
|
||||||
|
notInMongo: 0,
|
||||||
|
unknown: 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
const notInMongoProjectIds = new Set()
|
||||||
|
const notBackedUpProjectIds = new Set()
|
||||||
|
|
||||||
|
let stopping = false
|
||||||
|
|
||||||
|
process.on('SIGTERM', () => {
|
||||||
|
console.log('SIGTERM received')
|
||||||
|
stopping = true
|
||||||
|
})
|
||||||
|
|
||||||
|
process.on('SIGINT', () => {
|
||||||
|
console.log('SIGINT received')
|
||||||
|
stopping = true
|
||||||
|
})
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {string} historyId
|
||||||
|
* @param {string} path
|
||||||
|
* @param {string} hash
|
||||||
|
* @return {Promise<void>}
|
||||||
|
*/
|
||||||
|
async function checkPath(historyId, path, hash) {
|
||||||
|
try {
|
||||||
|
assert.mongoId(historyId)
|
||||||
|
} catch (error) {
|
||||||
|
throw InvalidHistoryIdError('invalid history id', { historyId })
|
||||||
|
}
|
||||||
|
if (notInMongoProjectIds.has(historyId)) {
|
||||||
|
throw new MissingProjectError('project not in mongo', { historyId })
|
||||||
|
}
|
||||||
|
if (notBackedUpProjectIds.has(historyId)) {
|
||||||
|
throw new MissingDEKError('project not backed up', { historyId })
|
||||||
|
}
|
||||||
|
|
||||||
|
const project = await projects.findOne({ _id: new ObjectId(historyId) })
|
||||||
|
if (!project) {
|
||||||
|
notInMongoProjectIds.add(historyId)
|
||||||
|
throw new MissingProjectError('project not in mongo', { historyId })
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await checkBlobExists(path, historyId)
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof NotFoundError) {
|
||||||
|
throw new MissingBlobError('missing blob', { historyId, hash })
|
||||||
|
}
|
||||||
|
if (error instanceof MissingDEKError) {
|
||||||
|
notBackedUpProjectIds.add(historyId)
|
||||||
|
}
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for await (const line of readCSV(input)) {
|
||||||
|
if (stopping) break
|
||||||
|
total++
|
||||||
|
if (total % 10_000 === 0) {
|
||||||
|
console.log(`checked ${total}`)
|
||||||
|
}
|
||||||
|
const { historyId, path, hash } = line
|
||||||
|
try {
|
||||||
|
await checkPath(historyId, path, hash)
|
||||||
|
if (verbose) {
|
||||||
|
console.log(`✓ Project ${historyId} has ${hash} backed up`)
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof InvalidHistoryIdError) {
|
||||||
|
errors.invalidProjectId++
|
||||||
|
console.warn(`invalid historyId ${historyId}`)
|
||||||
|
continue
|
||||||
|
} else if (error instanceof MissingProjectError) {
|
||||||
|
errors.notInMongo++
|
||||||
|
console.warn(`✗ project ${historyId} not in mongo`)
|
||||||
|
continue
|
||||||
|
} else if (error instanceof MissingDEKError) {
|
||||||
|
errors.notBackedUpProjectId++
|
||||||
|
console.error(`✗ Project DEK ${historyId} not found`)
|
||||||
|
continue
|
||||||
|
} else if (error instanceof MissingBlobError) {
|
||||||
|
errors.missingBlob++
|
||||||
|
console.error(`✗ missing blob ${hash} from project ${historyId}`)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
errors.unknown++
|
||||||
|
console.error(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`total checked: ${total}`)
|
||||||
|
console.log(`invalid project id: ${errors.invalidProjectId}`)
|
||||||
|
console.log(`not found in mongo: ${errors.notInMongo}`)
|
||||||
|
console.log(`missing blob: ${errors.missingBlob}`)
|
||||||
|
console.log(`project not backed up: ${errors.notBackedUpProjectId}`)
|
||||||
|
console.log(`unknown errors: ${errors.unknown}`)
|
||||||
|
|
||||||
|
await client.close()
|
||||||
|
await setTimeout(100)
|
||||||
|
process.exit()
|
33
services/history-v1/storage/scripts/verify_project.mjs
Normal file
33
services/history-v1/storage/scripts/verify_project.mjs
Normal file
|
@ -0,0 +1,33 @@
|
||||||
|
import commandLineArgs from 'command-line-args'
|
||||||
|
import { verifyProjectWithErrorContext } from '../lib/backupVerifier.mjs'
|
||||||
|
import knex from '../lib/knex.js'
|
||||||
|
import { client } from '../lib/mongodb.js'
|
||||||
|
import { setTimeout } from 'node:timers/promises'
|
||||||
|
import { loadGlobalBlobs } from '../lib/blob_store/index.js'
|
||||||
|
|
||||||
|
const { historyId } = commandLineArgs([{ name: 'historyId', type: String }])
|
||||||
|
|
||||||
|
async function gracefulShutdown(code = process.exitCode) {
|
||||||
|
await knex.destroy()
|
||||||
|
await client.close()
|
||||||
|
await setTimeout(1_000)
|
||||||
|
process.exit(code)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!historyId) {
|
||||||
|
console.error('missing --historyId')
|
||||||
|
process.exitCode = 1
|
||||||
|
await gracefulShutdown()
|
||||||
|
}
|
||||||
|
|
||||||
|
await loadGlobalBlobs()
|
||||||
|
|
||||||
|
try {
|
||||||
|
await verifyProjectWithErrorContext(historyId)
|
||||||
|
console.log('OK')
|
||||||
|
} catch (error) {
|
||||||
|
console.error('error verifying', error)
|
||||||
|
process.exitCode = 1
|
||||||
|
} finally {
|
||||||
|
await gracefulShutdown()
|
||||||
|
}
|
215
services/history-v1/storage/scripts/verify_sampled_projects.mjs
Normal file
215
services/history-v1/storage/scripts/verify_sampled_projects.mjs
Normal file
|
@ -0,0 +1,215 @@
|
||||||
|
// @ts-check
|
||||||
|
import commandLineArgs from 'command-line-args'
|
||||||
|
import {
|
||||||
|
setWriteMetrics,
|
||||||
|
verifyProjectsCreatedInDateRange,
|
||||||
|
verifyRandomProjectSample,
|
||||||
|
verifyProjectsUpdatedInDateRange,
|
||||||
|
} from '../../backupVerifier/ProjectVerifier.mjs'
|
||||||
|
import knex from '../lib/knex.js'
|
||||||
|
import { client } from '../lib/mongodb.js'
|
||||||
|
import { setTimeout } from 'node:timers/promises'
|
||||||
|
import logger from '@overleaf/logger'
|
||||||
|
import { loadGlobalBlobs } from '../lib/blob_store/index.js'
|
||||||
|
import { getDatesBeforeRPO } from '../../backupVerifier/utils.mjs'
|
||||||
|
import { EventEmitter } from 'node:events'
|
||||||
|
import { mongodb } from '../index.js'
|
||||||
|
|
||||||
|
logger.logger.level('fatal')
|
||||||
|
|
||||||
|
const usageMessage = [
|
||||||
|
'Usage: node verify_sampled_projects.mjs [--startDate <start>] [--endDate <end>] [--nProjects <n>] [--verbose] [--usage] [--writeMetrics] [--concurrency <n>] [--strategy <range|random>]',
|
||||||
|
'strategy: defaults to "range"; startDate and endDate are required for "range" strategy',
|
||||||
|
].join('\n')
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gracefully shutdown the process
|
||||||
|
* @param code
|
||||||
|
* @return {Promise<void>}
|
||||||
|
*/
|
||||||
|
async function gracefulShutdown(code = process.exitCode) {
|
||||||
|
await knex.destroy()
|
||||||
|
await client.close()
|
||||||
|
await setTimeout(1_000)
|
||||||
|
process.exit(code)
|
||||||
|
}
|
||||||
|
|
||||||
|
const STATS = {
|
||||||
|
verifiable: 0,
|
||||||
|
unverifiable: 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {Object} CLIOptions
|
||||||
|
* @property {(signal: EventEmitter) => Promise<VerificationJobStatus>} projectVerifier
|
||||||
|
* @property {boolean} verbose
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @typedef {import('../../backupVerifier/types.d.ts').VerificationJobStatus} VerificationJobStatus
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @return {CLIOptions}
|
||||||
|
*/
|
||||||
|
function getOptions() {
|
||||||
|
const {
|
||||||
|
startDate,
|
||||||
|
endDate,
|
||||||
|
concurrency,
|
||||||
|
writeMetrics,
|
||||||
|
verbose,
|
||||||
|
nProjects,
|
||||||
|
strategy,
|
||||||
|
usage,
|
||||||
|
} = commandLineArgs([
|
||||||
|
{ name: 'startDate', type: String },
|
||||||
|
{ name: 'endDate', type: String },
|
||||||
|
{ name: 'concurrency', type: Number, defaultValue: 1 },
|
||||||
|
{ name: 'verbose', type: Boolean, defaultValue: false },
|
||||||
|
{ name: 'nProjects', type: Number, defaultValue: 10 },
|
||||||
|
{ name: 'usage', type: Boolean, defaultValue: false },
|
||||||
|
{ name: 'writeMetrics', type: Boolean, defaultValue: false },
|
||||||
|
{ name: 'strategy', type: String, defaultValue: 'range' },
|
||||||
|
])
|
||||||
|
|
||||||
|
if (usage) {
|
||||||
|
console.log(usageMessage)
|
||||||
|
process.exit(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!['range', 'random', 'recent'].includes(strategy)) {
|
||||||
|
throw new Error(`Invalid strategy: ${strategy}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
setWriteMetrics(writeMetrics)
|
||||||
|
|
||||||
|
switch (strategy) {
|
||||||
|
case 'random':
|
||||||
|
console.log('Verifying random projects')
|
||||||
|
return {
|
||||||
|
verbose,
|
||||||
|
projectVerifier: signal => verifyRandomProjectSample(nProjects, signal),
|
||||||
|
}
|
||||||
|
case 'recent':
|
||||||
|
return {
|
||||||
|
verbose,
|
||||||
|
projectVerifier: async signal => {
|
||||||
|
const { startDate, endDate } = getDatesBeforeRPO(3 * 3600)
|
||||||
|
return await verifyProjectsUpdatedInDateRange(
|
||||||
|
startDate,
|
||||||
|
endDate,
|
||||||
|
nProjects,
|
||||||
|
signal
|
||||||
|
)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
case 'range':
|
||||||
|
default: {
|
||||||
|
if (!startDate || !endDate) {
|
||||||
|
throw new Error(usageMessage)
|
||||||
|
}
|
||||||
|
const start = Date.parse(startDate)
|
||||||
|
const end = Date.parse(endDate)
|
||||||
|
if (Number.isNaN(start)) {
|
||||||
|
throw new Error(`Invalid start date: ${startDate}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Number.isNaN(end)) {
|
||||||
|
throw new Error(`Invalid end date: ${endDate}`)
|
||||||
|
}
|
||||||
|
if (verbose) {
|
||||||
|
console.log(`Verifying from ${startDate} to ${endDate}`)
|
||||||
|
console.log(`Concurrency: ${concurrency}`)
|
||||||
|
}
|
||||||
|
STATS.ranges = 0
|
||||||
|
return {
|
||||||
|
projectVerifier: signal =>
|
||||||
|
verifyProjectsCreatedInDateRange({
|
||||||
|
startDate: new Date(start),
|
||||||
|
endDate: new Date(end),
|
||||||
|
projectsPerRange: nProjects,
|
||||||
|
concurrency,
|
||||||
|
signal,
|
||||||
|
}),
|
||||||
|
verbose,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @type {CLIOptions}
|
||||||
|
*/
|
||||||
|
let options
|
||||||
|
try {
|
||||||
|
options = getOptions()
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error)
|
||||||
|
process.exitCode = 1
|
||||||
|
await gracefulShutdown(1)
|
||||||
|
process.exit() // just here so the type checker knows that the process will exit
|
||||||
|
}
|
||||||
|
|
||||||
|
const { projectVerifier, verbose } = options
|
||||||
|
|
||||||
|
if (verbose) {
|
||||||
|
logger.logger.level('debug')
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {Array<string>} array
|
||||||
|
* @param {string} matchString
|
||||||
|
* @return {*}
|
||||||
|
*/
|
||||||
|
function sumStringInstances(array, matchString) {
|
||||||
|
return array.reduce((total, string) => {
|
||||||
|
return string === matchString ? total + 1 : total
|
||||||
|
}, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* @param {VerificationJobStatus} stats
|
||||||
|
*/
|
||||||
|
function displayStats(stats) {
|
||||||
|
console.log(`Verified projects: ${stats.verified}`)
|
||||||
|
console.log(`Total projects sampled: ${stats.total}`)
|
||||||
|
if (stats.errorTypes.length > 0) {
|
||||||
|
console.log('Errors:')
|
||||||
|
for (const error of new Set(stats.errorTypes)) {
|
||||||
|
console.log(`${error}: ${sumStringInstances(stats.errorTypes, error)}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const shutdownEmitter = new EventEmitter()
|
||||||
|
|
||||||
|
shutdownEmitter.on('shutdown', async () => {
|
||||||
|
await gracefulShutdown()
|
||||||
|
})
|
||||||
|
|
||||||
|
process.on('SIGTERM', () => {
|
||||||
|
shutdownEmitter.emit('shutdown')
|
||||||
|
})
|
||||||
|
|
||||||
|
process.on('SIGINT', () => {
|
||||||
|
shutdownEmitter.emit('shutdown')
|
||||||
|
})
|
||||||
|
|
||||||
|
await loadGlobalBlobs()
|
||||||
|
|
||||||
|
try {
|
||||||
|
const stats = await projectVerifier(shutdownEmitter)
|
||||||
|
displayStats(stats)
|
||||||
|
console.log(`completed`)
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error)
|
||||||
|
console.log('completed with errors')
|
||||||
|
process.exitCode = 1
|
||||||
|
} finally {
|
||||||
|
console.log('shutting down')
|
||||||
|
await gracefulShutdown()
|
||||||
|
}
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue