DocumentUpdaterHandler.js 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393
  1. const request = require('request').defaults({ timeout: 30 * 100 })
  2. const OError = require('@overleaf/o-error')
  3. const settings = require('settings-sharelatex')
  4. const _ = require('underscore')
  5. const async = require('async')
  6. const logger = require('logger-sharelatex')
  7. const metrics = require('@overleaf/metrics')
  8. const { promisify } = require('util')
  9. module.exports = {
  10. flushProjectToMongo,
  11. flushMultipleProjectsToMongo,
  12. flushProjectToMongoAndDelete,
  13. flushDocToMongo,
  14. deleteDoc,
  15. getDocument,
  16. setDocument,
  17. getProjectDocsIfMatch,
  18. clearProjectState,
  19. acceptChanges,
  20. deleteThread,
  21. resyncProjectHistory,
  22. updateProjectStructure,
  23. promises: {
  24. flushProjectToMongo: promisify(flushProjectToMongo),
  25. flushMultipleProjectsToMongo: promisify(flushMultipleProjectsToMongo),
  26. flushProjectToMongoAndDelete: promisify(flushProjectToMongoAndDelete),
  27. flushDocToMongo: promisify(flushDocToMongo),
  28. deleteDoc: promisify(deleteDoc),
  29. getDocument: promisify(getDocument),
  30. setDocument: promisify(setDocument),
  31. getProjectDocsIfMatch: promisify(getProjectDocsIfMatch),
  32. clearProjectState: promisify(clearProjectState),
  33. acceptChanges: promisify(acceptChanges),
  34. deleteThread: promisify(deleteThread),
  35. resyncProjectHistory: promisify(resyncProjectHistory),
  36. updateProjectStructure: promisify(updateProjectStructure),
  37. },
  38. }
  39. function flushProjectToMongo(projectId, callback) {
  40. _makeRequest(
  41. {
  42. path: `/project/${projectId}/flush`,
  43. method: 'POST',
  44. },
  45. projectId,
  46. 'flushing.mongo.project',
  47. callback
  48. )
  49. }
  50. function flushMultipleProjectsToMongo(projectIds, callback) {
  51. const jobs = projectIds.map(projectId => callback => {
  52. flushProjectToMongo(projectId, callback)
  53. })
  54. async.series(jobs, callback)
  55. }
  56. function flushProjectToMongoAndDelete(projectId, callback) {
  57. _makeRequest(
  58. {
  59. path: `/project/${projectId}`,
  60. method: 'DELETE',
  61. },
  62. projectId,
  63. 'flushing.mongo.project',
  64. callback
  65. )
  66. }
  67. function flushDocToMongo(projectId, docId, callback) {
  68. _makeRequest(
  69. {
  70. path: `/project/${projectId}/doc/${docId}/flush`,
  71. method: 'POST',
  72. },
  73. projectId,
  74. 'flushing.mongo.doc',
  75. callback
  76. )
  77. }
  78. function deleteDoc(projectId, docId, callback) {
  79. _makeRequest(
  80. {
  81. path: `/project/${projectId}/doc/${docId}`,
  82. method: 'DELETE',
  83. },
  84. projectId,
  85. 'delete.mongo.doc',
  86. callback
  87. )
  88. }
  89. function getDocument(projectId, docId, fromVersion, callback) {
  90. _makeRequest(
  91. {
  92. path: `/project/${projectId}/doc/${docId}?fromVersion=${fromVersion}`,
  93. json: true,
  94. },
  95. projectId,
  96. 'get-document',
  97. function (error, doc) {
  98. if (error) {
  99. return callback(error)
  100. }
  101. callback(null, doc.lines, doc.version, doc.ranges, doc.ops)
  102. }
  103. )
  104. }
  105. function setDocument(projectId, docId, userId, docLines, source, callback) {
  106. _makeRequest(
  107. {
  108. path: `/project/${projectId}/doc/${docId}`,
  109. method: 'POST',
  110. json: {
  111. lines: docLines,
  112. source,
  113. user_id: userId,
  114. },
  115. },
  116. projectId,
  117. 'set-document',
  118. callback
  119. )
  120. }
  121. function getProjectDocsIfMatch(projectId, projectStateHash, callback) {
  122. // If the project state hasn't changed, we can get all the latest
  123. // docs from redis via the docupdater. Otherwise we will need to
  124. // fall back to getting them from mongo.
  125. const timer = new metrics.Timer('get-project-docs')
  126. const url = `${settings.apis.documentupdater.url}/project/${projectId}/get_and_flush_if_old?state=${projectStateHash}`
  127. request.post(url, function (error, res, body) {
  128. timer.done()
  129. if (error) {
  130. OError.tag(error, 'error getting project docs from doc updater', {
  131. url,
  132. projectId,
  133. })
  134. return callback(error)
  135. }
  136. if (res.statusCode === 409) {
  137. // HTTP response code "409 Conflict"
  138. // Docupdater has checked the projectStateHash and found that
  139. // it has changed. This means that the docs currently in redis
  140. // aren't the only change to the project and the full set of
  141. // docs/files should be retreived from docstore/filestore
  142. // instead.
  143. callback()
  144. } else if (res.statusCode >= 200 && res.statusCode < 300) {
  145. let docs
  146. try {
  147. docs = JSON.parse(body)
  148. } catch (error1) {
  149. return callback(OError.tag(error1))
  150. }
  151. callback(null, docs)
  152. } else {
  153. callback(
  154. new OError(
  155. `doc updater returned a non-success status code: ${res.statusCode}`,
  156. {
  157. projectId,
  158. url,
  159. }
  160. )
  161. )
  162. }
  163. })
  164. }
  165. function clearProjectState(projectId, callback) {
  166. _makeRequest(
  167. {
  168. path: `/project/${projectId}/clearState`,
  169. method: 'POST',
  170. },
  171. projectId,
  172. 'clear-project-state',
  173. callback
  174. )
  175. }
  176. function acceptChanges(projectId, docId, changeIds, callback) {
  177. _makeRequest(
  178. {
  179. path: `/project/${projectId}/doc/${docId}/change/accept`,
  180. json: { change_ids: changeIds },
  181. method: 'POST',
  182. },
  183. projectId,
  184. 'accept-changes',
  185. callback
  186. )
  187. }
  188. function deleteThread(projectId, docId, threadId, callback) {
  189. _makeRequest(
  190. {
  191. path: `/project/${projectId}/doc/${docId}/comment/${threadId}`,
  192. method: 'DELETE',
  193. },
  194. projectId,
  195. 'delete-thread',
  196. callback
  197. )
  198. }
  199. function resyncProjectHistory(
  200. projectId,
  201. projectHistoryId,
  202. docs,
  203. files,
  204. callback
  205. ) {
  206. _makeRequest(
  207. {
  208. path: `/project/${projectId}/history/resync`,
  209. json: { docs, files, projectHistoryId },
  210. method: 'POST',
  211. },
  212. projectId,
  213. 'resync-project-history',
  214. callback
  215. )
  216. }
  217. function updateProjectStructure(
  218. projectId,
  219. projectHistoryId,
  220. userId,
  221. changes,
  222. callback
  223. ) {
  224. if (
  225. settings.apis.project_history == null ||
  226. !settings.apis.project_history.sendProjectStructureOps
  227. ) {
  228. return callback()
  229. }
  230. const {
  231. deletes: docDeletes,
  232. adds: docAdds,
  233. renames: docRenames,
  234. } = _getUpdates('doc', changes.oldDocs, changes.newDocs)
  235. const {
  236. deletes: fileDeletes,
  237. adds: fileAdds,
  238. renames: fileRenames,
  239. } = _getUpdates('file', changes.oldFiles, changes.newFiles)
  240. const updates = [].concat(
  241. docDeletes,
  242. fileDeletes,
  243. docAdds,
  244. fileAdds,
  245. docRenames,
  246. fileRenames
  247. )
  248. const projectVersion =
  249. changes && changes.newProject && changes.newProject.version
  250. if (updates.length < 1) {
  251. return callback()
  252. }
  253. if (projectVersion == null) {
  254. logger.warn(
  255. { projectId, changes, projectVersion },
  256. 'did not receive project version in changes'
  257. )
  258. return callback(new Error('did not receive project version in changes'))
  259. }
  260. _makeRequest(
  261. {
  262. path: `/project/${projectId}`,
  263. json: {
  264. updates,
  265. userId,
  266. version: projectVersion,
  267. projectHistoryId,
  268. },
  269. method: 'POST',
  270. },
  271. projectId,
  272. 'update-project-structure',
  273. callback
  274. )
  275. }
  276. function _makeRequest(options, projectId, metricsKey, callback) {
  277. const timer = new metrics.Timer(metricsKey)
  278. request(
  279. {
  280. url: `${settings.apis.documentupdater.url}${options.path}`,
  281. json: options.json,
  282. method: options.method || 'GET',
  283. },
  284. function (error, res, body) {
  285. timer.done()
  286. if (error) {
  287. logger.warn(
  288. { error, projectId },
  289. 'error making request to document updater'
  290. )
  291. callback(error)
  292. } else if (res.statusCode >= 200 && res.statusCode < 300) {
  293. callback(null, body)
  294. } else {
  295. error = new Error(
  296. `document updater returned a failure status code: ${res.statusCode}`
  297. )
  298. logger.warn(
  299. { error, projectId },
  300. `document updater returned failure status code: ${res.statusCode}`
  301. )
  302. callback(error)
  303. }
  304. }
  305. )
  306. }
  307. function _getUpdates(entityType, oldEntities, newEntities) {
  308. if (!oldEntities) {
  309. oldEntities = []
  310. }
  311. if (!newEntities) {
  312. newEntities = []
  313. }
  314. const deletes = []
  315. const adds = []
  316. const renames = []
  317. const oldEntitiesHash = _.indexBy(oldEntities, entity =>
  318. entity[entityType]._id.toString()
  319. )
  320. const newEntitiesHash = _.indexBy(newEntities, entity =>
  321. entity[entityType]._id.toString()
  322. )
  323. // Send deletes before adds (and renames) to keep a 1:1 mapping between
  324. // paths and ids
  325. //
  326. // When a file is replaced, we first delete the old file and then add the
  327. // new file. If the 'add' operation is sent to project history before the
  328. // 'delete' then we would have two files with the same path at that point
  329. // in time.
  330. for (const id in oldEntitiesHash) {
  331. const oldEntity = oldEntitiesHash[id]
  332. const newEntity = newEntitiesHash[id]
  333. if (newEntity == null) {
  334. // entity deleted
  335. deletes.push({
  336. type: `rename-${entityType}`,
  337. id,
  338. pathname: oldEntity.path,
  339. newPathname: '',
  340. })
  341. }
  342. }
  343. for (const id in newEntitiesHash) {
  344. const newEntity = newEntitiesHash[id]
  345. const oldEntity = oldEntitiesHash[id]
  346. if (oldEntity == null) {
  347. // entity added
  348. adds.push({
  349. type: `add-${entityType}`,
  350. id,
  351. pathname: newEntity.path,
  352. docLines: newEntity.docLines,
  353. url: newEntity.url,
  354. hash: newEntity.file != null ? newEntity.file.hash : undefined,
  355. })
  356. } else if (newEntity.path !== oldEntity.path) {
  357. // entity renamed
  358. renames.push({
  359. type: `rename-${entityType}`,
  360. id,
  361. pathname: oldEntity.path,
  362. newPathname: newEntity.path,
  363. })
  364. }
  365. }
  366. return { deletes, adds, renames }
  367. }