compare_recurly_stripe_customers.mjs 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804
  1. #!/usr/bin/env node
  2. /**
  3. * This script compares existing Stripe customer records with data from Recurly
  4. * to detect any drift since the last migration run.
  5. *
  6. * It is a read-only companion to migrate_recurly_customers_to_stripe.mjs.
  7. * It uses the same normalization logic but makes no changes to Stripe.
  8. *
  9. * Input CSV format:
  10. * recurly_account_code,target_stripe_account,stripe_customer_id
  11. *
  12. * Output files:
  13. * --output (comparison file): One row per customer with match/mismatch status
  14. * Format: recurly_account_code,target_stripe_account,stripe_customer_id,status,diffs
  15. *
  16. * <output>_details.json: Detailed diff for each customer with mismatches
  17. * Format: Array of { recurly_account_code, stripe_customer_id, diffs: { field: { recurly, stripe } } }
  18. *
  19. * <output>_errors.csv: Records that failed
  20. * Format: recurly_account_code,target_stripe_account,stripe_customer_id,error
  21. *
  22. * Usage:
  23. * node scripts/recurly/compare_recurly_stripe_customers.mjs --input customers.csv --output comparison.csv --comparison-date 2026-03-19
  24. *
  25. * Options:
  26. * --input, -i <file> Path to input CSV file (required)
  27. * --output, -o <file> Path to output CSV file (required)
  28. * --comparison-date <date> Date in YYYY-MM-DD to compare against (required)
  29. * --limit, -l <n> Limit number of records processed (default: no limit)
  30. * --concurrency, -c <n> Number of customers to process concurrently (default: 10)
  31. * --recurly-rate-limit <n> Requests per second for Recurly (default: 10)
  32. * --recurly-api-retries <n> Number of retries on Recurly 429s (default: 5)
  33. * --recurly-retry-delay-ms <n> Delay between Recurly retries in ms (default: 1000)
  34. * --stripe-rate-limit <n> Requests per second for Stripe (default: 50)
  35. * --stripe-api-retries <n> Number of retries on Stripe 429s (default: 5)
  36. * --stripe-retry-delay-ms <n> Delay between Stripe retries in ms (default: 1000)
  37. * --verbose, -v Enable debug logging
  38. *
  39. * Note, prior to running this script, environment variables must have been loaded from config/local.env
  40. *
  41. * ```
  42. * set -a
  43. * source ../../config/local.env
  44. * set +a
  45. * ```
  46. */
  47. import Settings from '@overleaf/settings'
  48. import Stripe from 'stripe'
  49. import recurly from 'recurly'
  50. import minimist from 'minimist'
  51. import PQueue from 'p-queue'
  52. import fs from 'node:fs'
  53. import * as csv from 'csv'
  54. import { scriptRunner } from '../lib/ScriptRunner.mjs'
  55. import { compareAccountFields } from '../helpers/migrate_recurly_customers_to_stripe.helpers.mjs'
  56. import {
  57. createRateLimitedApiWrappers,
  58. DEFAULT_RECURLY_RATE_LIMIT,
  59. DEFAULT_STRIPE_RATE_LIMIT,
  60. DEFAULT_RECURLY_API_RETRIES,
  61. DEFAULT_RECURLY_RETRY_DELAY_MS,
  62. DEFAULT_STRIPE_API_RETRIES,
  63. DEFAULT_STRIPE_RETRY_DELAY_MS,
  64. } from '../stripe/RateLimiter.mjs'
  65. // =============================================================================
  66. // STRIPE CLIENT SETUP
  67. // =============================================================================
  68. const stripeClients = {}
  69. function getRegionClient(region) {
  70. const regionLower = String(region || '')
  71. .trim()
  72. .toLowerCase()
  73. if (regionLower !== 'us' && regionLower !== 'uk') {
  74. throw new Error(
  75. `Unknown Stripe region: ${region}. Expected stripe-us or stripe-uk.`
  76. )
  77. }
  78. if (stripeClients[regionLower]) return stripeClients[regionLower]
  79. const secretKey =
  80. regionLower === 'us'
  81. ? Settings.apis?.stripeUS?.secretKey ||
  82. process.env.STRIPE_OL_SECRET_KEY ||
  83. process.env.STRIPE_OL_US_SECRET_KEY
  84. : Settings.apis?.stripeUK?.secretKey ||
  85. process.env.STRIPE_OL_UK_SECRET_KEY
  86. if (!secretKey || !String(secretKey).trim()) {
  87. throw new Error(
  88. `No Stripe secret key configured for region ${regionLower}.`
  89. )
  90. }
  91. const client = new Stripe(secretKey, {
  92. httpClient: Stripe.createFetchHttpClient(),
  93. telemetry: false,
  94. })
  95. client.serviceName = `stripe-${regionLower}`
  96. stripeClients[regionLower] = client
  97. return client
  98. }
  99. // =============================================================================
  100. // RECURLY CLIENT SETUP
  101. // =============================================================================
  102. const recurlyApiKey =
  103. process.env.RECURLY_API_KEY || Settings.apis?.recurly?.apiKey
  104. if (!recurlyApiKey) {
  105. throw new Error(
  106. 'Recurly API key is not set. Set RECURLY_API_KEY env var or configure Settings.apis.recurly.apiKey'
  107. )
  108. }
  109. const recurlyClient = new recurly.Client(recurlyApiKey)
  110. // =============================================================================
  111. // LOGGING UTILITIES
  112. // =============================================================================
  113. function timestamp() {
  114. return new Date().toISOString()
  115. }
  116. function logWarn(message, context = {}) {
  117. const contextStr =
  118. Object.keys(context).length > 0 ? ` ${JSON.stringify(context)}` : ''
  119. console.warn(`[${timestamp()}] WARN: ${message}${contextStr}`)
  120. }
  121. function logError(message, error = null, context = {}) {
  122. const contextStr =
  123. Object.keys(context).length > 0 ? ` ${JSON.stringify(context)}` : ''
  124. console.error(`[${timestamp()}] ERROR: ${message}${contextStr}`)
  125. if (error?.stack) {
  126. console.error(`[${timestamp()}] STACK: ${error.stack}`)
  127. }
  128. }
  129. let DEBUG_MODE = false
  130. function logDebug(message, context = {}, { verboseOnly = false } = {}) {
  131. if (verboseOnly && !DEBUG_MODE) return
  132. const contextStr =
  133. Object.keys(context).length > 0 ? ` ${JSON.stringify(context)}` : ''
  134. const level = verboseOnly ? 'DEBUG' : 'INFO'
  135. console.log(`[${timestamp()}] ${level}: ${message}${contextStr}`)
  136. }
  137. // =============================================================================
  138. // DATA FETCHING
  139. // =============================================================================
  140. let rateLimiters
  141. async function fetchRecurlyData(accountCode, context) {
  142. return await rateLimiters.requestWithRetries(
  143. 'recurly',
  144. () => recurlyClient.getAccount(`code-${accountCode}`),
  145. context
  146. )
  147. }
  148. async function fetchRecurlySubscription(accountCode, context) {
  149. // Try live subscriptions first, then fall back to expired.
  150. for (const state of ['live', 'expired']) {
  151. const subscriptions = await rateLimiters.requestWithRetries(
  152. 'recurly',
  153. async () => {
  154. const pager = recurlyClient.listAccountSubscriptions(
  155. `code-${accountCode}`,
  156. { params: { state, order: 'desc', sort: 'updated_at' } }
  157. )
  158. const results = []
  159. for await (const subscription of pager.each()) {
  160. results.push(subscription)
  161. }
  162. return results
  163. },
  164. context
  165. )
  166. if (subscriptions.length > 0) {
  167. // Return the most recently updated subscription in this state
  168. return subscriptions[0]
  169. }
  170. }
  171. return null
  172. }
  173. async function fetchTargetStripeCustomer(
  174. stripeClient,
  175. stripeCustomerId,
  176. context
  177. ) {
  178. const customer = await rateLimiters.requestWithRetries(
  179. stripeClient.serviceName,
  180. () =>
  181. stripeClient.customers.retrieve(stripeCustomerId, {
  182. expand: ['tax_ids', 'invoice_settings.default_payment_method'],
  183. }),
  184. { ...context, stripeApi: 'customers.retrieve' }
  185. )
  186. if (customer.deleted) {
  187. throw new Error(`Stripe customer ${stripeCustomerId} has been deleted`)
  188. }
  189. return customer
  190. }
  191. async function fetchTargetStripeCustomerPaymentMethods(
  192. stripeClient,
  193. stripeCustomerId,
  194. context
  195. ) {
  196. const paymentMethods = await rateLimiters.requestWithRetries(
  197. stripeClient.serviceName,
  198. () => stripeClient.customers.listPaymentMethods(stripeCustomerId),
  199. { ...context, stripeApi: 'customers.listPaymentMethods' }
  200. )
  201. return paymentMethods.data
  202. }
  203. // =============================================================================
  204. // COMPARISON LOGIC
  205. // =============================================================================
  206. /**
  207. * Compare a single customer's Recurly data against Stripe.
  208. */
  209. async function compareCustomer(row, rowNumber, comparisonDate) {
  210. const {
  211. recurly_account_code: recurlyAccountCode,
  212. target_stripe_account: targetStripeAccount,
  213. stripe_customer_id: stripeCustomerId,
  214. } = row
  215. const context = {
  216. rowNumber,
  217. recurlyAccountCode,
  218. targetStripeAccount,
  219. stripeCustomerId,
  220. }
  221. const stripeContext = {
  222. rowNumber,
  223. stripeCustomerId,
  224. stripeAccount: targetStripeAccount,
  225. }
  226. const result = {
  227. recurly_account_code: recurlyAccountCode,
  228. target_stripe_account: targetStripeAccount,
  229. stripe_customer_id: stripeCustomerId,
  230. status: '', // 'match', 'mismatch', or 'error'
  231. diffs: '',
  232. error: '',
  233. diffDetails: null,
  234. }
  235. try {
  236. if (!recurlyAccountCode) throw new Error('Missing recurly_account_code')
  237. if (!targetStripeAccount) throw new Error('Missing target_stripe_account')
  238. if (!stripeCustomerId) throw new Error('Missing stripe_customer_id')
  239. const region = String(targetStripeAccount || '')
  240. .trim()
  241. .toLowerCase()
  242. .replace(/^stripe-/, '')
  243. const stripeClient = getRegionClient(region)
  244. const account = await fetchRecurlyData(recurlyAccountCode, context)
  245. const recurlyAccountUpdatedAt = account.updatedAt?.getTime() || 0
  246. // If Recurly account was not updated after the comparison date, consider it a match
  247. if (recurlyAccountUpdatedAt <= comparisonDate.getTime()) {
  248. result.status = 'match'
  249. result.diffs = ''
  250. return result
  251. }
  252. const stripeCustomer = await fetchTargetStripeCustomer(
  253. stripeClient,
  254. stripeCustomerId,
  255. stripeContext
  256. )
  257. const stripePaymentMethods = await fetchTargetStripeCustomerPaymentMethods(
  258. stripeClient,
  259. stripeCustomerId,
  260. stripeContext
  261. )
  262. const diffs = await compareAccountFields({
  263. account,
  264. stripeCustomer,
  265. overleafUserId: recurlyAccountCode,
  266. fetchCollectionMethod: async () => {
  267. const subscription = await fetchRecurlySubscription(
  268. recurlyAccountCode,
  269. context
  270. )
  271. return subscription?.collectionMethod || null
  272. },
  273. stripePaymentMethods,
  274. stripeServiceName: stripeClient.serviceName,
  275. })
  276. // Determine result
  277. const diffKeys = Object.keys(diffs)
  278. if (diffKeys.length === 0) {
  279. result.status = 'match'
  280. result.diffs = ''
  281. } else {
  282. result.status = 'mismatch'
  283. result.diffs = diffKeys.join('; ')
  284. result.diffDetails = {
  285. recurly_account_code: recurlyAccountCode,
  286. stripe_customer_id: stripeCustomerId,
  287. target_stripe_account: targetStripeAccount,
  288. comparison_date: comparisonDate.toISOString(),
  289. recurly_account_updated_at: account.updatedAt?.toISOString(),
  290. diffs,
  291. }
  292. logDebug('Customer has diffs', {
  293. ...context,
  294. diffFields: diffKeys,
  295. })
  296. }
  297. } catch (error) {
  298. result.status = 'error'
  299. const errorDetails = [error.message]
  300. if (error.code) errorDetails.push(`code=${error.code}`)
  301. if (error.type) errorDetails.push(`type=${error.type}`)
  302. if (error.statusCode) errorDetails.push(`statusCode=${error.statusCode}`)
  303. result.error = errorDetails.join('; ')
  304. logError('Failed to compare customer', error, context)
  305. }
  306. return result
  307. }
  308. // =============================================================================
  309. // CSV HELPERS
  310. // =============================================================================
  311. function formatCsvRow(columns, row) {
  312. const values = columns.map(col => {
  313. const raw = row[col]
  314. const val = raw == null ? '' : String(raw)
  315. if (val.includes(',') || val.includes('"') || val.includes('\n')) {
  316. return `"${val.replace(/"/g, '""')}"`
  317. }
  318. return val
  319. })
  320. return values.join(',') + '\n'
  321. }
  322. function createJsonArrayWriter(jsonPath) {
  323. const stream = fs.createWriteStream(jsonPath, { flags: 'w' })
  324. stream.write('[\n')
  325. let wroteAny = false
  326. function write(value) {
  327. const serialized = JSON.stringify(value, null, 2)
  328. if (wroteAny) stream.write(',\n')
  329. stream.write(serialized)
  330. wroteAny = true
  331. }
  332. async function close() {
  333. stream.write('\n]\n')
  334. stream.end()
  335. await new Promise((resolve, reject) => {
  336. stream.on('finish', resolve)
  337. stream.on('error', reject)
  338. })
  339. }
  340. return { write, close }
  341. }
  342. // =============================================================================
  343. // CLI
  344. // =============================================================================
  345. function usage() {
  346. console.error(
  347. 'Compare Recurly customer data against migrated Stripe customers'
  348. )
  349. console.error('')
  350. console.error('Usage:')
  351. console.error(
  352. ' node scripts/recurly/compare_recurly_stripe_customers.mjs [options]'
  353. )
  354. console.error('')
  355. console.error('Options:')
  356. console.error(' --input, -i <file> Path to input CSV file (required)')
  357. console.error(' --output, -o <file> Path to output CSV file (required)')
  358. console.error(
  359. ' --comparison-date <date> Date in YYYY-MM-DD to compare against (required)'
  360. )
  361. console.error(
  362. ' --limit, -l <n> Limit number of records processed (default: no limit)'
  363. )
  364. console.error(
  365. ' --concurrency, -c <n> Number of customers to process concurrently (default: 10)'
  366. )
  367. console.error(
  368. ' --recurly-rate-limit <n> Requests per second for Recurly (default: 10)'
  369. )
  370. console.error(
  371. ' --recurly-api-retries <n> Number of retries on Recurly 429s (default: 5)'
  372. )
  373. console.error(
  374. ' --recurly-retry-delay-ms <n> Delay between Recurly retries in ms (default: 1000)'
  375. )
  376. console.error(
  377. ' --stripe-rate-limit <n> Requests per second for Stripe (default: 50)'
  378. )
  379. console.error(
  380. ' --stripe-api-retries <n> Number of retries on Stripe 429s (default: 5)'
  381. )
  382. console.error(
  383. ' --stripe-retry-delay-ms <n> Delay between Stripe retries in ms (default: 1000)'
  384. )
  385. console.error(' --verbose, -v Enable debug logging')
  386. }
  387. function parseConcurrency(value, { defaultValue = 10 } = {}) {
  388. if (value === undefined || value === null || value === '') return defaultValue
  389. const parsed = Number(value)
  390. if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed < 1) {
  391. throw new Error(
  392. `Invalid --concurrency value: ${value}. Expected a positive integer.`
  393. )
  394. }
  395. return parsed
  396. }
  397. function parseRateLimit(value, { defaultValue, name }) {
  398. if (value === undefined || value === null || value === '') return defaultValue
  399. const parsed = Number(value)
  400. if (!Number.isFinite(parsed) || parsed <= 0) {
  401. throw new Error(
  402. `Invalid --${name} value: ${value}. Expected a positive number.`
  403. )
  404. }
  405. return parsed
  406. }
  407. function parseNonNegativeInt(value, { defaultValue, name }) {
  408. if (value === undefined || value === null || value === '') return defaultValue
  409. const parsed = Number(value)
  410. if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed < 0) {
  411. throw new Error(
  412. `Invalid --${name} value: ${value}. Expected a non-negative integer.`
  413. )
  414. }
  415. return parsed
  416. }
  417. function parseArgs() {
  418. return minimist(process.argv.slice(2), {
  419. alias: {
  420. i: 'input',
  421. o: 'output',
  422. h: 'help',
  423. v: 'verbose',
  424. c: 'concurrency',
  425. l: 'limit',
  426. },
  427. string: [
  428. 'input',
  429. 'output',
  430. 'comparison-date',
  431. 'limit',
  432. 'recurly-rate-limit',
  433. 'recurly-api-retries',
  434. 'recurly-retry-delay-ms',
  435. 'stripe-rate-limit',
  436. 'stripe-api-retries',
  437. 'stripe-retry-delay-ms',
  438. ],
  439. boolean: ['verbose', 'help'],
  440. default: {
  441. verbose: false,
  442. concurrency: 10,
  443. 'recurly-rate-limit': DEFAULT_RECURLY_RATE_LIMIT,
  444. 'recurly-api-retries': DEFAULT_RECURLY_API_RETRIES,
  445. 'recurly-retry-delay-ms': DEFAULT_RECURLY_RETRY_DELAY_MS,
  446. 'stripe-rate-limit': DEFAULT_STRIPE_RATE_LIMIT,
  447. 'stripe-api-retries': DEFAULT_STRIPE_API_RETRIES,
  448. 'stripe-retry-delay-ms': DEFAULT_STRIPE_RETRY_DELAY_MS,
  449. },
  450. })
  451. }
  452. // =============================================================================
  453. // MAIN
  454. // =============================================================================
  455. async function main(trackProgress) {
  456. const startTime = new Date()
  457. const args = parseArgs()
  458. const {
  459. input: inputPath,
  460. output: outputPath,
  461. 'comparison-date': comparisonDateRaw,
  462. verbose,
  463. help,
  464. concurrency: concurrencyRaw,
  465. limit: limitRaw,
  466. 'recurly-rate-limit': recurlyRateLimitRaw,
  467. 'recurly-api-retries': recurlyApiRetriesRaw,
  468. 'recurly-retry-delay-ms': recurlyRetryDelayMsRaw,
  469. 'stripe-rate-limit': stripeRateLimitRaw,
  470. 'stripe-api-retries': stripeApiRetriesRaw,
  471. 'stripe-retry-delay-ms': stripeRetryDelayMsRaw,
  472. } = args
  473. let concurrency,
  474. recurlyRateLimit,
  475. recurlyApiRetriesValue,
  476. recurlyRetryDelayMsValue,
  477. stripeRateLimitPerSecond,
  478. stripeApiRetriesValue,
  479. stripeRetryDelayMsValue,
  480. limit,
  481. comparisonDate
  482. try {
  483. concurrency = parseConcurrency(concurrencyRaw, { defaultValue: 10 })
  484. limit = parseNonNegativeInt(limitRaw, {
  485. defaultValue: null,
  486. name: 'limit',
  487. })
  488. recurlyRateLimit = parseRateLimit(recurlyRateLimitRaw, {
  489. defaultValue: DEFAULT_RECURLY_RATE_LIMIT,
  490. name: 'recurly-rate-limit',
  491. })
  492. recurlyApiRetriesValue = parseNonNegativeInt(recurlyApiRetriesRaw, {
  493. defaultValue: DEFAULT_RECURLY_API_RETRIES,
  494. name: 'recurly-api-retries',
  495. })
  496. recurlyRetryDelayMsValue = parseNonNegativeInt(recurlyRetryDelayMsRaw, {
  497. defaultValue: DEFAULT_RECURLY_RETRY_DELAY_MS,
  498. name: 'recurly-retry-delay-ms',
  499. })
  500. stripeRateLimitPerSecond = parseRateLimit(stripeRateLimitRaw, {
  501. defaultValue: DEFAULT_STRIPE_RATE_LIMIT,
  502. name: 'stripe-rate-limit',
  503. })
  504. stripeApiRetriesValue = parseNonNegativeInt(stripeApiRetriesRaw, {
  505. defaultValue: DEFAULT_STRIPE_API_RETRIES,
  506. name: 'stripe-api-retries',
  507. })
  508. stripeRetryDelayMsValue = parseNonNegativeInt(stripeRetryDelayMsRaw, {
  509. defaultValue: DEFAULT_STRIPE_RETRY_DELAY_MS,
  510. name: 'stripe-retry-delay-ms',
  511. })
  512. if (!comparisonDateRaw) {
  513. throw new Error('--comparison-date is required')
  514. }
  515. const dateMatch = comparisonDateRaw.match(/^(\d{4})-(\d{2})-(\d{2})$/)
  516. if (!dateMatch) {
  517. throw new Error('--comparison-date must be in YYYY-MM-DD format')
  518. }
  519. const [, year, month, day] = dateMatch
  520. comparisonDate = new Date(`${year}-${month}-${day}T00:00:00.000Z`)
  521. if (isNaN(comparisonDate.getTime())) {
  522. throw new Error('Invalid date provided for --comparison-date')
  523. }
  524. } catch (error) {
  525. logError(error.message)
  526. usage()
  527. process.exit(1)
  528. }
  529. rateLimiters = createRateLimitedApiWrappers({
  530. recurlyRateLimit,
  531. recurlyApiRetries: recurlyApiRetriesValue,
  532. recurlyRetryDelayMs: recurlyRetryDelayMsValue,
  533. stripeRateLimit: stripeRateLimitPerSecond,
  534. stripeApiRetries: stripeApiRetriesValue,
  535. stripeRetryDelayMs: stripeRetryDelayMsValue,
  536. logDebug,
  537. logWarn,
  538. })
  539. DEBUG_MODE = !!verbose
  540. if (help || !inputPath || !outputPath) {
  541. usage()
  542. process.exit(help ? 0 : 1)
  543. }
  544. const errorsPath = outputPath.replace(/\.csv$/, '_errors.csv')
  545. const detailsJsonPath = outputPath.replace(/\.csv$/, '_details.json')
  546. logDebug('Starting comparison', {
  547. inputPath,
  548. outputPath,
  549. errorsPath,
  550. detailsJsonPath,
  551. comparisonDate: comparisonDate.toISOString(),
  552. concurrency,
  553. ...(limit != null ? { limit } : {}),
  554. })
  555. await trackProgress('Starting comparison')
  556. // Output CSV columns
  557. const outputColumns = [
  558. 'recurly_account_code',
  559. 'target_stripe_account',
  560. 'stripe_customer_id',
  561. 'status',
  562. 'diffs',
  563. ]
  564. const errorColumns = [
  565. 'recurly_account_code',
  566. 'target_stripe_account',
  567. 'stripe_customer_id',
  568. 'error',
  569. ]
  570. const outputStream = fs.createWriteStream(outputPath, { flags: 'w' })
  571. outputStream.write(outputColumns.join(',') + '\n')
  572. const errorsStream = fs.createWriteStream(errorsPath, { flags: 'w' })
  573. errorsStream.write(errorColumns.join(',') + '\n')
  574. const detailsWriter = createJsonArrayWriter(detailsJsonPath)
  575. try {
  576. let totalInInput = 0
  577. let processedCount = 0
  578. let matchCount = 0
  579. let mismatchCount = 0
  580. let errorCount = 0
  581. let queuedCount = 0
  582. const inputStream = fs.createReadStream(inputPath)
  583. const parser = csv.parse({
  584. columns: true,
  585. trim: true,
  586. bom: true,
  587. skip_empty_lines: true,
  588. relax_column_count: true,
  589. relax_column_count_less: true,
  590. })
  591. inputStream.pipe(parser)
  592. const queue = new PQueue({ concurrency })
  593. const maxQueueSize = concurrency
  594. let rowNumber = 0
  595. let limitReached = false
  596. try {
  597. for await (const row of parser) {
  598. rowNumber++
  599. totalInInput++
  600. const thisRowNumber = rowNumber
  601. if (limit != null && queuedCount >= limit) {
  602. limitReached = true
  603. logDebug('Record limit reached', { limit, queuedCount })
  604. break
  605. }
  606. if (queue.size >= maxQueueSize) {
  607. await queue.onSizeLessThan(maxQueueSize)
  608. }
  609. queuedCount++
  610. queue.add(async () => {
  611. let result
  612. try {
  613. result = await compareCustomer(row, thisRowNumber, comparisonDate)
  614. } catch (error) {
  615. result = {
  616. ...row,
  617. status: 'error',
  618. diffs: '',
  619. error: error?.message || String(error),
  620. diffDetails: null,
  621. }
  622. logError('Unhandled error', error, {
  623. rowNumber: thisRowNumber,
  624. accountCode: row.recurly_account_code,
  625. })
  626. }
  627. processedCount++
  628. if (result.status === 'match') {
  629. matchCount++
  630. outputStream.write(formatCsvRow(outputColumns, result))
  631. } else if (result.status === 'mismatch') {
  632. mismatchCount++
  633. outputStream.write(formatCsvRow(outputColumns, result))
  634. if (result.diffDetails) {
  635. detailsWriter.write(result.diffDetails)
  636. }
  637. } else {
  638. errorCount++
  639. errorsStream.write(formatCsvRow(errorColumns, result))
  640. }
  641. const progressInterval = DEBUG_MODE ? 100 : 1000
  642. if (processedCount % progressInterval === 0) {
  643. logDebug('Progress', {
  644. processedCount,
  645. matchCount,
  646. mismatchCount,
  647. errorCount,
  648. })
  649. await trackProgress(
  650. `Progress: ${processedCount} processed, ${matchCount} match, ${mismatchCount} mismatch, ${errorCount} errors`
  651. )
  652. }
  653. })
  654. }
  655. } finally {
  656. await queue.onIdle()
  657. }
  658. if (limitReached) {
  659. await trackProgress(`Limit reached (${limit}).`)
  660. }
  661. // Final summary
  662. const endTime = new Date()
  663. const durationMs = endTime.getTime() - startTime.getTime()
  664. const durationSeconds = Math.floor(durationMs / 1000)
  665. const finalStats = rateLimiters.getRateLimiterStats()
  666. await trackProgress('=== COMPARISON SUMMARY ===')
  667. await trackProgress(`Total in input: ${totalInInput}`)
  668. await trackProgress(`Processed: ${processedCount}`)
  669. await trackProgress(` Match: ${matchCount}`)
  670. await trackProgress(` Mismatch: ${mismatchCount}`)
  671. await trackProgress(` Error: ${errorCount}`)
  672. await trackProgress(`Duration: ${durationSeconds}s`)
  673. await trackProgress(`Output: ${outputPath}`)
  674. await trackProgress(`Errors: ${errorsPath} (${errorCount} records)`)
  675. await trackProgress(
  676. `Details: ${detailsJsonPath} (${mismatchCount} records)`
  677. )
  678. await trackProgress(
  679. `API calls - Recurly: ${finalStats.recurly.totalRequests}, Stripe: ${finalStats.stripe.totalRequests}`
  680. )
  681. logDebug('Comparison complete', {
  682. totalInInput,
  683. processedCount,
  684. matchCount,
  685. mismatchCount,
  686. errorCount,
  687. })
  688. return errorCount === 0 && mismatchCount === 0 ? 0 : 1
  689. } finally {
  690. outputStream.end()
  691. errorsStream.end()
  692. const results = await Promise.allSettled([
  693. new Promise((resolve, reject) => {
  694. outputStream.on('finish', resolve)
  695. outputStream.on('error', reject)
  696. }),
  697. new Promise((resolve, reject) => {
  698. errorsStream.on('finish', resolve)
  699. errorsStream.on('error', reject)
  700. }),
  701. detailsWriter.close(),
  702. ])
  703. for (const result of results) {
  704. if (result.status === 'rejected') {
  705. logWarn('Failed to close output stream', {
  706. error: result.reason?.message || String(result.reason),
  707. })
  708. }
  709. }
  710. }
  711. }
  712. try {
  713. const exitCode = await scriptRunner(main)
  714. process.exit(exitCode ?? 0)
  715. } catch (error) {
  716. logError('Script failed with unhandled error', error)
  717. process.exit(1)
  718. }