From f3cace6a8b988097c8e550317343cc8b4ce93361 Mon Sep 17 00:00:00 2001 From: logurita Date: Wed, 16 Jun 2021 01:08:21 +0200 Subject: [PATCH 1/3] fix: large files into chunks, fix memory issue --- lib/helpers/chopfile.js | 41 ++++++++++++++++++++++++++++++++++++ lib/helpers/parse-config.js | 1 + lib/helpers/replace-async.js | 35 +++++++++++++++--------------- lib/helpers/replace-sync.js | 31 +++++++++++++++------------ 4 files changed, 77 insertions(+), 31 deletions(-) create mode 100644 lib/helpers/chopfile.js diff --git a/lib/helpers/chopfile.js b/lib/helpers/chopfile.js new file mode 100644 index 0000000..dd37ef9 --- /dev/null +++ b/lib/helpers/chopfile.js @@ -0,0 +1,41 @@ +const fs = require('fs'); + +module.exports = { + chopByFileSizeSync(filePath, encoding, chunkSize) { + const file = fs.readFileSync(filePath, encoding); + var chunks = Math.ceil(fs.statSync(filePath).size / chunkSize, chunkSize); + var chunk = 0; + const files = []; + while (chunk <= chunks) { + var offset = chunk * chunkSize; + console.log('current chunk..', chunk); + console.log('offset...', chunk * chunkSize); + console.log('file blob from offset...', offset); + files.push((file.slice(offset, offset + chunkSize))); + chunk++; + } + return files.filter(v => v); + }, + chopByFileSizeAsync(filePath, encoding, chunkSize) { + return new Promise((resolve, reject) => { + fs.readFile(filePath, encoding, (error, file) => { + //istanbul ignore if + if (error) { + return reject(error); + } + var chunks = Math.ceil(fs.statSync(filePath).size / chunkSize, chunkSize); + var chunk = 0; + const files = []; + while (chunk <= chunks) { + var offset = chunk * chunkSize; + console.log('current chunk..', chunk); + console.log('offset...', chunk * chunkSize); + console.log('file blob from offset...', offset); + files.push((file.slice(offset, offset + chunkSize))); + chunk++; + } + return resolve(files.filter(v => v)); + }); + }); + } +} diff --git a/lib/helpers/parse-config.js b/lib/helpers/parse-config.js index 41d87a1..20e5c3c 100644 --- a/lib/helpers/parse-config.js +++ b/lib/helpers/parse-config.js @@ -13,6 +13,7 @@ const defaults = { verbose: false, quiet: false, dry: false, + chunkSize: 300 * 300, glob: {}, cwd: null, }; diff --git a/lib/helpers/replace-async.js b/lib/helpers/replace-async.js index 5167d4b..2904513 100644 --- a/lib/helpers/replace-async.js +++ b/lib/helpers/replace-async.js @@ -5,41 +5,40 @@ */ const fs = require('fs'); const makeReplacements = require('./make-replacements'); +const chopByFileSize = require('./chopfile').chopByFileSizeAsync; /** * Helper to replace in a single file (async) */ -module.exports = function replaceAsync(file, from, to, config) { +module.exports = function replaceAsync(filePath, from, to, config) { //Extract relevant config - const {encoding, dry, countMatches} = config; + const { encoding, dry, countMatches, chunkSize } = config; //Wrap in promise return new Promise((resolve, reject) => { - fs.readFile(file, encoding, (error, contents) => { - //istanbul ignore if - if (error) { - return reject(error); - } - - //Make replacements - const [result, newContents] = makeReplacements( - contents, from, to, file, countMatches - ); - + chopByFileSize(filePath, encoding, chunkSize).then(chunks => { + let result = []; + chunks.forEach((chunk, i) => { + //Make replacements + const response = makeReplacements( + chunk, from, to, chunk, countMatches + ); + chunks[i] = response[1]; + result.push(response[0].hasChanged); + }); //Not changed or dry run? - if (!result.hasChanged || dry) { + if (result.filter(v => v).length === 0 || dry) { return resolve(result); } - //Write to file - fs.writeFile(file, newContents, encoding, error => { + fs.writeFile(filePath, chunks.join(), encoding, error => { //istanbul ignore if if (error) { return reject(error); } resolve(result); }); - }); + }, err => reject(err)); }); -}; +} diff --git a/lib/helpers/replace-sync.js b/lib/helpers/replace-sync.js index ada3519..2f1886c 100644 --- a/lib/helpers/replace-sync.js +++ b/lib/helpers/replace-sync.js @@ -5,26 +5,31 @@ */ const fs = require('fs'); const makeReplacements = require('./make-replacements'); +const chopByFileSize = require('./chopfile').chopByFileSizeSync; /** * Helper to replace in a single file (sync) */ -module.exports = function replaceSync(file, from, to, config) { - + module.exports = function replaceSync(filePath, from, to, config) { + debugger; //Extract relevant config and read file contents - const {encoding, dry, countMatches} = config; - const contents = fs.readFileSync(file, encoding); - - //Replace contents and check if anything changed - const [result, newContents] = makeReplacements( - contents, from, to, file, countMatches - ); + const { encoding, dry, countMatches, chunkSize } = config; + const chunks = chopByFileSize(filePath, encoding, chunkSize); + let result; + let newContents; + chunks.forEach((chunk, i) => { - //Contents changed and not a dry run? Write to file - if (result.hasChanged && !dry) { - fs.writeFileSync(file, newContents, encoding); - } + //Replace contents and check if anything changed + [result, newContents] = makeReplacements( + chunk, from, to, chunk, countMatches + ); + //Contents changed and not a dry run? Write to file + if (result.hasChanged && !dry) { + chunks[i] = newContents; + } + }); + fs.writeFileSync(filePath, chunks.join(), encoding); //Return result return result; }; From e8e2d3b2e0e1be968f61fc16fd4cdcd46132440f Mon Sep 17 00:00:00 2001 From: logurita Date: Wed, 16 Jun 2021 01:10:50 +0200 Subject: [PATCH 2/3] logs out --- lib/helpers/chopfile.js | 7 ------- lib/helpers/replace-sync.js | 1 - 2 files changed, 8 deletions(-) diff --git a/lib/helpers/chopfile.js b/lib/helpers/chopfile.js index dd37ef9..7afe327 100644 --- a/lib/helpers/chopfile.js +++ b/lib/helpers/chopfile.js @@ -8,9 +8,6 @@ module.exports = { const files = []; while (chunk <= chunks) { var offset = chunk * chunkSize; - console.log('current chunk..', chunk); - console.log('offset...', chunk * chunkSize); - console.log('file blob from offset...', offset); files.push((file.slice(offset, offset + chunkSize))); chunk++; } @@ -19,7 +16,6 @@ module.exports = { chopByFileSizeAsync(filePath, encoding, chunkSize) { return new Promise((resolve, reject) => { fs.readFile(filePath, encoding, (error, file) => { - //istanbul ignore if if (error) { return reject(error); } @@ -28,9 +24,6 @@ module.exports = { const files = []; while (chunk <= chunks) { var offset = chunk * chunkSize; - console.log('current chunk..', chunk); - console.log('offset...', chunk * chunkSize); - console.log('file blob from offset...', offset); files.push((file.slice(offset, offset + chunkSize))); chunk++; } diff --git a/lib/helpers/replace-sync.js b/lib/helpers/replace-sync.js index 2f1886c..d521f6f 100644 --- a/lib/helpers/replace-sync.js +++ b/lib/helpers/replace-sync.js @@ -11,7 +11,6 @@ const chopByFileSize = require('./chopfile').chopByFileSizeSync; * Helper to replace in a single file (sync) */ module.exports = function replaceSync(filePath, from, to, config) { - debugger; //Extract relevant config and read file contents const { encoding, dry, countMatches, chunkSize } = config; const chunks = chopByFileSize(filePath, encoding, chunkSize); From 35392f3c71b485aed51d8c0393a99703a4589bd5 Mon Sep 17 00:00:00 2001 From: logurita Date: Wed, 16 Jun 2021 13:24:23 +0200 Subject: [PATCH 3/3] fix: chunks join blank string --- lib/helpers/replace-async.js | 2 +- lib/helpers/replace-sync.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/helpers/replace-async.js b/lib/helpers/replace-async.js index 2904513..0368314 100644 --- a/lib/helpers/replace-async.js +++ b/lib/helpers/replace-async.js @@ -32,7 +32,7 @@ module.exports = function replaceAsync(filePath, from, to, config) { return resolve(result); } //Write to file - fs.writeFile(filePath, chunks.join(), encoding, error => { + fs.writeFile(filePath, chunks.join(''), encoding, error => { //istanbul ignore if if (error) { return reject(error); diff --git a/lib/helpers/replace-sync.js b/lib/helpers/replace-sync.js index d521f6f..f4d4ff3 100644 --- a/lib/helpers/replace-sync.js +++ b/lib/helpers/replace-sync.js @@ -28,7 +28,7 @@ const chopByFileSize = require('./chopfile').chopByFileSizeSync; chunks[i] = newContents; } }); - fs.writeFileSync(filePath, chunks.join(), encoding); + fs.writeFileSync(filePath, chunks.join(''), encoding); //Return result return result; };