#!/usr/bin/env node const {open, readdir, stat, rmdir} = require('node:fs/promises'); const {join} = require('node:path'); const {unlink} = require('node:fs/promises'); const {Buffer} = require('node:buffer'); const Kilobyte = 1024; const Megabyte = Kilobyte * 1024; const Gigabyte = Megabyte * 1024; const BLOCK_SIZE = 16 * Megabyte; const locale = 'en-US'; /** @type {Buffer} */ const BufferA = Buffer.alloc(BLOCK_SIZE); /** @type {Buffer} */ const BufferB = Buffer.alloc(BLOCK_SIZE); const Terminal = Object.freeze({ fg: { red: (text) => `\u001b[31m${text}\u001b[0m`, green: (text) => `\u001b[32m${text}\u001b[0m`, yellow: (text) => `\u001b[33m${text}\u001b[0m`, blue: (text) => `\u001b[34m${text}\u001b[0m`, } }); /** * @typedef {{bySize: {[key: string]: string[]}}} Cache */ /** * @param {function(typeof Terminal)|string} param */ function log(param) { console.log(typeof param === 'string' ? param : param(Terminal)) } /** * @param {Number} size * @returns {*} */ function formatSize(size) { let suffix = ''; let digits = 0; if (size >= Megabyte) { digits = 1; if (size >= Gigabyte) { size /= Gigabyte; suffix = ' G'; } else { size /= Megabyte; suffix = ' M'; } } else if (size >= Kilobyte) { digits = 1; size /= Kilobyte; suffix = ' K' } return `${size.toLocaleString(locale, { minimumFractionDigits: digits, maximumFractionDigits: digits, useGrouping: false })}${suffix}` } /** * @param path * @returns {Promise<{files: Dirent[], dirs: Dirent[]}>} */ async function dirEntries(path) { return (await readdir(path, {withFileTypes: true})) .reduce((acc, file) => { (file.isDirectory() ? acc.dirs : acc.files).push(file) return acc; }, {files: [], dirs: []}) } async function compareFiles(localPath, remotePath, size) { const localFile = await open(localPath), remoteFile = await open(remotePath); try { for (let i = 0; ;) { const start = process.hrtime.bigint(); const a = await localFile.read(BufferA, 0, BLOCK_SIZE); const b = await remoteFile.read(BufferB, 0, BLOCK_SIZE); if (a.bytesRead !== b.bytesRead || BufferA.compare(BufferB) !== 0) { log(f => f.fg.yellow(` files are different`)); console.log(); return false; } if (a.bytesRead === 0) { console.log(); return true; } const seconds = Number(process.hrtime.bigint() - start) / 1_000_000_000; i += a.bytesRead; process.stdout.write(`\r${(i * 100 / size).toLocaleString(locale, { minimumFractionDigits: 2, maximumFractionDigits: 2 })}% ${formatSize((a.bytesRead + b.bytesRead) / seconds)}Bytes/sec`) } console.log('') } finally { Promise.allSettled([localFile.close(), remoteFile.close()]); } } /** * @param local * @param {Cache} cache * @param options * @returns {Promise} */ async function compareDirs(local, cache, options) { try { let canRemoveDir = true; const entries = await dirEntries(local) loop: for (const file of entries.files) { const localPath = join(local, file.name); const localStat = await stat(localPath); const sameSize = cache.bySize[localStat.size]; if (!sameSize) { canRemoveDir = false; continue; } log(f => `${f.fg.green(localPath)} (${f.fg.green(`${formatSize(localStat.size)}B`)})`) for (let i = 0; i < sameSize.length; i++) { const remotePath = sameSize[i]; const remoteStat = await stat(remotePath); if (localStat.ino === remoteStat.ino) { log(f => f.fg.red('This is the same file')); } else { log(f => `Comparing with ${f.fg.blue(remotePath)}`) if (await compareFiles(localPath, remotePath, localStat.size)) { if (options.delete) { log(f => ` ${f.fg.red('files are the same - deleting')}`); await unlink(localPath) } continue loop; } } } canRemoveDir = false; } for (const dir of entries.dirs) { if (!await compareDirs(join(local, dir.name), cache, options)) canRemoveDir = false; } if (canRemoveDir && options.delete) { log(f => `Directory ${f.fg.yellow(local)} is empty, removed`) await rmdir(local) } return canRemoveDir; } catch (err) { console.error(err); return false; } } const options = { delete: false }; if (process.argv[3] === 'delete') { options.delete = true; } /** * @param path * @param result * @returns {Promise} */ async function scanDir(path, result) { const entries = await dirEntries(path) for (const file of entries.files) { let absPath = join(path, file.name); let {size} = (await stat(absPath)); let bySize = result.bySize[size]; if (bySize) { bySize.push(absPath); } else { bySize = [absPath]; result.bySize[size] = bySize; } } for (const dir of entries.dirs) { await scanDir(join(path, dir.name), result) } return result; } (async function () { log('Scanning target directory...') const scan = await scanDir(process.argv[2], {bySize: {}}) log('Comparing...') await compareDirs(process.cwd(), scan, options); })();