diff --git a/bin/photos-diff.ts b/bin/photos-diff.ts index a7ebad3..7f1fcf4 100755 --- a/bin/photos-diff.ts +++ b/bin/photos-diff.ts @@ -1,10 +1,11 @@ #!/usr/bin/env node -import { execSync } from 'child_process'; +import { execSync, spawn } from 'child_process'; import { createHash } from 'crypto'; import { readFileSync, statSync, writeFileSync } from 'fs'; import { parseArgs } from 'util'; import Redis from 'iovalkey'; +import { createInterface } from 'readline'; const CACHE_TTL = 3600; // 1 hour const CACHE_PREFIX_ANDROID = 'photo-sync:android:'; @@ -19,7 +20,6 @@ interface FileInfo { interface Results { matched: number; missingInBackup: FileInfo[]; - missingOnPhone: FileInfo[]; duplicatesOnPhone: Record; duplicatesInBackup: Record; } @@ -43,8 +43,23 @@ function parseArguments() { async function getLocalFiles(dir: string, redis: Redis): Promise { console.log(`\nšŸ“ Scanning local directory: ${dir}`); - const output = execSync(`find "${dir}" -type f`, { encoding: 'utf-8', maxBuffer: 100 * 1024 * 1024 }); - const files = output.trim().split('\n').filter(Boolean); + // Stream files instead of buffering + const files: string[] = []; + const find = spawn('find', [dir, '-type', 'f']); + const rl = createInterface({ input: find.stdout }); + + for await (const line of rl) { + if (line.trim()) { + files.push(line.trim()); + } + } + + await new Promise((resolve, reject) => { + find.on('close', resolve); + find.on('error', reject); + }); + + console.log(`šŸ“Š Found ${files.length} files, getting sizes...`); const fileInfos: FileInfo[] = []; const startTime = Date.now(); @@ -94,12 +109,22 @@ async function getLocalFiles(dir: string, redis: Redis): Promise { async function getAndroidFiles(dir: string, redis: Redis): Promise { console.log(`\nšŸ“± Scanning Android directory: ${dir}`); - const output = execSync(`adb shell "find '${dir}' -type f 2>/dev/null"`, { - encoding: 'utf-8', - maxBuffer: 50 * 1024 * 1024 - }); + // Stream files instead of buffering + const files: string[] = []; + const adb = spawn('adb', ['shell', `find '${dir}' -type f 2>/dev/null`]); + const rl = createInterface({ input: adb.stdout }); - const files = output.trim().split('\n').filter(Boolean).map(f => f.trim()).filter(Boolean); + for await (const line of rl) { + const trimmed = line.trim(); + if (trimmed) { + files.push(trimmed); + } + } + + await new Promise((resolve, reject) => { + adb.on('close', resolve); + adb.on('error', reject); + }); console.log(`šŸ“Š Getting file sizes for ${files.length} files...`); @@ -190,16 +215,18 @@ async function getAndroidFiles(dir: string, redis: Redis): Promise { } // Progress and ETA - const elapsed = Date.now() - startTime; - const avgTime = elapsed / processed; - const remaining = files.length - processed; - const eta = Math.round((avgTime * remaining) / 1000); + if (processed > 0) { + const elapsed = Date.now() - startTime; + const avgTime = elapsed / processed; + const remaining = files.length - processed; + const eta = Math.round((avgTime * remaining) / 1000); - const etaStr = eta > 60 - ? `${Math.floor(eta / 60)}m ${eta % 60}s` - : `${eta}s`; + const etaStr = eta > 60 + ? `${Math.floor(eta / 60)}m ${eta % 60}s` + : `${eta}s`; - process.stdout.write(`\ršŸ“Š Progress: ${processed}/${files.length} files | ETA: ${etaStr} `); + process.stdout.write(`\ršŸ“Š Progress: ${processed}/${files.length} files | ETA: ${etaStr} `); + } } console.log(`\nāœ… Found ${fileInfos.length} Android files`); @@ -485,7 +512,6 @@ async function main() { console.log('\nšŸ” Comparing files...'); const missingInBackup: FileInfo[] = []; - const missingOnPhone: FileInfo[] = []; let matched = 0; // Files on Android but not in backup @@ -497,13 +523,6 @@ async function main() { } } - // Files in backup but not on Android - for (const [hash, localGroup] of localHashes.entries()) { - if (!androidHashes.has(hash)) { - missingOnPhone.push(...localGroup); - } - } - // Step 7: Find duplicates const duplicatesOnPhone = findDuplicates(androidFiles); const duplicatesInBackup = findDuplicates(localFiles); @@ -512,7 +531,6 @@ async function main() { const results: Results = { matched, missingInBackup, - missingOnPhone, duplicatesOnPhone, duplicatesInBackup, }; @@ -520,11 +538,15 @@ async function main() { console.log('\n' + '='.repeat(60)); console.log('šŸ“Š RESULTS'); console.log('='.repeat(60)); - console.log(`āœ… Matched files: ${results.matched}`); - console.log(`āŒ Missing in backup: ${results.missingInBackup.length}`); - console.log(`ā„¹ļø Missing on phone: ${results.missingOnPhone.length}`); - console.log(`šŸ”„ Duplicates on phone: ${Object.keys(results.duplicatesOnPhone).length} hashes`); - console.log(`šŸ”„ Duplicates in backup: ${Object.keys(results.duplicatesInBackup).length} hashes`); + console.log(`šŸ“± Total files on phone: ${androidFiles.length}`); + console.log(`āœ… Matched in backup: ${results.matched}`); + console.log(`āŒ MISSING in backup: ${results.missingInBackup.length}`); + if (results.missingInBackup.length > 0) { + console.log(`\n Missing files:`); + results.missingInBackup.forEach(f => console.log(` - ${f.path} (${f.size} bytes)`)); + } + console.log(`\nšŸ”„ Duplicates on phone: ${Object.keys(results.duplicatesOnPhone).length} groups`); + console.log(`šŸ”„ Duplicates in backup: ${Object.keys(results.duplicatesInBackup).length} groups`); console.log('='.repeat(60)); console.log('\nšŸ’¾ Writing results to results.json...'); diff --git a/bin/pull-missing.ts b/bin/pull-missing.ts new file mode 100755 index 0000000..999dc16 --- /dev/null +++ b/bin/pull-missing.ts @@ -0,0 +1,149 @@ +#!/usr/bin/env node + +import { execSync } from 'child_process'; +import { existsSync, mkdirSync } from 'fs'; +import { readFileSync } from 'fs'; +import { basename, dirname, join } from 'path'; + +interface FileInfo { + path: string; + size: number; + hash?: string; +} + +interface Results { + matched: number; + missingInBackup: FileInfo[]; + missingOnPhone: FileInfo[]; + duplicatesOnPhone: Record; + duplicatesInBackup: Record; +} + +function getUniqueFilename(targetPath: string): string { + if (!existsSync(targetPath)) { + return targetPath; + } + + const dir = dirname(targetPath); + const ext = targetPath.match(/\.[^.]+$/)?.[0] || ''; + const nameWithoutExt = basename(targetPath, ext); + + let counter = 1; + let newPath: string; + + do { + newPath = join(dir, `${nameWithoutExt}-${counter}${ext}`); + counter++; + } while (existsSync(newPath)); + + return newPath; +} + +async function pullFile(androidPath: string, localDir: string): Promise { + const filename = basename(androidPath); + const targetPath = join(localDir, filename); + const finalPath = getUniqueFilename(targetPath); + + console.log(`šŸ“„ Pulling: ${androidPath}`); + console.log(` -> ${finalPath}`); + + try { + execSync(`adb pull "${androidPath}" "${finalPath}"`, { + encoding: 'utf-8', + stdio: 'pipe' + }); + return finalPath; + } catch (err: any) { + console.error(`āŒ Failed to pull: ${androidPath}`); + console.error(` Error: ${err.message}`); + throw err; + } +} + +async function main() { + const resultsPath = 'results.json'; + + if (!existsSync(resultsPath)) { + console.error(`āŒ Error: ${resultsPath} not found`); + console.error('Run photos-diff.ts first to generate results.json'); + process.exit(1); + } + + console.log('šŸ“– Reading results.json...'); + const results: Results = JSON.parse(readFileSync(resultsPath, 'utf-8')); + + const missingFiles = results.missingInBackup; + + if (missingFiles.length === 0) { + console.log('āœ… No missing files! Backup is complete.'); + return; + } + + console.log(`\nšŸ” Found ${missingFiles.length} missing files`); + console.log(`šŸ“¦ Total size: ${(missingFiles.reduce((sum, f) => sum + f.size, 0) / 1024 / 1024).toFixed(2)} MB`); + + // Create target directory if needed + const targetDir = './pulled-files'; + if (!existsSync(targetDir)) { + console.log(`\nšŸ“ Creating directory: ${targetDir}`); + mkdirSync(targetDir, { recursive: true }); + } + + console.log(`\nšŸš€ Starting download to: ${targetDir}\n`); + + const startTime = Date.now(); + const pulled: string[] = []; + const failed: string[] = []; + + for (let i = 0; i < missingFiles.length; i++) { + const file = missingFiles[i]; + + try { + const localPath = await pullFile(file.path, targetDir); + pulled.push(localPath); + } catch (err) { + failed.push(file.path); + } + + // Progress + const processed = i + 1; + const elapsed = Date.now() - startTime; + const avgTime = elapsed / processed; + const remaining = missingFiles.length - processed; + const eta = Math.round((avgTime * remaining) / 1000); + + const etaStr = eta > 60 + ? `${Math.floor(eta / 60)}m ${eta % 60}s` + : `${eta}s`; + + console.log(`\nšŸ“Š Progress: ${processed}/${missingFiles.length} | ETA: ${etaStr}`); + console.log(` āœ… Success: ${pulled.length} | āŒ Failed: ${failed.length}\n`); + } + + // Final report + const totalTime = Math.round((Date.now() - startTime) / 1000); + const timeStr = totalTime > 60 + ? `${Math.floor(totalTime / 60)}m ${totalTime % 60}s` + : `${totalTime}s`; + + console.log('='.repeat(60)); + console.log('šŸ“Š DOWNLOAD COMPLETE'); + console.log('='.repeat(60)); + console.log(`āœ… Successfully pulled: ${pulled.length} files`); + console.log(`āŒ Failed: ${failed.length} files`); + console.log(`ā±ļø Total time: ${timeStr}`); + console.log(`šŸ“ Files saved to: ${targetDir}`); + console.log('='.repeat(60)); + + if (failed.length > 0) { + console.log('\nāŒ Failed files:'); + failed.forEach(f => console.log(` - ${f}`)); + process.exit(1); + } +} + +main().catch(err => { + console.error('šŸ’„ Fatal error:', err); + process.exit(1); +}); + diff --git a/bun.lock b/bun.lock index 07f3dd5..ebc702d 100644 --- a/bun.lock +++ b/bun.lock @@ -4,6 +4,7 @@ "": { "dependencies": { "chalk": "^5.5.0", + "iovalkey": "^0.3.3", "ollama": "^0.5.17", }, "devDependencies": { @@ -12,12 +13,34 @@ }, }, "packages": { + "@iovalkey/commands": ["@iovalkey/commands@0.1.0", "", {}, "sha512-/B9W4qKSSITDii5nkBCHyPkIkAi+ealUtr1oqBJsLxjSRLka4pxun2VvMNSmcwgAMxgXtQfl0qRv7TE+udPJzg=="], + "@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], "chalk": ["chalk@5.5.0", "", {}, "sha512-1tm8DTaJhPBG3bIkVeZt1iZM9GfSX2lzOeDVZH9R9ffRHpmHvxZ/QhgQH/aDTkswQVt+YHdXAdS/In/30OjCbg=="], + "cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="], + + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + + "denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="], + + "iovalkey": ["iovalkey@0.3.3", "", { "dependencies": { "@iovalkey/commands": "^0.1.0", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" } }, "sha512-4rTJX6Q5wTYEvxboXi8DsEiUo+OvqJGtLYOSGm37KpdRXsG5XJjbVtYKGJpPSWP+QT7rWscA4vsrdmzbEbenpw=="], + + "lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="], + + "lodash.isarguments": ["lodash.isarguments@3.1.0", "", {}, "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "ollama": ["ollama@0.5.17", "", { "dependencies": { "whatwg-fetch": "^3.6.20" } }, "sha512-q5LmPtk6GLFouS+3aURIVl+qcAOPC4+Msmx7uBb3pd+fxI55WnGjmLZ0yijI/CYy79x0QPGx3BwC3u5zv9fBvQ=="], + "redis-errors": ["redis-errors@1.2.0", "", {}, "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="], + + "redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="], + + "standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="], + "undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], "whatwg-fetch": ["whatwg-fetch@3.6.20", "", {}, "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg=="], diff --git a/package.json b/package.json index 4edb3d4..06cc554 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ }, "dependencies": { "chalk": "^5.5.0", - "ollama": "^0.5.17" + "ollama": "^0.5.17", + "iovalkey": "^0.3.3" } }