pull-missing.ts
parent
6010b5a24c
commit
5579c329f8
|
|
@ -1,10 +1,11 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import { execSync } from 'child_process';
|
||||
import { execSync, spawn } from 'child_process';
|
||||
import { createHash } from 'crypto';
|
||||
import { readFileSync, statSync, writeFileSync } from 'fs';
|
||||
import { parseArgs } from 'util';
|
||||
import Redis from 'iovalkey';
|
||||
import { createInterface } from 'readline';
|
||||
|
||||
const CACHE_TTL = 3600; // 1 hour
|
||||
const CACHE_PREFIX_ANDROID = 'photo-sync:android:';
|
||||
|
|
@ -19,7 +20,6 @@ interface FileInfo {
|
|||
interface Results {
|
||||
matched: number;
|
||||
missingInBackup: FileInfo[];
|
||||
missingOnPhone: FileInfo[];
|
||||
duplicatesOnPhone: Record<string, string[]>;
|
||||
duplicatesInBackup: Record<string, string[]>;
|
||||
}
|
||||
|
|
@ -43,8 +43,23 @@ function parseArguments() {
|
|||
async function getLocalFiles(dir: string, redis: Redis): Promise<FileInfo[]> {
|
||||
console.log(`\n📁 Scanning local directory: ${dir}`);
|
||||
|
||||
const output = execSync(`find "${dir}" -type f`, { encoding: 'utf-8', maxBuffer: 100 * 1024 * 1024 });
|
||||
const files = output.trim().split('\n').filter(Boolean);
|
||||
// Stream files instead of buffering
|
||||
const files: string[] = [];
|
||||
const find = spawn('find', [dir, '-type', 'f']);
|
||||
const rl = createInterface({ input: find.stdout });
|
||||
|
||||
for await (const line of rl) {
|
||||
if (line.trim()) {
|
||||
files.push(line.trim());
|
||||
}
|
||||
}
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
find.on('close', resolve);
|
||||
find.on('error', reject);
|
||||
});
|
||||
|
||||
console.log(`📊 Found ${files.length} files, getting sizes...`);
|
||||
|
||||
const fileInfos: FileInfo[] = [];
|
||||
const startTime = Date.now();
|
||||
|
|
@ -94,12 +109,22 @@ async function getLocalFiles(dir: string, redis: Redis): Promise<FileInfo[]> {
|
|||
async function getAndroidFiles(dir: string, redis: Redis): Promise<FileInfo[]> {
|
||||
console.log(`\n📱 Scanning Android directory: ${dir}`);
|
||||
|
||||
const output = execSync(`adb shell "find '${dir}' -type f 2>/dev/null"`, {
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 50 * 1024 * 1024
|
||||
});
|
||||
// Stream files instead of buffering
|
||||
const files: string[] = [];
|
||||
const adb = spawn('adb', ['shell', `find '${dir}' -type f 2>/dev/null`]);
|
||||
const rl = createInterface({ input: adb.stdout });
|
||||
|
||||
const files = output.trim().split('\n').filter(Boolean).map(f => f.trim()).filter(Boolean);
|
||||
for await (const line of rl) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed) {
|
||||
files.push(trimmed);
|
||||
}
|
||||
}
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
adb.on('close', resolve);
|
||||
adb.on('error', reject);
|
||||
});
|
||||
|
||||
console.log(`📊 Getting file sizes for ${files.length} files...`);
|
||||
|
||||
|
|
@ -190,16 +215,18 @@ async function getAndroidFiles(dir: string, redis: Redis): Promise<FileInfo[]> {
|
|||
}
|
||||
|
||||
// Progress and ETA
|
||||
const elapsed = Date.now() - startTime;
|
||||
const avgTime = elapsed / processed;
|
||||
const remaining = files.length - processed;
|
||||
const eta = Math.round((avgTime * remaining) / 1000);
|
||||
if (processed > 0) {
|
||||
const elapsed = Date.now() - startTime;
|
||||
const avgTime = elapsed / processed;
|
||||
const remaining = files.length - processed;
|
||||
const eta = Math.round((avgTime * remaining) / 1000);
|
||||
|
||||
const etaStr = eta > 60
|
||||
? `${Math.floor(eta / 60)}m ${eta % 60}s`
|
||||
: `${eta}s`;
|
||||
const etaStr = eta > 60
|
||||
? `${Math.floor(eta / 60)}m ${eta % 60}s`
|
||||
: `${eta}s`;
|
||||
|
||||
process.stdout.write(`\r📊 Progress: ${processed}/${files.length} files | ETA: ${etaStr} `);
|
||||
process.stdout.write(`\r📊 Progress: ${processed}/${files.length} files | ETA: ${etaStr} `);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n✅ Found ${fileInfos.length} Android files`);
|
||||
|
|
@ -485,7 +512,6 @@ async function main() {
|
|||
console.log('\n🔍 Comparing files...');
|
||||
|
||||
const missingInBackup: FileInfo[] = [];
|
||||
const missingOnPhone: FileInfo[] = [];
|
||||
let matched = 0;
|
||||
|
||||
// Files on Android but not in backup
|
||||
|
|
@ -497,13 +523,6 @@ async function main() {
|
|||
}
|
||||
}
|
||||
|
||||
// Files in backup but not on Android
|
||||
for (const [hash, localGroup] of localHashes.entries()) {
|
||||
if (!androidHashes.has(hash)) {
|
||||
missingOnPhone.push(...localGroup);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 7: Find duplicates
|
||||
const duplicatesOnPhone = findDuplicates(androidFiles);
|
||||
const duplicatesInBackup = findDuplicates(localFiles);
|
||||
|
|
@ -512,7 +531,6 @@ async function main() {
|
|||
const results: Results = {
|
||||
matched,
|
||||
missingInBackup,
|
||||
missingOnPhone,
|
||||
duplicatesOnPhone,
|
||||
duplicatesInBackup,
|
||||
};
|
||||
|
|
@ -520,11 +538,15 @@ async function main() {
|
|||
console.log('\n' + '='.repeat(60));
|
||||
console.log('📊 RESULTS');
|
||||
console.log('='.repeat(60));
|
||||
console.log(`✅ Matched files: ${results.matched}`);
|
||||
console.log(`❌ Missing in backup: ${results.missingInBackup.length}`);
|
||||
console.log(`ℹ️ Missing on phone: ${results.missingOnPhone.length}`);
|
||||
console.log(`🔄 Duplicates on phone: ${Object.keys(results.duplicatesOnPhone).length} hashes`);
|
||||
console.log(`🔄 Duplicates in backup: ${Object.keys(results.duplicatesInBackup).length} hashes`);
|
||||
console.log(`📱 Total files on phone: ${androidFiles.length}`);
|
||||
console.log(`✅ Matched in backup: ${results.matched}`);
|
||||
console.log(`❌ MISSING in backup: ${results.missingInBackup.length}`);
|
||||
if (results.missingInBackup.length > 0) {
|
||||
console.log(`\n Missing files:`);
|
||||
results.missingInBackup.forEach(f => console.log(` - ${f.path} (${f.size} bytes)`));
|
||||
}
|
||||
console.log(`\n🔄 Duplicates on phone: ${Object.keys(results.duplicatesOnPhone).length} groups`);
|
||||
console.log(`🔄 Duplicates in backup: ${Object.keys(results.duplicatesInBackup).length} groups`);
|
||||
console.log('='.repeat(60));
|
||||
|
||||
console.log('\n💾 Writing results to results.json...');
|
||||
|
|
|
|||
|
|
@ -0,0 +1,149 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import { execSync } from 'child_process';
|
||||
import { existsSync, mkdirSync } from 'fs';
|
||||
import { readFileSync } from 'fs';
|
||||
import { basename, dirname, join } from 'path';
|
||||
|
||||
interface FileInfo {
|
||||
path: string;
|
||||
size: number;
|
||||
hash?: string;
|
||||
}
|
||||
|
||||
interface Results {
|
||||
matched: number;
|
||||
missingInBackup: FileInfo[];
|
||||
missingOnPhone: FileInfo[];
|
||||
duplicatesOnPhone: Record<string, string[]>;
|
||||
duplicatesInBackup: Record<string, string[]>;
|
||||
}
|
||||
|
||||
function getUniqueFilename(targetPath: string): string {
|
||||
if (!existsSync(targetPath)) {
|
||||
return targetPath;
|
||||
}
|
||||
|
||||
const dir = dirname(targetPath);
|
||||
const ext = targetPath.match(/\.[^.]+$/)?.[0] || '';
|
||||
const nameWithoutExt = basename(targetPath, ext);
|
||||
|
||||
let counter = 1;
|
||||
let newPath: string;
|
||||
|
||||
do {
|
||||
newPath = join(dir, `${nameWithoutExt}-${counter}${ext}`);
|
||||
counter++;
|
||||
} while (existsSync(newPath));
|
||||
|
||||
return newPath;
|
||||
}
|
||||
|
||||
async function pullFile(androidPath: string, localDir: string): Promise<string> {
|
||||
const filename = basename(androidPath);
|
||||
const targetPath = join(localDir, filename);
|
||||
const finalPath = getUniqueFilename(targetPath);
|
||||
|
||||
console.log(`📥 Pulling: ${androidPath}`);
|
||||
console.log(` -> ${finalPath}`);
|
||||
|
||||
try {
|
||||
execSync(`adb pull "${androidPath}" "${finalPath}"`, {
|
||||
encoding: 'utf-8',
|
||||
stdio: 'pipe'
|
||||
});
|
||||
return finalPath;
|
||||
} catch (err: any) {
|
||||
console.error(`❌ Failed to pull: ${androidPath}`);
|
||||
console.error(` Error: ${err.message}`);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const resultsPath = 'results.json';
|
||||
|
||||
if (!existsSync(resultsPath)) {
|
||||
console.error(`❌ Error: ${resultsPath} not found`);
|
||||
console.error('Run photos-diff.ts first to generate results.json');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('📖 Reading results.json...');
|
||||
const results: Results = JSON.parse(readFileSync(resultsPath, 'utf-8'));
|
||||
|
||||
const missingFiles = results.missingInBackup;
|
||||
|
||||
if (missingFiles.length === 0) {
|
||||
console.log('✅ No missing files! Backup is complete.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`\n🔍 Found ${missingFiles.length} missing files`);
|
||||
console.log(`📦 Total size: ${(missingFiles.reduce((sum, f) => sum + f.size, 0) / 1024 / 1024).toFixed(2)} MB`);
|
||||
|
||||
// Create target directory if needed
|
||||
const targetDir = './pulled-files';
|
||||
if (!existsSync(targetDir)) {
|
||||
console.log(`\n📁 Creating directory: ${targetDir}`);
|
||||
mkdirSync(targetDir, { recursive: true });
|
||||
}
|
||||
|
||||
console.log(`\n🚀 Starting download to: ${targetDir}\n`);
|
||||
|
||||
const startTime = Date.now();
|
||||
const pulled: string[] = [];
|
||||
const failed: string[] = [];
|
||||
|
||||
for (let i = 0; i < missingFiles.length; i++) {
|
||||
const file = missingFiles[i];
|
||||
|
||||
try {
|
||||
const localPath = await pullFile(file.path, targetDir);
|
||||
pulled.push(localPath);
|
||||
} catch (err) {
|
||||
failed.push(file.path);
|
||||
}
|
||||
|
||||
// Progress
|
||||
const processed = i + 1;
|
||||
const elapsed = Date.now() - startTime;
|
||||
const avgTime = elapsed / processed;
|
||||
const remaining = missingFiles.length - processed;
|
||||
const eta = Math.round((avgTime * remaining) / 1000);
|
||||
|
||||
const etaStr = eta > 60
|
||||
? `${Math.floor(eta / 60)}m ${eta % 60}s`
|
||||
: `${eta}s`;
|
||||
|
||||
console.log(`\n📊 Progress: ${processed}/${missingFiles.length} | ETA: ${etaStr}`);
|
||||
console.log(` ✅ Success: ${pulled.length} | ❌ Failed: ${failed.length}\n`);
|
||||
}
|
||||
|
||||
// Final report
|
||||
const totalTime = Math.round((Date.now() - startTime) / 1000);
|
||||
const timeStr = totalTime > 60
|
||||
? `${Math.floor(totalTime / 60)}m ${totalTime % 60}s`
|
||||
: `${totalTime}s`;
|
||||
|
||||
console.log('='.repeat(60));
|
||||
console.log('📊 DOWNLOAD COMPLETE');
|
||||
console.log('='.repeat(60));
|
||||
console.log(`✅ Successfully pulled: ${pulled.length} files`);
|
||||
console.log(`❌ Failed: ${failed.length} files`);
|
||||
console.log(`⏱️ Total time: ${timeStr}`);
|
||||
console.log(`📁 Files saved to: ${targetDir}`);
|
||||
console.log('='.repeat(60));
|
||||
|
||||
if (failed.length > 0) {
|
||||
console.log('\n❌ Failed files:');
|
||||
failed.forEach(f => console.log(` - ${f}`));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('💥 Fatal error:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
23
bun.lock
23
bun.lock
|
|
@ -4,6 +4,7 @@
|
|||
"": {
|
||||
"dependencies": {
|
||||
"chalk": "^5.5.0",
|
||||
"iovalkey": "^0.3.3",
|
||||
"ollama": "^0.5.17",
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
@ -12,12 +13,34 @@
|
|||
},
|
||||
},
|
||||
"packages": {
|
||||
"@iovalkey/commands": ["@iovalkey/commands@0.1.0", "", {}, "sha512-/B9W4qKSSITDii5nkBCHyPkIkAi+ealUtr1oqBJsLxjSRLka4pxun2VvMNSmcwgAMxgXtQfl0qRv7TE+udPJzg=="],
|
||||
|
||||
"@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="],
|
||||
|
||||
"chalk": ["chalk@5.5.0", "", {}, "sha512-1tm8DTaJhPBG3bIkVeZt1iZM9GfSX2lzOeDVZH9R9ffRHpmHvxZ/QhgQH/aDTkswQVt+YHdXAdS/In/30OjCbg=="],
|
||||
|
||||
"cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="],
|
||||
|
||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||
|
||||
"denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="],
|
||||
|
||||
"iovalkey": ["iovalkey@0.3.3", "", { "dependencies": { "@iovalkey/commands": "^0.1.0", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" } }, "sha512-4rTJX6Q5wTYEvxboXi8DsEiUo+OvqJGtLYOSGm37KpdRXsG5XJjbVtYKGJpPSWP+QT7rWscA4vsrdmzbEbenpw=="],
|
||||
|
||||
"lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="],
|
||||
|
||||
"lodash.isarguments": ["lodash.isarguments@3.1.0", "", {}, "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="],
|
||||
|
||||
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
||||
|
||||
"ollama": ["ollama@0.5.17", "", { "dependencies": { "whatwg-fetch": "^3.6.20" } }, "sha512-q5LmPtk6GLFouS+3aURIVl+qcAOPC4+Msmx7uBb3pd+fxI55WnGjmLZ0yijI/CYy79x0QPGx3BwC3u5zv9fBvQ=="],
|
||||
|
||||
"redis-errors": ["redis-errors@1.2.0", "", {}, "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="],
|
||||
|
||||
"redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="],
|
||||
|
||||
"standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="],
|
||||
|
||||
"undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],
|
||||
|
||||
"whatwg-fetch": ["whatwg-fetch@3.6.20", "", {}, "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg=="],
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"chalk": "^5.5.0",
|
||||
"ollama": "^0.5.17"
|
||||
"ollama": "^0.5.17",
|
||||
"iovalkey": "^0.3.3"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue