pull-missing.ts

bin/photos-diff.ts

@@ -1,10 +1,11 @@
 #!/usr/bin/env node
 
-import { execSync } from 'child_process';
+import { execSync, spawn } from 'child_process';
 import { createHash } from 'crypto';
 import { readFileSync, statSync, writeFileSync } from 'fs';
 import { parseArgs } from 'util';
 import Redis from 'iovalkey';
+import { createInterface } from 'readline';
 
 const CACHE_TTL = 3600; // 1 hour
 const CACHE_PREFIX_ANDROID = 'photo-sync:android:';
@@ -19,7 +20,6 @@ interface FileInfo {
 interface Results {
   matched: number;
   missingInBackup: FileInfo[];
-  missingOnPhone: FileInfo[];
   duplicatesOnPhone: Record<string, string[]>;
   duplicatesInBackup: Record<string, string[]>;
 }
@@ -43,8 +43,23 @@ function parseArguments() {
 async function getLocalFiles(dir: string, redis: Redis): Promise<FileInfo[]> {
   console.log(`\n📁 Scanning local directory: ${dir}`);
 
-  const output = execSync(`find "${dir}" -type f`, { encoding: 'utf-8', maxBuffer: 100 * 1024 * 1024 });
-  const files = output.trim().split('\n').filter(Boolean);
+  // Stream files instead of buffering
+  const files: string[] = [];
+  const find = spawn('find', [dir, '-type', 'f']);
+  const rl = createInterface({ input: find.stdout });
+
+  for await (const line of rl) {
+    if (line.trim()) {
+      files.push(line.trim());
+    }
+  }
+
+  await new Promise((resolve, reject) => {
+    find.on('close', resolve);
+    find.on('error', reject);
+  });
+
+  console.log(`📊 Found ${files.length} files, getting sizes...`);
 
   const fileInfos: FileInfo[] = [];
   const startTime = Date.now();
@@ -94,12 +109,22 @@ async function getLocalFiles(dir: string, redis: Redis): Promise<FileInfo[]> {
 async function getAndroidFiles(dir: string, redis: Redis): Promise<FileInfo[]> {
   console.log(`\n📱 Scanning Android directory: ${dir}`);
 
-  const output = execSync(`adb shell "find '${dir}' -type f 2>/dev/null"`, {
-    encoding: 'utf-8',
-    maxBuffer: 50 * 1024 * 1024
-  });
+  // Stream files instead of buffering
+  const files: string[] = [];
+  const adb = spawn('adb', ['shell', `find '${dir}' -type f 2>/dev/null`]);
+  const rl = createInterface({ input: adb.stdout });
 
-  const files = output.trim().split('\n').filter(Boolean).map(f => f.trim()).filter(Boolean);
+  for await (const line of rl) {
+    const trimmed = line.trim();
+    if (trimmed) {
+      files.push(trimmed);
+    }
+  }
+
+  await new Promise((resolve, reject) => {
+    adb.on('close', resolve);
+    adb.on('error', reject);
+  });
 
   console.log(`📊 Getting file sizes for ${files.length} files...`);
 
@@ -190,16 +215,18 @@ async function getAndroidFiles(dir: string, redis: Redis): Promise<FileInfo[]> {
     }
 
     // Progress and ETA
-    const elapsed = Date.now() - startTime;
-    const avgTime = elapsed / processed;
-    const remaining = files.length - processed;
-    const eta = Math.round((avgTime * remaining) / 1000);
+    if (processed > 0) {
+      const elapsed = Date.now() - startTime;
+      const avgTime = elapsed / processed;
+      const remaining = files.length - processed;
+      const eta = Math.round((avgTime * remaining) / 1000);
 
-    const etaStr = eta > 60
-      ? `${Math.floor(eta / 60)}m ${eta % 60}s`
-      : `${eta}s`;
+      const etaStr = eta > 60
+        ? `${Math.floor(eta / 60)}m ${eta % 60}s`
+        : `${eta}s`;
 
-    process.stdout.write(`\r📊 Progress: ${processed}/${files.length} files | ETA: ${etaStr}     `);
+      process.stdout.write(`\r📊 Progress: ${processed}/${files.length} files | ETA: ${etaStr}     `);
+    }
   }
 
   console.log(`\n✅ Found ${fileInfos.length} Android files`);
@@ -485,7 +512,6 @@ async function main() {
     console.log('\n🔍 Comparing files...');
 
     const missingInBackup: FileInfo[] = [];
-    const missingOnPhone: FileInfo[] = [];
     let matched = 0;
 
     // Files on Android but not in backup
@@ -497,13 +523,6 @@ async function main() {
       }
     }
 
-    // Files in backup but not on Android
-    for (const [hash, localGroup] of localHashes.entries()) {
-      if (!androidHashes.has(hash)) {
-        missingOnPhone.push(...localGroup);
-      }
-    }
-
     // Step 7: Find duplicates
     const duplicatesOnPhone = findDuplicates(androidFiles);
     const duplicatesInBackup = findDuplicates(localFiles);
@@ -512,7 +531,6 @@ async function main() {
     const results: Results = {
       matched,
       missingInBackup,
-      missingOnPhone,
       duplicatesOnPhone,
       duplicatesInBackup,
     };
@@ -520,11 +538,15 @@ async function main() {
     console.log('\n' + '='.repeat(60));
     console.log('📊 RESULTS');
     console.log('='.repeat(60));
-    console.log(`✅ Matched files: ${results.matched}`);
-    console.log(`❌ Missing in backup: ${results.missingInBackup.length}`);
-    console.log(`ℹ️  Missing on phone: ${results.missingOnPhone.length}`);
-    console.log(`🔄 Duplicates on phone: ${Object.keys(results.duplicatesOnPhone).length} hashes`);
-    console.log(`🔄 Duplicates in backup: ${Object.keys(results.duplicatesInBackup).length} hashes`);
+    console.log(`📱 Total files on phone: ${androidFiles.length}`);
+    console.log(`✅ Matched in backup: ${results.matched}`);
+    console.log(`❌ MISSING in backup: ${results.missingInBackup.length}`);
+    if (results.missingInBackup.length > 0) {
+      console.log(`\n   Missing files:`);
+      results.missingInBackup.forEach(f => console.log(`   - ${f.path} (${f.size} bytes)`));
+    }
+    console.log(`\n🔄 Duplicates on phone: ${Object.keys(results.duplicatesOnPhone).length} groups`);
+    console.log(`🔄 Duplicates in backup: ${Object.keys(results.duplicatesInBackup).length} groups`);
     console.log('='.repeat(60));
 
     console.log('\n💾 Writing results to results.json...');

bin/pull-missing.ts

@@ -0,0 +1,149 @@
+#!/usr/bin/env node
+
+import { execSync } from 'child_process';
+import { existsSync, mkdirSync } from 'fs';
+import { readFileSync } from 'fs';
+import { basename, dirname, join } from 'path';
+
+interface FileInfo {
+  path: string;
+  size: number;
+  hash?: string;
+}
+
+interface Results {
+  matched: number;
+  missingInBackup: FileInfo[];
+  missingOnPhone: FileInfo[];
+  duplicatesOnPhone: Record<string, string[]>;
+  duplicatesInBackup: Record<string, string[]>;
+}
+
+function getUniqueFilename(targetPath: string): string {
+  if (!existsSync(targetPath)) {
+    return targetPath;
+  }
+
+  const dir = dirname(targetPath);
+  const ext = targetPath.match(/\.[^.]+$/)?.[0] || '';
+  const nameWithoutExt = basename(targetPath, ext);
+
+  let counter = 1;
+  let newPath: string;
+
+  do {
+    newPath = join(dir, `${nameWithoutExt}-${counter}${ext}`);
+    counter++;
+  } while (existsSync(newPath));
+
+  return newPath;
+}
+
+async function pullFile(androidPath: string, localDir: string): Promise<string> {
+  const filename = basename(androidPath);
+  const targetPath = join(localDir, filename);
+  const finalPath = getUniqueFilename(targetPath);
+
+  console.log(`📥 Pulling: ${androidPath}`);
+  console.log(`   -> ${finalPath}`);
+
+  try {
+    execSync(`adb pull "${androidPath}" "${finalPath}"`, {
+      encoding: 'utf-8',
+      stdio: 'pipe'
+    });
+    return finalPath;
+  } catch (err: any) {
+    console.error(`❌ Failed to pull: ${androidPath}`);
+    console.error(`   Error: ${err.message}`);
+    throw err;
+  }
+}
+
+async function main() {
+  const resultsPath = 'results.json';
+
+  if (!existsSync(resultsPath)) {
+    console.error(`❌ Error: ${resultsPath} not found`);
+    console.error('Run photos-diff.ts first to generate results.json');
+    process.exit(1);
+  }
+
+  console.log('📖 Reading results.json...');
+  const results: Results = JSON.parse(readFileSync(resultsPath, 'utf-8'));
+
+  const missingFiles = results.missingInBackup;
+  
+  if (missingFiles.length === 0) {
+    console.log('✅ No missing files! Backup is complete.');
+    return;
+  }
+
+  console.log(`\n🔍 Found ${missingFiles.length} missing files`);
+  console.log(`📦 Total size: ${(missingFiles.reduce((sum, f) => sum + f.size, 0) / 1024 / 1024).toFixed(2)} MB`);
+
+  // Create target directory if needed
+  const targetDir = './pulled-files';
+  if (!existsSync(targetDir)) {
+    console.log(`\n📁 Creating directory: ${targetDir}`);
+    mkdirSync(targetDir, { recursive: true });
+  }
+
+  console.log(`\n🚀 Starting download to: ${targetDir}\n`);
+
+  const startTime = Date.now();
+  const pulled: string[] = [];
+  const failed: string[] = [];
+
+  for (let i = 0; i < missingFiles.length; i++) {
+    const file = missingFiles[i];
+    
+    try {
+      const localPath = await pullFile(file.path, targetDir);
+      pulled.push(localPath);
+    } catch (err) {
+      failed.push(file.path);
+    }
+
+    // Progress
+    const processed = i + 1;
+    const elapsed = Date.now() - startTime;
+    const avgTime = elapsed / processed;
+    const remaining = missingFiles.length - processed;
+    const eta = Math.round((avgTime * remaining) / 1000);
+
+    const etaStr = eta > 60
+      ? `${Math.floor(eta / 60)}m ${eta % 60}s`
+      : `${eta}s`;
+
+    console.log(`\n📊 Progress: ${processed}/${missingFiles.length} | ETA: ${etaStr}`);
+    console.log(`   ✅ Success: ${pulled.length} | ❌ Failed: ${failed.length}\n`);
+  }
+
+  // Final report
+  const totalTime = Math.round((Date.now() - startTime) / 1000);
+  const timeStr = totalTime > 60
+    ? `${Math.floor(totalTime / 60)}m ${totalTime % 60}s`
+    : `${totalTime}s`;
+
+  console.log('='.repeat(60));
+  console.log('📊 DOWNLOAD COMPLETE');
+  console.log('='.repeat(60));
+  console.log(`✅ Successfully pulled: ${pulled.length} files`);
+  console.log(`❌ Failed: ${failed.length} files`);
+  console.log(`⏱️  Total time: ${timeStr}`);
+  console.log(`📁 Files saved to: ${targetDir}`);
+  console.log('='.repeat(60));
+
+  if (failed.length > 0) {
+    console.log('\n❌ Failed files:');
+    failed.forEach(f => console.log(`   - ${f}`));
+    process.exit(1);
+  }
+}
+
+main().catch(err => {
+  console.error('💥 Fatal error:', err);
+  process.exit(1);
+});
+