Merge pull request #59 from torarnv/improve-file-backups

Improve file backups

package-lock.json

@@ -5111,6 +5111,11 @@
       "integrity": "sha1-1PM6tU6OOHeLDKXP07OvsS22hiA=",
       "dev": true
     },
+    "stat-mode": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/stat-mode/-/stat-mode-1.0.0.tgz",
+      "integrity": "sha512-jH9EhtKIjuXZ2cWxmXS8ZP80XyC3iasQxMDV8jzhNJpfDb7VbQLVW4Wvsxz9QZvzV+G4YoSfBUVKDOyxLzi/sg=="
+    },
     "string-width": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",

package.json

@@ -23,6 +23,7 @@
     "json2csv": "^3.11.5",
     "plist": "^2.1.0",
     "sqlite3": "^4.0.0",
+    "stat-mode": "^1.0.0",
     "strip-ansi": "^4.0.0"
   },
   "devDependencies": {

tools/reports/backup/files.js

@@ -2,6 +2,8 @@ const fs = require('fs-extra')
 const path = require('path')
 const log = require('../../util/log')
 const manifestMBDBParse = require('../../util/manifest_mbdb_parse')
+const plist = require('../../util/plist')
+const Mode = require('stat-mode');
 
 module.exports = {
   version: 4,
@@ -30,7 +32,9 @@ module.exports = {
     id: el => el.fileID,
     domain: el => el.domain,
     path: el => el.filename,
-    size: el => el.filelen || 0
+    size: el => el.filelen || 0,
+    mtime: el => el.mtime || 0,
+    mode: el => new Mode(el).toString()
   }
 }
 
@@ -39,9 +43,18 @@ function getSqliteFileManifest (backup) {
   return new Promise(async (resolve, reject) => {
     backup.openDatabase('Manifest.db', true)
       .then(db => {
-        db.all('SELECT fileID, domain, relativePath as filename from FILES', async function (err, rows) {
+        db.all('SELECT fileID, domain, relativePath as filename, file from FILES', async function (err, rows) {
           if (err) reject(err)
 
+          // Extract binary plist metadata
+          for (var row of rows) {
+            let data = plist.parseBuffer(row.file)
+            let metadata = data['$objects'][1];
+            row.filelen = metadata.Size
+            row.mode = metadata.Mode
+            row.mtime = row.atime = metadata.LastModified
+          }
+
           resolve(rows)
         })
       })
@@ -83,11 +96,12 @@ function getManifest (backup) {
 }
 
 /// Filter exclusion check
-function isIncludedByFilter (filter, item) {
+function isIncludedByFilter (filter, item, filePath) {
   return filter === 'all' ||
     filter === undefined ||
     (filter && item.domain.indexOf(filter) > -1) ||
-    (filter && item.filename.indexOf(filter) > -1)
+    (filter && item.filename.indexOf(filter) > -1) ||
+    (filePath.indexOf(filter) > -1)
 }
 
 /// Extract files
@@ -97,36 +111,56 @@ function isIncludedByFilter (filter, item) {
 /// - items: list of files.
 function extractFiles (backup, destination, filter, items) {
   for (var item of items) {
-    // Filter by the domain.
-    // Simple "Contains" Search
-    if (!isIncludedByFilter(filter, item)) {
-      // Skip to the next iteration of the loop.
-      log.action('skipped', item.filename)
-      continue
-    }
-
     try {
-      let sourceFile = backup.getFileName(item.fileID)
-      var stat = fs.lstatSync(sourceFile)
+      var domainPath = item.domain
+      if (domainPath.match(/^AppDomain.*-/)) {
+        // Extract sub-domain from app domain
+        domainPath = domainPath.replace('-', path.sep)
+      }
 
-      // Only process files that exist.
-      if (stat.isFile() && fs.existsSync(sourceFile)) {
-        log.action('export', item.filename)
+      domainPath = domainPath.replace('Domain', '')
 
-        // Calculate the output dir.
-        var outDir = path.join(destination, item.domain, item.filename)
+      var filePath = path.join(domainPath, item.filename)
 
-        // Create the directory and copy
-        fs.ensureDirSync(path.dirname(outDir))
-        fs.copySync(sourceFile, outDir)
+      // Skip items not included by the filter
+      if (!isIncludedByFilter(filter, item, filePath)) {
+        // Skip to the next iteration of the loop.
+        log.action('skipped', filePath)
+        continue
+      }
 
-        // Save output info to the data item.
-        item.output_dir = outDir
-      } else if (stat.isDirectory()) {
-      // Do nothing..
+      var stat = new Mode(item)
+
+      if (stat.isSymbolicLink()) {
+        log.warning('skipping symlink', filePath, 'to', item.linktarget)
+        // FIXME: Restore symlinks
+        continue
+      }
+
+      // Calculate the output path
+      var outPath = path.join(destination, filePath)
+
+      if (stat.isDirectory()) {
+        log.action('mkdir', filePath)
+        fs.ensureDirSync(outPath)
+      } else if (stat.isFile()) {
+        let sourceFile = backup.getFileName(item.fileID)
+
+        // Only process files that exist.
+        if (fs.existsSync(sourceFile)) {
+          log.action('export', filePath)
+          fs.copySync(sourceFile, outPath)
+          fs.utimesSync(outPath, item.atime, item.mtime)
+        } else {
+          log.error('not found', sourceFile)
+        }
       } else {
-        log.error('not found', sourceFile)
+        throw new Error('unknown filetype')
       }
+
+      // Save output info to the data item.
+      item.output_dir = outPath
+
     } catch (e) {
       log.error(item.fileID, item.filename, e.toString())
     }

tools/reports/backup/info.js

@@ -1,9 +1,8 @@
 const fs = require('fs')
 const path = require('path')
-const plist = require('plist')
-const bplist = require('bplist-parser')
 
 const log = require('../../util/log')
+const plist = require('../../util/plist')
 
 module.exports = {
   version: 4,
@@ -16,28 +15,11 @@ module.exports = {
     // Get the path for the info plist.
     let infoPath = path.join(backup.path, 'Info.plist')
 
-    let fd = fs.openSync(infoPath, 'r')
-    let buffer = Buffer.alloc(7)
-    // Read the first 7 bytes into the buffer.
-    fs.readSync(fd, buffer, 0, 7, 0)
-    fs.closeSync(fd)
+    log.verbose('parsing info', infoPath)
+    var data = plist.parseFile(infoPath)
 
-    var data
-    // Binary plists have the marker 'bplist0'
-    if (buffer.toString('ascii') === 'bplist0') {
-      // Parse as binary plist
-      log.verbose('parsing manifest', infoPath)
-      data = bplist.parseBuffer(fs.readFileSync(infoPath))[0]
-
-      // Remove this data, it's kind of useless.
-      delete data['iTunes Files']
-    } else {
-      // Parse as normal plist.
-      log.verbose('parsing info', infoPath)
-      data = plist.parse(fs.readFileSync(infoPath, 'utf8'))
-
-      delete data['iTunes Files']
-    }
+    // Remove this data, it's kind of useless.
+    delete data['iTunes Files']
 
     return data
   },

tools/reports/backup/manifest.js

@@ -2,7 +2,7 @@ const fs = require('fs')
 const path = require('path')
 
 const log = require('../../util/log')
-const bplist = require('bplist-parser')
+const plist = require('../../util/plist')
 
 module.exports = {
   version: 4,
@@ -14,7 +14,7 @@ module.exports = {
   async run (lib, { backup }) {
     // Load and parse the maniest for the backup.
     log.verbose('parsing manifest', backup.path)
-    let data = bplist.parseBuffer(fs.readFileSync(path.join(backup.path, 'Manifest.plist')))[0]
+    let data = plist.parseFile(path.join(backup.path, 'Manifest.plist'))
 
     // Remove this data, it's kind of useless.
     delete data['BackupKeyBag']

tools/reports/backup/status.js

@@ -2,7 +2,7 @@ const fs = require('fs')
 const path = require('path')
 
 const log = require('../../util/log')
-const bplist = require('bplist-parser')
+const plist = require('../../util/plist')
 
 module.exports = {
   version: 4,
@@ -14,9 +14,7 @@ module.exports = {
   async run (lib, { backup }) {
     // Load and parse status for the backup.
     log.verbose('parsing status', backup.path)
-    let data = bplist.parseBuffer(fs.readFileSync(path.join(backup.path, 'Status.plist')))[0]
-
-    return data
+    return plist.parseFile(path.join(backup.path, 'Status.plist'))
   },
 
   // Status fields.

tools/reports/backups/list.js

@@ -23,6 +23,11 @@ module.exports = {
         results.push(result)
       }
 
+      // Sort by descending dates
+      results.sort(function(a, b) {
+        return b.status.date - a.status.date;
+      });
+
       resolve(results)
     })
   },
@@ -31,8 +36,8 @@ module.exports = {
     udid: el => el.id,
     encrypted: el => el.manifest ? (!!el.manifest.IsEncrypted) : false,
     date: el => el.status ? new Date(el.status.date).toLocaleString() : '',
-    deviceName: el => el.manifest && el.manifest.Lockdown ? el.manifest.Lockdown.DeviceName : 'Unknown Device',
-    serialNumber: el => el.manifest && el.manifest.Lockdown ? el.manifest.Lockdown.SerialNumber : 'Unknown Serial #',
+    deviceName: el => el.info ? el.info.deviceName : 'Unknown Device',
+    serialNumber: el => el.info ? el.info.serialNumber : 'Unknown Serial #',
     iOSVersion: el => el.manifest && el.manifest.Lockdown ? el.manifest.Lockdown.ProductVersion : '?',
     backupVersion: el => el.status ? el.status.version : '?'
   }

tools/reports/messages/conversations.js

@@ -1,4 +1,4 @@
-const bplist = require('bplist-parser')
+const plist = require('../../util/plist')
 
 const fileHash = require('../../util/backup_filehash')
 const log = require('../../util/log')
@@ -39,7 +39,7 @@ function getConversationsiOS9 (backup) {
           // The timestamp information is stored in a binary blob named `properties`
           // Which is formatted as a binary PLIST.
           for (var el of rows) {
-            if (el.properties) el.properties = bplist.parseBuffer(el.properties)[0]
+            if (el.properties) el.properties = plist.parseBuffer(el.properties)
 
             // Interestingly, some of these do not have dates attached.
             if (el.properties) {

tools/reports/phone/speed_dial.js

@@ -1,5 +1,5 @@
 
-const bplist = require('bplist-parser')
+const plist = require('../../util/plist')
 const fs = require('fs')
 
 // Derive filenames based on domain + file path
@@ -41,7 +41,7 @@ const speedDialReport = (backup) => {
   return new Promise((resolve, reject) => {
     try {
       var filename = backup.getFileName(file)
-      let speeddialPlist = bplist.parseBuffer(fs.readFileSync(filename))[0]
+      let speeddialPlist = plist.parseFile(filename)
 
       resolve(speeddialPlist)
     } catch (e) {

tools/reports/safari/recent_searches.js

@@ -1,5 +1,5 @@
 const fs = require('fs')
-const bplist = require('bplist-parser')
+const plist = require('../../util/plist')
 const fileHash = require('../../util/backup_filehash')
 
 const SAFARI_PLIST = fileHash('Library/Preferences/com.apple.mobilesafari.plist', 'AppDomain-com.apple.mobilesafari')
@@ -28,7 +28,7 @@ const safariRecentSearches = (backup) => {
     try {
       // Get the filename of the ID
       var filename = backup.getFileName(SAFARI_PLIST)
-      let mobilesafariPlist = bplist.parseBuffer(fs.readFileSync(filename))[0]
+      let mobilesafariPlist = plist.parseFile(filename)
 
       resolve(mobilesafariPlist['RecentWebSearches'])
     } catch (e) {

tools/reports/system/pushstore.js

@@ -1,5 +1,5 @@
 const fs = require('fs')
-const bplist = require('bplist-parser')
+const plist = require('../../util/plist')
 const pushstoreParse = require('../../util/pushstore_parse')
 
 module.exports = {
@@ -36,8 +36,8 @@ module.exports = {
 
         // For each file, run a parse on the plist.
         files.forEach((file) => {
-          let plist = bplist.parseBuffer(fs.readFileSync(backup.getFileName(file.id)))[0]
-          pushstores.push(...pushstoreParse.run(plist))
+          let data = plist.parseFile(backup.getFileName(file.id))
+          pushstores.push(...pushstoreParse.run(data))
         })
 
         resolve(pushstores)

tools/reports/system/wifi.js

@@ -1,4 +1,4 @@
-const bplist = require('bplist-parser')
+const plist = require('../../util/plist')
 const fs = require('fs')
 
 // Normalize mac addresses in wifi output
@@ -23,7 +23,7 @@ module.exports = {
         var filename = backup.getFileName(WIFI_PLIST)
 
         // Attempt to parse it
-        let wifiList = bplist.parseBuffer(fs.readFileSync(filename))[0]
+        let wifiList = plist.parseFile(filename)
         let result = wifiList['List of known networks']
           .map(el => {
             if (el.BSSID) {

tools/reports/thirdparty/facebook/profile.js

@@ -1,4 +1,4 @@
-const bplist = require('bplist-parser')
+const plist = require('../../../util/plist')
 const fs = require('fs')
 
 // Derive filenames based on domain + file path
@@ -28,7 +28,7 @@ const facebookProfileReport = (backup) => {
   return new Promise((resolve, reject) => {
     var filename = backup.getFileName(file)
     try {
-      let facebookPlist = bplist.parseBuffer(fs.readFileSync(filename))[0]
+      let facebookPlist = plist.parseFile(filename)
       let facebookUserIds = Object.keys(facebookPlist['kUserGlobalSettings'])
       facebookUserIds = facebookUserIds.map((fbid) => ({
         fbid: fbid

tools/reports/thirdparty/gmail/accounts.js

@@ -1,4 +1,4 @@
-const bplist = require('bplist-parser')
+const plist = require('../../../util/plist')
 const fs = require('fs')
 
 // Derive filenames based on domain + file path
@@ -32,7 +32,7 @@ const gmailAccountsReport = (backup) => {
   return new Promise((resolve, reject) => {
     var filename = backup.getFileName(file)
     try {
-      let gmailPlist = bplist.parseBuffer(fs.readFileSync(filename))[0]
+      let gmailPlist = plist.parseFile(filename)
       let gmailAccountIds = Object.keys(gmailPlist).filter(key => key.indexOf('kIdToEmailMapKey') !== -1)
       let gmailAvatars = Object.keys(gmailPlist).filter(key => key.indexOf('kCurrentAvatarUrlKey') !== -1)
       gmailAvatars = gmailAvatars.map(avatarKey => {

tools/reports/thirdparty/gmail/shared_contacts.js

@@ -1,4 +1,4 @@
-const bplist = require('bplist-parser')
+const plist = require('../../../util/plist')
 const fs = require('fs')
 
 // Derive filenames based on domain + file path
@@ -33,7 +33,7 @@ const gmailAccountsReport = (backup) => {
   return new Promise((resolve, reject) => {
     var filename = backup.getFileName(file)
     try {
-      let gmailPlist = bplist.parseBuffer(fs.readFileSync(filename))[0]
+      let gmailPlist = plist.parseFile(filename)
       let gmailAccountIds = Object.keys(gmailPlist).filter(key => key.indexOf('kIdToEmailMapKey') !== -1)
       let gmailContactsByAccount = Object.keys(gmailPlist).filter(key => key.indexOf('kInboxSharedStorageContacts') !== -1)
       gmailContactsByAccount = gmailContactsByAccount.map(contactsKey => {

tools/reports/thirdparty/instagram/fb_friends.js

@@ -1,4 +1,4 @@
-const bplist = require('bplist-parser')
+const plist = require('../../../util/plist')
 const fs = require('fs')
 
 // Derive filenames based on domain + file path
@@ -33,7 +33,7 @@ const instagramRecentSearchesReport = (backup) => {
     var results = []
     var filename = backup.getFileName(file)
     try {
-      let instagramPlist = bplist.parseBuffer(fs.readFileSync(filename))[0]
+      let instagramPlist = plist.parseFile(filename)
       let regex = /[0-9]*-fb-friends$/g
       let fbFriendsKey = Object.keys(instagramPlist).filter(key => regex.test(key))
       console.log(fbFriendsKey)

tools/reports/thirdparty/instagram/following_users_coded.js

@@ -1,4 +1,4 @@
-const bplist = require('bplist-parser')
+const plist = require('../../../util/plist')
 const fs = require('fs')
 
 // Derive filenames based on domain + file path
@@ -28,7 +28,7 @@ const instagramRecentSearchesReport = (backup) => {
     var results = []
     var filename = backup.getFileName(file)
     try {
-      let instagramPlist = bplist.parseBuffer(fs.readFileSync(filename))[0]
+      let instagramPlist = plist.parseFile(filename)
       let followingUsersKey = Object.keys(instagramPlist).filter(key => key.indexOf('-following-users.coded') !== -1)
       followingUsersKey.forEach(key => {
         let followingUsers = instagramPlist[key]

tools/reports/thirdparty/instagram/profile.js

@@ -1,4 +1,4 @@
-const bplist = require('bplist-parser')
+const plist = require('../../../util/plist')
 const fs = require('fs')
 
 // Derive filenames based on domain + file path
@@ -35,7 +35,7 @@ const instagramProfileReport = (backup) => {
     var results = []
     var filename = backup.getFileName(file)
     try {
-      let instagramPlist = bplist.parseBuffer(fs.readFileSync(filename))[0]
+      let instagramPlist = plist.parseFile(filename)
 
       results.push(new KeyValue('last-logged-in-username', instagramPlist))
       results.push(new KeyValue('prefill_fb_email', instagramPlist))

tools/reports/thirdparty/instagram/recent_searches.js

@@ -1,4 +1,4 @@
-const bplist = require('bplist-parser')
+const plist = require('../../../util/plist')
 const fs = require('fs')
 
 // Derive filenames based on domain + file path
@@ -31,7 +31,7 @@ const instagramRecentSearchesReport = (backup) => {
     var results = []
     var filename = backup.getFileName(file)
     try {
-      let instagramPlist = bplist.parseBuffer(fs.readFileSync(filename))[0]
+      let instagramPlist = plist.parseFile(filename)
       let recentSearchesKey = Object.keys(instagramPlist).filter(key => key.indexOf('-blended-search-recent-item-order') !== -1)
       recentSearchesKey.forEach(key => {
         let recentSearches = instagramPlist[key]

tools/reports/thirdparty/spotify/searches.js

@@ -1,4 +1,4 @@
-const bplist = require('bplist-parser')
+const plist = require('../../../util/plist')
 const fs = require('fs')
 
 // Derive filenames based on domain + file path
@@ -31,7 +31,7 @@ const spotifyReport = (backup) => {
   return new Promise((resolve, reject) => {
     var filename = backup.getFileName(database)
     try {
-      let spotifyData = bplist.parseBuffer(fs.readFileSync(filename))[0]
+      let spotifyData = plist.parseFile(filename)
       let spotifyResult = []
 
       console.log('spotifyData', spotifyData)

tools/reports/thirdparty/waze/favorites.js

@@ -1,9 +1,7 @@
 const log = require('../../../util/log')
 const path = require('path')
 const sqlite3 = require('sqlite3')
-const bplist = require('bplist-parser')
 const fs = require('fs')
-const plist = require('plist')
 
 // Derive filenames based on domain + file path
 const fileHash = require('../../../util/backup_filehash')

tools/reports/thirdparty/waze/places.js

@@ -1,10 +1,7 @@
 const log = require('../../../util/log')
 const path = require('path')
 const sqlite3 = require('sqlite3')
-const bplist = require('bplist-parser')
 const fs = require('fs')
-const plist = require('plist')
-
 
 // Derive filenames based on domain + file path
 const fileHash = require('../../../util/backup_filehash')

tools/reports/thirdparty/waze/recents.js

@@ -1,9 +1,7 @@
 const log = require('../../../util/log')
 const path = require('path')
 const sqlite3 = require('sqlite3')
-const bplist = require('bplist-parser')
 const fs = require('fs')
-const plist = require('plist')
 
 // Derive filenames based on domain + file path
 const fileHash = require('../../../util/backup_filehash')

tools/util/iphone_backup.js

@@ -1,9 +1,8 @@
 const log = require('./log')
 const path = require('path')
 const sqlite3 = require('sqlite3')
-const bplist = require('bplist-parser')
 const fs = require('fs')
-const plist = require('plist')
+const plist = require('./plist')
 
 // Cookie Parser
 const cookieParser = require('./cookies.js')
@@ -17,7 +16,7 @@ const fileHash = require('./backup_filehash')
 // Manifest.mbdb parser
 const manifestMBDBParse = require('./manifest_mbdb_parse')
 
-// Pushstore bplist parser
+// Pushstore plist parser
 const pushstoreParse = require('./pushstore_parse')
 
 const databases = {
@@ -65,22 +64,22 @@ class IPhoneBackup {
       base = path.join(process.env.HOME, '/Library/Application Support/MobileSync/Backup/', id)
     }
 
-    // Parse manifest bplist files
+    // Parse manifest plist files
     try {
       log.verbose('parsing status', base)
-      var status = bplist.parseBuffer(fs.readFileSync(path.join(base, 'Status.plist')))[0]
+      var status = plist.parseFile(path.join(base, 'Status.plist'))
     } catch (e) {
       log.error('Cannot open Status.plist', e)
     }
     try {
       log.verbose('parsing manifest', base)
-      var manifest = bplist.parseBuffer(fs.readFileSync(path.join(base, 'Manifest.plist')))[0]
+      var manifest = plist.parseFile(path.join(base, 'Manifest.plist'))
     } catch (e) {
       log.error('Cannot open Manifest.plist', e)
     }
     try {
       log.verbose('parsing status', base)
-      var info = plist.parse(fs.readFileSync(path.join(base, 'Info.plist'), 'utf8'))
+      var info = plist.parseFile(path.join(base, 'Info.plist'))
     } catch (e) {
       log.error('Cannot open Info.plist', e)
     }
@@ -329,7 +328,7 @@ class IPhoneBackup {
         // The timestamp information is stored in a binary blob named `properties`
         // Which is formatted as a binary PLIST.
         for (var el of rows) {
-          if (el.properties) el.properties = bplist.parseBuffer(el.properties)[0]
+          if (el.properties) el.properties = plist.parseBuffer(el.properties)
 
           // Interestingly, some of these do not have dates attached.
           if (el.properties) {
@@ -571,7 +570,7 @@ class IPhoneBackup {
       var filename = this.getFileName(databases.WiFi)
 
       try {
-        let wifiList = bplist.parseBuffer(fs.readFileSync(filename))[0]
+        let wifiList = plist.parseFile(filename)
         wifiList['List of known networks'] = wifiList['List of known networks']
           .map(el => {
             if (el.BSSID) {
@@ -816,8 +815,8 @@ class IPhoneBackup {
           const pushstores = []
 
           files.forEach((file) => {
-            let plist = bplist.parseBuffer(fs.readFileSync(this.getFileName(file.fileID)))[0]
-            pushstores.push(...pushstoreParse.run(plist))
+            let data = plist.parseFile(this.getFileName(file.fileID))
+            pushstores.push(...pushstoreParse.run(data))
           })
           resolve(pushstores)
         } catch (e) {
@@ -840,8 +839,8 @@ class IPhoneBackup {
 
           let pushstores = []
           files.forEach((file) => {
-            let plist = bplist.parseBuffer(fs.readFileSync(this.getFileName(file.fileID)))[0]
-            pushstores.push(...pushstoreParse.run(plist))
+            let data = plist.parseFile(this.getFileName(file.fileID))
+            pushstores.push(...pushstoreParse.run(data))
           })
           resolve(pushstores)
         } catch (e) {

tools/util/plist.js

@@ -0,0 +1,21 @@
+const fs = require('fs')
+const plist = require('plist')
+const bplist = require('bplist-parser')
+
+function parseBuffer (buffer) {
+  // Binary plists have the marker 'bplist0'
+  if (buffer.slice(0, 7).toString('ascii') === 'bplist0') {
+    // Parse as binary plist
+    data = bplist.parseBuffer(buffer)[0]
+  } else {
+    // Parse as normal plist
+    data = plist.parse(buffer.toString('utf8'))
+  }
+  return data
+}
+
+function parseFile (filePath) {
+  return parseBuffer(fs.readFileSync(filePath))
+}
+
+module.exports = { parseBuffer, parseFile }