Add updated tools version

Readme.md

@@ -1,20 +1,86 @@
 # iPhone backup tools
-Are _you_ storing unencrypted iPhone backups on your personal computer? With very little effort, we can dump *all* the saved messages from the backup, as well as contacts and other data.
+Are _you_ storing unencrypted iPhone backups on your personal computer? With very little effort, we can dump **all** the saved messages from the backup, as well as notes, photo locations, and other data. 
+
+Check out my recently updated post about my work on backups here: [Reverse Engineering the iOS Backup](/2017/3/16/reverse-engineering-the-ios-backup)
+
+**This tool is also still fairly experimental, so use at your own risk! Even though the tool opens the backup files as read-only, you should still make a copy of your backups before using this if they are important.**
 
 Currently works on macOS, If someone wants to make the changes nessecary for windows, send a PR.
 
+## iOS Support
+- iOS 9 (possibly earlier?) backup type: `2.4`
+- iOS 10 (no contact lookup) backup type: `3.2`
+- iOS 11 (no contact lookup) backup type: `3.2`
+
 ## Installing
-clone this repo, then run:
+
 ```bash
+# Install directly from NPM
+npm i -g ibackuptool
+
+# If you prefer, you can do this manually:
+# Clone this repo, then run:
+# Install Globally on your system.
+npm i -g 
+
+# Or, If you really want:
+# Clone this repo, then run: 
 npm install
+run `node tools/index.js` # use this instead of ibackuptool
+```
+
+### Usage
+```bash
+# List all the backups on the system
+ibackuptool -l 
+
+# I'm using "0c1bc52c50016933679b0980ccff3680e5831162" as a placeholder.
+# The list of backups contains the different UDIDs in the first column.
+UDID="0c1bc52c50016933679b0980ccff3680e5831162"
 ```
 
-## Usage
+### Reports
+- run using `ibackuptool -b <udid> --report <type>`
+- Current types:
+    - `apps`: List all installed bundle ids and groups.
+    - `notes`: List all notes data
+    - `webhistory`: List recent web history
+    - `photolocations`: List photo locations (lat/lng) and timestamp.
+    - `manifest`: List all files in the backup manifest and what they are.
+
+```bash
+# Using a UDID from the previous step, now you can run:
+# List Installed Apps
+ibackuptool -b $UDID --report apps
+
+# List Recent Web History
+ibackuptool -b $UDID --report webhistory
+
+# List Recent Photos Geolocations
+ibackuptool -b $UDID --report photolocations
+
+# List iOS Notes
+ibackuptool -b $UDID --report notes
+```
+
+### Messages Access
+
 ```bash
-npm start
+# List of all conversations, indexed by ID.
+# Each row starts with an ID number, which is needed for the next step.
+ibackuptool -b $UDID --conversations
+
+# Now, Fetch the messages with the following command
+# Replace $CONVERSATION_ID with a row ID from `ibackuptool -b $UDID --conversations`
+ibackuptool -b $UDID --messages $CONVERSATION_ID
 ```
 
+## Need More Data?
+- !! This will cause the program to output **Everything** to STDOUT as formatted JSON. !!
+- Append the `--dump` flag to the program.
+- I'd recommend piping this output to a file.
+
 - You should make a backup of the backups you look at using this tool, even though they are opened as read-only, you should still do that do you don't accidentally do something to lose data.
 
 ## TODO
-- Contact name lookup for newer iOS10 backups
+- Contact name lookup for newer iOS10, iOS11 backups

package.json

@@ -1,19 +1,25 @@
 {
-  "name": "iphonebackuptools",
-  "version": "1.0.0",
-  "description": "Read Messages from iOS Backups.",
+  "name": "ibackuptool",
+  "version": "2.0.0",
+  "description": "Read Messages and info from iOS Backups",
   "main": "tools/index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "start" : "node tools/index.js"
+    "start": "node tools/index.js"
   },
+  "bin": {
+    "ibackuptool": "tools/index.js"
+  },
+  "preferGlobal": true,
+  "repository" : "https://github.com/richinfante/iphonebackuptools",
   "author": "@richinfante",
   "license": "MIT",
   "dependencies": {
     "bplist-parser": "^0.1.1",
     "chalk": "^1.1.3",
-    "inquirer": "^3.0.6",
+    "commander": "^2.12.2",
     "plist": "^2.0.1",
-    "sqlite3": "^3.1.8"
+    "sqlite3": "^3.1.8",
+    "strip-ansi": "^4.0.0"
   }
 }

tools/index.js

@@ -1,61 +1,288 @@
-const backup = require('./util/iphone_backup')
-const inquirer = require('inquirer')
+#!/usr/bin/env node
+
 const chalk = require('chalk')
+const fs = require('fs')
+const program = require('commander')
+const path = require('path')
+const { URL } = require('url')
+const stripAnsi = require('strip-ansi')
+const iPhoneBackup = require('./util/iphone_backup.js').iPhoneBackup
+const normalizeCols = require('./util/normalize.js')
+var base = path.join(process.env.HOME, '/Library/Application Support/MobileSync/Backup/')
+
+program
+    .version('2.0.0')
+    .option('-l, --list', 'List Backups')
+    .option('-c, --conversations', 'List Conversations')
+    .option('-m, --messages <conversation_id>', 'List Conversations')
+    .option('-r, --report <report_type>', 'Report types: apps, notes, webhistory, photolocations, manifest')
+    .option(`-d, --dir <directory>`, `Backup Directory (default: ${base})`)
+    .option(`-u, --device <device>`, 'Device UUID')
+    .option(`-b, --backup <backup>`, 'Backup ID')
+    .option(`-v, --verbose`, 'Verbose debugging output')
+    .option(`-x, --no-color`, 'Disable colorized output')
+    .option('-z, --dump', 'Dump a ton of raw JSON formatted data instead of formatted output')
+    
+program.on('--help', function(){
+    console.log('')
+    console.log("If you're interested to know how this works, check out my post:")
+    console.log("https://www.richinfante.com/2017/3/16/reverse-engineering-the-ios-backup")
+    console.log('')
+})
+    
+program.parse(process.argv);
+
+if(!process.stdout.isTTY) { program.color = false }
+
+base = program.dir || base
+
+if(program.verbose) console.log('Using source:', base)
+
+if(program.list) {
+    var items = fs.readdirSync(base, { encoding: 'utf8' })
+        .filter(el => (el.length == 40))
+        .map(file => iPhoneBackup.fromID(file, base))
+        
+        
+        // Possibly dump output
+        if(program.dump) {
+            console.log(JSON.stringify(items, null, 4))
+            return
+        }
+
+    items = items.map(el => { 
+            return {
+            encrypted: el.manifest ? el.manifest.IsEncrypted 
+                                        ? chalk.green('encrypted') 
+                                        : chalk.red('not encrypted')
+                                   : 'unknown encryption',
+            device_name: el.manifest ? el.manifest.Lockdown.DeviceName : 'Unknown Device',
+            device_id: el.id,
+            serial: el.manifest.Lockdown.SerialNumber,
+            iOSVersion: el.manifest.Lockdown.ProductVersion + '(' + el.manifest.Lockdown.BuildVersion + ')',
+            backupVersion: el.status ? el.status.Version : '?',
+            date: el.status ? new Date(el.status.Date).toLocaleString() : ''
+        }})
+        .map(el => [
+            chalk.gray(el.device_id), 
+            el.encrypted,
+            el.date, 
+            el.device_name,
+            el.serial,
+            el.iOSVersion,
+            el.backupVersion
+        ])
+
+    items = [
+        ['UDID', 'Encryption', 'Date', 'Device Name', 'Serial #', 'iOS Version', 'Backup Version'],
+        ['-','-','-','-','-','-','-'],
+            ...items
+    ]
+    items = normalizeCols(items)
+    items = items.map(el => el.join(' | ')).join('\n')
+
+    if(!program.color) { items = stripAnsi(items) }
+
+    console.log('BACKUPS LIST')
+    console.log(items)
+} else if (program.conversations) {
+    if(!program.backup) {
+        console.log('use -b or --backup <id> to specify backup.')
+        process.exit(1)
+    }
+
+    // Grab the backup
+    var backup = iPhoneBackup.fromID(program.backup, base)
+
 
-async function main() {
-  try {
-  var backups = await backup.availableBackups()
-
-  var result = await inquirer.prompt({
-    type: 'list',
-    name: 'backupid',
-    message: 'Select Backup:',
-    choices: backups.map(el => {
-      console.log(el)
-      return {
-        name: el.manifest ? 
-          `${el.manifest.Lockdown.DeviceName} <${el.id}> ${el.status ? new Date(el.status.Date).toLocaleString() : ''}` : 
-          `Unknown Device ${el.id} ${el.status ? new Date(el.status.Date).toLocaleString() : ''}`,
-        value: el.id
-      }
+    backup.getConversations(program.dump)
+    .then((items) => {
+        if(program.dump) return 
+
+        var items = items.map(el => [ 
+            el.ROWID + '', 
+            chalk.gray(el.date ? el.date.toLocaleString() : '??'),
+            el.chat_identifier + '',
+            el.display_name + ''
+        ])
+
+        items = [['ID', 'DATE', 'Chat Name', 'Display Name'], ['-', '-', '-', '-',], ...items]
+        items = normalizeCols(items).map(el => el.join(' | ')).join('\n')
+        
+        if(!program.color) { items = stripAnsi(items) }
+
+        console.log(items)
     })
-  })
-  
-  const selectedBackup = backup.iPhoneBackup.fromID(result.backupid)
-  
-  const conversations = await selectedBackup.getConversations()
-
-  var conversation = await inquirer.prompt({
-    type: 'list',
-    name: 'chat_id',
-    message: 'Select Conversation:',
-    choices: conversations.map(el => {
-      return {
-        name: chalk.gray(el.date ? el.date.toLocaleString() : '??') + ' ' + el.display_name + ' ' +  el.chat_identifier,
-        value: el.ROWID
-      }
+} else if(program.messages) {
+    if(!program.backup) {
+        console.log('use -b or --backup <id> to specify backup.')
+        process.exit(1)
+    }
+
+    // Grab the backup
+    var backup = iPhoneBackup.fromID(program.backup, base)
+
+
+    backup.getMessages(program.messages, program.dump)
+    .then((items) => {
+        if(program.dump) return 
+
+        items = items.map(el => [
+            chalk.gray(el.date ? el.date.toLocaleString() : ''),
+            chalk.blue(el.sender + ': '),
+            el.text || ''
+        ])
+
+        items = normalizeCols(items).map(el => el.join(' | ')).join('\n')
+
+        if(!program.color) { items = stripAnsi(items) }
+
+        console.log(items)
     })
-  })
-
-  //console.log(conversation)
-
-  const messages = await selectedBackup.getMessages(conversation.chat_id)
-  console.log(
-    messages.map(el => chalk.gray(el.date ? el.date.toLocaleString() : '') + ' ' + chalk.blue(el.sender + ': ') + el.text)
-    .join('\n')
-  )
-  } catch(e) {
-    console.log(e)
-  }
-}
+} else if(program.report) {
+    ///
+    /// APPS REPORT
+    ///
+    if(program.report == 'apps') {
+        if(!program.backup) {
+            console.log('use -b or --backup <id> to specify backup.')
+            process.exit(1)
+        }
 
-process.on('unhandledRejection', (reason, p) => {
-  console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
-  // application specific logging, throwing an error, or other logic here
-})
+        // Grab the backup
+        var backup = iPhoneBackup.fromID(program.backup, base)
 
-try {
-  main()
-} catch (e) {
-  console.log(e)
-}
+        if (!backup.manifest) return {}
+
+        // Possibly dump output
+        if(program.dump) {
+            console.log(JSON.stringify(backup.manifest, null, 4))
+            return
+        }
+
+        // Enumerate the apps in the backup
+        var apps = []
+        for (var key in backup.manifest.Applications) {
+            apps.push(key)
+        }
+
+        console.log(`Apps installed inside backup: ${backup.id}`)
+        console.log(apps.map(el => '- ' + el).join('\n'))
+    } else if(program.report == 'notes') {
+        if(!program.backup) {
+            console.log('use -b or --backup <id> to specify backup.')
+            process.exit(1)
+        }
+
+        // Grab the backup
+        var backup = iPhoneBackup.fromID(program.backup, base)
+        backup.getNotes(program.dump)
+            .then((items) => {
+                // Dump if needed
+                if(program.dump) {
+                    console.log(JSON.stringify(items, null, 4))
+                    return
+                }
+                
+                // Otherwise, format table
+                items = items.map(el => [el.ZMODIFICATIONDATE + '', (el.Z_PK + ''), (el.ZTITLE + '').substring(0, 128)])
+                items = [['Modified', 'ID', 'Title'], ['-', '-', '-'], ...items]
+                items = normalizeCols(items).map(el => el.join(' | ')).join('\n')
+                
+                if(!program.color) { items = stripAnsi(items) }
+
+                console.log(items)
+            })
+    }  else if(program.report == 'webhistory') {
+        if(!program.backup) {
+            console.log('use -b or --backup <id> to specify backup.')
+            process.exit(1)
+        }
+
+        // Grab the backup
+        var backup = iPhoneBackup.fromID(program.backup, base)
+        backup.getWebHistory(program.dump)
+            .then((history) => {
+
+                if(program.dump) {
+                    console.log(JSON.stringify(history, null, 4))
+                    return
+                }
+
+                var items = history.map(el => [
+                    el.visit_time + '' || '',
+                    new URL(el.url || '').origin || '',
+                    (el.title || '').substring(0, 64)
+                ])
+
+                items = [['Time', 'URL', 'Title'], ['-', '-', '-'], ...items]
+                items = normalizeCols(items).map(el => el.join(' | ').replace(/\n/g, '')).join('\n')
+                
+                if(!program.color) { items = stripAnsi(items) }
+
+                console.log(items)
+            })
+    } else if(program.report == 'photolocations') {
+        if(!program.backup) {
+            console.log('use -b or --backup <id> to specify backup.')
+            process.exit(1)
+        }
+
+
+        // Grab the backup
+        var backup = iPhoneBackup.fromID(program.backup, base)
+        backup.getPhotoLocationHistory(program.dump)
+            .then((history) => {
+
+                if(program.dump) {
+                    console.log(JSON.stringify(history, null, 4))
+                    return
+                }
+
+                var items = history.map(el => [
+                    el.ZDATECREATED + '' || '',
+                    el.ZLATITUDE + '' || '',
+                    el.ZLONGITUDE  + '' || '',
+                    el.ZFILENAME + '' || ''
+                ])
+
+                items = [['Time', 'Latitude', 'Longitude', 'Photo Name'], ['-', '-', '-'], ...items]
+                items = normalizeCols(items).map(el => el.join(' | ').replace(/\n/g, '')).join('\n')
+
+                if(!program.color) { items = stripAnsi(items) }
+
+                console.log(items)
+            })
+    }  else if(program.report == 'manifest') {
+        if(!program.backup) {
+            console.log('use -b or --backup <id> to specify backup.')
+            process.exit(1)
+        }
+
+
+        // Grab the backup
+        var backup = iPhoneBackup.fromID(program.backup, base)
+        backup.getFileManifest()
+            .then((items) => {
+
+                if(program.dump) {
+                    console.log(JSON.stringify(items, null, 4))
+                    return
+                }
+
+                var items = items.map(el => [
+                    el.fileID + '',
+                    el.relativePath + '' 
+                ])
+
+                items = [['ID', 'Path'], ['-', '-'], ...items]
+                items = normalizeCols(items).map(el => el.join(' | ').replace(/\n/g, '')).join('\n')
+                
+                if(!program.color) { items = stripAnsi(items) }
+
+                console.log(items)
+            })
+    }
+} else {
+    program.outputHelp()
+}

tools/util/iphone_backup.js

@@ -12,7 +12,9 @@ const databases = {
   Reminders: '2041457d5fe04d39d0ab481178355df6781e6858',
   Notes: 'ca3bc056d4da0bbf88b5fb3be254f3b7147e639c',
   Calls: '2b2b0084a1bc3a5ac8c27afdf14afb42c61a19ca',
-  Locations: '4096c9ec676f2847dc283405900e284a7c815836'
+  Locations: '4096c9ec676f2847dc283405900e284a7c815836',
+  WebHistory: 'e74113c185fd8297e140cfcf9c99436c5cc06b57',
+  Photos: '12b144c0bd44f2b3dffd9186d3f9c05b917cee25'
 }
 
 var cache = {}
@@ -25,33 +27,43 @@ class iPhoneBackup {
     this.manifest = manifest;
   }
 
-
-  static fromID(id) {
+  // Open a backup with a specified ID
+  // base is optional and will be computed if not used.
+  static fromID(id, base) {
     // Get the path of the folder.
-    const base = path.join(process.env.HOME, '/Library/Application Support/MobileSync/Backup/', id)
+    if(base) {
+      base = path.join(base, id)
+    } else {
+      base = path.join(process.env.HOME, '/Library/Application Support/MobileSync/Backup/', id)
+    }
 
     // Parse manifest bplist files
     try {
       var status = bplist.parseBuffer(fs.readFileSync(path.join(base, 'Status.plist')))[0];
     } catch (e) {
+      console.log('Cannot open Status.plist', e)
     }
     try {
       var manifest = bplist.parseBuffer(fs.readFileSync(path.join(base, 'Manifest.plist')))[0];
     } catch (e) {
+      console.log('Cannot open Manifest.plist', e)
     }
     try {
       var info = plist.parse(fs.readFileSync(path.join(base, 'Info.plist'), 'utf8'));
     } catch (e) {
+      console.log('Cannot open Info.plist', e)
     }
 
     return new iPhoneBackup(id, status, info, manifest)
   }
 
-  getDatabase(fileID) {
+  getDatabase(fileID, isAbsoulte) {
+    isAbsoulte = isAbsoulte || false
+
     // Get the backup folder
     const base = path.join(process.env.HOME, '/Library/Application Support/MobileSync/Backup/', this.id)
     // Return v2 filename
-    if (this.status.Version < 3) {
+    if (this.status.Version < 3 || isAbsoulte) {
       return new sqlite3.Database(path.join(base, fileID), sqlite3.OPEN_READONLY)
     } else {
       // v3 has folders
@@ -60,7 +72,7 @@ class iPhoneBackup {
   }
 
   getName(messageDest) {
-
+    
     return new Promise((resolve, reject) => {
       if(messageDest.indexOf('@') === -1) {
         messageDest = messageDest.replace(/[\s+\-()]*/g, '')
@@ -83,7 +95,6 @@ class iPhoneBackup {
         from ABPersonFullTextSearch_content WHERE c15Phone like '%${messageDest}%'`, 
       (err, row) => {
           if(err) return resolve()
-        
           if(!row) return resolve()
 
           var result = {
@@ -99,7 +110,7 @@ class iPhoneBackup {
     })
   }
 
-  getMessages(chat_id) {
+  getMessages(chat_id, dumpAll) {
     var backup = this;
     return new Promise((resolve, reject) => {
       var messagedb = this.getDatabase(databases.SMS)
@@ -115,10 +126,21 @@ class iPhoneBackup {
           ON handle.rowid = message.handle_id
         WHERE chat_message_join.chat_id = ?`, [parseInt(chat_id)], 
      async function (err, chats) {
+      if(err) return reject(err)
+
+       chats = chats || []
         var offset = new Date('2001-01-01 00:00:00').getTime()
         
+        if(dumpAll) console.log(JSON.stringify(chats, null, 4))
+
         for(var i in chats) {
           var el = chats[i]
+          
+          // Some of the dates are of much larger precision
+          if(el.date > 100000000000000000) {
+            el.date = el.date / 1000000000
+          }
+
           var date = new Date(offset + el.date * 1000 - tz_offset * 60 * 60 * 1000)
           var text = el.text
           var sender = el.is_from_me ? 'Me' : el.sender_name
@@ -134,20 +156,27 @@ class iPhoneBackup {
           chats[i] = { sender, text, date }
         }
 
+        
+
         resolve(chats)
       })
     })
   }
 
-  getConversations() {
+  getConversations(dumpAll) {
     var backup = this
     return new Promise((resolve, reject) => {
       var messagedb = this.getDatabase(databases.SMS)
 
       messagedb.all('SELECT * FROM chat', async function (err, rows) {
+        if(err) return reject(err)
         rows = rows || []
+
         for(var el of rows) {
           if (el.properties) el.properties = bplist.parseBuffer(el.properties)[0]
+
+
+          // Interestingly, some of these do not have dates attached.
           if (el.properties) {
             el.date = new Date(el.properties.CKChatWatermarkTime * 1000)
           } else {
@@ -167,8 +196,58 @@ class iPhoneBackup {
           return new Date(b.date) - new Date(a.date);
         });
 
+        if(dumpAll) console.log(JSON.stringify(rows, null, 4))
+
+        resolve(rows)
+      })
+    })
+  }
+
+  getFileManifest() {
+    return new Promise((resolve, reject) => {
+      var messagedb = this.getDatabase('Manifest.db', true)
+      messagedb.all('SELECT * from FILES', async function (err, rows) {
+        if (err) reject(err)
+
+        resolve(rows)
+      })
+    
+    })
+  }
+
+  getNotes() {
+    return new Promise((resolve, reject) => {
+      var messagedb = this.getDatabase(databases.Notes)
+      messagedb.all('SELECT * from ZNOTE LEFT JOIN ZNOTEBODY ON ZBODY = ZNOTEBODY.Z_PK', async function (err, rows) {
+        if (err) reject(err)
+        
+        resolve(rows)
+      })
+    
+    })
+  }
+
+  getWebHistory() {
+    return new Promise((resolve, reject) => {
+      var messagedb = this.getDatabase(databases.WebHistory)
+      messagedb.all('SELECT * from history_visits LEFT JOIN history_items ON history_items.ROWID = history_visits.history_item', async function (err, rows) {
+        if(err) reject(err)
+
+        resolve(rows)
+      })
+    
+    })
+  }
+
+  getPhotoLocationHistory() {
+    return new Promise((resolve, reject) => {
+      var messagedb = this.getDatabase(databases.Photos)
+      messagedb.all('SELECT ZDATECREATED, ZLATITUDE, ZLONGITUDE, ZFILENAME FROM ZGENERICASSET ORDER BY ZDATECREATED ASC', async function (err, rows) {
+        if(err) reject(err)
+
         resolve(rows)
       })
+    
     })
   }
 }

tools/util/normalize.js

@@ -0,0 +1,34 @@
+const stripAnsi = require('strip-ansi')
+
+module.exports = function normalizeOutput(rows) {
+    function padEnd(string, maxLength, fillString) {
+        while(stripAnsi(string).length < maxLength) {
+            string = string + fillString;
+        }    
+    
+        return string   
+    }
+
+    var widths = []
+
+    for(var i = 0; i < rows.length; i++) {
+        for(var j = 0; j < rows[i].length; j++) {
+            if(!widths[j] || widths[j] < stripAnsi(rows[i][j]).length) {
+                widths[j] = stripAnsi(rows[i][j]).length
+            }
+        }
+    }
+
+    for(var i = 0; i < rows.length; i++) {
+        for(var j = 0; j < rows[i].length; j++) {
+            if(rows[i][j] == '-') {
+                rows[i][j] = padEnd(rows[i][j], widths[j], '-')
+            } else {
+                rows[i][j] = padEnd(rows[i][j], widths[j], ' ')
+            }
+        }
+    }
+
+    return rows
+    
+}