Merge branch 'lzma-updater'

20 files changed, 276 insertions, 118 deletions

diff --git a/config/external/moz.build b/config/external/moz.build
index dc094b6af1..47db9e1526 100644
--- a/config/external/moz.build
+++ b/config/external/moz.build
@@ -54,7 +54,7 @@ external_dirs += [
     'media/psshparser'
 ]
 
-if CONFIG['MOZ_LINKER']:
+if CONFIG['MOZ_UPDATER']:
     external_dirs += ['modules/xz-embedded']
 
 DIRS += ['../../' + i for i in external_dirs]
diff --git a/modules/libmar/src/mar.h b/modules/libmar/src/mar.h
index 776daf648d..e9c8d555e4 100644
--- a/modules/libmar/src/mar.h
+++ b/modules/libmar/src/mar.h
@@ -109,7 +109,7 @@ int mar_enum_items(MarFile *mar, MarItemCallback callback, void *data);
  * @return          The number of bytes written or a negative value if an
  *                  error occurs.
  */
-int mar_read(MarFile *mar, const MarItem *item, int offset, char *buf,
+int mar_read(MarFile *mar, const MarItem *item, int offset, uint8_t *buf,
              int bufsize);
 
 /**
diff --git a/modules/libmar/src/mar_extract.c b/modules/libmar/src/mar_extract.c
index ec1cd6c534..969697ee29 100644
--- a/modules/libmar/src/mar_extract.c
+++ b/modules/libmar/src/mar_extract.c
@@ -37,7 +37,7 @@ static int mar_ensure_parent_dir(const char *path)
 
 static int mar_test_callback(MarFile *mar, const MarItem *item, void *unused) {
   FILE *fp;
-  char buf[BLOCKSIZE];
+  uint8_t buf[BLOCKSIZE];
   int fd, len, offset = 0;
 
   if (mar_ensure_parent_dir(item->name))
diff --git a/modules/libmar/src/mar_read.c b/modules/libmar/src/mar_read.c
index 241d0c08e4..89b9da27d6 100644
--- a/modules/libmar/src/mar_read.c
+++ b/modules/libmar/src/mar_read.c
@@ -530,7 +530,7 @@ int mar_enum_items(MarFile *mar, MarItemCallback callback, void *closure) {
   return 0;
 }
 
-int mar_read(MarFile *mar, const MarItem *item, int offset, char *buf,
+int mar_read(MarFile *mar, const MarItem *item, int offset, uint8_t *buf,
              int bufsize) {
   int nr;
 
diff --git a/modules/xz-embedded/moz.build b/modules/xz-embedded/moz.build
index fa1e5fc468..9806dca1ea 100644
--- a/modules/xz-embedded/moz.build
+++ b/modules/xz-embedded/moz.build
@@ -29,4 +29,7 @@ elif '86' in CONFIG['TARGET_CPU']:
 
 DEFINES['XZ_USE_CRC64'] = 1
 
+if CONFIG['OS_ARCH'] == 'WINNT':
+    USE_STATIC_LIBS = True
+
 Library('xz-embedded')
diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py
index a2983b48c2..4f58a2a33d 100644
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1140,8 +1140,13 @@ class Mar(MachCommandBase):
 
     @Command('mar', category='post-build',
         description='Create the mar file for the built product for distribution.')
-    def mar(self):
-        return self._run_make(directory="./tools/update-packaging/", target='', ensure_exit_code=False)
+    @CommandArgument('--bz2', action='store_true',
+        help='Compress the mar package with old-style bz2 instead of xz')
+    def mar(self, bz2):
+        if bz2:
+          return self._run_make(directory="./tools/update-packaging/", target='mar-package-bz2', ensure_exit_code=False)
+        else:
+          return self._run_make(directory="./tools/update-packaging/", target='mar-package', ensure_exit_code=False)
 
 @CommandProvider
 class Install(MachCommandBase):
diff --git a/toolkit/mozapps/installer/package-name.mk b/toolkit/mozapps/installer/package-name.mk
index 680590b3c5..548a93b47f 100644
--- a/toolkit/mozapps/installer/package-name.mk
+++ b/toolkit/mozapps/installer/package-name.mk
@@ -80,6 +80,7 @@ PKG_UPDATE_BASENAME = $(PKG_BASENAME)
 CHECKSUMS_FILE_BASENAME = $(PKG_BASENAME)
 MOZ_INFO_BASENAME = $(PKG_BASENAME)
 PKG_UPDATE_PATH = $(PKG_PATH)
+BARE_MAR = $(PKG_UPDATE_PATH)$(PKG_UPDATE_BASENAME).mar
 COMPLETE_MAR = $(PKG_UPDATE_PATH)$(PKG_UPDATE_BASENAME).complete.mar
 # PARTIAL_MAR needs to be processed by $(wildcard) before you use it.
 PARTIAL_MAR = $(PKG_UPDATE_PATH)$(PKG_UPDATE_BASENAME).partial.*.mar
diff --git a/toolkit/mozapps/update/common/errors.h b/toolkit/mozapps/update/common/errors.h
index 846118d55d..de1782b8e5 100644
--- a/toolkit/mozapps/update/common/errors.h
+++ b/toolkit/mozapps/update/common/errors.h
@@ -46,7 +46,7 @@
 #define WRITE_ERROR_ACCESS_DENIED 35
 // #define WRITE_ERROR_SHARING_VIOLATION 36 // Replaced with errors 46-48
 #define WRITE_ERROR_CALLBACK_APP 37
-#define UNEXPECTED_BZIP_ERROR 39
+#define UNEXPECTED_XZ_ERROR 39
 #define UNEXPECTED_MAR_ERROR 40
 #define UNEXPECTED_BSPATCH_ERROR 41
 #define UNEXPECTED_FILE_OPERATION_ERROR 42
diff --git a/toolkit/mozapps/update/updater/archivereader.cpp b/toolkit/mozapps/update/updater/archivereader.cpp
index 90cf45c3db..72debbe82e 100644
--- a/toolkit/mozapps/update/updater/archivereader.cpp
+++ b/toolkit/mozapps/update/updater/archivereader.cpp
@@ -1,5 +1,4 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
-/* vim:set ts=2 sw=2 sts=2 et cindent: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@@ -15,6 +14,9 @@
 #include "updatehelper.h"
 #endif
 
+#define XZ_USE_CRC64
+#include "xz.h"
+
 // These are generated at compile time based on the DER file for the channel
 // being used
 #ifdef MOZ_VERIFY_MAR_SIGNATURE
@@ -36,10 +38,10 @@
 # include <io.h>
 #endif
 
-static int inbuf_size  = 262144;
-static int outbuf_size = 262144;
-static char *inbuf  = nullptr;
-static char *outbuf = nullptr;
+static size_t inbuf_size  = 262144;
+static size_t outbuf_size = 262144;
+static uint8_t *inbuf  = nullptr;
+static uint8_t *outbuf = nullptr;
 
 /**
  * Performs a verification on the opened MAR file with the passed in
@@ -182,22 +184,22 @@ ArchiveReader::Open(const NS_tchar *path)
     Close();
 
   if (!inbuf) {
-    inbuf = (char *)malloc(inbuf_size);
+    inbuf = (uint8_t *)malloc(inbuf_size);
     if (!inbuf) {
       // Try again with a smaller buffer.
       inbuf_size = 1024;
-      inbuf = (char *)malloc(inbuf_size);
+      inbuf = (uint8_t *)malloc(inbuf_size);
       if (!inbuf)
         return ARCHIVE_READER_MEM_ERROR;
     }
   }
 
   if (!outbuf) {
-    outbuf = (char *)malloc(outbuf_size);
+    outbuf = (uint8_t *)malloc(outbuf_size);
     if (!outbuf) {
       // Try again with a smaller buffer.
       outbuf_size = 1024;
-      outbuf = (char *)malloc(outbuf_size);
+      outbuf = (uint8_t *)malloc(outbuf_size);
       if (!outbuf)
         return ARCHIVE_READER_MEM_ERROR;
     }
@@ -211,6 +213,9 @@ ArchiveReader::Open(const NS_tchar *path)
   if (!mArchive)
     return READ_ERROR;
 
+  xz_crc32_init();
+  xz_crc64_init();
+
   return OK;
 }
 
@@ -273,12 +278,21 @@ ArchiveReader::ExtractItemToStream(const MarItem *item, FILE *fp)
 {
   /* decompress the data chunk by chunk */
 
-  bz_stream strm;
-  int offset, inlen, outlen, ret = OK;
+  int offset, inlen, ret = OK;
+  struct xz_buf strm = { 0 };
+  enum xz_ret xz_rv = XZ_OK;
+
+  struct xz_dec * dec = xz_dec_init(XZ_DYNALLOC, 64 * 1024 * 1024);
+  if (!dec) {
+    return UNEXPECTED_XZ_ERROR;
+  }
 
-  memset(&strm, 0, sizeof(strm));
-  if (BZ2_bzDecompressInit(&strm, 0, 0) != BZ_OK)
-    return UNEXPECTED_BZIP_ERROR;
+  strm.in = inbuf;
+  strm.in_pos = 0;
+  strm.in_size = 0;
+  strm.out = outbuf;
+  strm.out_pos = 0;
+  strm.out_size = outbuf_size;
 
   offset = 0;
   for (;;) {
@@ -287,38 +301,50 @@ ArchiveReader::ExtractItemToStream(const MarItem *item, FILE *fp)
       break;
     }
 
-    if (offset < (int) item->length && strm.avail_in == 0) {
+    if (offset < (int) item->length && strm.in_pos == strm.in_size) {
       inlen = mar_read(mArchive, item, offset, inbuf, inbuf_size);
-      if (inlen <= 0)
-        return READ_ERROR;
+      if (inlen <= 0) {
+        ret = READ_ERROR;
+        break;
+      }
       offset += inlen;
-      strm.next_in = inbuf;
-      strm.avail_in = inlen;
+      strm.in_size = inlen;
+      strm.in_pos = 0;
     }
 
-    strm.next_out = outbuf;
-    strm.avail_out = outbuf_size;
+    xz_rv = xz_dec_run(dec, &strm);
 
-    ret = BZ2_bzDecompress(&strm);
-    if (ret != BZ_OK && ret != BZ_STREAM_END) {
-      ret = UNEXPECTED_BZIP_ERROR;
-      break;
-    }
-
-    outlen = outbuf_size - strm.avail_out;
-    if (outlen) {
-      if (fwrite(outbuf, outlen, 1, fp) != 1) {
+    if (strm.out_pos == outbuf_size) {
+      if (fwrite(outbuf, 1, strm.out_pos, fp) != strm.out_pos) {
         ret = WRITE_ERROR_EXTRACT;
         break;
       }
+
+      strm.out_pos = 0;
+    }
+
+    if (xz_rv == XZ_OK) {
+      // There is still more data to decompress.
+      continue;
     }
 
-    if (ret == BZ_STREAM_END) {
-      ret = OK;
+    // The return value of xz_dec_run is not XZ_OK and if it isn't XZ_STREAM_END
+    // an error has occured.
+    if (xz_rv != XZ_STREAM_END) {
+      ret = UNEXPECTED_XZ_ERROR;
       break;
     }
+
+    // Write out the remainder of the decompressed data. In the case of
+    // strm.out_pos == 0 this is needed to create empty files included in the
+    // mar file.
+    if (fwrite(outbuf, 1, strm.out_pos, fp) != strm.out_pos) {
+      ret = WRITE_ERROR_EXTRACT;
+    }
+
+    break;
   }
 
-  BZ2_bzDecompressEnd(&strm);
+  xz_dec_end(dec);
   return ret;
 }
diff --git a/toolkit/mozapps/update/updater/updater-common.build b/toolkit/mozapps/update/updater/updater-common.build
index a46c1388cc..1ace8fcc71 100644
--- a/toolkit/mozapps/update/updater/updater-common.build
+++ b/toolkit/mozapps/update/updater/updater-common.build
@@ -57,6 +57,7 @@ else:
 
 USE_LIBS += [
     'mar',
+    'xz-embedded',
 ]
 
 if CONFIG['MOZ_SYSTEM_BZ2']:
diff --git a/tools/update-packaging/Makefile.in b/tools/update-packaging/Makefile.in
index 4788a55752..a751ae3ce5 100644
--- a/tools/update-packaging/Makefile.in
+++ b/tools/update-packaging/Makefile.in
@@ -31,7 +31,7 @@ MAR_BIN	= $(DIST)/host/bin/mar$(HOST_BIN_SUFFIX)
 MBSDIFF_BIN	= $(DIST)/host/bin/mbsdiff$(HOST_BIN_SUFFIX)
 
 OVERRIDE_DEFAULT_GOAL := full-update
-full-update:: complete-patch
+full-update:: mar-package
 
 ifeq ($(OS_TARGET), WINNT)
 ifdef AB_CD
@@ -70,3 +70,23 @@ partial-patch:: $(dir-stage)
 ifdef MOZ_SIGN_CMD
 	$(MOZ_SIGN_CMD) -f mar '$(STAGE_DIR)/$(PKG_UPDATE_BASENAME).partial.$(SRC_BUILD_ID)-$(DST_BUILD_ID).mar'
 endif
+
+mar-package: $(call mkdir_deps,$(ABS_DIST)/$(PKG_STAGE_DIR))
+	@echo 'Creating update mar (xz compressed)...'
+	MAR=$(MAR_BIN) \
+	  $(srcdir)/make_full_update.sh \
+	  '$(DIST)/$(BARE_MAR)' \
+	  '$(PACKAGE_DIR)'
+ifdef MOZ_SIGN_CMD
+	$(MOZ_SIGN_CMD) -f mar '$(DIST)/$(BARE_MAR)'
+endif
+
+mar-package-bz2: $(call mkdir_deps,$(ABS_DIST)/$(PKG_STAGE_DIR))
+	@echo 'Creating update mar (bz2 compressed)...'
+	MAR_OLD_FORMAT=1 MAR=$(MAR_BIN) \
+	  $(srcdir)/make_full_update.sh \
+	  '$(DIST)/$(COMPLETE_MAR)' \
+	  '$(PACKAGE_DIR)'
+ifdef MOZ_SIGN_CMD
+	$(MOZ_SIGN_CMD) -f mar '$(DIST)/$(COMPLETE_MAR)'
+endif
\ No newline at end of file
diff --git a/tools/update-packaging/common.sh b/tools/update-packaging/common.sh
index 24f1ca6eb4..3df2cc4768 100755
--- a/tools/update-packaging/common.sh
+++ b/tools/update-packaging/common.sh
@@ -9,10 +9,15 @@
 #
 
 # -----------------------------------------------------------------------------
-# By default just assume that these tools exist on our path
+# By default just assume that these tools exist in our path
 MAR=${MAR:-mar}
-BZIP2=${BZIP2:-bzip2}
 MBSDIFF=${MBSDIFF:-mbsdiff}
+if [[ -z "${MAR_OLD_FORMAT}" ]]; then
+  XZ=${XZ:-xz}
+else
+  MAR_OLD_FORMAT=1
+  BZIP2=${BZIP2:-bzip2}
+fi
 
 # -----------------------------------------------------------------------------
 # Helper routines
diff --git a/tools/update-packaging/make_full_update.sh b/tools/update-packaging/make_full_update.sh
index ba1404474f..c480f8ee66 100755
--- a/tools/update-packaging/make_full_update.sh
+++ b/tools/update-packaging/make_full_update.sh
@@ -93,7 +93,11 @@ for ((i=0; $i<$num_files; i=$i+1)); do
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
-  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  if [[ -n $MAR_OLD_FORMAT ]]; then
+    $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  else
+    $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
+  fi
   copy_perm "$targetdir/$f" "$workdir/$f"
 
   targetfiles="$targetfiles \"$f\""
@@ -104,8 +108,13 @@ notice ""
 notice "Adding file and directory remove instructions from file 'removed-files'"
 append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
 
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
+if [[ -n $MAR_OLD_FORMAT ]]; then
+  $BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
+  $BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
+else
+  $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
+  $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
+fi
 
 eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
diff --git a/tools/update-packaging/make_incremental_update.sh b/tools/update-packaging/make_incremental_update.sh
index 90372c017e..af3af854fd 100755
--- a/tools/update-packaging/make_incremental_update.sh
+++ b/tools/update-packaging/make_incremental_update.sh
@@ -169,7 +169,11 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
     if check_for_add_if_not_update "$f"; then
       # The full workdir may not exist yet, so create it if necessary.
       mkdir -p `dirname "$workdir/$f"`
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      if [[ -n $MAR_OLD_FORMAT ]]; then
+        $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      else
+        $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
+      fi
       copy_perm "$newdir/$f" "$workdir/$f"
       make_add_if_not_instruction "$f" "$updatemanifestv3"
       archivefiles="$archivefiles \"$f\""
@@ -179,7 +183,11 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
     if check_for_forced_update "$requested_forced_updates" "$f"; then
       # The full workdir may not exist yet, so create it if necessary.
       mkdir -p `dirname "$workdir/$f"`
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      if [[ -n $MAR_OLD_FORMAT ]]; then
+        $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      else
+        $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
+      fi
       copy_perm "$newdir/$f" "$workdir/$f"
       make_add_instruction "$f" "$updatemanifestv2" "$updatemanifestv3" 1
       archivefiles="$archivefiles \"$f\""
@@ -207,21 +215,44 @@ for ((i=0; $i<$num_oldfiles; i=$i+1)); do
       # if service is not enabled then default to old behavior
       if [ -z "$MBSDIFF_HOOK" ]; then
         $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
-        $BZIP2 -z9 "$workdir/$f.patch"
+        if [[ -n $MAR_OLD_FORMAT ]]; then
+          $BZIP2 -z9 "$workdir/$f.patch"
+        else
+          $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
+        fi
       else
         # if service enabled then check patch existence for retrieval
-        if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
-          notice "file \"$f\" found in funsize, diffing skipped"
+        if [[ -n $MAR_OLD_FORMAT ]]; then
+          if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"; then
+            notice "file \"$f\" found in funsize, diffing skipped"
+          else
+            # if not found already - compute it and cache it for future use
+            $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
+            $BZIP2 -z9 "$workdir/$f.patch"
+            $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
+          fi
         else
-          # if not found already - compute it and cache it for future use
-          $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
-          $BZIP2 -z9 "$workdir/$f.patch"
-          $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.bz2"
+          if $MBSDIFF_HOOK -g "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"; then
+            notice "file \"$f\" found in funsize, diffing skipped"
+          else
+            # if not found already - compute it and cache it for future use
+            $MBSDIFF "$olddir/$f" "$newdir/$f" "$workdir/$f.patch"
+            $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$workdir/$f.patch"
+            $MBSDIFF_HOOK -u "$olddir/$f" "$newdir/$f" "$workdir/$f.patch.xz"
+          fi
         fi
       fi
-      $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      if [[ -n $MAR_OLD_FORMAT ]]; then
+        $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+      else
+        $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
+      fi
       copy_perm "$newdir/$f" "$workdir/$f"
-      patchfile="$workdir/$f.patch.bz2"
+      if [[ -n $MAR_OLD_FORMAT ]]; then
+        patchfile="$workdir/$f.patch.bz2"
+      else
+        patchfile="$workdir/$f.patch.xz"
+      fi
       patchsize=$(get_file_size "$patchfile")
       fullsize=$(get_file_size "$workdir/$f")
 
@@ -262,7 +293,11 @@ for ((i=0; $i<$num_newfiles; i=$i+1)); do
   dir=$(dirname "$workdir/$f")
   mkdir -p "$dir"
 
-  $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+  if [[ -n $MAR_OLD_FORMAT ]]; then
+    $BZIP2 -cz9 "$newdir/$f" > "$workdir/$f"
+  else
+    $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$newdir/$f" > "$workdir/$f"
+  fi
   copy_perm "$newdir/$f" "$workdir/$f"
 
   if check_for_add_if_not_update "$f"; then
@@ -303,8 +338,13 @@ for ((i=0; $i<$num_olddirs; i=$i+1)); do
   fi
 done
 
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
+if [[ -n $MAR_OLD_FORMAT ]]; then
+  $BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
+  $BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
+else
+  $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
+  $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
+fi
 
 mar_command="$MAR"
 if [[ -n $MOZ_PRODUCT_VERSION ]]
diff --git a/tools/update-packaging/make_incremental_updates.py b/tools/update-packaging/make_incremental_updates.py
index d346ee2df0..dc7b3a01ea 100755
--- a/tools/update-packaging/make_incremental_updates.py
+++ b/tools/update-packaging/make_incremental_updates.py
@@ -4,7 +4,7 @@
 
 import os
 import shutil
-import sha
+import hashlib
 from os.path import join, getsize
 from stat import *
 import re
@@ -12,9 +12,9 @@ import sys
 import getopt
 import time
 import datetime
-import bz2
 import string
 import tempfile
+import io
 
 class PatchInfo:
     """ Represents the meta-data associated with a patch
@@ -107,23 +107,23 @@ class PatchInfo:
         """ Create the v2 manifest file in the root of the work_dir """
         manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
         manifest_file = open(manifest_file_path, "wb")
-        manifest_file.writelines("type \"partial\"\n")
-        manifest_file.writelines(string.join(self.manifestv2, '\n'))
-        manifest_file.writelines("\n")
+        manifest_file.writelines(io.BytesIO(b"type \"partial\"\n"))
+        manifest_file.writelines(io.BytesIO('\n'.join(self.manifestv2).encode('ascii')))
+        manifest_file.writelines(io.BytesIO(b"\n"))
         manifest_file.close()
 
-        bzip_file(manifest_file_path)
+        xz_file(manifest_file_path)
         self.archive_files.append('"updatev2.manifest"')
 
         """ Create the v3 manifest file in the root of the work_dir """
         manifest_file_path = os.path.join(self.work_dir,"updatev3.manifest")
         manifest_file = open(manifest_file_path, "wb")
-        manifest_file.writelines("type \"partial\"\n")
-        manifest_file.writelines(string.join(self.manifestv3, '\n'))
-        manifest_file.writelines("\n")
+        manifest_file.writelines(io.BytesIO(b"type \"partial\"\n"))
+        manifest_file.writelines(io.BytesIO('\n'.join(self.manifestv3).encode('ascii')))
+        manifest_file.writelines(io.BytesIO(b"\n"))
         manifest_file.close()
 
-        bzip_file(manifest_file_path)
+        xz_file(manifest_file_path)
         self.archive_files.append('"updatev3.manifest"')
 
     def build_marfile_entry_hash(self, root_path):
@@ -176,8 +176,8 @@ class MarFileEntry:
 
     def calc_file_sha_digest(self, filename):
         """ Returns sha digest of given filename"""
-        file_content = open(filename, 'r').read()
-        return sha.new(file_content).digest()
+        file_content = open(filename, 'rb').read()
+        return hashlib.sha1(file_content).digest()
 
     def sha(self):
         """ Returns sha digest of file repreesnted by this _marfile_entry"""
@@ -188,7 +188,7 @@ class MarFileEntry:
 def exec_shell_cmd(cmd):
     """Execs shell cmd and raises an exception if the cmd fails"""
     if (os.system(cmd)):
-        raise Exception, "cmd failed "+cmd
+        raise Exception("cmd failed "+cmd)
 
 
 def copy_file(src_file_abs_path, dst_file_abs_path):
@@ -199,19 +199,19 @@ def copy_file(src_file_abs_path, dst_file_abs_path):
     # Copy the file over
     shutil.copy2(src_file_abs_path, dst_file_abs_path)
 
-def bzip_file(filename):
-    """ Bzip's the file in place.  The original file is replaced with a bzip'd version of itself
+def xz_file(filename):
+    """ XZ compresses the file in place.  The original file is replaced with the xz compressed version of itself
         assumes the path is absolute"""
-    exec_shell_cmd('bzip2 -z9 "' + filename+'"')
-    os.rename(filename+".bz2",filename)
+    exec_shell_cmd('xz --compress --x86 --lzma2 --format=xz --check=crc64 "' + filename+'"')
+    os.rename(filename+".xz",filename)
 
-def bunzip_file(filename):
-    """ Bzip's the file in palce.   The original file is replaced with a bunzip'd version of itself.
-        doesn't matter if the filename ends in .bz2 or not"""
-    if not filename.endswith(".bz2"):
-        os.rename(filename, filename+".bz2")
-        filename=filename+".bz2"
-    exec_shell_cmd('bzip2 -d "' + filename+'"')
+def xzunzip_file(filename):
+    """ xz decompresses the file in palce.  The original file is replaced with a xz decompressed version of itself.
+        doesn't matter if the filename ends in .xz or not"""
+    if not filename.endswith(".xz"):
+        os.rename(filename, filename+".xz")
+        filename=filename+".xz"
+    exec_shell_cmd('xz -d "' + filename+'"')
 
 
 def extract_mar(filename, work_dir):
@@ -241,14 +241,14 @@ def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, pa
         if not os.path.exists(patch_file_dir):
             os.makedirs(patch_file_dir)
 
-        # Create bzip'd patch file
+        # Create xz'd patch file
         exec_shell_cmd("mbsdiff "+from_marfile_entry.abs_path+" "+to_marfile_entry.abs_path+" "+patch_file_abs_path)
-        bzip_file(patch_file_abs_path)
+        xz_file(patch_file_abs_path)
 
-        # Create bzip's full file
+        # Create xz full file
         full_file_abs_path =  os.path.join(patch_info.work_dir, to_marfile_entry.name)
         shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path)
-        bzip_file(full_file_abs_path)
+        xz_file(full_file_abs_path)
 
         if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path):
             # Patch is smaller than file.  Remove the file and add patch to manifest
@@ -304,13 +304,17 @@ def process_explicit_remove_files(dir_path, patch_info):
         list_file_path = os.path.join(dir_path, "Contents/Resources/removed-files")
 
     if (os.path.exists(list_file_path)):
-        list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
+        fd, tmppath = tempfile.mkstemp('', 'tmp', os.getcwd())
+        os.close(fd)
+        exec_shell_cmd('xz -k -d --stdout "' + list_file_path + '" > "'+tmppath+'"')
+        list_file = open(tmppath)
 
         lines = []
         for line in list_file:
             lines.append(line.strip())
-        list_file.close()
 
+        list_file.close()
+        os.remove(tmppath)
         lines.sort(reverse=True)
         for line in lines:
             # Exclude any blank and comment lines.
@@ -406,7 +410,7 @@ def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch
     patch_info.create_manifest_files()
 
     # And construct the mar
-    mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')
+    mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+' '.join(patch_info.archive_files)
     exec_shell_cmd(mar_cmd)
 
     # Copy mar to final destination
@@ -431,11 +435,18 @@ def get_buildid(work_dir):
             print 'WARNING: application.ini not found, cannot find build ID'
             return ''
 
-    file = bz2.BZ2File(ini)
+    fd, tmppath = tempfile.mkstemp('', 'tmp', os.getcwd())
+    os.close(fd)
+    exec_shell_cmd('xz -k -d --stdout "' + ini + '" > "'+tmppath+'"')
+    file = open(tmppath)
     for line in file:
         if line.find('BuildID') == 0:
+            file.close()
+            os.remove(tmppath)
             return line.strip().split('=')[1]
-    print 'WARNING: cannot find build ID in application.ini'
+    print('WARNING: cannot find build ID in application.ini')
+    file.close()
+    os.remove(tmppath)
     return ''
 
 def decode_filename(filepath):
@@ -486,7 +497,7 @@ def create_partial_patches(patches):
             extract_mar(from_filename,work_dir_from)
             from_decoded = decode_filename(from_filename)
             from_buildid = get_buildid(work_dir_from)
-            from_shasum = sha.sha(open(from_filename).read()).hexdigest()
+            from_shasum = hashlib.sha1(open(from_filename, "rb").read()).hexdigest()
             from_size = str(os.path.getsize(to_filename))
 
             # Extract to mar into to dir
@@ -495,14 +506,14 @@ def create_partial_patches(patches):
             extract_mar(to_filename, work_dir_to)
             to_decoded = decode_filename(from_filename)
             to_buildid = get_buildid(work_dir_to)
-            to_shasum = sha.sha(open(to_filename).read()).hexdigest()
+            to_shasum = hashlib.sha1(open(to_filename, 'rb').read()).hexdigest()
             to_size = str(os.path.getsize(to_filename))
 
             mar_extract_time = time.time()
 
             partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['update.manifest','updatev2.manifest','updatev3.manifest'],[]),forced_updates,['channel-prefs.js','update-settings.ini'])
             partial_buildid = to_buildid
-            partial_shasum = sha.sha(open(partial_filename).read()).hexdigest()
+            partial_shasum = hashlib.sha1(open(partial_filename, "rb").read()).hexdigest()
             partial_size = str(os.path.getsize(partial_filename))
 
             metadata.append({
diff --git a/tools/update-packaging/test/catmanifest.sh b/tools/update-packaging/test/catmanifest.sh
index 354c3ff411..e6349d47a3 100755
--- a/tools/update-packaging/test/catmanifest.sh
+++ b/tools/update-packaging/test/catmanifest.sh
@@ -9,6 +9,6 @@ mkdir -p "$workdir"
 cp "$1" "$workdir"
 cd "$workdir"
 mar -x "$1"
-mv updatev2.manifest updatev2.manifest.bz2
-bzip2 -d updatev2.manifest.bz2
+mv updatev2.manifest updatev2.manifest.xz
+xz -d updatev2.manifest.xz
 cat updatev2.manifest
diff --git a/tools/update-packaging/test/common.sh b/tools/update-packaging/test/common.sh
index 7c133724e5..a9c904f037 100755
--- a/tools/update-packaging/test/common.sh
+++ b/tools/update-packaging/test/common.sh
@@ -10,9 +10,9 @@
 # In here to use the local common.sh to allow the full mars to have unfiltered files
 
 # -----------------------------------------------------------------------------
-# By default just assume that these tools exist on our path
+# By default just assume that these tools exist in our path
 MAR=${MAR:-mar}
-BZIP2=${BZIP2:-bzip2}
+XZ=${XZ:-xz}
 MBSDIFF=${MBSDIFF:-mbsdiff}
 
 # -----------------------------------------------------------------------------
diff --git a/tools/update-packaging/test/diffmar.sh b/tools/update-packaging/test/diffmar.sh
index 771b6b0ce1..f1dfc7fbdd 100755
--- a/tools/update-packaging/test/diffmar.sh
+++ b/tools/update-packaging/test/diffmar.sh
@@ -30,19 +30,19 @@ cp "$2" "$todir"
 cd "$fromdir"
 mar -x "$1"
 rm "$1"
-mv updatev2.manifest updatev2.manifest.bz2
-bzip2 -d updatev2.manifest.bz2
-mv updatev3.manifest updatev3.manifest.bz2
-bzip2 -d updatev3.manifest.bz2
+mv updatev2.manifest updatev2.manifest.xz
+xz -d updatev2.manifest.xz
+mv updatev3.manifest updatev3.manifest.xz
+xz -d updatev3.manifest.xz
 ls $lsargs > files.txt
 
 cd "$todir"
 mar -x "$2"
 rm "$2"
-mv updatev2.manifest updatev2.manifest.bz2
-bzip2 -d updatev2.manifest.bz2
-mv updatev3.manifest updatev3.manifest.bz2
-bzip2 -d updatev3.manifest.bz2
+mv updatev2.manifest updatev2.manifest.xz
+xz -d updatev2.manifest.xz
+mv updatev3.manifest updatev3.manifest.xz
+xz -d updatev3.manifest.xz
 ls $lsargs > files.txt
 
 echo "diffing $fromdir and $todir"
diff --git a/tools/update-packaging/test/make_full_update.sh b/tools/update-packaging/test/make_full_update.sh
index f2907da9ee..d6a01dcb76 100755
--- a/tools/update-packaging/test/make_full_update.sh
+++ b/tools/update-packaging/test/make_full_update.sh
@@ -94,7 +94,7 @@ for ((i=0; $i<$num_files; i=$i+1)); do
 
   dir=$(dirname "$f")
   mkdir -p "$workdir/$dir"
-  $BZIP2 -cz9 "$targetdir/$f" > "$workdir/$f"
+  $XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force --stdout "$targetdir/$f" > "$workdir/$f"
   copy_perm "$targetdir/$f" "$workdir/$f"
 
   targetfiles="$targetfiles \"$f\""
@@ -105,8 +105,8 @@ notice ""
 notice "Adding file and directory remove instructions from file 'removed-files'"
 append_remove_instructions "$targetdir" "$updatemanifestv2" "$updatemanifestv3"
 
-$BZIP2 -z9 "$updatemanifestv2" && mv -f "$updatemanifestv2.bz2" "$updatemanifestv2"
-$BZIP2 -z9 "$updatemanifestv3" && mv -f "$updatemanifestv3.bz2" "$updatemanifestv3"
+$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv2" && mv -f "$updatemanifestv2.xz" "$updatemanifestv2"
+$XZ --compress --x86 --lzma2 --format=xz --check=crc64 --force "$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
 
 eval "$MAR -C \"$workdir\" -c output.mar $targetfiles"
 mv -f "$workdir/output.mar" "$archive"
diff --git a/tools/update-packaging/unwrap_full_update.pl b/tools/update-packaging/unwrap_full_update.pl
index ead1157db9..90aa8ee726 100755
--- a/tools/update-packaging/unwrap_full_update.pl
+++ b/tools/update-packaging/unwrap_full_update.pl
@@ -13,7 +13,7 @@
 
 use Getopt::Std;
 
-my ($MAR, $BZIP2, $archive, @marentries, @marfiles);
+my ($MAR, $XZ, $BZIP2, $MAR_OLD_FORMAT, $archive, @marentries, @marfiles);
 
 if (defined($ENV{"MAR"})) {
     $MAR = $ENV{"MAR"};
@@ -29,6 +29,18 @@ else {
     $BZIP2 = "bzip2";
 }
 
+if (defined($ENV{"XZ"})) {
+    $XZ = $ENV{"XZ"};
+}
+else {
+    if (system("xz --version > /dev/null 2>&1") != 0) {
+        die("The xz executable must be in the path.");
+    }
+    else {
+        $XZ = "xz";
+    }
+}
+
 sub print_usage
 {
     print "Usage: unwrap_full_update.pl [OPTIONS] ARCHIVE\n\n";
@@ -50,18 +62,43 @@ $archive = $ARGV[0];
 
 $? && die("Couldn't run \"$MAR\" -t");
 
-shift @marentries;
+if (system("$MAR -x \"$archive\"") != 0) {
+  die "Couldn't run $MAR -x";
+}
+
+# Try to determine if the mar file contains bzip2 compressed files and if not
+# assume that the mar file contains lzma compressed files. The updatev3.manifest
+# file is checked since a valid mar file must have this file in the root path.
+open(my $testfilename, "updatev3.manifest") or die $!;
+binmode($testfilename);
+read($testfilename, my $bytes, 3);
+if ($bytes eq "BZh") {
+    $MAR_OLD_FORMAT = 1;
+} else {
+    undef $MAR_OLD_FORMAT;
+}
+close $testfilename;
 
-system("$MAR -x \"$archive\"") == 0 || die "Couldn't run $MAR -x";
+shift @marentries;
 
 foreach (@marentries) {
     tr/\n\r//d;
     my @splits = split(/\t/,$_);
     my $file = $splits[2];
 
-    system("mv \"$file\" \"$file.bz2\"") == 0 ||
-      die "Couldn't mv \"$file\"";
-    system("\"$BZIP2\" -d \"$file.bz2\"") == 0 ||
-      die "Couldn't decompress \"$file\"";
+    print "Decompressing: " . $file . "\n";
+    if ($MAR_OLD_FORMAT == 1) {
+      system("mv \"$file\" \"$file.bz2\"") == 0 ||
+        die "Couldn't mv \"$file\"";
+      system("\"$BZIP2\" -d \"$file.bz2\"") == 0 ||
+        die "Couldn't decompress \"$file\"";
+    }
+    else {
+      system("mv \"$file\" \"$file.xz\"") == 0 ||
+        die "Couldn't mv \"$file\"";
+      system("\"$XZ\" -d \"$file.xz\"") == 0 ||
+        die "Couldn't decompress \"$file\"";
+    }
 }
 
+print "Finished\n";