Issue #2065 - Part 1: Restore FasterMake build backend

I use this for quickly rebuilding and preprocessing the JS/XUL/CSS assets them without the need to recompile the IDL interfaces and other unnecessary items.

6 files changed, 290 insertions, 4 deletions

diff --git a/Makefile.in b/Makefile.in
index aec100ea0d..365dbd4a8a 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -173,6 +173,9 @@ endif
 $(addprefix install-,$(install_manifests)): install-%: $(install_manifest_depends)
 	$(addprefix $(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$*) ,$(wildcard _build_manifests/install/$(subst /,_,$*)))
 
+# Dummy wrapper rule to allow the faster backend to piggy back
+$(addprefix install-,$(subst /,_,$(filter dist/%,$(install_manifests)))): install-dist_%: install-dist/% ;
+
 .PHONY: install-tests
 install-tests: install-test-files
 
diff --git a/config/faster/rules.mk b/config/faster/rules.mk
new file mode 100644
index 0000000000..9d7b322fa2
--- /dev/null
+++ b/config/faster/rules.mk
@@ -0,0 +1,110 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# /!\ Please make sure to update the following comment when you touch this
+# file. Thank you /!\
+
+# The traditional Mozilla build system relied on going through the entire
+# build tree a number of times with different targets, and many of the
+# things happening at each step required other things happening in previous
+# steps without any documentation of those dependencies.
+#
+# This new build system tries to start afresh by establishing what files or
+# operations are needed for the build, and applying the necessary rules to
+# have those in place, relying on make dependencies to get them going.
+#
+# As of writing, only building non-compiled parts of Firefox is supported
+# here (a few other things are also left out). This is a starting point, with
+# the intent to grow this build system to make it more complete.
+#
+# This file contains rules and dependencies to get things working. The intent
+# is for a Makefile to define some dependencies and variables, and include
+# this file. What needs to be defined there, and ends up being generated by
+# python/mozbuild/mozbuild/backend/fastermake.py is the following:
+# - TOPSRCDIR/TOPOBJDIR, respectively the top source directory and the top
+#   object directory
+# - PYTHON, the path to the python executable
+# - ACDEFINES, which contains a set of -Dvar=name to be used during
+#   preprocessing
+# - INSTALL_MANIFESTS, which defines the list of base directories handled
+#   by install manifests, see further below
+#
+# A convention used between this file and the Makefile including it is that
+# global Make variables names are uppercase, while "local" Make variables
+# applied to specific targets are lowercase.
+
+# Targets to be triggered for a default build
+default: $(addprefix install-,$(INSTALL_MANIFESTS))
+
+ifndef NO_XPIDL
+# Targets from the recursive make backend to be built for a default build
+default: $(TOPOBJDIR)/config/makefiles/xpidl/xpidl
+endif
+
+# Mac builds require to copy things in dist/bin/*.app
+# TODO: remove the MOZ_WIDGET_TOOLKIT and MOZ_BUILD_APP variables from
+# faster/Makefile and python/mozbuild/mozbuild/test/backend/test_build.py
+# when this is not required anymore.
+# We however don't need to do this when using the hybrid
+# FasterMake/RecursiveMake backend (FASTER_RECURSIVE_MAKE is set when
+# recursing from the RecursiveMake backend)
+ifndef FASTER_RECURSIVE_MAKE
+ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
+default:
+	$(MAKE) -C $(TOPOBJDIR)/$(MOZ_BUILD_APP)/app repackage
+endif
+endif
+
+.PHONY: FORCE
+
+# Extra define to trigger some workarounds. We should strive to limit the
+# use of those. As of writing the only ones are in
+# toolkit/content/buildconfig.html and browser/locales/jar.mn.
+ACDEFINES += -DBUILD_FASTER
+
+# Files under the faster/ sub-directory, however, are not meant to use the
+# fallback
+$(TOPOBJDIR)/faster/%: ;
+
+# Generic rule to fall back to the recursive make backend.
+# This needs to stay after other $(TOPOBJDIR)/* rules because GNU Make
+# <3.82 apply pattern rules in definition order, not stem length like
+# modern GNU Make.
+$(TOPOBJDIR)/%: FORCE
+	$(MAKE) -C $(dir $@) $(notdir $@)
+
+# Install files using install manifests
+#
+# The list of base directories is given in INSTALL_MANIFESTS. The
+# corresponding install manifests are named correspondingly, with forward
+# slashes replaced with underscores, and prefixed with `install_`. That is,
+# the install manifest for `dist/bin` would be `install_dist_bin`.
+$(addprefix install-,$(INSTALL_MANIFESTS)): install-%: $(addprefix $(TOPOBJDIR)/,buildid.h source-repo.h)
+	@# For now, force preprocessed files to be reprocessed every time.
+	@# The overhead is not that big, and this avoids waiting for proper
+	@# support for defines tracking in process_install_manifest.
+	@touch install_$(subst /,_,$*)
+	@# BOOKMARKS_INCLUDE_DIR is for bookmarks.html only.
+	$(PYTHON) -m mozbuild.action.process_install_manifest \
+		--track install_$(subst /,_,$*).track \
+		$(TOPOBJDIR)/$* \
+		-DAB_CD=en-US \
+		-DBOOKMARKS_INCLUDE_DIR=$(TOPSRCDIR)/browser/locales/en-US/profile \
+		$(ACDEFINES) \
+		install_$(subst /,_,$*)
+
+# ============================================================================
+# Below is a set of additional dependencies and variables used to build things
+# that are not supported by data in moz.build.
+
+# The xpidl target in config/makefiles/xpidl requires the install manifest for
+# dist/idl to have been processed. When using the hybrid
+# FasterMake/RecursiveMake backend, this dependency is handled in the top-level
+# Makefile.
+ifndef FASTER_RECURSIVE_MAKE
+$(TOPOBJDIR)/config/makefiles/xpidl/xpidl: $(TOPOBJDIR)/install-dist_idl
+endif
+# It also requires all the install manifests for dist/bin to have been processed
+# because it adds interfaces.manifest references with buildlist.py.
+$(TOPOBJDIR)/config/makefiles/xpidl/xpidl: $(addprefix install-,$(filter dist/bin%,$(INSTALL_MANIFESTS)))
diff --git a/moz.configure b/moz.configure
index e3ee68bac3..f3718f9493 100644
--- a/moz.configure
+++ b/moz.configure
@@ -46,8 +46,8 @@ include('build/moz.configure/warnings.configure',
 
 include(include_project_configure)
 
-# We only support one build-backend, namely RecursiveMake.
-set_config('BUILD_BACKENDS', tuple(['RecursiveMake']))
+# Supported build back-ends: RecursiveMake and FasterMake
+set_config('BUILD_BACKENDS', tuple(['RecursiveMake', 'FasterMake']))
 
 
 # Awk detection
diff --git a/python/mozbuild/mozbuild/backend/__init__.py b/python/mozbuild/mozbuild/backend/__init__.py
index fede9cf9c7..448e97fa48 100644
--- a/python/mozbuild/mozbuild/backend/__init__.py
+++ b/python/mozbuild/mozbuild/backend/__init__.py
@@ -4,6 +4,7 @@
 
 backends = {
     'CompileDB': 'mozbuild.compilation.database',
+    'FasterMake': 'mozbuild.backend.fastermake',
     'RecursiveMake': 'mozbuild.backend.recursivemake',
 }
 
diff --git a/python/mozbuild/mozbuild/backend/base.py b/python/mozbuild/mozbuild/backend/base.py
index c46a3b1397..7ec5fe43d2 100644
--- a/python/mozbuild/mozbuild/backend/base.py
+++ b/python/mozbuild/mozbuild/backend/base.py
@@ -125,11 +125,13 @@ class BuildBackend(LoggingMixin):
 
         for obj in objs:
             obj_start = time.time()
-            if (not self.consume_object(obj)):
+            if (not self.consume_object(obj) and
+                    not isinstance(self, PartialBackend)):
                 raise Exception('Unhandled object of type %s' % type(obj))
             self._execution_time += time.time() - obj_start
 
-            if (isinstance(obj, ContextDerived)):
+            if (isinstance(obj, ContextDerived) and
+                    not isinstance(self, PartialBackend)):
                 self.backend_input_files |= obj.context_all_paths
 
         # Pull in all loaded Python as dependencies so any Python changes that
@@ -264,3 +266,8 @@ class BuildBackend(LoggingMixin):
         with self._write_file(obj.output_path) as fh:
             pp.out = fh
             yield pp
+
+class PartialBackend(BuildBackend):
+    """A PartialBackend is a BuildBackend declaring that its consume_object
+    method may not handle all build configuration objects it's passed, and
+    that it's fine."""
diff --git a/python/mozbuild/mozbuild/backend/fastermake.py b/python/mozbuild/mozbuild/backend/fastermake.py
new file mode 100644
index 0000000000..d55928e8c7
--- /dev/null
+++ b/python/mozbuild/mozbuild/backend/fastermake.py
@@ -0,0 +1,165 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals, print_function
+
+from mozbuild.backend.base import PartialBackend
+from mozbuild.backend.common import CommonBackend
+from mozbuild.frontend.context import (
+    ObjDirPath,
+)
+from mozbuild.frontend.data import (
+    ChromeManifestEntry,
+    FinalTargetPreprocessedFiles,
+    FinalTargetFiles,
+    JARManifest,
+    XPIDLFile,
+)
+from mozbuild.makeutil import Makefile
+from mozbuild.util import OrderedDefaultDict
+from mozpack.manifests import InstallManifest
+import mozpack.path as mozpath
+
+
+class FasterMakeBackend(CommonBackend, PartialBackend):
+    def _init(self):
+        super(FasterMakeBackend, self)._init()
+
+        self._manifest_entries = OrderedDefaultDict(set)
+
+        self._install_manifests = OrderedDefaultDict(InstallManifest)
+
+        self._dependencies = OrderedDefaultDict(list)
+
+        self._has_xpidl = False
+
+    def _add_preprocess(self, obj, path, dest, target=None, **kwargs):
+        if target is None:
+            target = mozpath.basename(path)
+        # This matches what PP_TARGETS do in config/rules.
+        if target.endswith('.in'):
+            target = target[:-3]
+        if target.endswith('.css'):
+            kwargs['marker'] = '%'
+        depfile = mozpath.join(
+            self.environment.topobjdir, 'faster', '.deps',
+            mozpath.join(obj.install_target, dest, target).replace('/', '_'))
+        self._install_manifests[obj.install_target].add_preprocess(
+            mozpath.join(obj.srcdir, path),
+            mozpath.join(dest, target),
+            depfile,
+            **kwargs)
+
+    def consume_object(self, obj):
+        if isinstance(obj, JARManifest) and \
+                obj.install_target.startswith('dist/bin'):
+            self._consume_jar_manifest(obj)
+
+        elif isinstance(obj, (FinalTargetFiles,
+                              FinalTargetPreprocessedFiles)) and \
+                obj.install_target.startswith('dist/bin'):
+            defines = obj.defines or {}
+            if defines:
+                defines = defines.defines
+            for path, files in obj.files.walk():
+                for f in files:
+                    if isinstance(obj, FinalTargetPreprocessedFiles):
+                        self._add_preprocess(obj, f.full_path, path,
+                                             target=f.target_basename,
+                                             defines=defines)
+                    elif '*' in f:
+                        def _prefix(s):
+                            for p in mozpath.split(s):
+                                if '*' not in p:
+                                    yield p + '/'
+                        prefix = ''.join(_prefix(f.full_path))
+
+                        self._install_manifests[obj.install_target] \
+                            .add_pattern_symlink(
+                                prefix,
+                                f.full_path[len(prefix):],
+                                mozpath.join(path, f.target_basename))
+                    else:
+                        self._install_manifests[obj.install_target].add_symlink(
+                            f.full_path,
+                            mozpath.join(path, f.target_basename)
+                        )
+                    if isinstance(f, ObjDirPath):
+                        dep_target = 'install-%s' % obj.install_target
+                        self._dependencies[dep_target].append(
+                            mozpath.relpath(f.full_path,
+                                            self.environment.topobjdir))
+
+        elif isinstance(obj, ChromeManifestEntry) and \
+                obj.install_target.startswith('dist/bin'):
+            top_level = mozpath.join(obj.install_target, 'chrome.manifest')
+            if obj.path != top_level:
+                entry = 'manifest %s' % mozpath.relpath(obj.path,
+                                                        obj.install_target)
+                self._manifest_entries[top_level].add(entry)
+            self._manifest_entries[obj.path].add(str(obj.entry))
+
+        elif isinstance(obj, XPIDLFile):
+            self._has_xpidl = True
+            # We're not actually handling XPIDL files.
+            return False
+
+        else:
+            return False
+
+        return True
+
+    def consume_finished(self):
+        mk = Makefile()
+        # Add the default rule at the very beginning.
+        mk.create_rule(['default'])
+        mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
+        mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
+        if not self._has_xpidl:
+            mk.add_statement('NO_XPIDL = 1')
+
+        # Add a few necessary variables inherited from configure
+        for var in (
+            'PYTHON',
+            'ACDEFINES',
+            'MOZ_BUILD_APP',
+            'MOZ_WIDGET_TOOLKIT',
+        ):
+            value = self.environment.substs.get(var)
+            if value is not None:
+                mk.add_statement('%s = %s' % (var, value))
+
+        install_manifests_bases = self._install_manifests.keys()
+
+        # Add information for chrome manifest generation
+        manifest_targets = []
+
+        for target, entries in self._manifest_entries.iteritems():
+            manifest_targets.append(target)
+            install_target = mozpath.basedir(target, install_manifests_bases)
+            self._install_manifests[install_target].add_content(
+                ''.join('%s\n' % e for e in sorted(entries)),
+                mozpath.relpath(target, install_target))
+
+        # Add information for install manifests.
+        mk.add_statement('INSTALL_MANIFESTS = %s'
+                         % ' '.join(self._install_manifests.keys()))
+
+        # Add dependencies we infered:
+        for target, deps in self._dependencies.iteritems():
+            mk.create_rule([target]).add_dependencies(
+                '$(TOPOBJDIR)/%s' % d for d in deps)
+
+        mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
+
+        for base, install_manifest in self._install_manifests.iteritems():
+            with self._write_file(
+                    mozpath.join(self.environment.topobjdir, 'faster',
+                                 'install_%s' % base.replace('/', '_'))) as fh:
+                install_manifest.write(fileobj=fh)
+
+        with self._write_file(
+                mozpath.join(self.environment.topobjdir, 'faster',
+                             'Makefile')) as fh:
+            mk.dump(fh, removal_guard=False)