Issue MoonchildProductions/UXP#1577 - Update client.mk to be more consistent with the original platform version

1 files changed, 132 insertions, 123 deletions

diff --git a/client.mk b/client.mk
index 564f569..89100d3 100644
--- a/client.mk
+++ b/client.mk
@@ -1,25 +1,24 @@
+# -*- makefile -*-
+# vim:set ts=8 sw=8 sts=8 noet:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# Build a comm application (Mozilla calendar, mail or suite).
+# Build a UXP application.
 #
 # To build a tree,
-#    1. hg clone http://hg.mozilla.org/comm-central comm
-#    2. cd comm
-#    3. python client.py checkout
-#    4. create your .mozconfig file with
-#       ac_add_options --enable-application=suite
-#         (or mail, or calendar)
-#    5. gmake -f client.mk
+#    1. git clone https://url.of/repo.git name-of-source-directory
+#    2. cd name-of-source-directory
+#    3. git submodule init && git submodule update
+#    4. create your .mozconfig file
+#    5. (g|moz)make -f client.mk or use mach
 #
-# Other targets (gmake -f client.mk [targets...]),
+# Other targets ((g|moz)make -f client.mk [targets...]),
 #    build
-#    clean
+#    clean (realclean is now the same as clean)
 #    distclean
 #
-# See http://developer.mozilla.org/en/Build_Documentation for
-# more information.
+# See relevant build documention for more information.
 #
 # Options:
 #   MOZ_BUILD_PROJECTS   - Build multiple projects in subdirectories
@@ -38,6 +37,16 @@
 
 comma := ,
 
+ifdef MACH
+ifndef NO_BUILDSTATUS_MESSAGES
+define BUILDSTATUS
+@echo 'BUILDSTATUS $1'
+
+endef
+endif
+endif
+
+
 CWD := $(CURDIR)
 ifneq (1,$(words $(CWD)))
 $(error The platform directory cannot be located in a path with spaces.)
@@ -73,19 +82,14 @@ ifneq (,$(findstring mingw,$(CONFIG_GUESS)))
 
 # check for CRLF line endings
 ifneq (0,$(shell $(PERL) -e 'binmode(STDIN); while (<STDIN>) { if (/\r/) { print "1"; exit } } print "0"' < $(TOPSRCDIR)/client.mk))
-$(error This source tree appears to have Windows-style line endings. To \
-convert it to Unix-style line endings, check \
-"https://developer.mozilla.org/en-US/docs/Developer_Guide/Mozilla_build_FAQ\#Win32-specific_questions" \
-for a workaround of this issue.)
+$(error This source tree appears to have Windows-style line endings. Please convert it to Unix-style line endings.)
 endif
 endif
 
 ####################################
 # Load mozconfig Options
 
-# See build pages, http://www.mozilla.org/build/ for how to set up mozconfig.
-
-MOZCONFIG_LOADER := build/autoconf/mozconfig2client-mk
+# See relevant build pages for how to set up mozconfig.
 
 define CR
 
@@ -98,7 +102,8 @@ endef
 # followed by a space (since sed doesn't remove newlines), except on the
 # last line, so replace both '|| ' and '||'.
 # Also, make MOZ_PGO available to mozconfig when passed on make command line.
-MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell MOZ_PGO=$(MOZ_PGO) $(TOPSRCDIR)/$(MOZCONFIG_LOADER) $(TOPSRCDIR) | sed 's/$$/||/')))
+# Likewise for MOZ_CURRENT_PROJECT.
+MOZCONFIG_CONTENT := $(subst ||,$(CR),$(subst || ,$(CR),$(shell $(addprefix MOZ_CURRENT_PROJECT=,$(MOZ_CURRENT_PROJECT)) MOZ_PGO=$(MOZ_PGO) $(TOPSRCDIR)/mach environment --format=client.mk | sed 's/$$/||/')))
 $(eval $(MOZCONFIG_CONTENT))
 
 export FOUND_MOZCONFIG
@@ -119,6 +124,11 @@ ifdef MOZ_PGO
 export MOZ_PGO
 endif
 
+ifdef MOZ_PARALLEL_BUILD
+  MOZ_MAKE_FLAGS := $(filter-out -j%,$(MOZ_MAKE_FLAGS))
+  MOZ_MAKE_FLAGS += -j$(MOZ_PARALLEL_BUILD)
+endif
+
 # Automatically add -jN to make flags if not defined. N defaults to number of cores.
 ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS)))
   cores=$(shell $(PYTHON) -c 'import multiprocessing; print(multiprocessing.cpu_count())')
@@ -126,40 +136,17 @@ ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS)))
 endif
 
 
-ifndef MOZ_OBJDIR
-  MOZ_OBJDIR = obj-$(CONFIG_GUESS)
-else
-# On Windows Pymake builds check MOZ_OBJDIR doesn't start with "/"
-  ifneq (,$(findstring mingw,$(CONFIG_GUESS)))
-  ifeq (1_a,$(.PYMAKE)_$(firstword a$(subst /, ,$(MOZ_OBJDIR))))
-  $(error For Windows Pymake builds, MOZ_OBJDIR must be a Windows [and not MSYS] style path.)
-  endif
-  endif
-endif
-
 ifdef MOZ_BUILD_PROJECTS
 
 ifdef MOZ_CURRENT_PROJECT
-  OBJDIR = $(MOZ_OBJDIR)/$(MOZ_CURRENT_PROJECT)
-  MOZ_MAKE = $(MAKE) $(MOZ_MAKE_FLAGS) -C $(OBJDIR)
   BUILD_PROJECT_ARG = MOZ_BUILD_APP=$(MOZ_CURRENT_PROJECT)
+  export MOZ_CURRENT_PROJECT
 else
-  OBJDIR = $(error Cannot find the OBJDIR when MOZ_CURRENT_PROJECT is not set.)
   MOZ_MAKE = $(error Cannot build in the OBJDIR when MOZ_CURRENT_PROJECT is not set.)
 endif
-
-else # MOZ_BUILD_PROJECTS
-
-OBJDIR = $(MOZ_OBJDIR)
-MOZ_MAKE = $(MAKE) $(MOZ_MAKE_FLAGS) -C $(OBJDIR)
-
 endif # MOZ_BUILD_PROJECTS
 
-# If we have a MOZ_OBJDIR that's set from the environment, ensure that it is an
-# absolute path.
-ifdef MOZ_OBJDIR
-MOZ_OBJDIR := $(shell $(PYTHON) -c "import os.path; print(os.path.join(\"$(TOPSRCDIR)\", \"$(MOZ_OBJDIR)\").replace('\\\\','/'))")
-endif
+MOZ_MAKE = $(MAKE) $(MOZ_MAKE_FLAGS) -C $(OBJDIR)
 
 # 'configure' scripts generated by autoconf.
 CONFIGURES := $(TOPSRCDIR)/configure
@@ -174,6 +161,7 @@ OBJDIR_TARGETS = install export libs clean realclean distclean maybe_clobber_pro
 
 # The default rule is build
 build::
+	$(MAKE) -f $(TOPSRCDIR)/client.mk $(if $(MOZ_PGO),profiledbuild,realbuild) CREATE_MOZCONFIG_JSON=
 
 # Define mkdir
 include $(TOPSRCDIR)/config/makefiles/makeutils.mk
@@ -192,10 +180,11 @@ WANT_MOZCONFIG_MK = 1
 endif
 
 ifdef WANT_MOZCONFIG_MK
-# For now, only output "export" lines from mozconfig2client-mk output.
-MOZCONFIG_MK_LINES := $(filter export||%,$(MOZCONFIG_OUT_LINES))
-$(OBJDIR)/.mozconfig.mk: $(FOUND_MOZCONFIG) $(call mkdir_deps,$(OBJDIR))
-	$(if $(MOZCONFIG_MK_LINES),( $(foreach line,$(MOZCONFIG_MK_LINES), echo "$(subst ||, ,$(line))";) )) > $@
+# For now, only output "export" lines and lines containing UPLOAD_EXTRA_FILES
+# from mach environment --format=client.mk output.
+MOZCONFIG_MK_LINES := $(filter export||% UPLOAD_EXTRA_FILES% %UPLOAD_EXTRA_FILES%,$(MOZCONFIG_OUT_LINES))
+$(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG) $(call mkdir_deps,$(OBJDIR)) $(OBJDIR)/CLOBBER
+	$(if $(MOZCONFIG_MK_LINES),( $(foreach line,$(MOZCONFIG_MK_LINES), echo '$(subst ||, ,$(line))';) )) > $@
 ifdef MOZ_CURRENT_PROJECT
 	echo export MOZ_CURRENT_PROJECT=$(MOZ_CURRENT_PROJECT) >> $@
 endif
@@ -206,23 +195,8 @@ endif
 include $(OBJDIR)/.mozconfig.mk
 endif
 
-# UPLOAD_EXTRA_FILES is appended to and exported from mozconfig, which makes
-# submakes as well as configure add even more to that, so just unexport it
-# for submakes to pick it from .mozconfig.mk and for configure to pick it
-# from mach environment.
-unexport UPLOAD_EXTRA_FILES
-
-# These targets are candidates for auto-running client.py
-
-ifeq (01,$(MAKELEVEL)$(if $(ALWAYS_RUN_CLIENT_PY),1,))
-
-build profiledbuild configure:: run_client_py
-	$(MAKE) -f $(TOPSRCDIR)/client.mk $@
-else
-
-
 # Print out any options loaded from mozconfig.
-all build clean distclean export libs install realclean::
+all realbuild clean distclean export libs install realclean::
 ifneq (,$(strip $(MOZCONFIG_OUT_FILTERED)))
 	$(info Adding client.mk options from $(FOUND_MOZCONFIG):)
 	$(foreach line,$(MOZCONFIG_OUT_FILTERED),$(info $(NULL) $(NULL) $(NULL) $(NULL) $(subst ||, ,$(line))))
@@ -232,32 +206,35 @@ endif
 build_all: build
 clobber clobber_all: clean
 
+# helper target for mobile
+build_and_deploy: build package install
+
 # Do everything from scratch
 everything: clean build
 
 ####################################
 # Profile-Guided Optimization
-#  To use this, you should set the following variables in your mozconfig
-#    mk_add_options PROFILE_GEN_SCRIPT=/path/to/profile-script
-#
-#  The profile script should exercise the functionality to be included
-#  in the profile feedback.
-#
 #  This is up here, outside of the MOZ_CURRENT_PROJECT logic so that this
 #  is usable in multi-pass builds, where you might not have a runnable
 #  application until all the build passes and postflight scripts have run.
-ifdef MOZ_OBJDIR
-  PGO_OBJDIR = $(MOZ_OBJDIR)
-else
-  PGO_OBJDIR := $(TOPSRCDIR)
-endif
-
 profiledbuild::
-	$(MAKE) -f $(TOPSRCDIR)/client.mk build MOZ_PROFILE_GENERATE=1
-	$(MAKE) -C $(PGO_OBJDIR) stage-package
-	OBJDIR=${PGO_OBJDIR} $(PROFILE_GEN_SCRIPT)
-	$(MAKE) -f $(TOPSRCDIR)/client.mk maybe_clobber_profiledbuild
-	$(MAKE) -f $(TOPSRCDIR)/client.mk build MOZ_PROFILE_USE=1
+	$(call BUILDSTATUS,TIERS pgo_profile_generate pgo_package pgo_profile pgo_clobber pgo_profile_use)
+	$(call BUILDSTATUS,TIER_START pgo_profile_generate)
+	$(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_GENERATE=1 MOZ_PGO_INSTRUMENTED=1 CREATE_MOZCONFIG_JSON=
+	$(call BUILDSTATUS,TIER_FINISH pgo_profile_generate)
+	$(call BUILDSTATUS,TIER_START pgo_package)
+	$(MAKE) -C $(OBJDIR) package MOZ_PGO_INSTRUMENTED=1 MOZ_INTERNAL_SIGNING_FORMAT= MOZ_EXTERNAL_SIGNING_FORMAT=
+	rm -f $(OBJDIR)/jarlog/en-US.log
+	$(call BUILDSTATUS,TIER_FINISH pgo_package)
+	$(call BUILDSTATUS,TIER_START pgo_profile)
+	MOZ_PGO_INSTRUMENTED=1 JARLOG_FILE=jarlog/en-US.log EXTRA_TEST_ARGS=10 $(MAKE) -C $(OBJDIR) pgo-profile-run
+	$(call BUILDSTATUS,TIER_FINISH pgo_profile)
+	$(call BUILDSTATUS,TIER_START pgo_clobber)
+	$(MAKE) -f $(TOPSRCDIR)/client.mk maybe_clobber_profiledbuild CREATE_MOZCONFIG_JSON=
+	$(call BUILDSTATUS,TIER_FINISH pgo_clobber)
+	$(call BUILDSTATUS,TIER_START pgo_profile_use)
+	$(MAKE) -f $(TOPSRCDIR)/client.mk realbuild MOZ_PROFILE_USE=1 CREATE_MOZCONFIG_JSON=
+	$(call BUILDSTATUS,TIER_FINISH pgo_profile_use)
 
 #####################################################
 # Build date unification
@@ -274,7 +251,7 @@ endif
 #####################################################
 # Preflight, before building any project
 
-build preflight_all::
+realbuild preflight_all::
 ifeq (,$(MOZ_CURRENT_PROJECT)$(if $(MOZ_PREFLIGHT_ALL),,1))
 # Don't run preflight_all for individual projects in multi-project builds
 # (when MOZ_CURRENT_PROJECT is set.)
@@ -298,7 +275,7 @@ endif
 # loop through them.
 
 ifeq (,$(MOZ_CURRENT_PROJECT)$(if $(MOZ_BUILD_PROJECTS),,1))
-configure build preflight postflight $(OBJDIR_TARGETS)::
+configure realbuild preflight postflight $(OBJDIR_TARGETS)::
 	set -e; \
 	for app in $(MOZ_BUILD_PROJECTS); do \
 	  $(MAKE) -f $(TOPSRCDIR)/client.mk $@ MOZ_CURRENT_PROJECT=$$app; \
@@ -317,37 +294,38 @@ CONFIG_STATUS = $(wildcard $(OBJDIR)/config.status)
 CONFIG_CACHE  = $(wildcard $(OBJDIR)/config.cache)
 
 EXTRA_CONFIG_DEPS := \
-	$(TOPSRCDIR)/aclocal.m4 \
-	$(TOPSRCDIR)/platform/aclocal.m4 \
-	$(TOPSRCDIR)/platform/old-configure.in \
-	$(wildcard $(TOPSRCDIR)/platform/build/autoconf/*.m4) \
-	$(TOPSRCDIR)/platform/js/src/aclocal.m4 \
-	$(TOPSRCDIR)/platform/js/src/old-configure.in \
-	$(NULL)
+  $(TOPSRCDIR)/aclocal.m4 \
+  $(TOPSRCDIR)/platform/aclocal.m4 \
+  $(TOPSRCDIR)/platform/old-configure.in \
+  $(wildcard $(TOPSRCDIR)/platform/build/autoconf/*.m4) \
+  $(TOPSRCDIR)/platform/js/src/aclocal.m4 \
+  $(TOPSRCDIR)/platform/js/src/old-configure.in \
+  $(NULL)
 
 $(CONFIGURES): %: %.in $(EXTRA_CONFIG_DEPS)
 	@echo Generating $@
-	sed '1,/^divert/d' $< > $@
+	cp -f $< $@
 	chmod +x $@
 
 CONFIG_STATUS_DEPS := \
-	$(wildcard $(TOPSRCDIR)/*/confvars.sh) \
-	$(wildcard $(TOPSRCDIR)/*/configure.in) \
-	$(wildcard $(TOPSRCDIR)/*/config/version.txt) \
-	$(wildcard $(CONFIGURES)) \
-	$(TOPSRCDIR)/platform/CLOBBER \
-	$(wildcard $(TOPSRCDIR)/platform/nsprpub/configure) \
-	$(wildcard $(TOPSRCDIR)/platform/config/milestone.txt) \
-	$(wildcard $(TOPSRCDIR)/platform/ldap/sdks/c-sdk/configure) \
-	$(wildcard $(addsuffix confvars.sh,$(wildcard $(TOPSRCDIR)/*/))) \
-	$(NULL)
+  $(wildcard $(TOPSRCDIR)/platform/ldap/sdks/c-sdk/configure) \
+  $(wildcard $(TOPSRCDIR)/*/confvars.sh) \
+  $(CONFIGURES) \
+  $(TOPSRCDIR)/platform/CLOBBER \
+  $(TOPSRCDIR)/platform/nsprpub/configure \
+  $(TOPSRCDIR)/platform/config/milestone.txt \
+  $(TOPSRCDIR)/platform/build/virtualenv_packages.txt \
+  $(TOPSRCDIR)/platform/python/mozbuild/mozbuild/virtualenv.py \
+  $(TOPSRCDIR)/platform/testing/mozbase/packages.txt \
+  $(OBJDIR)/.mozconfig.json \
+  $(NULL)
 
 CONFIGURE_ENV_ARGS += \
   MAKE='$(MAKE)' \
   $(NULL)
 
 # configure uses the program name to determine @srcdir@. Calling it without
-#   $(TOPSRCDIR) will set @srcdir@ to '.'; otherwise, it is set to the full
+#   $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full
 #   path of $(TOPSRCDIR).
 ifeq ($(TOPSRCDIR),$(OBJDIR))
   CONFIGURE = ./configure
@@ -357,7 +335,7 @@ endif
 
 $(OBJDIR)/CLOBBER: $(TOPSRCDIR)/platform/CLOBBER
 	$(PYTHON) $(TOPSRCDIR)/platform/config/pythonpath.py -I $(TOPSRCDIR)/platform/testing/mozbase/mozfile \
-			$(TOPSRCDIR)/platform/python/mozbuild/mozbuild/controller/clobber.py $(TOPSRCDIR)/platform $(OBJDIR)
+	    $(TOPSRCDIR)/platform/python/mozbuild/mozbuild/controller/clobber.py $(TOPSRCDIR)/platform $(OBJDIR)
 
 configure-files: $(CONFIGURES)
 
@@ -366,15 +344,34 @@ configure-preqs = \
   configure-files \
   $(call mkdir_deps,$(OBJDIR)) \
   $(if $(MOZ_BUILD_PROJECTS),$(call mkdir_deps,$(MOZ_OBJDIR))) \
+  save-mozconfig \
+  $(OBJDIR)/.mozconfig.json \
   $(NULL)
 
+CREATE_MOZCONFIG_JSON = $(shell $(TOPSRCDIR)/mach environment --format=json -o $(OBJDIR)/.mozconfig.json)
+# Force CREATE_MOZCONFIG_JSON above to be resolved, without side effects in
+# case the result is non empty, and allowing an override on the make command
+# line not running the command (using := $(shell) still runs the shell command).
+ifneq (,$(CREATE_MOZCONFIG_JSON))
+endif
+
+$(OBJDIR)/.mozconfig.json: $(call mkdir_deps,$(OBJDIR)) ;
+
+save-mozconfig: $(FOUND_MOZCONFIG)
+ifdef FOUND_MOZCONFIG
+	-cp $(FOUND_MOZCONFIG) $(OBJDIR)/.mozconfig
+endif
+
 configure:: $(configure-preqs)
+	$(call BUILDSTATUS,TIERS configure)
+	$(call BUILDSTATUS,TIER_START configure)
 	@echo cd $(OBJDIR);
 	@echo $(CONFIGURE) $(CONFIGURE_ARGS)
 	@cd $(OBJDIR) && $(BUILD_PROJECT_ARG) $(CONFIGURE_ENV_ARGS) $(CONFIGURE) $(CONFIGURE_ARGS) \
 	  || ( echo '*** Fix above errors and then restart with\
                "$(MAKE) -f client.mk build"' && exit 1 )
 	@touch $(OBJDIR)/Makefile
+	$(call BUILDSTATUS,TIER_FINISH configure)
 
 ifneq (,$(MAKEFILE))
 $(OBJDIR)/Makefile: $(OBJDIR)/config.status
@@ -383,7 +380,7 @@ $(OBJDIR)/config.status: $(CONFIG_STATUS_DEPS)
 else
 $(OBJDIR)/Makefile: $(CONFIG_STATUS_DEPS)
 endif
-	@$(MAKE) -f $(TOPSRCDIR)/client.mk configure
+	@$(MAKE) -f $(TOPSRCDIR)/client.mk configure CREATE_MOZCONFIG_JSON=
 
 ifneq (,$(CONFIG_STATUS))
 $(OBJDIR)/config/autoconf.mk: $(TOPSRCDIR)/config/autoconf.mk.in
@@ -394,7 +391,7 @@ endif
 ####################################
 # Preflight
 
-build preflight::
+realbuild preflight::
 ifdef MOZ_PREFLIGHT
 	set -e; \
 	for mkfile in $(MOZ_PREFLIGHT); do \
@@ -405,7 +402,7 @@ endif
 ####################################
 # Build it
 
-build::  $(OBJDIR)/Makefile $(OBJDIR)/config.status
+realbuild::  $(OBJDIR)/Makefile $(OBJDIR)/config.status
 	+$(MOZ_MAKE)
 
 ####################################
@@ -418,7 +415,7 @@ $(OBJDIR_TARGETS):: $(OBJDIR)/Makefile $(OBJDIR)/config.status
 ####################################
 # Postflight
 
-build postflight::
+realbuild postflight::
 ifdef MOZ_POSTFLIGHT
 	set -e; \
 	for mkfile in $(MOZ_POSTFLIGHT); do \
@@ -427,12 +424,20 @@ ifdef MOZ_POSTFLIGHT
 endif
 
 endif # MOZ_CURRENT_PROJECT
-endif # RAN_CLIENT_PY
 
 ####################################
 # Postflight, after building all projects
 
-build postflight_all::
+ifdef MOZ_AUTOMATION
+ifndef MOZ_CURRENT_PROJECT
+$(if $(MOZ_PGO),profiledbuild,realbuild)::
+# Only run the automation/build target for the first project.
+# (i.e. first platform of universal builds)
+	$(MAKE) -f $(TOPSRCDIR)/client.mk automation/build $(addprefix MOZ_CURRENT_PROJECT=,$(firstword $(MOZ_BUILD_PROJECTS)))
+endif
+endif
+
+realbuild postflight_all::
 ifeq (,$(MOZ_CURRENT_PROJECT)$(if $(MOZ_POSTFLIGHT_ALL),,1))
 # Don't run postflight_all for individual projects in multi-project builds
 # (when MOZ_CURRENT_PROJECT is set.)
@@ -452,18 +457,6 @@ else
 endif
 endif
 
-cleansrcdir:
-	@cd $(TOPSRCDIR); \
-	if [ -f Makefile ]; then \
-	  $(MAKE) distclean ; \
-	else \
-	  echo 'Removing object files from srcdir...'; \
-	  rm -fr `find . -type d \( -name .deps -print -o -name CVS \
-	          -o -exec test ! -d {}/CVS \; \) -prune \
-	          -o \( -name '*.[ao]' -o -name '*.so' \) -type f -print`; \
-	   build/autoconf/clean-config.sh; \
-	fi;
-
 echo-variable-%:
 	@echo $($*)
 
@@ -472,4 +465,20 @@ echo-variable-%:
 # in parallel.
 .NOTPARALLEL:
 
-.PHONY: checkout co real_checkout build profiledbuild cleansrcdir pull_all build_all clobber clobber_all pull_and_build_all everything configure preflight_all preflight postflight postflight_all $(OBJDIR_TARGETS)
+.PHONY: checkout \
+    real_checkout \
+    realbuild \
+    build \
+    profiledbuild \
+    pull_all \
+    build_all \
+    clobber \
+    clobber_all \
+    pull_and_build_all \
+    everything \
+    configure \
+    preflight_all \
+    preflight \
+    postflight \
+    postflight_all \
+    $(OBJDIR_TARGETS)