From e4ed223f80260aa735a77bc4cfff25b76f3e7a94 Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Wed, 5 Oct 2022 14:14:19 -0500 Subject: [RES2] Move pymake to python/ and remove a bunch of mozconfig and other half-out-of-date MozInfra items from build/ --- build/mozconfig.automation | 33 - build/mozconfig.cache | 135 -- build/mozconfig.clang-cl | 7 - build/mozconfig.common | 21 - build/mozconfig.common.override | 11 - build/mozconfig.vs-common | 4 - build/mozconfig.win-common | 16 - build/pymake/LICENSE | 21 - build/pymake/README | 64 - build/pymake/make.py | 35 - build/pymake/mkformat.py | 13 - build/pymake/mkparse.py | 12 - build/pymake/pymake/__init__.py | 0 build/pymake/pymake/builtins.py | 120 -- build/pymake/pymake/command.py | 278 --- build/pymake/pymake/data.py | 1842 -------------------- build/pymake/pymake/functions.py | 873 ---------- build/pymake/pymake/globrelative.py | 68 - build/pymake/pymake/implicit.py | 14 - build/pymake/pymake/parser.py | 822 --------- build/pymake/pymake/parserdata.py | 1006 ----------- build/pymake/pymake/process.py | 556 ------ build/pymake/pymake/util.py | 150 -- build/pymake/pymake/win32process.py | 28 - build/pymake/tests/automatic-variables.mk | 79 - build/pymake/tests/bad-command-continuation.mk | 3 - build/pymake/tests/call.mk | 12 - build/pymake/tests/cmd-stripdotslash.mk | 5 - build/pymake/tests/cmdgoals.mk | 9 - build/pymake/tests/commandmodifiers.mk | 21 - build/pymake/tests/comment-parsing.mk | 29 - build/pymake/tests/continuations-in-functions.mk | 6 - build/pymake/tests/datatests.py | 237 --- build/pymake/tests/default-goal-set-first.mk | 7 - build/pymake/tests/default-goal.mk | 8 - build/pymake/tests/default-target.mk | 14 - build/pymake/tests/default-target2.mk | 6 - build/pymake/tests/define-directive.mk | 69 - build/pymake/tests/depfailed.mk | 4 - build/pymake/tests/depfailedj.mk | 10 - build/pymake/tests/diamond-deps.mk | 13 - build/pymake/tests/dotslash-dir.mk | 8 - build/pymake/tests/dotslash-parse.mk | 4 - build/pymake/tests/dotslash-phony.mk | 3 - build/pymake/tests/dotslash.mk | 9 - build/pymake/tests/doublecolon-exists.mk | 16 - build/pymake/tests/doublecolon-priordeps.mk | 19 - build/pymake/tests/doublecolon-remake.mk | 4 - build/pymake/tests/dynamic-var.mk | 18 - build/pymake/tests/empty-arg.mk | 2 - build/pymake/tests/empty-command-semicolon.mk | 5 - build/pymake/tests/empty-with-deps.mk | 4 - build/pymake/tests/env-var-append.mk | 7 - build/pymake/tests/env-var-append2.mk | 8 - build/pymake/tests/eof-continuation.mk | 5 - build/pymake/tests/escape-chars.mk | 26 - build/pymake/tests/escaped-continuation.mk | 6 - build/pymake/tests/eval-duringexecute.mk | 12 - build/pymake/tests/eval.mk | 7 - build/pymake/tests/exit-code.mk | 5 - build/pymake/tests/file-functions-symlinks.mk | 22 - build/pymake/tests/file-functions.mk | 19 - build/pymake/tests/foreach-local-variable.mk | 8 - build/pymake/tests/formattingtests.py | 289 --- build/pymake/tests/func-refs.mk | 11 - build/pymake/tests/functions.mk | 36 - build/pymake/tests/functiontests.py | 54 - build/pymake/tests/if-syntaxerr.mk | 6 - build/pymake/tests/ifdefs-nesting.mk | 13 - build/pymake/tests/ifdefs.mk | 127 -- build/pymake/tests/ignore-error.mk | 13 - build/pymake/tests/implicit-chain.mk | 12 - build/pymake/tests/implicit-dir.mk | 16 - build/pymake/tests/implicit-terminal.mk | 16 - build/pymake/tests/implicitsubdir.mk | 12 - build/pymake/tests/include-dynamic.mk | 21 - build/pymake/tests/include-file.inc | 1 - build/pymake/tests/include-missing.mk | 9 - build/pymake/tests/include-notfound.mk | 19 - build/pymake/tests/include-optional-warning.mk | 4 - build/pymake/tests/include-regen.mk | 10 - build/pymake/tests/include-regen2.mk | 10 - build/pymake/tests/include-regen3.mk | 10 - build/pymake/tests/include-test.mk | 8 - build/pymake/tests/includedeps-norebuild.mk | 15 - build/pymake/tests/includedeps-sideeffects.mk | 10 - build/pymake/tests/includedeps-stripdotslash.deps | 1 - build/pymake/tests/includedeps-stripdotslash.mk | 8 - build/pymake/tests/includedeps-variables.deps | 1 - build/pymake/tests/includedeps-variables.mk | 10 - build/pymake/tests/includedeps.deps | 1 - build/pymake/tests/includedeps.mk | 9 - build/pymake/tests/info.mk | 8 - build/pymake/tests/justprint-native.mk | 28 - build/pymake/tests/justprint.mk | 5 - build/pymake/tests/keep-going-doublecolon.mk | 16 - build/pymake/tests/keep-going-parallel.mk | 11 - build/pymake/tests/keep-going.mk | 14 - build/pymake/tests/line-continuations.mk | 24 - build/pymake/tests/link-search.mk | 7 - build/pymake/tests/makeflags.mk | 7 - build/pymake/tests/matchany.mk | 14 - build/pymake/tests/matchany2.mk | 13 - build/pymake/tests/matchany3.mk | 10 - build/pymake/tests/mkdir-fail.mk | 7 - build/pymake/tests/mkdir.mk | 27 - .../tests/multiple-rules-prerequisite-merge.mk | 25 - build/pymake/tests/native-command-delay-load.mk | 12 - build/pymake/tests/native-command-raise.mk | 9 - build/pymake/tests/native-command-return-fail1.mk | 8 - build/pymake/tests/native-command-return-fail2.mk | 8 - build/pymake/tests/native-command-return.mk | 11 - build/pymake/tests/native-command-shell-glob.mk | 11 - .../pymake/tests/native-command-sys-exit-fail1.mk | 8 - .../pymake/tests/native-command-sys-exit-fail2.mk | 8 - build/pymake/tests/native-command-sys-exit.mk | 11 - build/pymake/tests/native-environment.mk | 11 - build/pymake/tests/native-pycommandpath-sep.mk | 21 - build/pymake/tests/native-pycommandpath.mk | 15 - build/pymake/tests/native-simple.mk | 12 - build/pymake/tests/native-touch.mk | 15 - build/pymake/tests/newlines.mk | 30 - build/pymake/tests/no-remake.mk | 7 - build/pymake/tests/nosuchfile.mk | 4 - build/pymake/tests/notargets.mk | 5 - build/pymake/tests/notparallel.mk | 8 - .../pymake/tests/oneline-command-continuations.mk | 5 - build/pymake/tests/override-propagate.mk | 37 - build/pymake/tests/parallel-dep-resolution.mk | 8 - build/pymake/tests/parallel-dep-resolution2.mk | 9 - build/pymake/tests/parallel-native.mk | 21 - build/pymake/tests/parallel-simple.mk | 27 - build/pymake/tests/parallel-submake.mk | 17 - build/pymake/tests/parallel-toserial.mk | 31 - build/pymake/tests/parallel-waiting.mk | 21 - build/pymake/tests/parentheses.mk | 2 - build/pymake/tests/parsertests.py | 314 ---- build/pymake/tests/path-length.mk | 9 - build/pymake/tests/pathdir/pathtest | 2 - build/pymake/tests/pathdir/pathtest.exe | Bin 45056 -> 0 bytes build/pymake/tests/pathdir/src/Makefile | 2 - build/pymake/tests/pathdir/src/pathtest.cpp | 6 - build/pymake/tests/patsubst.mk | 7 - build/pymake/tests/phony.mk | 10 - build/pymake/tests/pycmd.py | 38 - build/pymake/tests/recursive-set.mk | 7 - build/pymake/tests/recursive-set2.mk | 8 - build/pymake/tests/remake-mtime.mk | 14 - build/pymake/tests/rm-fail.mk | 7 - build/pymake/tests/rm.mk | 21 - build/pymake/tests/runtests.py | 215 --- build/pymake/tests/serial-dep-resolution.mk | 5 - build/pymake/tests/serial-doublecolon-execution.mk | 18 - build/pymake/tests/serial-rule-execution.mk | 5 - build/pymake/tests/serial-rule-execution2.mk | 13 - build/pymake/tests/serial-toparallel.mk | 5 - build/pymake/tests/shellfunc.mk | 7 - build/pymake/tests/simple-makeflags.mk | 10 - build/pymake/tests/sort.mk | 4 - build/pymake/tests/specified-target.mk | 7 - build/pymake/tests/static-pattern.mk | 5 - build/pymake/tests/static-pattern2.mk | 10 - build/pymake/tests/subdir/delayload.py | 1 - build/pymake/tests/subdir/pymod.py | 5 - build/pymake/tests/subdir/testmodule.py | 3 - build/pymake/tests/submake-path.makefile2 | 11 - build/pymake/tests/submake-path.mk | 16 - build/pymake/tests/submake.makefile2 | 24 - build/pymake/tests/submake.mk | 16 - build/pymake/tests/subprocess-path.mk | 32 - build/pymake/tests/tab-intro.mk | 16 - build/pymake/tests/target-specific.mk | 30 - build/pymake/tests/unexport.mk | 15 - build/pymake/tests/unexport.submk | 15 - build/pymake/tests/unterminated-dollar.mk | 6 - build/pymake/tests/var-change-flavor.mk | 12 - build/pymake/tests/var-commandline.mk | 8 - build/pymake/tests/var-overrides.mk | 21 - build/pymake/tests/var-ref.mk | 19 - build/pymake/tests/var-set.mk | 55 - build/pymake/tests/var-substitutions.mk | 49 - build/pymake/tests/vpath-directive-dynamic.mk | 12 - build/pymake/tests/vpath-directive.mk | 31 - build/pymake/tests/vpath.mk | 18 - build/pymake/tests/vpath2.mk | 18 - build/pymake/tests/wildcards.mk | 22 - build/pymake/tests/windows-paths.mk | 5 - build/virtualenv_packages.txt | 2 +- build/win32/mozconfig.vs-latest | 1 - build/win32/mozconfig.vs2015-win64 | 25 - build/win64/mozconfig.vs-latest | 1 - build/win64/mozconfig.vs2015 | 24 - python/pymake/LICENSE | 21 + python/pymake/README | 64 + python/pymake/make.py | 35 + python/pymake/mkformat.py | 13 + python/pymake/mkparse.py | 12 + python/pymake/pymake/__init__.py | 0 python/pymake/pymake/builtins.py | 120 ++ python/pymake/pymake/command.py | 278 +++ python/pymake/pymake/data.py | 1842 ++++++++++++++++++++ python/pymake/pymake/functions.py | 873 ++++++++++ python/pymake/pymake/globrelative.py | 68 + python/pymake/pymake/implicit.py | 14 + python/pymake/pymake/parser.py | 822 +++++++++ python/pymake/pymake/parserdata.py | 1006 +++++++++++ python/pymake/pymake/process.py | 556 ++++++ python/pymake/pymake/util.py | 150 ++ python/pymake/pymake/win32process.py | 28 + python/pymake/tests/automatic-variables.mk | 79 + python/pymake/tests/bad-command-continuation.mk | 3 + python/pymake/tests/call.mk | 12 + python/pymake/tests/cmd-stripdotslash.mk | 5 + python/pymake/tests/cmdgoals.mk | 9 + python/pymake/tests/commandmodifiers.mk | 21 + python/pymake/tests/comment-parsing.mk | 29 + python/pymake/tests/continuations-in-functions.mk | 6 + python/pymake/tests/datatests.py | 237 +++ python/pymake/tests/default-goal-set-first.mk | 7 + python/pymake/tests/default-goal.mk | 8 + python/pymake/tests/default-target.mk | 14 + python/pymake/tests/default-target2.mk | 6 + python/pymake/tests/define-directive.mk | 69 + python/pymake/tests/depfailed.mk | 4 + python/pymake/tests/depfailedj.mk | 10 + python/pymake/tests/diamond-deps.mk | 13 + python/pymake/tests/dotslash-dir.mk | 8 + python/pymake/tests/dotslash-parse.mk | 4 + python/pymake/tests/dotslash-phony.mk | 3 + python/pymake/tests/dotslash.mk | 9 + python/pymake/tests/doublecolon-exists.mk | 16 + python/pymake/tests/doublecolon-priordeps.mk | 19 + python/pymake/tests/doublecolon-remake.mk | 4 + python/pymake/tests/dynamic-var.mk | 18 + python/pymake/tests/empty-arg.mk | 2 + python/pymake/tests/empty-command-semicolon.mk | 5 + python/pymake/tests/empty-with-deps.mk | 4 + python/pymake/tests/env-var-append.mk | 7 + python/pymake/tests/env-var-append2.mk | 8 + python/pymake/tests/eof-continuation.mk | 5 + python/pymake/tests/escape-chars.mk | 26 + python/pymake/tests/escaped-continuation.mk | 6 + python/pymake/tests/eval-duringexecute.mk | 12 + python/pymake/tests/eval.mk | 7 + python/pymake/tests/exit-code.mk | 5 + python/pymake/tests/file-functions-symlinks.mk | 22 + python/pymake/tests/file-functions.mk | 19 + python/pymake/tests/foreach-local-variable.mk | 8 + python/pymake/tests/formattingtests.py | 289 +++ python/pymake/tests/func-refs.mk | 11 + python/pymake/tests/functions.mk | 36 + python/pymake/tests/functiontests.py | 54 + python/pymake/tests/if-syntaxerr.mk | 6 + python/pymake/tests/ifdefs-nesting.mk | 13 + python/pymake/tests/ifdefs.mk | 127 ++ python/pymake/tests/ignore-error.mk | 13 + python/pymake/tests/implicit-chain.mk | 12 + python/pymake/tests/implicit-dir.mk | 16 + python/pymake/tests/implicit-terminal.mk | 16 + python/pymake/tests/implicitsubdir.mk | 12 + python/pymake/tests/include-dynamic.mk | 21 + python/pymake/tests/include-file.inc | 1 + python/pymake/tests/include-missing.mk | 9 + python/pymake/tests/include-notfound.mk | 19 + python/pymake/tests/include-optional-warning.mk | 4 + python/pymake/tests/include-regen.mk | 10 + python/pymake/tests/include-regen2.mk | 10 + python/pymake/tests/include-regen3.mk | 10 + python/pymake/tests/include-test.mk | 8 + python/pymake/tests/includedeps-norebuild.mk | 15 + python/pymake/tests/includedeps-sideeffects.mk | 10 + python/pymake/tests/includedeps-stripdotslash.deps | 1 + python/pymake/tests/includedeps-stripdotslash.mk | 8 + python/pymake/tests/includedeps-variables.deps | 1 + python/pymake/tests/includedeps-variables.mk | 10 + python/pymake/tests/includedeps.deps | 1 + python/pymake/tests/includedeps.mk | 9 + python/pymake/tests/info.mk | 8 + python/pymake/tests/justprint-native.mk | 28 + python/pymake/tests/justprint.mk | 5 + python/pymake/tests/keep-going-doublecolon.mk | 16 + python/pymake/tests/keep-going-parallel.mk | 11 + python/pymake/tests/keep-going.mk | 14 + python/pymake/tests/line-continuations.mk | 24 + python/pymake/tests/link-search.mk | 7 + python/pymake/tests/makeflags.mk | 7 + python/pymake/tests/matchany.mk | 14 + python/pymake/tests/matchany2.mk | 13 + python/pymake/tests/matchany3.mk | 10 + python/pymake/tests/mkdir-fail.mk | 7 + python/pymake/tests/mkdir.mk | 27 + .../tests/multiple-rules-prerequisite-merge.mk | 25 + python/pymake/tests/native-command-delay-load.mk | 12 + python/pymake/tests/native-command-raise.mk | 9 + python/pymake/tests/native-command-return-fail1.mk | 8 + python/pymake/tests/native-command-return-fail2.mk | 8 + python/pymake/tests/native-command-return.mk | 11 + python/pymake/tests/native-command-shell-glob.mk | 11 + .../pymake/tests/native-command-sys-exit-fail1.mk | 8 + .../pymake/tests/native-command-sys-exit-fail2.mk | 8 + python/pymake/tests/native-command-sys-exit.mk | 11 + python/pymake/tests/native-environment.mk | 11 + python/pymake/tests/native-pycommandpath-sep.mk | 21 + python/pymake/tests/native-pycommandpath.mk | 15 + python/pymake/tests/native-simple.mk | 12 + python/pymake/tests/native-touch.mk | 15 + python/pymake/tests/newlines.mk | 30 + python/pymake/tests/no-remake.mk | 7 + python/pymake/tests/nosuchfile.mk | 4 + python/pymake/tests/notargets.mk | 5 + python/pymake/tests/notparallel.mk | 8 + .../pymake/tests/oneline-command-continuations.mk | 5 + python/pymake/tests/override-propagate.mk | 37 + python/pymake/tests/parallel-dep-resolution.mk | 8 + python/pymake/tests/parallel-dep-resolution2.mk | 9 + python/pymake/tests/parallel-native.mk | 21 + python/pymake/tests/parallel-simple.mk | 27 + python/pymake/tests/parallel-submake.mk | 17 + python/pymake/tests/parallel-toserial.mk | 31 + python/pymake/tests/parallel-waiting.mk | 21 + python/pymake/tests/parentheses.mk | 2 + python/pymake/tests/parsertests.py | 314 ++++ python/pymake/tests/path-length.mk | 9 + python/pymake/tests/pathdir/pathtest | 2 + python/pymake/tests/pathdir/pathtest.exe | Bin 0 -> 45056 bytes python/pymake/tests/pathdir/src/Makefile | 2 + python/pymake/tests/pathdir/src/pathtest.cpp | 6 + python/pymake/tests/patsubst.mk | 7 + python/pymake/tests/phony.mk | 10 + python/pymake/tests/pycmd.py | 38 + python/pymake/tests/recursive-set.mk | 7 + python/pymake/tests/recursive-set2.mk | 8 + python/pymake/tests/remake-mtime.mk | 14 + python/pymake/tests/rm-fail.mk | 7 + python/pymake/tests/rm.mk | 21 + python/pymake/tests/runtests.py | 215 +++ python/pymake/tests/serial-dep-resolution.mk | 5 + .../pymake/tests/serial-doublecolon-execution.mk | 18 + python/pymake/tests/serial-rule-execution.mk | 5 + python/pymake/tests/serial-rule-execution2.mk | 13 + python/pymake/tests/serial-toparallel.mk | 5 + python/pymake/tests/shellfunc.mk | 7 + python/pymake/tests/simple-makeflags.mk | 10 + python/pymake/tests/sort.mk | 4 + python/pymake/tests/specified-target.mk | 7 + python/pymake/tests/static-pattern.mk | 5 + python/pymake/tests/static-pattern2.mk | 10 + python/pymake/tests/subdir/delayload.py | 1 + python/pymake/tests/subdir/pymod.py | 5 + python/pymake/tests/subdir/testmodule.py | 3 + python/pymake/tests/submake-path.makefile2 | 11 + python/pymake/tests/submake-path.mk | 16 + python/pymake/tests/submake.makefile2 | 24 + python/pymake/tests/submake.mk | 16 + python/pymake/tests/subprocess-path.mk | 32 + python/pymake/tests/tab-intro.mk | 16 + python/pymake/tests/target-specific.mk | 30 + python/pymake/tests/unexport.mk | 15 + python/pymake/tests/unexport.submk | 15 + python/pymake/tests/unterminated-dollar.mk | 6 + python/pymake/tests/var-change-flavor.mk | 12 + python/pymake/tests/var-commandline.mk | 8 + python/pymake/tests/var-overrides.mk | 21 + python/pymake/tests/var-ref.mk | 19 + python/pymake/tests/var-set.mk | 55 + python/pymake/tests/var-substitutions.mk | 49 + python/pymake/tests/vpath-directive-dynamic.mk | 12 + python/pymake/tests/vpath-directive.mk | 31 + python/pymake/tests/vpath.mk | 18 + python/pymake/tests/vpath2.mk | 18 + python/pymake/tests/wildcards.mk | 22 + python/pymake/tests/windows-paths.mk | 5 + 372 files changed, 9221 insertions(+), 9499 deletions(-) delete mode 100644 build/mozconfig.automation delete mode 100644 build/mozconfig.cache delete mode 100644 build/mozconfig.clang-cl delete mode 100644 build/mozconfig.common delete mode 100644 build/mozconfig.common.override delete mode 100644 build/mozconfig.vs-common delete mode 100644 build/mozconfig.win-common delete mode 100644 build/pymake/LICENSE delete mode 100644 build/pymake/README delete mode 100755 build/pymake/make.py delete mode 100755 build/pymake/mkformat.py delete mode 100755 build/pymake/mkparse.py delete mode 100644 build/pymake/pymake/__init__.py delete mode 100644 build/pymake/pymake/builtins.py delete mode 100644 build/pymake/pymake/command.py delete mode 100644 build/pymake/pymake/data.py delete mode 100644 build/pymake/pymake/functions.py delete mode 100644 build/pymake/pymake/globrelative.py delete mode 100644 build/pymake/pymake/implicit.py delete mode 100644 build/pymake/pymake/parser.py delete mode 100644 build/pymake/pymake/parserdata.py delete mode 100644 build/pymake/pymake/process.py delete mode 100644 build/pymake/pymake/util.py delete mode 100644 build/pymake/pymake/win32process.py delete mode 100644 build/pymake/tests/automatic-variables.mk delete mode 100644 build/pymake/tests/bad-command-continuation.mk delete mode 100644 build/pymake/tests/call.mk delete mode 100644 build/pymake/tests/cmd-stripdotslash.mk delete mode 100644 build/pymake/tests/cmdgoals.mk delete mode 100644 build/pymake/tests/commandmodifiers.mk delete mode 100644 build/pymake/tests/comment-parsing.mk delete mode 100644 build/pymake/tests/continuations-in-functions.mk delete mode 100644 build/pymake/tests/datatests.py delete mode 100644 build/pymake/tests/default-goal-set-first.mk delete mode 100644 build/pymake/tests/default-goal.mk delete mode 100644 build/pymake/tests/default-target.mk delete mode 100644 build/pymake/tests/default-target2.mk delete mode 100644 build/pymake/tests/define-directive.mk delete mode 100644 build/pymake/tests/depfailed.mk delete mode 100644 build/pymake/tests/depfailedj.mk delete mode 100644 build/pymake/tests/diamond-deps.mk delete mode 100644 build/pymake/tests/dotslash-dir.mk delete mode 100644 build/pymake/tests/dotslash-parse.mk delete mode 100644 build/pymake/tests/dotslash-phony.mk delete mode 100644 build/pymake/tests/dotslash.mk delete mode 100644 build/pymake/tests/doublecolon-exists.mk delete mode 100644 build/pymake/tests/doublecolon-priordeps.mk delete mode 100644 build/pymake/tests/doublecolon-remake.mk delete mode 100644 build/pymake/tests/dynamic-var.mk delete mode 100644 build/pymake/tests/empty-arg.mk delete mode 100644 build/pymake/tests/empty-command-semicolon.mk delete mode 100644 build/pymake/tests/empty-with-deps.mk delete mode 100644 build/pymake/tests/env-var-append.mk delete mode 100644 build/pymake/tests/env-var-append2.mk delete mode 100644 build/pymake/tests/eof-continuation.mk delete mode 100644 build/pymake/tests/escape-chars.mk delete mode 100644 build/pymake/tests/escaped-continuation.mk delete mode 100644 build/pymake/tests/eval-duringexecute.mk delete mode 100644 build/pymake/tests/eval.mk delete mode 100644 build/pymake/tests/exit-code.mk delete mode 100644 build/pymake/tests/file-functions-symlinks.mk delete mode 100644 build/pymake/tests/file-functions.mk delete mode 100644 build/pymake/tests/foreach-local-variable.mk delete mode 100644 build/pymake/tests/formattingtests.py delete mode 100644 build/pymake/tests/func-refs.mk delete mode 100644 build/pymake/tests/functions.mk delete mode 100644 build/pymake/tests/functiontests.py delete mode 100644 build/pymake/tests/if-syntaxerr.mk delete mode 100644 build/pymake/tests/ifdefs-nesting.mk delete mode 100644 build/pymake/tests/ifdefs.mk delete mode 100644 build/pymake/tests/ignore-error.mk delete mode 100644 build/pymake/tests/implicit-chain.mk delete mode 100644 build/pymake/tests/implicit-dir.mk delete mode 100644 build/pymake/tests/implicit-terminal.mk delete mode 100644 build/pymake/tests/implicitsubdir.mk delete mode 100644 build/pymake/tests/include-dynamic.mk delete mode 100644 build/pymake/tests/include-file.inc delete mode 100644 build/pymake/tests/include-missing.mk delete mode 100644 build/pymake/tests/include-notfound.mk delete mode 100644 build/pymake/tests/include-optional-warning.mk delete mode 100644 build/pymake/tests/include-regen.mk delete mode 100644 build/pymake/tests/include-regen2.mk delete mode 100644 build/pymake/tests/include-regen3.mk delete mode 100644 build/pymake/tests/include-test.mk delete mode 100644 build/pymake/tests/includedeps-norebuild.mk delete mode 100644 build/pymake/tests/includedeps-sideeffects.mk delete mode 100644 build/pymake/tests/includedeps-stripdotslash.deps delete mode 100644 build/pymake/tests/includedeps-stripdotslash.mk delete mode 100644 build/pymake/tests/includedeps-variables.deps delete mode 100644 build/pymake/tests/includedeps-variables.mk delete mode 100644 build/pymake/tests/includedeps.deps delete mode 100644 build/pymake/tests/includedeps.mk delete mode 100644 build/pymake/tests/info.mk delete mode 100644 build/pymake/tests/justprint-native.mk delete mode 100644 build/pymake/tests/justprint.mk delete mode 100644 build/pymake/tests/keep-going-doublecolon.mk delete mode 100644 build/pymake/tests/keep-going-parallel.mk delete mode 100644 build/pymake/tests/keep-going.mk delete mode 100644 build/pymake/tests/line-continuations.mk delete mode 100644 build/pymake/tests/link-search.mk delete mode 100644 build/pymake/tests/makeflags.mk delete mode 100644 build/pymake/tests/matchany.mk delete mode 100644 build/pymake/tests/matchany2.mk delete mode 100644 build/pymake/tests/matchany3.mk delete mode 100644 build/pymake/tests/mkdir-fail.mk delete mode 100644 build/pymake/tests/mkdir.mk delete mode 100644 build/pymake/tests/multiple-rules-prerequisite-merge.mk delete mode 100644 build/pymake/tests/native-command-delay-load.mk delete mode 100644 build/pymake/tests/native-command-raise.mk delete mode 100644 build/pymake/tests/native-command-return-fail1.mk delete mode 100644 build/pymake/tests/native-command-return-fail2.mk delete mode 100644 build/pymake/tests/native-command-return.mk delete mode 100644 build/pymake/tests/native-command-shell-glob.mk delete mode 100644 build/pymake/tests/native-command-sys-exit-fail1.mk delete mode 100644 build/pymake/tests/native-command-sys-exit-fail2.mk delete mode 100644 build/pymake/tests/native-command-sys-exit.mk delete mode 100644 build/pymake/tests/native-environment.mk delete mode 100644 build/pymake/tests/native-pycommandpath-sep.mk delete mode 100644 build/pymake/tests/native-pycommandpath.mk delete mode 100644 build/pymake/tests/native-simple.mk delete mode 100644 build/pymake/tests/native-touch.mk delete mode 100644 build/pymake/tests/newlines.mk delete mode 100644 build/pymake/tests/no-remake.mk delete mode 100644 build/pymake/tests/nosuchfile.mk delete mode 100644 build/pymake/tests/notargets.mk delete mode 100644 build/pymake/tests/notparallel.mk delete mode 100644 build/pymake/tests/oneline-command-continuations.mk delete mode 100644 build/pymake/tests/override-propagate.mk delete mode 100644 build/pymake/tests/parallel-dep-resolution.mk delete mode 100644 build/pymake/tests/parallel-dep-resolution2.mk delete mode 100644 build/pymake/tests/parallel-native.mk delete mode 100644 build/pymake/tests/parallel-simple.mk delete mode 100644 build/pymake/tests/parallel-submake.mk delete mode 100644 build/pymake/tests/parallel-toserial.mk delete mode 100644 build/pymake/tests/parallel-waiting.mk delete mode 100644 build/pymake/tests/parentheses.mk delete mode 100644 build/pymake/tests/parsertests.py delete mode 100644 build/pymake/tests/path-length.mk delete mode 100755 build/pymake/tests/pathdir/pathtest delete mode 100644 build/pymake/tests/pathdir/pathtest.exe delete mode 100644 build/pymake/tests/pathdir/src/Makefile delete mode 100644 build/pymake/tests/pathdir/src/pathtest.cpp delete mode 100644 build/pymake/tests/patsubst.mk delete mode 100644 build/pymake/tests/phony.mk delete mode 100644 build/pymake/tests/pycmd.py delete mode 100644 build/pymake/tests/recursive-set.mk delete mode 100644 build/pymake/tests/recursive-set2.mk delete mode 100644 build/pymake/tests/remake-mtime.mk delete mode 100644 build/pymake/tests/rm-fail.mk delete mode 100644 build/pymake/tests/rm.mk delete mode 100644 build/pymake/tests/runtests.py delete mode 100644 build/pymake/tests/serial-dep-resolution.mk delete mode 100644 build/pymake/tests/serial-doublecolon-execution.mk delete mode 100644 build/pymake/tests/serial-rule-execution.mk delete mode 100644 build/pymake/tests/serial-rule-execution2.mk delete mode 100644 build/pymake/tests/serial-toparallel.mk delete mode 100644 build/pymake/tests/shellfunc.mk delete mode 100644 build/pymake/tests/simple-makeflags.mk delete mode 100644 build/pymake/tests/sort.mk delete mode 100644 build/pymake/tests/specified-target.mk delete mode 100644 build/pymake/tests/static-pattern.mk delete mode 100644 build/pymake/tests/static-pattern2.mk delete mode 100644 build/pymake/tests/subdir/delayload.py delete mode 100644 build/pymake/tests/subdir/pymod.py delete mode 100644 build/pymake/tests/subdir/testmodule.py delete mode 100644 build/pymake/tests/submake-path.makefile2 delete mode 100644 build/pymake/tests/submake-path.mk delete mode 100644 build/pymake/tests/submake.makefile2 delete mode 100644 build/pymake/tests/submake.mk delete mode 100644 build/pymake/tests/subprocess-path.mk delete mode 100644 build/pymake/tests/tab-intro.mk delete mode 100644 build/pymake/tests/target-specific.mk delete mode 100644 build/pymake/tests/unexport.mk delete mode 100644 build/pymake/tests/unexport.submk delete mode 100644 build/pymake/tests/unterminated-dollar.mk delete mode 100644 build/pymake/tests/var-change-flavor.mk delete mode 100644 build/pymake/tests/var-commandline.mk delete mode 100644 build/pymake/tests/var-overrides.mk delete mode 100644 build/pymake/tests/var-ref.mk delete mode 100644 build/pymake/tests/var-set.mk delete mode 100644 build/pymake/tests/var-substitutions.mk delete mode 100644 build/pymake/tests/vpath-directive-dynamic.mk delete mode 100644 build/pymake/tests/vpath-directive.mk delete mode 100644 build/pymake/tests/vpath.mk delete mode 100644 build/pymake/tests/vpath2.mk delete mode 100644 build/pymake/tests/wildcards.mk delete mode 100644 build/pymake/tests/windows-paths.mk delete mode 100644 build/win32/mozconfig.vs-latest delete mode 100644 build/win32/mozconfig.vs2015-win64 delete mode 100644 build/win64/mozconfig.vs-latest delete mode 100644 build/win64/mozconfig.vs2015 create mode 100644 python/pymake/LICENSE create mode 100644 python/pymake/README create mode 100644 python/pymake/make.py create mode 100644 python/pymake/mkformat.py create mode 100644 python/pymake/mkparse.py create mode 100644 python/pymake/pymake/__init__.py create mode 100644 python/pymake/pymake/builtins.py create mode 100644 python/pymake/pymake/command.py create mode 100644 python/pymake/pymake/data.py create mode 100644 python/pymake/pymake/functions.py create mode 100644 python/pymake/pymake/globrelative.py create mode 100644 python/pymake/pymake/implicit.py create mode 100644 python/pymake/pymake/parser.py create mode 100644 python/pymake/pymake/parserdata.py create mode 100644 python/pymake/pymake/process.py create mode 100644 python/pymake/pymake/util.py create mode 100644 python/pymake/pymake/win32process.py create mode 100644 python/pymake/tests/automatic-variables.mk create mode 100644 python/pymake/tests/bad-command-continuation.mk create mode 100644 python/pymake/tests/call.mk create mode 100644 python/pymake/tests/cmd-stripdotslash.mk create mode 100644 python/pymake/tests/cmdgoals.mk create mode 100644 python/pymake/tests/commandmodifiers.mk create mode 100644 python/pymake/tests/comment-parsing.mk create mode 100644 python/pymake/tests/continuations-in-functions.mk create mode 100644 python/pymake/tests/datatests.py create mode 100644 python/pymake/tests/default-goal-set-first.mk create mode 100644 python/pymake/tests/default-goal.mk create mode 100644 python/pymake/tests/default-target.mk create mode 100644 python/pymake/tests/default-target2.mk create mode 100644 python/pymake/tests/define-directive.mk create mode 100644 python/pymake/tests/depfailed.mk create mode 100644 python/pymake/tests/depfailedj.mk create mode 100644 python/pymake/tests/diamond-deps.mk create mode 100644 python/pymake/tests/dotslash-dir.mk create mode 100644 python/pymake/tests/dotslash-parse.mk create mode 100644 python/pymake/tests/dotslash-phony.mk create mode 100644 python/pymake/tests/dotslash.mk create mode 100644 python/pymake/tests/doublecolon-exists.mk create mode 100644 python/pymake/tests/doublecolon-priordeps.mk create mode 100644 python/pymake/tests/doublecolon-remake.mk create mode 100644 python/pymake/tests/dynamic-var.mk create mode 100644 python/pymake/tests/empty-arg.mk create mode 100644 python/pymake/tests/empty-command-semicolon.mk create mode 100644 python/pymake/tests/empty-with-deps.mk create mode 100644 python/pymake/tests/env-var-append.mk create mode 100644 python/pymake/tests/env-var-append2.mk create mode 100644 python/pymake/tests/eof-continuation.mk create mode 100644 python/pymake/tests/escape-chars.mk create mode 100644 python/pymake/tests/escaped-continuation.mk create mode 100644 python/pymake/tests/eval-duringexecute.mk create mode 100644 python/pymake/tests/eval.mk create mode 100644 python/pymake/tests/exit-code.mk create mode 100644 python/pymake/tests/file-functions-symlinks.mk create mode 100644 python/pymake/tests/file-functions.mk create mode 100644 python/pymake/tests/foreach-local-variable.mk create mode 100644 python/pymake/tests/formattingtests.py create mode 100644 python/pymake/tests/func-refs.mk create mode 100644 python/pymake/tests/functions.mk create mode 100644 python/pymake/tests/functiontests.py create mode 100644 python/pymake/tests/if-syntaxerr.mk create mode 100644 python/pymake/tests/ifdefs-nesting.mk create mode 100644 python/pymake/tests/ifdefs.mk create mode 100644 python/pymake/tests/ignore-error.mk create mode 100644 python/pymake/tests/implicit-chain.mk create mode 100644 python/pymake/tests/implicit-dir.mk create mode 100644 python/pymake/tests/implicit-terminal.mk create mode 100644 python/pymake/tests/implicitsubdir.mk create mode 100644 python/pymake/tests/include-dynamic.mk create mode 100644 python/pymake/tests/include-file.inc create mode 100644 python/pymake/tests/include-missing.mk create mode 100644 python/pymake/tests/include-notfound.mk create mode 100644 python/pymake/tests/include-optional-warning.mk create mode 100644 python/pymake/tests/include-regen.mk create mode 100644 python/pymake/tests/include-regen2.mk create mode 100644 python/pymake/tests/include-regen3.mk create mode 100644 python/pymake/tests/include-test.mk create mode 100644 python/pymake/tests/includedeps-norebuild.mk create mode 100644 python/pymake/tests/includedeps-sideeffects.mk create mode 100644 python/pymake/tests/includedeps-stripdotslash.deps create mode 100644 python/pymake/tests/includedeps-stripdotslash.mk create mode 100644 python/pymake/tests/includedeps-variables.deps create mode 100644 python/pymake/tests/includedeps-variables.mk create mode 100644 python/pymake/tests/includedeps.deps create mode 100644 python/pymake/tests/includedeps.mk create mode 100644 python/pymake/tests/info.mk create mode 100644 python/pymake/tests/justprint-native.mk create mode 100644 python/pymake/tests/justprint.mk create mode 100644 python/pymake/tests/keep-going-doublecolon.mk create mode 100644 python/pymake/tests/keep-going-parallel.mk create mode 100644 python/pymake/tests/keep-going.mk create mode 100644 python/pymake/tests/line-continuations.mk create mode 100644 python/pymake/tests/link-search.mk create mode 100644 python/pymake/tests/makeflags.mk create mode 100644 python/pymake/tests/matchany.mk create mode 100644 python/pymake/tests/matchany2.mk create mode 100644 python/pymake/tests/matchany3.mk create mode 100644 python/pymake/tests/mkdir-fail.mk create mode 100644 python/pymake/tests/mkdir.mk create mode 100644 python/pymake/tests/multiple-rules-prerequisite-merge.mk create mode 100644 python/pymake/tests/native-command-delay-load.mk create mode 100644 python/pymake/tests/native-command-raise.mk create mode 100644 python/pymake/tests/native-command-return-fail1.mk create mode 100644 python/pymake/tests/native-command-return-fail2.mk create mode 100644 python/pymake/tests/native-command-return.mk create mode 100644 python/pymake/tests/native-command-shell-glob.mk create mode 100644 python/pymake/tests/native-command-sys-exit-fail1.mk create mode 100644 python/pymake/tests/native-command-sys-exit-fail2.mk create mode 100644 python/pymake/tests/native-command-sys-exit.mk create mode 100644 python/pymake/tests/native-environment.mk create mode 100644 python/pymake/tests/native-pycommandpath-sep.mk create mode 100644 python/pymake/tests/native-pycommandpath.mk create mode 100644 python/pymake/tests/native-simple.mk create mode 100644 python/pymake/tests/native-touch.mk create mode 100644 python/pymake/tests/newlines.mk create mode 100644 python/pymake/tests/no-remake.mk create mode 100644 python/pymake/tests/nosuchfile.mk create mode 100644 python/pymake/tests/notargets.mk create mode 100644 python/pymake/tests/notparallel.mk create mode 100644 python/pymake/tests/oneline-command-continuations.mk create mode 100644 python/pymake/tests/override-propagate.mk create mode 100644 python/pymake/tests/parallel-dep-resolution.mk create mode 100644 python/pymake/tests/parallel-dep-resolution2.mk create mode 100644 python/pymake/tests/parallel-native.mk create mode 100644 python/pymake/tests/parallel-simple.mk create mode 100644 python/pymake/tests/parallel-submake.mk create mode 100644 python/pymake/tests/parallel-toserial.mk create mode 100644 python/pymake/tests/parallel-waiting.mk create mode 100644 python/pymake/tests/parentheses.mk create mode 100644 python/pymake/tests/parsertests.py create mode 100644 python/pymake/tests/path-length.mk create mode 100644 python/pymake/tests/pathdir/pathtest create mode 100644 python/pymake/tests/pathdir/pathtest.exe create mode 100644 python/pymake/tests/pathdir/src/Makefile create mode 100644 python/pymake/tests/pathdir/src/pathtest.cpp create mode 100644 python/pymake/tests/patsubst.mk create mode 100644 python/pymake/tests/phony.mk create mode 100644 python/pymake/tests/pycmd.py create mode 100644 python/pymake/tests/recursive-set.mk create mode 100644 python/pymake/tests/recursive-set2.mk create mode 100644 python/pymake/tests/remake-mtime.mk create mode 100644 python/pymake/tests/rm-fail.mk create mode 100644 python/pymake/tests/rm.mk create mode 100644 python/pymake/tests/runtests.py create mode 100644 python/pymake/tests/serial-dep-resolution.mk create mode 100644 python/pymake/tests/serial-doublecolon-execution.mk create mode 100644 python/pymake/tests/serial-rule-execution.mk create mode 100644 python/pymake/tests/serial-rule-execution2.mk create mode 100644 python/pymake/tests/serial-toparallel.mk create mode 100644 python/pymake/tests/shellfunc.mk create mode 100644 python/pymake/tests/simple-makeflags.mk create mode 100644 python/pymake/tests/sort.mk create mode 100644 python/pymake/tests/specified-target.mk create mode 100644 python/pymake/tests/static-pattern.mk create mode 100644 python/pymake/tests/static-pattern2.mk create mode 100644 python/pymake/tests/subdir/delayload.py create mode 100644 python/pymake/tests/subdir/pymod.py create mode 100644 python/pymake/tests/subdir/testmodule.py create mode 100644 python/pymake/tests/submake-path.makefile2 create mode 100644 python/pymake/tests/submake-path.mk create mode 100644 python/pymake/tests/submake.makefile2 create mode 100644 python/pymake/tests/submake.mk create mode 100644 python/pymake/tests/subprocess-path.mk create mode 100644 python/pymake/tests/tab-intro.mk create mode 100644 python/pymake/tests/target-specific.mk create mode 100644 python/pymake/tests/unexport.mk create mode 100644 python/pymake/tests/unexport.submk create mode 100644 python/pymake/tests/unterminated-dollar.mk create mode 100644 python/pymake/tests/var-change-flavor.mk create mode 100644 python/pymake/tests/var-commandline.mk create mode 100644 python/pymake/tests/var-overrides.mk create mode 100644 python/pymake/tests/var-ref.mk create mode 100644 python/pymake/tests/var-set.mk create mode 100644 python/pymake/tests/var-substitutions.mk create mode 100644 python/pymake/tests/vpath-directive-dynamic.mk create mode 100644 python/pymake/tests/vpath-directive.mk create mode 100644 python/pymake/tests/vpath.mk create mode 100644 python/pymake/tests/vpath2.mk create mode 100644 python/pymake/tests/wildcards.mk create mode 100644 python/pymake/tests/windows-paths.mk diff --git a/build/mozconfig.automation b/build/mozconfig.automation deleted file mode 100644 index 057a4a0b5..000000000 --- a/build/mozconfig.automation +++ /dev/null @@ -1,33 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# Common mozconfig for automation builds. -# -# We export MOZ_AUTOMATION_* variables here to trigger various steps in -# automation builds. For example, if MOZ_AUTOMATION_PACKAGE is set, then the -# package step will run. This file contains the default settings, which can be -# overridden by setting them earlier in the appropriate mozconfig. - -mk_add_options "export MOZ_AUTOMATION_BUILD_SYMBOLS=${MOZ_AUTOMATION_BUILD_SYMBOLS-1}" -mk_add_options "export MOZ_AUTOMATION_L10N_CHECK=${MOZ_AUTOMATION_L10N_CHECK-1}" -mk_add_options "export MOZ_AUTOMATION_PACKAGE=${MOZ_AUTOMATION_PACKAGE-1}" -mk_add_options "export MOZ_AUTOMATION_PACKAGE_TESTS=${MOZ_AUTOMATION_PACKAGE_TESTS-1}" -mk_add_options "export MOZ_AUTOMATION_INSTALLER=${MOZ_AUTOMATION_INSTALLER-0}" -mk_add_options "export MOZ_AUTOMATION_UPDATE_PACKAGING=${MOZ_AUTOMATION_UPDATE_PACKAGING-0}" -mk_add_options "export MOZ_AUTOMATION_UPLOAD=${MOZ_AUTOMATION_UPLOAD-1}" -mk_add_options "export MOZ_AUTOMATION_UPLOAD_SYMBOLS=${MOZ_AUTOMATION_UPLOAD_SYMBOLS-0}" -mk_add_options "export MOZ_AUTOMATION_SDK=${MOZ_AUTOMATION_SDK-0}" - -# If we are also building with MOZ_PKG_PRETTYNAMES, set the corresponding -# stages. -if test "$MOZ_AUTOMATION_PRETTY" = "1"; then - mk_add_options "export MOZ_AUTOMATION_PRETTY_PACKAGE=${MOZ_AUTOMATION_PACKAGE-1}" - mk_add_options "export MOZ_AUTOMATION_PRETTY_PACKAGE_TESTS=${MOZ_AUTOMATION_PACKAGE_TESTS-1}" - mk_add_options "export MOZ_AUTOMATION_PRETTY_L10N_CHECK=${MOZ_AUTOMATION_L10N_CHECK-1}" - mk_add_options "export MOZ_AUTOMATION_PRETTY_INSTALLER=${MOZ_AUTOMATION_INSTALLER-0}" - - # Note that we always build the update packaging with pretty names even if - # we don't build it without, so this is set to 1. - mk_add_options "export MOZ_AUTOMATION_PRETTY_UPDATE_PACKAGING=1" -fi diff --git a/build/mozconfig.cache b/build/mozconfig.cache deleted file mode 100644 index be740e293..000000000 --- a/build/mozconfig.cache +++ /dev/null @@ -1,135 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# Setup for build cache - -# Avoid duplication if the file happens to be included twice. -if test -z "$bucket" -a -z "$NO_CACHE"; then - -# buildbot (or builders that use buildprops.json): -if [ -f $topsrcdir/../buildprops.json ]; then -read branch platform master < /dev/null) -EOF - -bucket= -if test -z "$SCCACHE_DISABLE" -a -z "$no_sccache" -a -z "$MOZ_PGO_IS_SET" -a -z "$MOZ_PGO"; then - case "${branch}" in - try) - case "${master}" in - *scl1.mozilla.com*|*.scl3.mozilla.com*) - bucket=mozilla-releng-s3-cache-us-west-1-try - ;; - *use1.mozilla.com*) - bucket=mozilla-releng-s3-cache-us-east-1-try - ;; - *usw2.mozilla.com*) - bucket=mozilla-releng-s3-cache-us-west-2-try - ;; - esac - ;; - autoland|mozilla-inbound) - case "${master}" in - *use1.mozilla.com*) - bucket=mozilla-releng-s3-cache-us-east-1-prod - ;; - *usw2.mozilla.com*) - bucket=mozilla-releng-s3-cache-us-west-2-prod - ;; - esac - ;; - esac -fi - -# builds without buildprops (eg: taskcluster or non-buildbot) and without ccache env config and without sccache disabled: -elif test -z "$CCACHE_DIR" -a -z "$SCCACHE_DISABLE" -a -z "$no_sccache" -a -z "$MOZ_PGO_IS_SET" -a -z "$MOZ_PGO"; then - - # prevent rerun if az is set, or wget is not available - if test -z "$availability_zone" -a -x "$(command -v wget)"; then - # timeout after 1 second, and don't retry (failure indicates instance is not in ec2 or network issue) - # availability_zone is of the form where region is e.g. us-west-2, and az is us-west-2a - availability_zone=$(wget -T 1 -t 1 -q -O - http://169.254.169.254/latest/meta-data/placement/availability-zone || true) - if test -z "$availability_zone" -o "$availability_zone" = "not-ec2"; then - availability_zone=not-ec2 - else - # region is az with last letter trimmed - region=${availability_zone%?} - # set S3 bucket according to tree (level) - case "${GECKO_HEAD_REPOSITORY}" in - *hg.mozilla.org/try*) - bucket=taskcluster-level-1-sccache-${region} - ;; - *hg.mozilla.org/integration/autoland*|*hg.mozilla.org/integration/mozilla-inbound*) - bucket=taskcluster-level-3-sccache-${region} - ;; - esac - - # set a dummy master - case "${region}" in - eu-central-1) - master=dummy.euc1.mozilla.com - ;; - us-east-1) - master=dummy.use1.mozilla.com - ;; - us-west-1) - master=dummy.usw1.mozilla.com - ;; - us-west-2) - master=dummy.usw2.mozilla.com - ;; - esac - fi - fi -fi - -# if platform hasn't been determined from buildprops, and we're on windows, -# it must be set to prevent adding ac_add_options --with-ccache below -if test -z "$platform"; then - # set platform based on the SYSTEMROOT env var - case "${SYSTEMROOT}" in - *Windows) - platform=windows - ;; - esac -fi - -if test -z "$bucket"; then - case "$platform" in - win*) : ;; - *) - ac_add_options --with-ccache - esac -else - if ! test -e $topsrcdir/sccache/sccache.py; then - echo "Sccache missing in the tooltool manifest" >&2 - exit 1 - fi - mk_add_options "export SCCACHE_BUCKET=$bucket" - case "$master" in - *us[ew][12].mozilla.com*|*euc1.mozilla.com*) - mk_add_options "export SCCACHE_NAMESERVER=169.254.169.253" - ;; - esac - ac_add_options "--with-compiler-wrapper=python2.7 $topsrcdir/sccache/sccache.py" - mk_add_options MOZ_PREFLIGHT_ALL+=build/sccache.mk - mk_add_options MOZ_POSTFLIGHT_ALL+=build/sccache.mk - mk_add_options "UPLOAD_EXTRA_FILES+=sccache.log.gz" - case "$platform" in - win*) - # sccache supports a special flag to create depfiles. - export _DEPEND_CFLAGS='-deps$(MDDEPDIR)/$(@F).pp' - # Windows builds have a default wrapper that needs to be overridden - mk_add_options "export CC_WRAPPER=" - mk_add_options "export CXX_WRAPPER=" - # For now, sccache doesn't support separate PDBs so force debug info to be - # in object files. - mk_add_options "export COMPILE_PDB_FLAG=" - mk_add_options "export HOST_PDB_FLAG=" - mk_add_options "export MOZ_DEBUG_FLAGS=-Z7" - ;; - esac -fi - -fi diff --git a/build/mozconfig.clang-cl b/build/mozconfig.clang-cl deleted file mode 100644 index 5fa1007bd..000000000 --- a/build/mozconfig.clang-cl +++ /dev/null @@ -1,7 +0,0 @@ -CLANG_DIR=`cd "$topsrcdir/clang/bin" ; pwd` -export PATH="${CLANG_DIR}:${PATH}" - -mk_export_correct_style PATH - -export CC=clang-cl -export CXX=clang-cl diff --git a/build/mozconfig.common b/build/mozconfig.common deleted file mode 100644 index 1bdf69ddc..000000000 --- a/build/mozconfig.common +++ /dev/null @@ -1,21 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# Common mozconfig for official builds. -# -# Add options to this file that will be inherited by all in-tree mozconfigs. -# This is useful for eg try builds with nondefault options that apply to all -# architectures, though note that if you want to override options set in -# another mozconfig file, you'll need to use mozconfig.common.override instead -# of this file. - -mk_add_options AUTOCLOBBER=1 - -ac_add_options --enable-crashreporter - -ac_add_options --enable-release - -ac_add_options --enable-js-shell - -. "$topsrcdir/build/mozconfig.automation" diff --git a/build/mozconfig.common.override b/build/mozconfig.common.override deleted file mode 100644 index 7285aa9dd..000000000 --- a/build/mozconfig.common.override +++ /dev/null @@ -1,11 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# Common mozconfig for all users -# -# Add options to this file that will be inherited by all in-tree mozconfigs. -# This file is included at the *end* of the mozconfigs, and so may be used -# to override anything done previously. -# -# The common expected usage is for try builds with nondefault options. diff --git a/build/mozconfig.vs-common b/build/mozconfig.vs-common deleted file mode 100644 index ca5df2f3a..000000000 --- a/build/mozconfig.vs-common +++ /dev/null @@ -1,4 +0,0 @@ -# Pymake needs Windows-style paths. Use cmd.exe to hack around this. -mk_export_correct_style() { - mk_add_options "export $1=$(cmd.exe //c echo %$1%)" -} diff --git a/build/mozconfig.win-common b/build/mozconfig.win-common deleted file mode 100644 index 6e25b7ce4..000000000 --- a/build/mozconfig.win-common +++ /dev/null @@ -1,16 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -if [ "x$IS_NIGHTLY" = "xyes" ]; then - # Some nightlies (eg: Mulet) don't want these set. - MOZ_AUTOMATION_UPLOAD_SYMBOLS=${MOZ_AUTOMATION_UPLOAD_SYMBOLS-1} - MOZ_AUTOMATION_UPDATE_PACKAGING=${MOZ_AUTOMATION_UPDATE_PACKAGING-1} - MOZ_AUTOMATION_SDK=${MOZ_AUTOMATION_SDK-1} -fi - -# Some builds (eg: Mulet) don't want the installer, so only set this if it -# hasn't already been set. -MOZ_AUTOMATION_INSTALLER=${MOZ_AUTOMATION_INSTALLER-1} - -export SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE=c:/builds/crash-stats-api.token diff --git a/build/pymake/LICENSE b/build/pymake/LICENSE deleted file mode 100644 index 04a7d641d..000000000 --- a/build/pymake/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License - -Copyright (c) 2009 The Mozilla Foundation - -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. diff --git a/build/pymake/README b/build/pymake/README deleted file mode 100644 index 4f0fdfea4..000000000 --- a/build/pymake/README +++ /dev/null @@ -1,64 +0,0 @@ -INTRODUCTION - -make.py (and the pymake modules that support it) are an implementation of the make tool -which are mostly compatible with makefiles written for GNU make. - -PURPOSE - -The Mozilla project inspired this tool with several goals: - -* Improve build speeds, especially on Windows. This can be done by reducing the total number - of processes that are launched, especially MSYS shell processes which are expensive. - -* Allow writing some complicated build logic directly in Python instead of in shell. - -* Allow computing dependencies for special targets, such as members within ZIP files. - -* Enable experiments with build system. By writing a makefile parser, we can experiment - with converting in-tree makefiles to another build system, such as SCons, waf, ant, ...insert - your favorite build tool here. Or we could experiment along the lines of makepp, keeping - our existing makefiles, but change the engine to build a global dependency graph. - -KNOWN INCOMPATIBILITIES - -* Order-only prerequisites are not yet supported - -* Secondary expansion is not yet supported. - -* Target-specific variables behave differently than in GNU make: in pymake, the target-specific - variable only applies to the specific target that is mentioned, and does not apply recursively - to all dependencies which are remade. This is an intentional change: the behavior of GNU make - is neither deterministic nor intuitive. - -* $(eval) is only supported during the parse phase. Any attempt to recursively expand - an $(eval) function during command execution will fail. This is an intentional incompatibility. - -* There is a subtle difference in execution order that can cause unexpected changes in the - following circumstance: -** A file `foo.c` exists on the VPATH -** A rule for `foo.c` exists with a dependency on `tool` and no commands -** `tool` is remade for some other reason earlier in the file - In this case, pymake resets the VPATH of `foo.c`, while GNU make does not. This shouldn't - happen in the real world, since a target found on the VPATH without commands is silly. But - mozilla/js/src happens to have a rule, which I'm patching. - -* pymake does not implement any of the builtin implicit rules or the related variables. Mozilla - only cares because pymake doesn't implicitly define $(RM), which I'm also fixing in the Mozilla - code. - -ISSUES - -* Speed is a problem. - -FUTURE WORK - -* implement a new type of command which is implemented in python. This would allow us -to replace the current `nsinstall` binary (and execution costs for the shell and binary) with an -in-process python solution. - -AUTHOR - -Initial code was written by Benjamin Smedberg . For future releases see -http://benjamin.smedbergs.us/pymake/ - -See the LICENSE file for license information (MIT license) diff --git a/build/pymake/make.py b/build/pymake/make.py deleted file mode 100755 index 0857f3f8c..000000000 --- a/build/pymake/make.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python - -""" -make.py - -A drop-in or mostly drop-in replacement for GNU make. -""" - -import sys, os -import pymake.command, pymake.process - -import gc - -if __name__ == '__main__': - if 'TINDERBOX_OUTPUT' in os.environ: - # When building on mozilla build slaves, execute mozmake instead. Until bug - # 978211, this is the easiest, albeit hackish, way to do this. - import subprocess - mozmake = os.path.join(os.path.dirname(__file__), '..', '..', - 'mozmake.exe') - cmd = [mozmake] - cmd.extend(sys.argv[1:]) - shell = os.environ.get('SHELL') - if shell and not shell.lower().endswith('.exe'): - cmd += ['SHELL=%s.exe' % shell] - sys.exit(subprocess.call(cmd)) - - sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) - sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0) - - gc.disable() - - pymake.command.main(sys.argv[1:], os.environ, os.getcwd(), cb=sys.exit) - pymake.process.ParallelContext.spin() - assert False, "Not reached" diff --git a/build/pymake/mkformat.py b/build/pymake/mkformat.py deleted file mode 100755 index 41dd761b2..000000000 --- a/build/pymake/mkformat.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python - -import sys -import pymake.parser - -filename = sys.argv[1] -source = None - -with open(filename, 'rU') as fh: - source = fh.read() - -statements = pymake.parser.parsestring(source, filename) -print statements.to_source() diff --git a/build/pymake/mkparse.py b/build/pymake/mkparse.py deleted file mode 100755 index 253683948..000000000 --- a/build/pymake/mkparse.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python - -import sys -import pymake.parser - -for f in sys.argv[1:]: - print "Parsing %s" % f - fd = open(f, 'rU') - s = fd.read() - fd.close() - stmts = pymake.parser.parsestring(s, f) - print stmts diff --git a/build/pymake/pymake/__init__.py b/build/pymake/pymake/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/build/pymake/pymake/builtins.py b/build/pymake/pymake/builtins.py deleted file mode 100644 index eb6f2e11b..000000000 --- a/build/pymake/pymake/builtins.py +++ /dev/null @@ -1,120 +0,0 @@ -# Basic commands implemented in Python -import errno, sys, os, shutil, time -from getopt import getopt, GetoptError - -from process import PythonException - -__all__ = ["mkdir", "rm", "sleep", "touch"] - -def mkdir(args): - """ - Emulate some of the behavior of mkdir(1). - Only supports the -p (--parents) argument. - """ - try: - opts, args = getopt(args, "p", ["parents"]) - except GetoptError, e: - raise PythonException, ("mkdir: %s" % e, 1) - parents = False - for o, a in opts: - if o in ('-p', '--parents'): - parents = True - for f in args: - try: - if parents: - os.makedirs(f) - else: - os.mkdir(f) - except OSError, e: - if e.errno == errno.EEXIST and parents: - pass - else: - raise PythonException, ("mkdir: %s" % e, 1) - -def rm(args): - """ - Emulate most of the behavior of rm(1). - Only supports the -r (--recursive) and -f (--force) arguments. - """ - try: - opts, args = getopt(args, "rRf", ["force", "recursive"]) - except GetoptError, e: - raise PythonException, ("rm: %s" % e, 1) - force = False - recursive = False - for o, a in opts: - if o in ('-f', '--force'): - force = True - elif o in ('-r', '-R', '--recursive'): - recursive = True - for f in args: - if os.path.isdir(f): - if not recursive: - raise PythonException, ("rm: cannot remove '%s': Is a directory" % f, 1) - else: - shutil.rmtree(f, force) - elif os.path.exists(f): - try: - os.unlink(f) - except: - if not force: - raise PythonException, ("rm: failed to remove '%s': %s" % (f, sys.exc_info()[0]), 1) - elif not force: - raise PythonException, ("rm: cannot remove '%s': No such file or directory" % f, 1) - -def sleep(args): - """ - Emulate the behavior of sleep(1). - """ - total = 0 - values = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400} - for a in args: - multiplier = 1 - for k, v in values.iteritems(): - if a.endswith(k): - a = a[:-1] - multiplier = v - break - try: - f = float(a) - total += f * multiplier - except ValueError: - raise PythonException, ("sleep: invalid time interval '%s'" % a, 1) - time.sleep(total) - -def touch(args): - """ - Emulate the behavior of touch(1). - """ - try: - opts, args = getopt(args, "t:") - except GetoptError, e: - raise PythonException, ("touch: %s" % e, 1) - opts = dict(opts) - times = None - if '-t' in opts: - import re - from time import mktime, localtime - m = re.match('^(?P(?:\d\d)?\d\d)?(?P\d\d)(?P\d\d)(?P\d\d)(?P\d\d)(?:\.(?P\d\d))?$', opts['-t']) - if not m: - raise PythonException, ("touch: invalid date format '%s'" % opts['-t'], 1) - def normalized_field(m, f): - if f == 'Y': - if m.group(f) is None: - return localtime()[0] - y = int(m.group(f)) - if y < 69: - y += 2000 - elif y < 100: - y += 1900 - return y - if m.group(f) is None: - return localtime()[0] if f == 'Y' else 0 - return int(m.group(f)) - time = [normalized_field(m, f) for f in ['Y', 'M', 'D', 'h', 'm', 's']] + [0, 0, -1] - time = mktime(time) - times = (time, time) - for f in args: - if not os.path.exists(f): - open(f, 'a').close() - os.utime(f, times) diff --git a/build/pymake/pymake/command.py b/build/pymake/pymake/command.py deleted file mode 100644 index cd68e4fdb..000000000 --- a/build/pymake/pymake/command.py +++ /dev/null @@ -1,278 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Makefile execution. - -Multiple `makes` can be run within the same process. Each one has an entirely data.Makefile and .Target -structure, environment, and working directory. Typically they will all share a parallel execution context, -except when a submake specifies -j1 when the parent make is building in parallel. -""" - -import os, subprocess, sys, logging, time, traceback, re -from optparse import OptionParser -import data, parserdata, process, util - -# TODO: If this ever goes from relocatable package to system-installed, this may need to be -# a configured-in path. - -makepypath = util.normaljoin(os.path.dirname(__file__), '../make.py') - -_simpleopts = re.compile(r'^[a-zA-Z]+(\s|$)') -def parsemakeflags(env): - """ - Parse MAKEFLAGS from the environment into a sequence of command-line arguments. - """ - - makeflags = env.get('MAKEFLAGS', '') - makeflags = makeflags.strip() - - if makeflags == '': - return [] - - if _simpleopts.match(makeflags): - makeflags = '-' + makeflags - - opts = [] - curopt = '' - - i = 0 - while i < len(makeflags): - c = makeflags[i] - if c.isspace(): - opts.append(curopt) - curopt = '' - i += 1 - while i < len(makeflags) and makeflags[i].isspace(): - i += 1 - continue - - if c == '\\': - i += 1 - if i == len(makeflags): - raise data.DataError("MAKEFLAGS has trailing backslash") - c = makeflags[i] - - curopt += c - i += 1 - - if curopt != '': - opts.append(curopt) - - return opts - -def _version(*args): - print """pymake: GNU-compatible make program -Copyright (C) 2009 The Mozilla Foundation -This is free software; see the source for copying conditions. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE.""" - -_log = logging.getLogger('pymake.execution') - -class _MakeContext(object): - def __init__(self, makeflags, makelevel, workdir, context, env, targets, options, ostmts, overrides, cb): - self.makeflags = makeflags - self.makelevel = makelevel - - self.workdir = workdir - self.context = context - self.env = env - self.targets = targets - self.options = options - self.ostmts = ostmts - self.overrides = overrides - self.cb = cb - - self.restarts = 0 - - self.remakecb(True) - - def remakecb(self, remade, error=None): - if error is not None: - print error - self.context.defer(self.cb, 2) - return - - if remade: - if self.restarts > 0: - _log.info("make.py[%i]: Restarting makefile parsing", self.makelevel) - - self.makefile = data.Makefile(restarts=self.restarts, - make='%s %s' % (sys.executable.replace('\\', '/'), makepypath.replace('\\', '/')), - makeflags=self.makeflags, - makeoverrides=self.overrides, - workdir=self.workdir, - context=self.context, - env=self.env, - makelevel=self.makelevel, - targets=self.targets, - keepgoing=self.options.keepgoing, - silent=self.options.silent, - justprint=self.options.justprint) - - self.restarts += 1 - - try: - self.ostmts.execute(self.makefile) - for f in self.options.makefiles: - self.makefile.include(f) - self.makefile.finishparsing() - self.makefile.remakemakefiles(self.remakecb) - except util.MakeError, e: - print e - self.context.defer(self.cb, 2) - - return - - if len(self.targets) == 0: - if self.makefile.defaulttarget is None: - print "No target specified and no default target found." - self.context.defer(self.cb, 2) - return - - _log.info("Making default target %s", self.makefile.defaulttarget) - self.realtargets = [self.makefile.defaulttarget] - self.tstack = [''] - else: - self.realtargets = self.targets - self.tstack = [''] - - self.makefile.gettarget(self.realtargets.pop(0)).make(self.makefile, self.tstack, cb=self.makecb) - - def makecb(self, error, didanything): - assert error in (True, False) - - if error: - self.context.defer(self.cb, 2) - return - - if not len(self.realtargets): - if self.options.printdir: - print "make.py[%i]: Leaving directory '%s'" % (self.makelevel, self.workdir) - sys.stdout.flush() - - self.context.defer(self.cb, 0) - else: - self.makefile.gettarget(self.realtargets.pop(0)).make(self.makefile, self.tstack, self.makecb) - -def main(args, env, cwd, cb): - """ - Start a single makefile execution, given a command line, working directory, and environment. - - @param cb a callback to notify with an exit code when make execution is finished. - """ - - try: - makelevel = int(env.get('MAKELEVEL', '0')) - - op = OptionParser() - op.add_option('-f', '--file', '--makefile', - action='append', - dest='makefiles', - default=[]) - op.add_option('-d', - action="store_true", - dest="verbose", default=False) - op.add_option('-k', '--keep-going', - action="store_true", - dest="keepgoing", default=False) - op.add_option('--debug-log', - dest="debuglog", default=None) - op.add_option('-C', '--directory', - dest="directory", default=None) - op.add_option('-v', '--version', action="store_true", - dest="printversion", default=False) - op.add_option('-j', '--jobs', type="int", - dest="jobcount", default=1) - op.add_option('-w', '--print-directory', action="store_true", - dest="printdir") - op.add_option('--no-print-directory', action="store_false", - dest="printdir", default=True) - op.add_option('-s', '--silent', action="store_true", - dest="silent", default=False) - op.add_option('-n', '--just-print', '--dry-run', '--recon', - action="store_true", - dest="justprint", default=False) - - options, arguments1 = op.parse_args(parsemakeflags(env)) - options, arguments2 = op.parse_args(args, values=options) - - op.destroy() - - arguments = arguments1 + arguments2 - - if options.printversion: - _version() - cb(0) - return - - shortflags = [] - longflags = [] - - if options.keepgoing: - shortflags.append('k') - - if options.printdir: - shortflags.append('w') - - if options.silent: - shortflags.append('s') - options.printdir = False - - if options.justprint: - shortflags.append('n') - - loglevel = logging.WARNING - if options.verbose: - loglevel = logging.DEBUG - shortflags.append('d') - - logkwargs = {} - if options.debuglog: - logkwargs['filename'] = options.debuglog - longflags.append('--debug-log=%s' % options.debuglog) - - if options.directory is None: - workdir = cwd - else: - workdir = util.normaljoin(cwd, options.directory) - - if options.jobcount != 1: - longflags.append('-j%i' % (options.jobcount,)) - - makeflags = ''.join(shortflags) - if len(longflags): - makeflags += ' ' + ' '.join(longflags) - - logging.basicConfig(level=loglevel, **logkwargs) - - context = process.getcontext(options.jobcount) - - if options.printdir: - print "make.py[%i]: Entering directory '%s'" % (makelevel, workdir) - sys.stdout.flush() - - if len(options.makefiles) == 0: - if os.path.exists(util.normaljoin(workdir, 'Makefile')): - options.makefiles.append('Makefile') - else: - print "No makefile found" - cb(2) - return - - ostmts, targets, overrides = parserdata.parsecommandlineargs(arguments) - - _MakeContext(makeflags, makelevel, workdir, context, env, targets, options, ostmts, overrides, cb) - except (util.MakeError), e: - print e - if options.printdir: - print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir) - sys.stdout.flush() - cb(2) - return diff --git a/build/pymake/pymake/data.py b/build/pymake/pymake/data.py deleted file mode 100644 index dcd7e9225..000000000 --- a/build/pymake/pymake/data.py +++ /dev/null @@ -1,1842 +0,0 @@ -""" -A representation of makefile data structures. -""" - -import logging, re, os, sys -import parserdata, parser, functions, process, util, implicit -from cStringIO import StringIO - -if sys.version_info[0] < 3: - str_type = basestring -else: - str_type = str - -_log = logging.getLogger('pymake.data') - -class DataError(util.MakeError): - pass - -class ResolutionError(DataError): - """ - Raised when dependency resolution fails, either due to recursion or to missing - prerequisites.This is separately catchable so that implicit rule search can try things - without having to commit. - """ - pass - -def withoutdups(it): - r = set() - for i in it: - if not i in r: - r.add(i) - yield i - -def mtimeislater(deptime, targettime): - """ - Is the mtime of the dependency later than the target? - """ - - if deptime is None: - return True - if targettime is None: - return False - # int(1000*x) because of http://bugs.python.org/issue10148 - return int(1000 * deptime) > int(1000 * targettime) - -def getmtime(path): - try: - s = os.stat(path) - return s.st_mtime - except OSError: - return None - -def stripdotslash(s): - if s.startswith('./'): - st = s[2:] - return st if st != '' else '.' - return s - -def stripdotslashes(sl): - for s in sl: - yield stripdotslash(s) - -def getindent(stack): - return ''.ljust(len(stack) - 1) - -def _if_else(c, t, f): - if c: - return t() - return f() - - -class BaseExpansion(object): - """Base class for expansions. - - A make expansion is the parsed representation of a string, which may - contain references to other elements. - """ - - @property - def is_static_string(self): - """Returns whether the expansion is composed of static string content. - - This is always True for StringExpansion. It will be True for Expansion - only if all elements of that Expansion are static strings. - """ - raise Exception('Must be implemented in child class.') - - def functions(self, descend=False): - """Obtain all functions inside this expansion. - - This is a generator for pymake.functions.Function instances. - - By default, this only returns functions existing as the primary - elements of this expansion. If `descend` is True, it will descend into - child expansions and extract all functions in the tree. - """ - # An empty generator. Yeah, it's weird. - for x in []: - yield x - - def variable_references(self, descend=False): - """Obtain all variable references in this expansion. - - This is a generator for pymake.functionsVariableRef instances. - - To retrieve the names of variables, simply query the `vname` field on - the returned instances. Most of the time these will be StringExpansion - instances. - """ - for f in self.functions(descend=descend): - if not isinstance(f, functions.VariableRef): - continue - - yield f - - @property - def is_filesystem_dependent(self): - """Whether this expansion may query the filesystem for evaluation. - - This effectively asks "is any function in this expansion dependent on - the filesystem. - """ - for f in self.functions(descend=True): - if f.is_filesystem_dependent: - return True - - return False - - @property - def is_shell_dependent(self): - """Whether this expansion may invoke a shell for evaluation.""" - - for f in self.functions(descend=True): - if isinstance(f, functions.ShellFunction): - return True - - return False - - -class StringExpansion(BaseExpansion): - """An Expansion representing a static string. - - This essentially wraps a single str instance. - """ - - __slots__ = ('loc', 's',) - simple = True - - def __init__(self, s, loc): - assert isinstance(s, str_type) - self.s = s - self.loc = loc - - def lstrip(self): - self.s = self.s.lstrip() - - def rstrip(self): - self.s = self.s.rstrip() - - def isempty(self): - return self.s == '' - - def resolve(self, i, j, fd, k=None): - fd.write(self.s) - - def resolvestr(self, i, j, k=None): - return self.s - - def resolvesplit(self, i, j, k=None): - return self.s.split() - - def clone(self): - e = Expansion(self.loc) - e.appendstr(self.s) - return e - - @property - def is_static_string(self): - return True - - def __len__(self): - return 1 - - def __getitem__(self, i): - assert i == 0 - return self.s, False - - def __repr__(self): - return "Exp<%s>(%r)" % (self.loc, self.s) - - def __eq__(self, other): - """We only compare the string contents.""" - return self.s == other - - def __ne__(self, other): - return not self.__eq__(other) - - def to_source(self, escape_variables=False, escape_comments=False): - s = self.s - - if escape_comments: - s = s.replace('#', '\\#') - - if escape_variables: - return s.replace('$', '$$') - - return s - - -class Expansion(BaseExpansion, list): - """A representation of expanded data. - - This is effectively an ordered list of StringExpansion and - pymake.function.Function instances. Every item in the collection appears in - the same context in a make file. - """ - - __slots__ = ('loc',) - simple = False - - def __init__(self, loc=None): - # A list of (element, isfunc) tuples - # element is either a string or a function - self.loc = loc - - @staticmethod - def fromstring(s, path): - return StringExpansion(s, parserdata.Location(path, 1, 0)) - - def clone(self): - e = Expansion() - e.extend(self) - return e - - def appendstr(self, s): - assert isinstance(s, str_type) - if s == '': - return - - self.append((s, False)) - - def appendfunc(self, func): - assert isinstance(func, functions.Function) - self.append((func, True)) - - def concat(self, o): - """Concatenate the other expansion on to this one.""" - if o.simple: - self.appendstr(o.s) - else: - self.extend(o) - - def isempty(self): - return (not len(self)) or self[0] == ('', False) - - def lstrip(self): - """Strip leading literal whitespace from this expansion.""" - while True: - i, isfunc = self[0] - if isfunc: - return - - i = i.lstrip() - if i != '': - self[0] = i, False - return - - del self[0] - - def rstrip(self): - """Strip trailing literal whitespace from this expansion.""" - while True: - i, isfunc = self[-1] - if isfunc: - return - - i = i.rstrip() - if i != '': - self[-1] = i, False - return - - del self[-1] - - def finish(self): - # Merge any adjacent literal strings: - strings = [] - elements = [] - for (e, isfunc) in self: - if isfunc: - if strings: - s = ''.join(strings) - if s: - elements.append((s, False)) - strings = [] - elements.append((e, True)) - else: - strings.append(e) - - if not elements: - # This can only happen if there were no function elements. - return StringExpansion(''.join(strings), self.loc) - - if strings: - s = ''.join(strings) - if s: - elements.append((s, False)) - - if len(elements) < len(self): - self[:] = elements - - return self - - def resolve(self, makefile, variables, fd, setting=[]): - """ - Resolve this variable into a value, by interpolating the value - of other variables. - - @param setting (Variable instance) the variable currently - being set, if any. Setting variables must avoid self-referential - loops. - """ - assert isinstance(makefile, Makefile) - assert isinstance(variables, Variables) - assert isinstance(setting, list) - - for e, isfunc in self: - if isfunc: - e.resolve(makefile, variables, fd, setting) - else: - assert isinstance(e, str_type) - fd.write(e) - - def resolvestr(self, makefile, variables, setting=[]): - fd = StringIO() - self.resolve(makefile, variables, fd, setting) - return fd.getvalue() - - def resolvesplit(self, makefile, variables, setting=[]): - return self.resolvestr(makefile, variables, setting).split() - - @property - def is_static_string(self): - """An Expansion is static if all its components are strings, not - functions.""" - for e, is_func in self: - if is_func: - return False - - return True - - def functions(self, descend=False): - for e, is_func in self: - if is_func: - yield e - - if descend: - for exp in e.expansions(descend=True): - for f in exp.functions(descend=True): - yield f - - def __repr__(self): - return "" % ([e for e, isfunc in self],) - - def to_source(self, escape_variables=False, escape_comments=False): - parts = [] - for e, is_func in self: - if is_func: - parts.append(e.to_source()) - continue - - if escape_variables: - parts.append(e.replace('$', '$$')) - continue - - parts.append(e) - - return ''.join(parts) - - def __eq__(self, other): - if not isinstance(other, (Expansion, StringExpansion)): - return False - - # Expansions are equivalent if adjacent string literals normalize to - # the same value. So, we must normalize before any comparisons are - # made. - a = self.clone().finish() - - if isinstance(other, StringExpansion): - if isinstance(a, StringExpansion): - return a == other - - # A normalized Expansion != StringExpansion. - return False - - b = other.clone().finish() - - # b could be a StringExpansion now. - if isinstance(b, StringExpansion): - if isinstance(a, StringExpansion): - return a == b - - # Our normalized Expansion != normalized StringExpansion. - return False - - if len(a) != len(b): - return False - - for i in xrange(len(self)): - e1, is_func1 = a[i] - e2, is_func2 = b[i] - - if is_func1 != is_func2: - return False - - if type(e1) != type(e2): - return False - - if e1 != e2: - return False - - return True - - def __ne__(self, other): - return not self.__eq__(other) - -class Variables(object): - """ - A mapping from variable names to variables. Variables have flavor, source, and value. The value is an - expansion object. - """ - - __slots__ = ('parent', '_map') - - FLAVOR_RECURSIVE = 0 - FLAVOR_SIMPLE = 1 - FLAVOR_APPEND = 2 - - SOURCE_OVERRIDE = 0 - SOURCE_COMMANDLINE = 1 - SOURCE_MAKEFILE = 2 - SOURCE_ENVIRONMENT = 3 - SOURCE_AUTOMATIC = 4 - SOURCE_IMPLICIT = 5 - - def __init__(self, parent=None): - self._map = {} # vname -> flavor, source, valuestr, valueexp - self.parent = parent - - def readfromenvironment(self, env): - for k, v in env.iteritems(): - self.set(k, self.FLAVOR_RECURSIVE, self.SOURCE_ENVIRONMENT, v) - - def get(self, name, expand=True): - """ - Get the value of a named variable. Returns a tuple (flavor, source, value) - - If the variable is not present, returns (None, None, None) - - @param expand If true, the value will be returned as an expansion. If false, - it will be returned as an unexpanded string. - """ - flavor, source, valuestr, valueexp = self._map.get(name, (None, None, None, None)) - if flavor is not None: - if expand and flavor != self.FLAVOR_SIMPLE and valueexp is None: - d = parser.Data.fromstring(valuestr, parserdata.Location("Expansion of variables '%s'" % (name,), 1, 0)) - valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata) - self._map[name] = flavor, source, valuestr, valueexp - - if flavor == self.FLAVOR_APPEND: - if self.parent: - pflavor, psource, pvalue = self.parent.get(name, expand) - else: - pflavor, psource, pvalue = None, None, None - - if pvalue is None: - flavor = self.FLAVOR_RECURSIVE - # fall through - else: - if source > psource: - # TODO: log a warning? - return pflavor, psource, pvalue - - if not expand: - return pflavor, psource, pvalue + ' ' + valuestr - - pvalue = pvalue.clone() - pvalue.appendstr(' ') - pvalue.concat(valueexp) - - return pflavor, psource, pvalue - - if not expand: - return flavor, source, valuestr - - if flavor == self.FLAVOR_RECURSIVE: - val = valueexp - else: - val = Expansion.fromstring(valuestr, "Expansion of variable '%s'" % (name,)) - - return flavor, source, val - - if self.parent is not None: - return self.parent.get(name, expand) - - return (None, None, None) - - def set(self, name, flavor, source, value, force=False): - assert flavor in (self.FLAVOR_RECURSIVE, self.FLAVOR_SIMPLE) - assert source in (self.SOURCE_OVERRIDE, self.SOURCE_COMMANDLINE, self.SOURCE_MAKEFILE, self.SOURCE_ENVIRONMENT, self.SOURCE_AUTOMATIC, self.SOURCE_IMPLICIT) - assert isinstance(value, str_type), "expected str, got %s" % type(value) - - prevflavor, prevsource, prevvalue = self.get(name) - if prevsource is not None and source > prevsource and not force: - # TODO: give a location for this warning - _log.info("not setting variable '%s', set by higher-priority source to value '%s'" % (name, prevvalue)) - return - - self._map[name] = flavor, source, value, None - - def append(self, name, source, value, variables, makefile): - assert source in (self.SOURCE_OVERRIDE, self.SOURCE_MAKEFILE, self.SOURCE_AUTOMATIC) - assert isinstance(value, str_type) - - if name not in self._map: - self._map[name] = self.FLAVOR_APPEND, source, value, None - return - - prevflavor, prevsource, prevvalue, valueexp = self._map[name] - if source > prevsource: - # TODO: log a warning? - return - - if prevflavor == self.FLAVOR_SIMPLE: - d = parser.Data.fromstring(value, parserdata.Location("Expansion of variables '%s'" % (name,), 1, 0)) - valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata) - - val = valueexp.resolvestr(makefile, variables, [name]) - self._map[name] = prevflavor, prevsource, prevvalue + ' ' + val, None - return - - newvalue = prevvalue + ' ' + value - self._map[name] = prevflavor, prevsource, newvalue, None - - def merge(self, other): - assert isinstance(other, Variables) - for k, flavor, source, value in other: - self.set(k, flavor, source, value) - - def __iter__(self): - for k, (flavor, source, value, valueexp) in self._map.iteritems(): - yield k, flavor, source, value - - def __contains__(self, item): - return item in self._map - -class Pattern(object): - """ - A pattern is a string, possibly with a % substitution character. From the GNU make manual: - - '%' characters in pattern rules can be quoted with precending backslashes ('\'). Backslashes that - would otherwise quote '%' charcters can be quoted with more backslashes. Backslashes that - quote '%' characters or other backslashes are removed from the pattern before it is compared t - file names or has a stem substituted into it. Backslashes that are not in danger of quoting '%' - characters go unmolested. For example, the pattern the\%weird\\%pattern\\ has `the%weird\' preceding - the operative '%' character, and 'pattern\\' following it. The final two backslashes are left alone - because they cannot affect any '%' character. - - This insane behavior probably doesn't matter, but we're compatible just for shits and giggles. - """ - - __slots__ = ('data') - - def __init__(self, s): - r = [] - i = 0 - slen = len(s) - while i < slen: - c = s[i] - if c == '\\': - nc = s[i + 1] - if nc == '%': - r.append('%') - i += 1 - elif nc == '\\': - r.append('\\') - i += 1 - else: - r.append(c) - elif c == '%': - self.data = (''.join(r), s[i+1:]) - return - else: - r.append(c) - i += 1 - - # This is different than (s,) because \% and \\ have been unescaped. Parsing patterns is - # context-sensitive! - self.data = (''.join(r),) - - def ismatchany(self): - return self.data == ('','') - - def ispattern(self): - return len(self.data) == 2 - - def __hash__(self): - return self.data.__hash__() - - def __eq__(self, o): - assert isinstance(o, Pattern) - return self.data == o.data - - def gettarget(self): - assert not self.ispattern() - return self.data[0] - - def hasslash(self): - return self.data[0].find('/') != -1 or self.data[1].find('/') != -1 - - def match(self, word): - """ - Match this search pattern against a word (string). - - @returns None if the word doesn't match, or the matching stem. - If this is a %-less pattern, the stem will always be '' - """ - d = self.data - if len(d) == 1: - if word == d[0]: - return word - return None - - d0, d1 = d - l1 = len(d0) - l2 = len(d1) - if len(word) >= l1 + l2 and word.startswith(d0) and word.endswith(d1): - if l2 == 0: - return word[l1:] - return word[l1:-l2] - - return None - - def resolve(self, dir, stem): - if self.ispattern(): - return dir + self.data[0] + stem + self.data[1] - - return self.data[0] - - def subst(self, replacement, word, mustmatch): - """ - Given a word, replace the current pattern with the replacement pattern, a la 'patsubst' - - @param mustmatch If true and this pattern doesn't match the word, throw a DataError. Otherwise - return word unchanged. - """ - assert isinstance(replacement, str_type) - - stem = self.match(word) - if stem is None: - if mustmatch: - raise DataError("target '%s' doesn't match pattern" % (word,)) - return word - - if not self.ispattern(): - # if we're not a pattern, the replacement is not parsed as a pattern either - return replacement - - return Pattern(replacement).resolve('', stem) - - def __repr__(self): - return "" % (self.data,) - - _backre = re.compile(r'[%\\]') - def __str__(self): - if not self.ispattern(): - return self._backre.sub(r'\\\1', self.data[0]) - - return self._backre.sub(r'\\\1', self.data[0]) + '%' + self.data[1] - -class RemakeTargetSerially(object): - __slots__ = ('target', 'makefile', 'indent', 'rlist') - - def __init__(self, target, makefile, indent, rlist): - self.target = target - self.makefile = makefile - self.indent = indent - self.rlist = rlist - self.commandscb(False) - - def resolvecb(self, error, didanything): - assert error in (True, False) - - if didanything: - self.target.didanything = True - - if error: - self.target.error = True - self.makefile.error = True - if not self.makefile.keepgoing: - self.target.notifydone(self.makefile) - return - else: - # don't run the commands! - del self.rlist[0] - self.commandscb(error=False) - else: - self.rlist.pop(0).runcommands(self.indent, self.commandscb) - - def commandscb(self, error): - assert error in (True, False) - - if error: - self.target.error = True - self.makefile.error = True - - if self.target.error and not self.makefile.keepgoing: - self.target.notifydone(self.makefile) - return - - if not len(self.rlist): - self.target.notifydone(self.makefile) - else: - self.rlist[0].resolvedeps(True, self.resolvecb) - -class RemakeTargetParallel(object): - __slots__ = ('target', 'makefile', 'indent', 'rlist', 'rulesremaining', 'currunning') - - def __init__(self, target, makefile, indent, rlist): - self.target = target - self.makefile = makefile - self.indent = indent - self.rlist = rlist - - self.rulesremaining = len(rlist) - self.currunning = False - - for r in rlist: - makefile.context.defer(self.doresolve, r) - - def doresolve(self, r): - if self.makefile.error and not self.makefile.keepgoing: - r.error = True - self.resolvecb(True, False) - else: - r.resolvedeps(False, self.resolvecb) - - def resolvecb(self, error, didanything): - assert error in (True, False) - - if error: - self.target.error = True - - if didanything: - self.target.didanything = True - - self.rulesremaining -= 1 - - # commandscb takes care of the details if we're currently building - # something - if self.currunning: - return - - self.runnext() - - def runnext(self): - assert not self.currunning - - if self.makefile.error and not self.makefile.keepgoing: - self.rlist = [] - else: - while len(self.rlist) and self.rlist[0].error: - del self.rlist[0] - - if not len(self.rlist): - if not self.rulesremaining: - self.target.notifydone(self.makefile) - return - - if self.rlist[0].depsremaining != 0: - return - - self.currunning = True - rule = self.rlist.pop(0) - self.makefile.context.defer(rule.runcommands, self.indent, self.commandscb) - - def commandscb(self, error): - assert error in (True, False) - if error: - self.target.error = True - self.makefile.error = True - - assert self.currunning - self.currunning = False - self.runnext() - -class RemakeRuleContext(object): - def __init__(self, target, makefile, rule, deps, - targetstack, avoidremakeloop): - self.target = target - self.makefile = makefile - self.rule = rule - self.deps = deps - self.targetstack = targetstack - self.avoidremakeloop = avoidremakeloop - - self.running = False - self.error = False - self.depsremaining = len(deps) + 1 - self.remake = False - - def resolvedeps(self, serial, cb): - self.resolvecb = cb - self.didanything = False - if serial: - self._resolvedepsserial() - else: - self._resolvedepsparallel() - - def _weakdepfinishedserial(self, error, didanything): - if error: - self.remake = True - self._depfinishedserial(False, didanything) - - def _depfinishedserial(self, error, didanything): - assert error in (True, False) - - if didanything: - self.didanything = True - - if error: - self.error = True - if not self.makefile.keepgoing: - self.resolvecb(error=True, didanything=self.didanything) - return - - if len(self.resolvelist): - dep, weak = self.resolvelist.pop(0) - self.makefile.context.defer(dep.make, - self.makefile, self.targetstack, weak and self._weakdepfinishedserial or self._depfinishedserial) - else: - self.resolvecb(error=self.error, didanything=self.didanything) - - def _resolvedepsserial(self): - self.resolvelist = list(self.deps) - self._depfinishedserial(False, False) - - def _startdepparallel(self, d): - dep, weak = d - if weak: - depfinished = self._weakdepfinishedparallel - else: - depfinished = self._depfinishedparallel - if self.makefile.error: - depfinished(True, False) - else: - dep.make(self.makefile, self.targetstack, depfinished) - - def _weakdepfinishedparallel(self, error, didanything): - if error: - self.remake = True - self._depfinishedparallel(False, didanything) - - def _depfinishedparallel(self, error, didanything): - assert error in (True, False) - - if error: - print "<%s>: Found error" % self.target.target - self.error = True - if didanything: - self.didanything = True - - self.depsremaining -= 1 - if self.depsremaining == 0: - self.resolvecb(error=self.error, didanything=self.didanything) - - def _resolvedepsparallel(self): - self.depsremaining -= 1 - if self.depsremaining == 0: - self.resolvecb(error=self.error, didanything=self.didanything) - return - - self.didanything = False - - for d in self.deps: - self.makefile.context.defer(self._startdepparallel, d) - - def _commandcb(self, error): - assert error in (True, False) - - if error: - self.runcb(error=True) - return - - if len(self.commands): - self.commands.pop(0)(self._commandcb) - else: - self.runcb(error=False) - - def runcommands(self, indent, cb): - assert not self.running - self.running = True - - self.runcb = cb - - if self.rule is None or not len(self.rule.commands): - if self.target.mtime is None: - self.target.beingremade() - else: - for d, weak in self.deps: - if mtimeislater(d.mtime, self.target.mtime): - if d.mtime is None: - self.target.beingremade() - else: - _log.info("%sNot remaking %s ubecause it would have no effect, even though %s is newer.", indent, self.target.target, d.target) - break - cb(error=False) - return - - if self.rule.doublecolon: - if len(self.deps) == 0: - if self.avoidremakeloop: - _log.info("%sNot remaking %s using rule at %s because it would introduce an infinite loop.", indent, self.target.target, self.rule.loc) - cb(error=False) - return - - remake = self.remake - if remake: - _log.info("%sRemaking %s using rule at %s: weak dependency was not found.", indent, self.target.target, self.rule.loc) - else: - if self.target.mtime is None: - remake = True - _log.info("%sRemaking %s using rule at %s: target doesn't exist or is a forced target", indent, self.target.target, self.rule.loc) - - if not remake: - if self.rule.doublecolon: - if len(self.deps) == 0: - _log.info("%sRemaking %s using rule at %s because there are no prerequisites listed for a double-colon rule.", indent, self.target.target, self.rule.loc) - remake = True - - if not remake: - for d, weak in self.deps: - if mtimeislater(d.mtime, self.target.mtime): - _log.info("%sRemaking %s using rule at %s because %s is newer.", indent, self.target.target, self.rule.loc, d.target) - remake = True - break - - if remake: - self.target.beingremade() - self.target.didanything = True - try: - self.commands = [c for c in self.rule.getcommands(self.target, self.makefile)] - except util.MakeError, e: - print e - sys.stdout.flush() - cb(error=True) - return - - self._commandcb(False) - else: - cb(error=False) - -MAKESTATE_NONE = 0 -MAKESTATE_FINISHED = 1 -MAKESTATE_WORKING = 2 - -class Target(object): - """ - An actual (non-pattern) target. - - It holds target-specific variables and a list of rules. It may also point to a parent - PatternTarget, if this target is being created by an implicit rule. - - The rules associated with this target may be Rule instances or, in the case of static pattern - rules, PatternRule instances. - """ - - wasremade = False - - def __init__(self, target, makefile): - assert isinstance(target, str_type) - self.target = target - self.vpathtarget = None - self.rules = [] - self.variables = Variables(makefile.variables) - self.explicit = False - self._state = MAKESTATE_NONE - - def addrule(self, rule): - assert isinstance(rule, (Rule, PatternRuleInstance)) - if len(self.rules) and rule.doublecolon != self.rules[0].doublecolon: - raise DataError("Cannot have single- and double-colon rules for the same target. Prior rule location: %s" % self.rules[0].loc, rule.loc) - - if isinstance(rule, PatternRuleInstance): - if len(rule.prule.targetpatterns) != 1: - raise DataError("Static pattern rules must only have one target pattern", rule.prule.loc) - if rule.prule.targetpatterns[0].match(self.target) is None: - raise DataError("Static pattern rule doesn't match target '%s'" % self.target, rule.loc) - - self.rules.append(rule) - - def isdoublecolon(self): - return self.rules[0].doublecolon - - def isphony(self, makefile): - """Is this a phony target? We don't check for existence of phony targets.""" - return makefile.gettarget('.PHONY').hasdependency(self.target) - - def hasdependency(self, t): - for rule in self.rules: - if t in rule.prerequisites: - return True - - return False - - def resolveimplicitrule(self, makefile, targetstack, rulestack): - """ - Try to resolve an implicit rule to build this target. - """ - # The steps in the GNU make manual Implicit-Rule-Search.html are very detailed. I hope they can be trusted. - - indent = getindent(targetstack) - - _log.info("%sSearching for implicit rule to make '%s'", indent, self.target) - - dir, s, file = util.strrpartition(self.target, '/') - dir = dir + s - - candidates = [] # list of PatternRuleInstance - - hasmatch = util.any((r.hasspecificmatch(file) for r in makefile.implicitrules)) - - for r in makefile.implicitrules: - if r in rulestack: - _log.info("%s %s: Avoiding implicit rule recursion", indent, r.loc) - continue - - if not len(r.commands): - continue - - for ri in r.matchesfor(dir, file, hasmatch): - candidates.append(ri) - - newcandidates = [] - - for r in candidates: - depfailed = None - for p in r.prerequisites: - t = makefile.gettarget(p) - t.resolvevpath(makefile) - if not t.explicit and t.mtime is None: - depfailed = p - break - - if depfailed is not None: - if r.doublecolon: - _log.info("%s Terminal rule at %s doesn't match: prerequisite '%s' not mentioned and doesn't exist.", indent, r.loc, depfailed) - else: - newcandidates.append(r) - continue - - _log.info("%sFound implicit rule at %s for target '%s'", indent, r.loc, self.target) - self.rules.append(r) - return - - # Try again, but this time with chaining and without terminal (double-colon) rules - - for r in newcandidates: - newrulestack = rulestack + [r.prule] - - depfailed = None - for p in r.prerequisites: - t = makefile.gettarget(p) - try: - t.resolvedeps(makefile, targetstack, newrulestack, True) - except ResolutionError: - depfailed = p - break - - if depfailed is not None: - _log.info("%s Rule at %s doesn't match: prerequisite '%s' could not be made.", indent, r.loc, depfailed) - continue - - _log.info("%sFound implicit rule at %s for target '%s'", indent, r.loc, self.target) - self.rules.append(r) - return - - _log.info("%sCouldn't find implicit rule to remake '%s'", indent, self.target) - - def ruleswithcommands(self): - "The number of rules with commands" - return reduce(lambda i, rule: i + (len(rule.commands) > 0), self.rules, 0) - - def resolvedeps(self, makefile, targetstack, rulestack, recursive): - """ - Resolve the actual path of this target, using vpath if necessary. - - Recursively resolve dependencies of this target. This means finding implicit - rules which match the target, if appropriate. - - Figure out whether this target needs to be rebuild, and set self.outofdate - appropriately. - - @param targetstack is the current stack of dependencies being resolved. If - this target is already in targetstack, bail to prevent infinite - recursion. - @param rulestack is the current stack of implicit rules being used to resolve - dependencies. A rule chain cannot use the same implicit rule twice. - """ - assert makefile.parsingfinished - - if self.target in targetstack: - raise ResolutionError("Recursive dependency: %s -> %s" % ( - " -> ".join(targetstack), self.target)) - - targetstack = targetstack + [self.target] - - indent = getindent(targetstack) - - _log.info("%sConsidering target '%s'", indent, self.target) - - self.resolvevpath(makefile) - - # Sanity-check our rules. If we're single-colon, only one rule should have commands - ruleswithcommands = self.ruleswithcommands() - if len(self.rules) and not self.isdoublecolon(): - if ruleswithcommands > 1: - # In GNU make this is a warning, not an error. I'm going to be stricter. - # TODO: provide locations - raise DataError("Target '%s' has multiple rules with commands." % self.target) - - if ruleswithcommands == 0: - self.resolveimplicitrule(makefile, targetstack, rulestack) - - # If a target is mentioned, but doesn't exist, has no commands and no - # prerequisites, it is special and exists just to say that targets which - # depend on it are always out of date. This is like .FORCE but more - # compatible with other makes. - # Otherwise, we don't know how to make it. - if not len(self.rules) and self.mtime is None and not util.any((len(rule.prerequisites) > 0 - for rule in self.rules)): - raise ResolutionError("No rule to make target '%s' needed by %r" % (self.target, - targetstack)) - - if recursive: - for r in self.rules: - newrulestack = rulestack + [r] - for d in r.prerequisites: - dt = makefile.gettarget(d) - if dt.explicit: - continue - - dt.resolvedeps(makefile, targetstack, newrulestack, True) - - for v in makefile.getpatternvariablesfor(self.target): - self.variables.merge(v) - - def resolvevpath(self, makefile): - if self.vpathtarget is not None: - return - - if self.isphony(makefile): - self.vpathtarget = self.target - self.mtime = None - return - - if self.target.startswith('-l'): - stem = self.target[2:] - f, s, e = makefile.variables.get('.LIBPATTERNS') - if e is not None: - libpatterns = [Pattern(stripdotslash(s)) for s in e.resolvesplit(makefile, makefile.variables)] - if len(libpatterns): - searchdirs = [''] - searchdirs.extend(makefile.getvpath(self.target)) - - for lp in libpatterns: - if not lp.ispattern(): - raise DataError('.LIBPATTERNS contains a non-pattern') - - libname = lp.resolve('', stem) - - for dir in searchdirs: - libpath = util.normaljoin(dir, libname).replace('\\', '/') - fspath = util.normaljoin(makefile.workdir, libpath) - mtime = getmtime(fspath) - if mtime is not None: - self.vpathtarget = libpath - self.mtime = mtime - return - - self.vpathtarget = self.target - self.mtime = None - return - - search = [self.target] - if not os.path.isabs(self.target): - search += [util.normaljoin(dir, self.target).replace('\\', '/') - for dir in makefile.getvpath(self.target)] - - targetandtime = self.searchinlocs(makefile, search) - if targetandtime is not None: - (self.vpathtarget, self.mtime) = targetandtime - return - - self.vpathtarget = self.target - self.mtime = None - - def searchinlocs(self, makefile, locs): - """ - Look in the given locations relative to the makefile working directory - for a file. Return a pair of the target and the mtime if found, None - if not. - """ - for t in locs: - fspath = util.normaljoin(makefile.workdir, t).replace('\\', '/') - mtime = getmtime(fspath) -# _log.info("Searching %s ... checking %s ... mtime %r" % (t, fspath, mtime)) - if mtime is not None: - return (t, mtime) - - return None - - def beingremade(self): - """ - When we remake ourself, we have to drop any vpath prefixes. - """ - self.vpathtarget = self.target - self.wasremade = True - - def notifydone(self, makefile): - assert self._state == MAKESTATE_WORKING, "State was %s" % self._state - # If we were remade then resolve mtime again - if self.wasremade: - targetandtime = self.searchinlocs(makefile, [self.target]) - if targetandtime is not None: - (_, self.mtime) = targetandtime - else: - self.mtime = None - - self._state = MAKESTATE_FINISHED - for cb in self._callbacks: - makefile.context.defer(cb, error=self.error, didanything=self.didanything) - del self._callbacks - - def make(self, makefile, targetstack, cb, avoidremakeloop=False, printerror=True): - """ - If we are out of date, asynchronously make ourself. This is a multi-stage process, mostly handled - by the helper objects RemakeTargetSerially, RemakeTargetParallel, - RemakeRuleContext. These helper objects should keep us from developing - any cyclical dependencies. - - * resolve dependencies (synchronous) - * gather a list of rules to execute and related dependencies (synchronous) - * for each rule (in parallel) - ** remake dependencies (asynchronous) - ** build list of commands to execute (synchronous) - ** execute each command (asynchronous) - * asynchronously notify when all rules are complete - - @param cb A callback function to notify when remaking is finished. It is called - thusly: callback(error=True/False, didanything=True/False) - If there is no asynchronous activity to perform, the callback may be called directly. - """ - - serial = makefile.context.jcount == 1 - - if self._state == MAKESTATE_FINISHED: - cb(error=self.error, didanything=self.didanything) - return - - if self._state == MAKESTATE_WORKING: - assert not serial - self._callbacks.append(cb) - return - - assert self._state == MAKESTATE_NONE - - self._state = MAKESTATE_WORKING - self._callbacks = [cb] - self.error = False - self.didanything = False - - indent = getindent(targetstack) - - try: - self.resolvedeps(makefile, targetstack, [], False) - except util.MakeError, e: - if printerror: - print e - self.error = True - self.notifydone(makefile) - return - - assert self.vpathtarget is not None, "Target was never resolved!" - if not len(self.rules): - self.notifydone(makefile) - return - - if self.isdoublecolon(): - rulelist = [RemakeRuleContext(self, makefile, r, [(makefile.gettarget(p), False) for p in r.prerequisites], targetstack, avoidremakeloop) for r in self.rules] - else: - alldeps = [] - - commandrule = None - for r in self.rules: - rdeps = [(makefile.gettarget(p), r.weakdeps) for p in r.prerequisites] - if len(r.commands): - assert commandrule is None - commandrule = r - # The dependencies of the command rule are resolved before other dependencies, - # no matter the ordering of the other no-command rules - alldeps[0:0] = rdeps - else: - alldeps.extend(rdeps) - - rulelist = [RemakeRuleContext(self, makefile, commandrule, alldeps, targetstack, avoidremakeloop)] - - targetstack = targetstack + [self.target] - - if serial: - RemakeTargetSerially(self, makefile, indent, rulelist) - else: - RemakeTargetParallel(self, makefile, indent, rulelist) - -def dirpart(p): - d, s, f = util.strrpartition(p, '/') - if d == '': - return '.' - - return d - -def filepart(p): - d, s, f = util.strrpartition(p, '/') - return f - -def setautomatic(v, name, plist): - v.set(name, Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, ' '.join(plist)) - v.set(name + 'D', Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, ' '.join((dirpart(p) for p in plist))) - v.set(name + 'F', Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, ' '.join((filepart(p) for p in plist))) - -def setautomaticvariables(v, makefile, target, prerequisites): - prtargets = [makefile.gettarget(p) for p in prerequisites] - prall = [pt.vpathtarget for pt in prtargets] - proutofdate = [pt.vpathtarget for pt in withoutdups(prtargets) - if target.mtime is None or mtimeislater(pt.mtime, target.mtime)] - - setautomatic(v, '@', [target.vpathtarget]) - if len(prall): - setautomatic(v, '<', [prall[0]]) - - setautomatic(v, '?', proutofdate) - setautomatic(v, '^', list(withoutdups(prall))) - setautomatic(v, '+', prall) - -def splitcommand(command): - """ - Using the esoteric rules, split command lines by unescaped newlines. - """ - start = 0 - i = 0 - while i < len(command): - c = command[i] - if c == '\\': - i += 1 - elif c == '\n': - yield command[start:i] - i += 1 - start = i - continue - - i += 1 - - if i > start: - yield command[start:i] - -def findmodifiers(command): - """ - Find any of +-@% prefixed on the command. - @returns (command, isHidden, isRecursive, ignoreErrors, isNative) - """ - - isHidden = False - isRecursive = False - ignoreErrors = False - isNative = False - - realcommand = command.lstrip(' \t\n@+-%') - modset = set(command[:-len(realcommand)]) - return realcommand, '@' in modset, '+' in modset, '-' in modset, '%' in modset - -class _CommandWrapper(object): - def __init__(self, cline, ignoreErrors, loc, context, **kwargs): - self.ignoreErrors = ignoreErrors - self.loc = loc - self.cline = cline - self.kwargs = kwargs - self.context = context - - def _cb(self, res): - if res != 0 and not self.ignoreErrors: - print "%s: command '%s' failed, return code %i" % (self.loc, self.cline, res) - self.usercb(error=True) - else: - self.usercb(error=False) - - def __call__(self, cb): - self.usercb = cb - process.call(self.cline, loc=self.loc, cb=self._cb, context=self.context, **self.kwargs) - -class _NativeWrapper(_CommandWrapper): - def __init__(self, cline, ignoreErrors, loc, context, - pycommandpath, **kwargs): - _CommandWrapper.__init__(self, cline, ignoreErrors, loc, context, - **kwargs) - if pycommandpath: - self.pycommandpath = re.split('[%s\s]+' % os.pathsep, - pycommandpath) - else: - self.pycommandpath = None - - def __call__(self, cb): - # get the module and method to call - parts, badchar = process.clinetoargv(self.cline, self.kwargs['cwd']) - if parts is None: - raise DataError("native command '%s': shell metacharacter '%s' in command line" % (self.cline, badchar), self.loc) - if len(parts) < 2: - raise DataError("native command '%s': no method name specified" % self.cline, self.loc) - module = parts[0] - method = parts[1] - cline_list = parts[2:] - self.usercb = cb - process.call_native(module, method, cline_list, - loc=self.loc, cb=self._cb, context=self.context, - pycommandpath=self.pycommandpath, **self.kwargs) - -def getcommandsforrule(rule, target, makefile, prerequisites, stem): - v = Variables(parent=target.variables) - setautomaticvariables(v, makefile, target, prerequisites) - if stem is not None: - setautomatic(v, '*', [stem]) - - env = makefile.getsubenvironment(v) - - for c in rule.commands: - cstring = c.resolvestr(makefile, v) - for cline in splitcommand(cstring): - cline, isHidden, isRecursive, ignoreErrors, isNative = findmodifiers(cline) - if (isHidden or makefile.silent) and not makefile.justprint: - echo = None - else: - echo = "%s$ %s" % (c.loc, cline) - if not isNative: - yield _CommandWrapper(cline, ignoreErrors=ignoreErrors, env=env, cwd=makefile.workdir, loc=c.loc, context=makefile.context, - echo=echo, justprint=makefile.justprint) - else: - f, s, e = v.get("PYCOMMANDPATH", True) - if e: - e = e.resolvestr(makefile, v, ["PYCOMMANDPATH"]) - yield _NativeWrapper(cline, ignoreErrors=ignoreErrors, - env=env, cwd=makefile.workdir, - loc=c.loc, context=makefile.context, - echo=echo, justprint=makefile.justprint, - pycommandpath=e) - -class Rule(object): - """ - A rule contains a list of prerequisites and a list of commands. It may also - contain rule-specific variables. This rule may be associated with multiple targets. - """ - - def __init__(self, prereqs, doublecolon, loc, weakdeps): - self.prerequisites = prereqs - self.doublecolon = doublecolon - self.commands = [] - self.loc = loc - self.weakdeps = weakdeps - - def addcommand(self, c): - assert isinstance(c, (Expansion, StringExpansion)) - self.commands.append(c) - - def getcommands(self, target, makefile): - assert isinstance(target, Target) - # Prerequisites are merged if the target contains multiple rules and is - # not a terminal (double colon) rule. See - # https://www.gnu.org/software/make/manual/make.html#Multiple-Targets. - prereqs = [] - prereqs.extend(self.prerequisites) - - if not self.doublecolon: - for rule in target.rules: - # The current rule comes first, which is already in prereqs so - # we don't need to add it again. - if rule != self: - prereqs.extend(rule.prerequisites) - - return getcommandsforrule(self, target, makefile, prereqs, stem=None) - # TODO: $* in non-pattern rules? - -class PatternRuleInstance(object): - weakdeps = False - - """ - A pattern rule instantiated for a particular target. It has the same API as Rule, but - different internals, forwarding most information on to the PatternRule. - """ - def __init__(self, prule, dir, stem, ismatchany): - assert isinstance(prule, PatternRule) - - self.dir = dir - self.stem = stem - self.prule = prule - self.prerequisites = prule.prerequisitesforstem(dir, stem) - self.doublecolon = prule.doublecolon - self.loc = prule.loc - self.ismatchany = ismatchany - self.commands = prule.commands - - def getcommands(self, target, makefile): - assert isinstance(target, Target) - return getcommandsforrule(self, target, makefile, self.prerequisites, stem=self.dir + self.stem) - - def __str__(self): - return "Pattern rule at %s with stem '%s', matchany: %s doublecolon: %s" % (self.loc, - self.dir + self.stem, - self.ismatchany, - self.doublecolon) - -class PatternRule(object): - """ - An implicit rule or static pattern rule containing target patterns, prerequisite patterns, - and a list of commands. - """ - - def __init__(self, targetpatterns, prerequisites, doublecolon, loc): - self.targetpatterns = targetpatterns - self.prerequisites = prerequisites - self.doublecolon = doublecolon - self.loc = loc - self.commands = [] - - def addcommand(self, c): - assert isinstance(c, (Expansion, StringExpansion)) - self.commands.append(c) - - def ismatchany(self): - return util.any((t.ismatchany() for t in self.targetpatterns)) - - def hasspecificmatch(self, file): - for p in self.targetpatterns: - if not p.ismatchany() and p.match(file) is not None: - return True - - return False - - def matchesfor(self, dir, file, skipsinglecolonmatchany): - """ - Determine all the target patterns of this rule that might match target t. - @yields a PatternRuleInstance for each. - """ - - for p in self.targetpatterns: - matchany = p.ismatchany() - if matchany: - if skipsinglecolonmatchany and not self.doublecolon: - continue - - yield PatternRuleInstance(self, dir, file, True) - else: - stem = p.match(dir + file) - if stem is not None: - yield PatternRuleInstance(self, '', stem, False) - else: - stem = p.match(file) - if stem is not None: - yield PatternRuleInstance(self, dir, stem, False) - - def prerequisitesforstem(self, dir, stem): - return [p.resolve(dir, stem) for p in self.prerequisites] - -class _RemakeContext(object): - def __init__(self, makefile, cb): - self.makefile = makefile - self.included = [(makefile.gettarget(f), required) - for f, required in makefile.included] - self.toremake = list(self.included) - self.cb = cb - - self.remakecb(error=False, didanything=False) - - def remakecb(self, error, didanything): - assert error in (True, False) - - if error and self.required: - print "Error remaking makefiles (ignored)" - - if len(self.toremake): - target, self.required = self.toremake.pop(0) - target.make(self.makefile, [], avoidremakeloop=True, cb=self.remakecb, printerror=False) - else: - for t, required in self.included: - if t.wasremade: - _log.info("Included file %s was remade, restarting make", t.target) - self.cb(remade=True) - return - elif required and t.mtime is None: - self.cb(remade=False, error=DataError("No rule to remake missing include file %s" % t.target)) - return - - self.cb(remade=False) - -class Makefile(object): - """ - The top-level data structure for makefile execution. It holds Targets, implicit rules, and other - state data. - """ - - def __init__(self, workdir=None, env=None, restarts=0, make=None, - makeflags='', makeoverrides='', - makelevel=0, context=None, targets=(), keepgoing=False, - silent=False, justprint=False): - self.defaulttarget = None - - if env is None: - env = os.environ - self.env = env - - self.variables = Variables() - self.variables.readfromenvironment(env) - - self.context = context - self.exportedvars = {} - self._targets = {} - self.keepgoing = keepgoing - self.silent = silent - self.justprint = justprint - self._patternvariables = [] # of (pattern, variables) - self.implicitrules = [] - self.parsingfinished = False - - self._patternvpaths = [] # of (pattern, [dir, ...]) - - if workdir is None: - workdir = os.getcwd() - workdir = os.path.realpath(workdir) - self.workdir = workdir - self.variables.set('CURDIR', Variables.FLAVOR_SIMPLE, - Variables.SOURCE_AUTOMATIC, workdir.replace('\\','/')) - - # the list of included makefiles, whether or not they existed - self.included = [] - - self.variables.set('MAKE_RESTARTS', Variables.FLAVOR_SIMPLE, - Variables.SOURCE_AUTOMATIC, restarts > 0 and str(restarts) or '') - - self.variables.set('.PYMAKE', Variables.FLAVOR_SIMPLE, - Variables.SOURCE_MAKEFILE, "1") - if make is not None: - self.variables.set('MAKE', Variables.FLAVOR_SIMPLE, - Variables.SOURCE_MAKEFILE, make) - - if makeoverrides != '': - self.variables.set('-*-command-variables-*-', Variables.FLAVOR_SIMPLE, - Variables.SOURCE_AUTOMATIC, makeoverrides) - makeflags += ' -- $(MAKEOVERRIDES)' - - self.variables.set('MAKEOVERRIDES', Variables.FLAVOR_RECURSIVE, - Variables.SOURCE_ENVIRONMENT, - '${-*-command-variables-*-}') - - self.variables.set('MAKEFLAGS', Variables.FLAVOR_RECURSIVE, - Variables.SOURCE_MAKEFILE, makeflags) - self.exportedvars['MAKEFLAGS'] = True - - self.makelevel = makelevel - self.variables.set('MAKELEVEL', Variables.FLAVOR_SIMPLE, - Variables.SOURCE_MAKEFILE, str(makelevel)) - - self.variables.set('MAKECMDGOALS', Variables.FLAVOR_SIMPLE, - Variables.SOURCE_AUTOMATIC, ' '.join(targets)) - - for vname, val in implicit.variables.iteritems(): - self.variables.set(vname, - Variables.FLAVOR_SIMPLE, - Variables.SOURCE_IMPLICIT, val) - - def foundtarget(self, t): - """ - Inform the makefile of a target which is a candidate for being the default target, - if there isn't already a default target. - """ - flavor, source, value = self.variables.get('.DEFAULT_GOAL') - if self.defaulttarget is None and t != '.PHONY' and value is None: - self.defaulttarget = t - self.variables.set('.DEFAULT_GOAL', Variables.FLAVOR_SIMPLE, - Variables.SOURCE_AUTOMATIC, t) - - def getpatternvariables(self, pattern): - assert isinstance(pattern, Pattern) - - for p, v in self._patternvariables: - if p == pattern: - return v - - v = Variables() - self._patternvariables.append( (pattern, v) ) - return v - - def getpatternvariablesfor(self, target): - for p, v in self._patternvariables: - if p.match(target): - yield v - - def hastarget(self, target): - return target in self._targets - - _globcheck = re.compile('[[*?]') - def gettarget(self, target): - assert isinstance(target, str_type) - - target = target.rstrip('/') - - assert target != '', "empty target?" - - assert not self._globcheck.match(target) - - t = self._targets.get(target, None) - if t is None: - t = Target(target, self) - self._targets[target] = t - return t - - def appendimplicitrule(self, rule): - assert isinstance(rule, PatternRule) - self.implicitrules.append(rule) - - def finishparsing(self): - """ - Various activities, such as "eval", are not allowed after parsing is - finished. In addition, various warnings and errors can only be issued - after the parsing data model is complete. All dependency resolution - and rule execution requires that parsing be finished. - """ - self.parsingfinished = True - - flavor, source, value = self.variables.get('GPATH') - if value is not None and value.resolvestr(self, self.variables, ['GPATH']).strip() != '': - raise DataError('GPATH was set: pymake does not support GPATH semantics') - - flavor, source, value = self.variables.get('VPATH') - if value is None: - self._vpath = [] - else: - self._vpath = filter(lambda e: e != '', - re.split('[%s\s]+' % os.pathsep, - value.resolvestr(self, self.variables, ['VPATH']))) - - targets = list(self._targets.itervalues()) - for t in targets: - t.explicit = True - for r in t.rules: - for p in r.prerequisites: - self.gettarget(p).explicit = True - - np = self.gettarget('.NOTPARALLEL') - if len(np.rules): - self.context = process.getcontext(1) - - flavor, source, value = self.variables.get('.DEFAULT_GOAL') - if value is not None: - self.defaulttarget = value.resolvestr(self, self.variables, ['.DEFAULT_GOAL']).strip() - - self.error = False - - def include(self, path, required=True, weak=False, loc=None): - """ - Include the makefile at `path`. - """ - self.included.append((path, required)) - fspath = util.normaljoin(self.workdir, path) - if os.path.exists(fspath): - if weak: - stmts = parser.parsedepfile(fspath) - else: - stmts = parser.parsefile(fspath) - self.variables.append('MAKEFILE_LIST', Variables.SOURCE_AUTOMATIC, path, None, self) - stmts.execute(self, weak=weak) - self.gettarget(path).explicit = True - - def addvpath(self, pattern, dirs): - """ - Add a directory to the vpath search for the given pattern. - """ - self._patternvpaths.append((pattern, dirs)) - - def clearvpath(self, pattern): - """ - Clear vpaths for the given pattern. - """ - self._patternvpaths = [(p, dirs) - for p, dirs in self._patternvpaths - if not p.match(pattern)] - - def clearallvpaths(self): - self._patternvpaths = [] - - def getvpath(self, target): - vp = list(self._vpath) - for p, dirs in self._patternvpaths: - if p.match(target): - vp.extend(dirs) - - return withoutdups(vp) - - def remakemakefiles(self, cb): - mlist = [] - for f, required in self.included: - t = self.gettarget(f) - t.explicit = True - t.resolvevpath(self) - oldmtime = t.mtime - - mlist.append((t, oldmtime)) - - _RemakeContext(self, cb) - - def getsubenvironment(self, variables): - env = dict(self.env) - for vname, v in self.exportedvars.iteritems(): - if v: - flavor, source, val = variables.get(vname) - if val is None: - strval = '' - else: - strval = val.resolvestr(self, variables, [vname]) - env[vname] = strval - else: - env.pop(vname, None) - - makeflags = '' - - env['MAKELEVEL'] = str(self.makelevel + 1) - return env diff --git a/build/pymake/pymake/functions.py b/build/pymake/pymake/functions.py deleted file mode 100644 index e53fb5472..000000000 --- a/build/pymake/pymake/functions.py +++ /dev/null @@ -1,873 +0,0 @@ -""" -Makefile functions. -""" - -import parser, util -import subprocess, os, logging, sys -from globrelative import glob -from cStringIO import StringIO - -log = logging.getLogger('pymake.data') - -def emit_expansions(descend, *expansions): - """Helper function to emit all expansions within an input set.""" - for expansion in expansions: - yield expansion - - if not descend or not isinstance(expansion, list): - continue - - for e, is_func in expansion: - if is_func: - for exp in e.expansions(True): - yield exp - else: - yield e - -class Function(object): - """ - An object that represents a function call. This class is always subclassed - with the following methods and attributes: - - minargs = minimum # of arguments - maxargs = maximum # of arguments (0 means unlimited) - - def resolve(self, makefile, variables, fd, setting) - Calls the function - calls fd.write() with strings - """ - - __slots__ = ('_arguments', 'loc') - - def __init__(self, loc): - self._arguments = [] - self.loc = loc - assert self.minargs > 0 - - def __getitem__(self, key): - return self._arguments[key] - - def setup(self): - argc = len(self._arguments) - - if argc < self.minargs: - raise data.DataError("Not enough arguments to function %s, requires %s" % (self.name, self.minargs), self.loc) - - assert self.maxargs == 0 or argc <= self.maxargs, "Parser screwed up, gave us too many args" - - def append(self, arg): - assert isinstance(arg, (data.Expansion, data.StringExpansion)) - self._arguments.append(arg) - - def to_source(self): - """Convert the function back to make file "source" code.""" - if not hasattr(self, 'name'): - raise Exception("%s must implement to_source()." % self.__class__) - - # The default implementation simply prints the function name and all - # the arguments joined by a comma. - # According to the GNU make manual Section 8.1, whitespace around - # arguments is *not* part of the argument's value. So, we trim excess - # white space so we have consistent behavior. - args = [] - curly = False - for i, arg in enumerate(self._arguments): - arg = arg.to_source() - - if i == 0: - arg = arg.lstrip() - - # Are balanced parens even OK? - if arg.count('(') != arg.count(')'): - curly = True - - args.append(arg) - - if curly: - return '${%s %s}' % (self.name, ','.join(args)) - - return '$(%s %s)' % (self.name, ','.join(args)) - - def expansions(self, descend=False): - """Obtain all expansions contained within this function. - - By default, only expansions directly part of this function are - returned. If descend is True, we will descend into child expansions and - return all of the composite parts. - - This is a generator for pymake.data.BaseExpansion instances. - """ - # Our default implementation simply returns arguments. More advanced - # functions like variable references may need their own implementation. - return emit_expansions(descend, *self._arguments) - - @property - def is_filesystem_dependent(self): - """Exposes whether this function depends on the filesystem for results. - - If True, the function touches the filesystem as part of evaluation. - - This only tests whether the function itself uses the filesystem. If - this function has arguments that are functions that touch the - filesystem, this will return False. - """ - return False - - def __len__(self): - return len(self._arguments) - - def __repr__(self): - return "%s<%s>(%r)" % ( - self.__class__.__name__, self.loc, - ','.join([repr(a) for a in self._arguments]), - ) - - def __eq__(self, other): - if not hasattr(self, 'name'): - raise Exception("%s must implement __eq__." % self.__class__) - - if type(self) != type(other): - return False - - if self.name != other.name: - return False - - if len(self._arguments) != len(other._arguments): - return False - - for i in xrange(len(self._arguments)): - # According to the GNU make manual Section 8.1, whitespace around - # arguments is *not* part of the argument's value. So, we do a - # whitespace-agnostic comparison. - if i == 0: - a = self._arguments[i] - a.lstrip() - - b = other._arguments[i] - b.lstrip() - - if a != b: - return False - - continue - - if self._arguments[i] != other._arguments[i]: - return False - - return True - - def __ne__(self, other): - return not self.__eq__(other) - -class VariableRef(Function): - AUTOMATIC_VARIABLES = set(['@', '%', '<', '?', '^', '+', '|', '*']) - - __slots__ = ('vname', 'loc') - - def __init__(self, loc, vname): - self.loc = loc - assert isinstance(vname, (data.Expansion, data.StringExpansion)) - self.vname = vname - - def setup(self): - assert False, "Shouldn't get here" - - def resolve(self, makefile, variables, fd, setting): - vname = self.vname.resolvestr(makefile, variables, setting) - if vname in setting: - raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc) - - flavor, source, value = variables.get(vname) - if value is None: - log.debug("%s: variable '%s' was not set" % (self.loc, vname)) - return - - value.resolve(makefile, variables, fd, setting + [vname]) - - def to_source(self): - if isinstance(self.vname, data.StringExpansion): - if self.vname.s in self.AUTOMATIC_VARIABLES: - return '$%s' % self.vname.s - - return '$(%s)' % self.vname.s - - return '$(%s)' % self.vname.to_source() - - def expansions(self, descend=False): - return emit_expansions(descend, self.vname) - - def __repr__(self): - return "VariableRef<%s>(%r)" % (self.loc, self.vname) - - def __eq__(self, other): - if not isinstance(other, VariableRef): - return False - - return self.vname == other.vname - -class SubstitutionRef(Function): - """$(VARNAME:.c=.o) and $(VARNAME:%.c=%.o)""" - - __slots__ = ('loc', 'vname', 'substfrom', 'substto') - - def __init__(self, loc, varname, substfrom, substto): - self.loc = loc - self.vname = varname - self.substfrom = substfrom - self.substto = substto - - def setup(self): - assert False, "Shouldn't get here" - - def resolve(self, makefile, variables, fd, setting): - vname = self.vname.resolvestr(makefile, variables, setting) - if vname in setting: - raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc) - - substfrom = self.substfrom.resolvestr(makefile, variables, setting) - substto = self.substto.resolvestr(makefile, variables, setting) - - flavor, source, value = variables.get(vname) - if value is None: - log.debug("%s: variable '%s' was not set" % (self.loc, vname)) - return - - f = data.Pattern(substfrom) - if not f.ispattern(): - f = data.Pattern('%' + substfrom) - substto = '%' + substto - - fd.write(' '.join([f.subst(substto, word, False) - for word in value.resolvesplit(makefile, variables, setting + [vname])])) - - def to_source(self): - return '$(%s:%s=%s)' % ( - self.vname.to_source(), - self.substfrom.to_source(), - self.substto.to_source()) - - def expansions(self, descend=False): - return emit_expansions(descend, self.vname, self.substfrom, - self.substto) - - def __repr__(self): - return "SubstitutionRef<%s>(%r:%r=%r)" % ( - self.loc, self.vname, self.substfrom, self.substto,) - - def __eq__(self, other): - if not isinstance(other, SubstitutionRef): - return False - - return self.vname == other.vname and self.substfrom == other.substfrom \ - and self.substto == other.substto - -class SubstFunction(Function): - name = 'subst' - minargs = 3 - maxargs = 3 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - s = self._arguments[0].resolvestr(makefile, variables, setting) - r = self._arguments[1].resolvestr(makefile, variables, setting) - d = self._arguments[2].resolvestr(makefile, variables, setting) - fd.write(d.replace(s, r)) - -class PatSubstFunction(Function): - name = 'patsubst' - minargs = 3 - maxargs = 3 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - s = self._arguments[0].resolvestr(makefile, variables, setting) - r = self._arguments[1].resolvestr(makefile, variables, setting) - - p = data.Pattern(s) - fd.write(' '.join([p.subst(r, word, False) - for word in self._arguments[2].resolvesplit(makefile, variables, setting)])) - -class StripFunction(Function): - name = 'strip' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - util.joiniter(fd, self._arguments[0].resolvesplit(makefile, variables, setting)) - -class FindstringFunction(Function): - name = 'findstring' - minargs = 2 - maxargs = 2 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - s = self._arguments[0].resolvestr(makefile, variables, setting) - r = self._arguments[1].resolvestr(makefile, variables, setting) - if r.find(s) == -1: - return - fd.write(s) - -class FilterFunction(Function): - name = 'filter' - minargs = 2 - maxargs = 2 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - plist = [data.Pattern(p) - for p in self._arguments[0].resolvesplit(makefile, variables, setting)] - - fd.write(' '.join([w for w in self._arguments[1].resolvesplit(makefile, variables, setting) - if util.any((p.match(w) for p in plist))])) - -class FilteroutFunction(Function): - name = 'filter-out' - minargs = 2 - maxargs = 2 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - plist = [data.Pattern(p) - for p in self._arguments[0].resolvesplit(makefile, variables, setting)] - - fd.write(' '.join([w for w in self._arguments[1].resolvesplit(makefile, variables, setting) - if not util.any((p.match(w) for p in plist))])) - -class SortFunction(Function): - name = 'sort' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - d = set(self._arguments[0].resolvesplit(makefile, variables, setting)) - util.joiniter(fd, sorted(d)) - -class WordFunction(Function): - name = 'word' - minargs = 2 - maxargs = 2 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - n = self._arguments[0].resolvestr(makefile, variables, setting) - # TODO: provide better error if this doesn't convert - n = int(n) - words = list(self._arguments[1].resolvesplit(makefile, variables, setting)) - if n < 1 or n > len(words): - return - fd.write(words[n - 1]) - -class WordlistFunction(Function): - name = 'wordlist' - minargs = 3 - maxargs = 3 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - nfrom = self._arguments[0].resolvestr(makefile, variables, setting) - nto = self._arguments[1].resolvestr(makefile, variables, setting) - # TODO: provide better errors if this doesn't convert - nfrom = int(nfrom) - nto = int(nto) - - words = list(self._arguments[2].resolvesplit(makefile, variables, setting)) - - if nfrom < 1: - nfrom = 1 - if nto < 1: - nto = 1 - - util.joiniter(fd, words[nfrom - 1:nto]) - -class WordsFunction(Function): - name = 'words' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - fd.write(str(len(self._arguments[0].resolvesplit(makefile, variables, setting)))) - -class FirstWordFunction(Function): - name = 'firstword' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - l = self._arguments[0].resolvesplit(makefile, variables, setting) - if len(l): - fd.write(l[0]) - -class LastWordFunction(Function): - name = 'lastword' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - l = self._arguments[0].resolvesplit(makefile, variables, setting) - if len(l): - fd.write(l[-1]) - -def pathsplit(path, default='./'): - """ - Splits a path into dirpart, filepart on the last slash. If there is no slash, dirpart - is ./ - """ - dir, slash, file = util.strrpartition(path, '/') - if dir == '': - return default, file - - return dir + slash, file - -class DirFunction(Function): - name = 'dir' - minargs = 1 - maxargs = 1 - - def resolve(self, makefile, variables, fd, setting): - fd.write(' '.join([pathsplit(path)[0] - for path in self._arguments[0].resolvesplit(makefile, variables, setting)])) - -class NotDirFunction(Function): - name = 'notdir' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - fd.write(' '.join([pathsplit(path)[1] - for path in self._arguments[0].resolvesplit(makefile, variables, setting)])) - -class SuffixFunction(Function): - name = 'suffix' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - @staticmethod - def suffixes(words): - for w in words: - dir, file = pathsplit(w) - base, dot, suffix = util.strrpartition(file, '.') - if base != '': - yield dot + suffix - - def resolve(self, makefile, variables, fd, setting): - util.joiniter(fd, self.suffixes(self._arguments[0].resolvesplit(makefile, variables, setting))) - -class BasenameFunction(Function): - name = 'basename' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - @staticmethod - def basenames(words): - for w in words: - dir, file = pathsplit(w, '') - base, dot, suffix = util.strrpartition(file, '.') - if dot == '': - base = suffix - - yield dir + base - - def resolve(self, makefile, variables, fd, setting): - util.joiniter(fd, self.basenames(self._arguments[0].resolvesplit(makefile, variables, setting))) - -class AddSuffixFunction(Function): - name = 'addsuffix' - minargs = 2 - maxargs = 2 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - suffix = self._arguments[0].resolvestr(makefile, variables, setting) - - fd.write(' '.join([w + suffix for w in self._arguments[1].resolvesplit(makefile, variables, setting)])) - -class AddPrefixFunction(Function): - name = 'addprefix' - minargs = 2 - maxargs = 2 - - def resolve(self, makefile, variables, fd, setting): - prefix = self._arguments[0].resolvestr(makefile, variables, setting) - - fd.write(' '.join([prefix + w for w in self._arguments[1].resolvesplit(makefile, variables, setting)])) - -class JoinFunction(Function): - name = 'join' - minargs = 2 - maxargs = 2 - - __slots__ = Function.__slots__ - - @staticmethod - def iterjoin(l1, l2): - for i in xrange(0, max(len(l1), len(l2))): - i1 = i < len(l1) and l1[i] or '' - i2 = i < len(l2) and l2[i] or '' - yield i1 + i2 - - def resolve(self, makefile, variables, fd, setting): - list1 = list(self._arguments[0].resolvesplit(makefile, variables, setting)) - list2 = list(self._arguments[1].resolvesplit(makefile, variables, setting)) - - util.joiniter(fd, self.iterjoin(list1, list2)) - -class WildcardFunction(Function): - name = 'wildcard' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - patterns = self._arguments[0].resolvesplit(makefile, variables, setting) - - fd.write(' '.join([x.replace('\\','/') - for p in patterns - for x in glob(makefile.workdir, p)])) - - @property - def is_filesystem_dependent(self): - return True - -class RealpathFunction(Function): - name = 'realpath' - minargs = 1 - maxargs = 1 - - def resolve(self, makefile, variables, fd, setting): - fd.write(' '.join([os.path.realpath(os.path.join(makefile.workdir, path)).replace('\\', '/') - for path in self._arguments[0].resolvesplit(makefile, variables, setting)])) - - def is_filesystem_dependent(self): - return True - -class AbspathFunction(Function): - name = 'abspath' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - assert os.path.isabs(makefile.workdir) - fd.write(' '.join([util.normaljoin(makefile.workdir, path).replace('\\', '/') - for path in self._arguments[0].resolvesplit(makefile, variables, setting)])) - -class IfFunction(Function): - name = 'if' - minargs = 1 - maxargs = 3 - - __slots__ = Function.__slots__ - - def setup(self): - Function.setup(self) - self._arguments[0].lstrip() - self._arguments[0].rstrip() - - def resolve(self, makefile, variables, fd, setting): - condition = self._arguments[0].resolvestr(makefile, variables, setting) - - if len(condition): - self._arguments[1].resolve(makefile, variables, fd, setting) - elif len(self._arguments) > 2: - return self._arguments[2].resolve(makefile, variables, fd, setting) - -class OrFunction(Function): - name = 'or' - minargs = 1 - maxargs = 0 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - for arg in self._arguments: - r = arg.resolvestr(makefile, variables, setting) - if r != '': - fd.write(r) - return - -class AndFunction(Function): - name = 'and' - minargs = 1 - maxargs = 0 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - r = '' - - for arg in self._arguments: - r = arg.resolvestr(makefile, variables, setting) - if r == '': - return - - fd.write(r) - -class ForEachFunction(Function): - name = 'foreach' - minargs = 3 - maxargs = 3 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - vname = self._arguments[0].resolvestr(makefile, variables, setting) - e = self._arguments[2] - - v = data.Variables(parent=variables) - firstword = True - - for w in self._arguments[1].resolvesplit(makefile, variables, setting): - if firstword: - firstword = False - else: - fd.write(' ') - - # The $(origin) of the local variable must be "automatic" to - # conform with GNU make. However, automatic variables have low - # priority. So, we must force its assignment to occur. - v.set(vname, data.Variables.FLAVOR_SIMPLE, - data.Variables.SOURCE_AUTOMATIC, w, force=True) - e.resolve(makefile, v, fd, setting) - -class CallFunction(Function): - name = 'call' - minargs = 1 - maxargs = 0 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - vname = self._arguments[0].resolvestr(makefile, variables, setting) - if vname in setting: - raise data.DataError("Recursively setting variable '%s'" % (vname,)) - - v = data.Variables(parent=variables) - v.set('0', data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, vname) - for i in xrange(1, len(self._arguments)): - param = self._arguments[i].resolvestr(makefile, variables, setting) - v.set(str(i), data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, param) - - flavor, source, e = variables.get(vname) - - if e is None: - return - - if flavor == data.Variables.FLAVOR_SIMPLE: - log.warning("%s: calling variable '%s' which is simply-expanded" % (self.loc, vname)) - - # but we'll do it anyway - e.resolve(makefile, v, fd, setting + [vname]) - -class ValueFunction(Function): - name = 'value' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - varname = self._arguments[0].resolvestr(makefile, variables, setting) - - flavor, source, value = variables.get(varname, expand=False) - if value is not None: - fd.write(value) - -class EvalFunction(Function): - name = 'eval' - minargs = 1 - maxargs = 1 - - def resolve(self, makefile, variables, fd, setting): - if makefile.parsingfinished: - # GNU make allows variables to be set by recursive expansion during - # command execution. This seems really dumb to me, so I don't! - raise data.DataError("$(eval) not allowed via recursive expansion after parsing is finished", self.loc) - - stmts = parser.parsestring(self._arguments[0].resolvestr(makefile, variables, setting), - 'evaluation from %s' % self.loc) - stmts.execute(makefile) - -class OriginFunction(Function): - name = 'origin' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - vname = self._arguments[0].resolvestr(makefile, variables, setting) - - flavor, source, value = variables.get(vname) - if source is None: - r = 'undefined' - elif source == data.Variables.SOURCE_OVERRIDE: - r = 'override' - - elif source == data.Variables.SOURCE_MAKEFILE: - r = 'file' - elif source == data.Variables.SOURCE_ENVIRONMENT: - r = 'environment' - elif source == data.Variables.SOURCE_COMMANDLINE: - r = 'command line' - elif source == data.Variables.SOURCE_AUTOMATIC: - r = 'automatic' - elif source == data.Variables.SOURCE_IMPLICIT: - r = 'default' - - fd.write(r) - -class FlavorFunction(Function): - name = 'flavor' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - varname = self._arguments[0].resolvestr(makefile, variables, setting) - - flavor, source, value = variables.get(varname) - if flavor is None: - r = 'undefined' - elif flavor == data.Variables.FLAVOR_RECURSIVE: - r = 'recursive' - elif flavor == data.Variables.FLAVOR_SIMPLE: - r = 'simple' - fd.write(r) - -class ShellFunction(Function): - name = 'shell' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - from process import prepare_command - cline = self._arguments[0].resolvestr(makefile, variables, setting) - executable, cline = prepare_command(cline, makefile.workdir, self.loc) - - # subprocess.Popen doesn't use the PATH set in the env argument for - # finding the executable on some platforms (but strangely it does on - # others!), so set os.environ['PATH'] explicitly. - oldpath = os.environ['PATH'] - if makefile.env is not None and 'PATH' in makefile.env: - os.environ['PATH'] = makefile.env['PATH'] - - log.debug("%s: running command '%s'" % (self.loc, ' '.join(cline))) - try: - p = subprocess.Popen(cline, executable=executable, env=makefile.env, shell=False, - stdout=subprocess.PIPE, cwd=makefile.workdir) - except OSError, e: - print >>sys.stderr, "Error executing command %s" % cline[0], e - return - finally: - os.environ['PATH'] = oldpath - - stdout, stderr = p.communicate() - stdout = stdout.replace('\r\n', '\n') - if stdout.endswith('\n'): - stdout = stdout[:-1] - stdout = stdout.replace('\n', ' ') - - fd.write(stdout) - -class ErrorFunction(Function): - name = 'error' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - v = self._arguments[0].resolvestr(makefile, variables, setting) - raise data.DataError(v, self.loc) - -class WarningFunction(Function): - name = 'warning' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - v = self._arguments[0].resolvestr(makefile, variables, setting) - log.warning(v) - -class InfoFunction(Function): - name = 'info' - minargs = 1 - maxargs = 1 - - __slots__ = Function.__slots__ - - def resolve(self, makefile, variables, fd, setting): - v = self._arguments[0].resolvestr(makefile, variables, setting) - print v - -functionmap = { - 'subst': SubstFunction, - 'patsubst': PatSubstFunction, - 'strip': StripFunction, - 'findstring': FindstringFunction, - 'filter': FilterFunction, - 'filter-out': FilteroutFunction, - 'sort': SortFunction, - 'word': WordFunction, - 'wordlist': WordlistFunction, - 'words': WordsFunction, - 'firstword': FirstWordFunction, - 'lastword': LastWordFunction, - 'dir': DirFunction, - 'notdir': NotDirFunction, - 'suffix': SuffixFunction, - 'basename': BasenameFunction, - 'addsuffix': AddSuffixFunction, - 'addprefix': AddPrefixFunction, - 'join': JoinFunction, - 'wildcard': WildcardFunction, - 'realpath': RealpathFunction, - 'abspath': AbspathFunction, - 'if': IfFunction, - 'or': OrFunction, - 'and': AndFunction, - 'foreach': ForEachFunction, - 'call': CallFunction, - 'value': ValueFunction, - 'eval': EvalFunction, - 'origin': OriginFunction, - 'flavor': FlavorFunction, - 'shell': ShellFunction, - 'error': ErrorFunction, - 'warning': WarningFunction, - 'info': InfoFunction, -} - -import data diff --git a/build/pymake/pymake/globrelative.py b/build/pymake/pymake/globrelative.py deleted file mode 100644 index 37ca28e06..000000000 --- a/build/pymake/pymake/globrelative.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -Filename globbing like the python glob module with minor differences: - -* glob relative to an arbitrary directory -* include . and .. -* check that link targets exist, not just links -""" - -import os, re, fnmatch -import util - -_globcheck = re.compile('[[*?]') - -def hasglob(p): - return _globcheck.search(p) is not None - -def glob(fsdir, path): - """ - Yield paths matching the path glob. Sorts as a bonus. Excludes '.' and '..' - """ - - dir, leaf = os.path.split(path) - if dir == '': - return globpattern(fsdir, leaf) - - if hasglob(dir): - dirsfound = glob(fsdir, dir) - else: - dirsfound = [dir] - - r = [] - - for dir in dirsfound: - fspath = util.normaljoin(fsdir, dir) - if not os.path.isdir(fspath): - continue - - r.extend((util.normaljoin(dir, found) for found in globpattern(fspath, leaf))) - - return r - -def globpattern(dir, pattern): - """ - Return leaf names in the specified directory which match the pattern. - """ - - if not hasglob(pattern): - if pattern == '': - if os.path.isdir(dir): - return [''] - return [] - - if os.path.exists(util.normaljoin(dir, pattern)): - return [pattern] - return [] - - leaves = os.listdir(dir) + ['.', '..'] - - # "hidden" filenames are a bit special - if not pattern.startswith('.'): - leaves = [leaf for leaf in leaves - if not leaf.startswith('.')] - - leaves = fnmatch.filter(leaves, pattern) - leaves = filter(lambda l: os.path.exists(util.normaljoin(dir, l)), leaves) - - leaves.sort() - return leaves diff --git a/build/pymake/pymake/implicit.py b/build/pymake/pymake/implicit.py deleted file mode 100644 index d73895cab..000000000 --- a/build/pymake/pymake/implicit.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Implicit variables; perhaps in the future this will also include some implicit -rules, at least match-anything cancellation rules. -""" - -variables = { - 'MKDIR': '%pymake.builtins mkdir', - 'RM': '%pymake.builtins rm -f', - 'SLEEP': '%pymake.builtins sleep', - 'TOUCH': '%pymake.builtins touch', - '.LIBPATTERNS': 'lib%.so lib%.a', - '.PYMAKE': '1', - } - diff --git a/build/pymake/pymake/parser.py b/build/pymake/pymake/parser.py deleted file mode 100644 index 4bff53368..000000000 --- a/build/pymake/pymake/parser.py +++ /dev/null @@ -1,822 +0,0 @@ -""" -Module for parsing Makefile syntax. - -Makefiles use a line-based parsing system. Continuations and substitutions are handled differently based on the -type of line being parsed: - -Lines with makefile syntax condense continuations to a single space, no matter the actual trailing whitespace -of the first line or the leading whitespace of the continuation. In other situations, trailing whitespace is -relevant. - -Lines with command syntax do not condense continuations: the backslash and newline are part of the command. -(GNU Make is buggy in this regard, at least on mac). - -Lines with an initial tab are commands if they can be (there is a rule or a command immediately preceding). -Otherwise, they are parsed as makefile syntax. - -This file parses into the data structures defined in the parserdata module. Those classes are what actually -do the dirty work of "executing" the parsed data into a data.Makefile. - -Four iterator functions are available: -* iterdata -* itermakefilechars -* itercommandchars - -The iterators handle line continuations and comments in different ways, but share a common calling -convention: - -Called with (data, startoffset, tokenlist, finditer) - -yield 4-tuples (flatstr, token, tokenoffset, afteroffset) -flatstr is data, guaranteed to have no tokens (may be '') -token, tokenoffset, afteroffset *may be None*. That means there is more text -coming. -""" - -import logging, re, os, sys -import data, functions, util, parserdata - -_log = logging.getLogger('pymake.parser') - -class SyntaxError(util.MakeError): - pass - -_skipws = re.compile('\S') -class Data(object): - """ - A single virtual "line", which can be multiple source lines joined with - continuations. - """ - - __slots__ = ('s', 'lstart', 'lend', 'loc') - - def __init__(self, s, lstart, lend, loc): - self.s = s - self.lstart = lstart - self.lend = lend - self.loc = loc - - @staticmethod - def fromstring(s, path): - return Data(s, 0, len(s), parserdata.Location(path, 1, 0)) - - def getloc(self, offset): - assert offset >= self.lstart and offset <= self.lend - return self.loc.offset(self.s, self.lstart, offset) - - def skipwhitespace(self, offset): - """ - Return the offset of the first non-whitespace character in data starting at offset, or None if there are - only whitespace characters remaining. - """ - m = _skipws.search(self.s, offset, self.lend) - if m is None: - return self.lend - - return m.start(0) - -_linere = re.compile(r'\\*\n') -def enumeratelines(s, filename): - """ - Enumerate lines in a string as Data objects, joining line - continuations. - """ - - off = 0 - lineno = 1 - curlines = 0 - for m in _linere.finditer(s): - curlines += 1 - start, end = m.span(0) - - if (start - end) % 2 == 0: - # odd number of backslashes is a continuation - continue - - yield Data(s, off, end - 1, parserdata.Location(filename, lineno, 0)) - - lineno += curlines - curlines = 0 - off = end - - yield Data(s, off, len(s), parserdata.Location(filename, lineno, 0)) - -_alltokens = re.compile(r'''\\*\# | # hash mark preceeded by any number of backslashes - := | - \+= | - \?= | - :: | - (?:\$(?:$|[\(\{](?:%s)\s+|.)) | # dollar sign followed by EOF, a function keyword with whitespace, or any character - :(?![\\/]) | # colon followed by anything except a slash (Windows path detection) - [=#{}();,|'"]''' % '|'.join(functions.functionmap.iterkeys()), re.VERBOSE) - -def iterdata(d, offset, tokenlist, it): - """ - Iterate over flat data without line continuations, comments, or any special escaped characters. - - Typically used to parse recursively-expanded variables. - """ - - assert len(tokenlist), "Empty tokenlist passed to iterdata is meaningless!" - assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) - - if offset == d.lend: - return - - s = d.s - for m in it: - mstart, mend = m.span(0) - token = s[mstart:mend] - if token in tokenlist or (token[0] == '$' and '$' in tokenlist): - yield s[offset:mstart], token, mstart, mend - else: - yield s[offset:mend], None, None, mend - offset = mend - - yield s[offset:d.lend], None, None, None - -# multiple backslashes before a newline are unescaped, halving their total number -_makecontinuations = re.compile(r'(?:\s*|((?:\\\\)+))\\\n\s*') -def _replacemakecontinuations(m): - start, end = m.span(1) - if start == -1: - return ' ' - return ' '.rjust((end - start) / 2 + 1, '\\') - -def itermakefilechars(d, offset, tokenlist, it, ignorecomments=False): - """ - Iterate over data in makefile syntax. Comments are found at unescaped # characters, and escaped newlines - are converted to single-space continuations. - """ - - assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) - - if offset == d.lend: - return - - s = d.s - for m in it: - mstart, mend = m.span(0) - token = s[mstart:mend] - - starttext = _makecontinuations.sub(_replacemakecontinuations, s[offset:mstart]) - - if token[-1] == '#' and not ignorecomments: - l = mend - mstart - # multiple backslashes before a hash are unescaped, halving their total number - if l % 2: - # found a comment - yield starttext + token[:(l - 1) / 2], None, None, None - return - else: - yield starttext + token[-l / 2:], None, None, mend - elif token in tokenlist or (token[0] == '$' and '$' in tokenlist): - yield starttext, token, mstart, mend - else: - yield starttext + token, None, None, mend - offset = mend - - yield _makecontinuations.sub(_replacemakecontinuations, s[offset:d.lend]), None, None, None - -_findcomment = re.compile(r'\\*\#') -def flattenmakesyntax(d, offset): - """ - A shortcut method for flattening line continuations and comments in makefile syntax without - looking for other tokens. - """ - - assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) - if offset == d.lend: - return '' - - s = _makecontinuations.sub(_replacemakecontinuations, d.s[offset:d.lend]) - - elements = [] - offset = 0 - for m in _findcomment.finditer(s): - mstart, mend = m.span(0) - elements.append(s[offset:mstart]) - if (mend - mstart) % 2: - # even number of backslashes... it's a comment - elements.append(''.ljust((mend - mstart - 1) / 2, '\\')) - return ''.join(elements) - - # odd number of backslashes - elements.append(''.ljust((mend - mstart - 2) / 2, '\\') + '#') - offset = mend - - elements.append(s[offset:]) - return ''.join(elements) - -def itercommandchars(d, offset, tokenlist, it): - """ - Iterate over command syntax. # comment markers are not special, and escaped newlines are included - in the output text. - """ - - assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) - - if offset == d.lend: - return - - s = d.s - for m in it: - mstart, mend = m.span(0) - token = s[mstart:mend] - starttext = s[offset:mstart].replace('\n\t', '\n') - - if token in tokenlist or (token[0] == '$' and '$' in tokenlist): - yield starttext, token, mstart, mend - else: - yield starttext + token, None, None, mend - offset = mend - - yield s[offset:d.lend].replace('\n\t', '\n'), None, None, None - -_redefines = re.compile('\s*define|\s*endef') -def iterdefinelines(it, startloc): - """ - Process the insides of a define. Most characters are included literally. Escaped newlines are treated - as they would be in makefile syntax. Internal define/endef pairs are ignored. - """ - - results = [] - - definecount = 1 - for d in it: - m = _redefines.match(d.s, d.lstart, d.lend) - if m is not None: - directive = m.group(0).strip() - if directive == 'endef': - definecount -= 1 - if definecount == 0: - return _makecontinuations.sub(_replacemakecontinuations, '\n'.join(results)) - else: - definecount += 1 - - results.append(d.s[d.lstart:d.lend]) - - # Falling off the end is an unterminated define! - raise SyntaxError("define without matching endef", startloc) - -def _ensureend(d, offset, msg): - """ - Ensure that only whitespace remains in this data. - """ - - s = flattenmakesyntax(d, offset) - if s != '' and not s.isspace(): - raise SyntaxError(msg, d.getloc(offset)) - -_eqargstokenlist = ('(', "'", '"') - -def ifeq(d, offset): - if offset > d.lend - 1: - raise SyntaxError("No arguments after conditional", d.getloc(offset)) - - # the variety of formats for this directive is rather maddening - token = d.s[offset] - if token not in _eqargstokenlist: - raise SyntaxError("No arguments after conditional", d.getloc(offset)) - - offset += 1 - - if token == '(': - arg1, t, offset = parsemakesyntax(d, offset, (',',), itermakefilechars) - if t is None: - raise SyntaxError("Expected two arguments in conditional", d.getloc(d.lend)) - - arg1.rstrip() - - offset = d.skipwhitespace(offset) - arg2, t, offset = parsemakesyntax(d, offset, (')',), itermakefilechars) - if t is None: - raise SyntaxError("Unexpected text in conditional", d.getloc(offset)) - - _ensureend(d, offset, "Unexpected text after conditional") - else: - arg1, t, offset = parsemakesyntax(d, offset, (token,), itermakefilechars) - if t is None: - raise SyntaxError("Unexpected text in conditional", d.getloc(d.lend)) - - offset = d.skipwhitespace(offset) - if offset == d.lend: - raise SyntaxError("Expected two arguments in conditional", d.getloc(offset)) - - token = d.s[offset] - if token not in '\'"': - raise SyntaxError("Unexpected text in conditional", d.getloc(offset)) - - arg2, t, offset = parsemakesyntax(d, offset + 1, (token,), itermakefilechars) - - _ensureend(d, offset, "Unexpected text after conditional") - - return parserdata.EqCondition(arg1, arg2) - -def ifneq(d, offset): - c = ifeq(d, offset) - c.expected = False - return c - -def ifdef(d, offset): - e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) - e.rstrip() - - return parserdata.IfdefCondition(e) - -def ifndef(d, offset): - c = ifdef(d, offset) - c.expected = False - return c - -_conditionkeywords = { - 'ifeq': ifeq, - 'ifneq': ifneq, - 'ifdef': ifdef, - 'ifndef': ifndef - } - -_conditiontokens = tuple(_conditionkeywords.iterkeys()) -_conditionre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_conditiontokens)) - -_directivestokenlist = _conditiontokens + \ - ('else', 'endif', 'define', 'endef', 'override', 'include', '-include', 'includedeps', '-includedeps', 'vpath', 'export', 'unexport') - -_directivesre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_directivestokenlist)) - -_varsettokens = (':=', '+=', '?=', '=') - -def _parsefile(pathname): - fd = open(pathname, "rU") - stmts = parsestring(fd.read(), pathname) - stmts.mtime = os.fstat(fd.fileno()).st_mtime - fd.close() - return stmts - -def _checktime(path, stmts): - mtime = os.path.getmtime(path) - if mtime != stmts.mtime: - _log.debug("Re-parsing makefile '%s': mtimes differ", path) - return False - - return True - -_parsecache = util.MostUsedCache(50, _parsefile, _checktime) - -def parsefile(pathname): - """ - Parse a filename into a parserdata.StatementList. A cache is used to avoid re-parsing - makefiles that have already been parsed and have not changed. - """ - - pathname = os.path.realpath(pathname) - return _parsecache.get(pathname) - -# colon followed by anything except a slash (Windows path detection) -_depfilesplitter = re.compile(r':(?![\\/])') -# simple variable references -_vars = re.compile('\$\((\w+)\)') - -def parsedepfile(pathname): - """ - Parse a filename listing only depencencies into a parserdata.StatementList. - Simple variable references are allowed in such files. - """ - def continuation_iter(lines): - current_line = [] - for line in lines: - line = line.rstrip() - if line.endswith("\\"): - current_line.append(line.rstrip("\\")) - continue - if not len(line): - continue - current_line.append(line) - yield ''.join(current_line) - current_line = [] - if current_line: - yield ''.join(current_line) - - def get_expansion(s): - if '$' in s: - expansion = data.Expansion() - # for an input like e.g. "foo $(bar) baz", - # _vars.split returns ["foo", "bar", "baz"] - # every other element is a variable name. - for i, element in enumerate(_vars.split(s)): - if i % 2: - expansion.appendfunc(functions.VariableRef(None, - data.StringExpansion(element, None))) - elif element: - expansion.appendstr(element) - - return expansion - - return data.StringExpansion(s, None) - - pathname = os.path.realpath(pathname) - stmts = parserdata.StatementList() - for line in continuation_iter(open(pathname).readlines()): - target, deps = _depfilesplitter.split(line, 1) - stmts.append(parserdata.Rule(get_expansion(target), - get_expansion(deps), False)) - return stmts - -def parsestring(s, filename): - """ - Parse a string containing makefile data into a parserdata.StatementList. - """ - - currule = False - condstack = [parserdata.StatementList()] - - fdlines = enumeratelines(s, filename) - for d in fdlines: - assert len(condstack) > 0 - - offset = d.lstart - - if currule and offset < d.lend and d.s[offset] == '\t': - e, token, offset = parsemakesyntax(d, offset + 1, (), itercommandchars) - assert token is None - assert offset is None - condstack[-1].append(parserdata.Command(e)) - continue - - # To parse Makefile syntax, we first strip leading whitespace and - # look for initial keywords. If there are no keywords, it's either - # setting a variable or writing a rule. - - offset = d.skipwhitespace(offset) - if offset is None: - continue - - m = _directivesre.match(d.s, offset, d.lend) - if m is not None: - kword = m.group(1) - offset = m.end(0) - - if kword == 'endif': - _ensureend(d, offset, "Unexpected data after 'endif' directive") - if len(condstack) == 1: - raise SyntaxError("unmatched 'endif' directive", - d.getloc(offset)) - - condstack.pop().endloc = d.getloc(offset) - continue - - if kword == 'else': - if len(condstack) == 1: - raise SyntaxError("unmatched 'else' directive", - d.getloc(offset)) - - m = _conditionre.match(d.s, offset, d.lend) - if m is None: - _ensureend(d, offset, "Unexpected data after 'else' directive.") - condstack[-1].addcondition(d.getloc(offset), parserdata.ElseCondition()) - else: - kword = m.group(1) - if kword not in _conditionkeywords: - raise SyntaxError("Unexpected condition after 'else' directive.", - d.getloc(offset)) - - startoffset = offset - offset = d.skipwhitespace(m.end(1)) - c = _conditionkeywords[kword](d, offset) - condstack[-1].addcondition(d.getloc(startoffset), c) - continue - - if kword in _conditionkeywords: - c = _conditionkeywords[kword](d, offset) - cb = parserdata.ConditionBlock(d.getloc(d.lstart), c) - condstack[-1].append(cb) - condstack.append(cb) - continue - - if kword == 'endef': - raise SyntaxError("endef without matching define", d.getloc(offset)) - - if kword == 'define': - currule = False - vname, t, i = parsemakesyntax(d, offset, (), itermakefilechars) - vname.rstrip() - - startloc = d.getloc(d.lstart) - value = iterdefinelines(fdlines, startloc) - condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=startloc, token='=', targetexp=None)) - continue - - if kword in ('include', '-include', 'includedeps', '-includedeps'): - if kword.startswith('-'): - required = False - kword = kword[1:] - else: - required = True - - deps = kword == 'includedeps' - - currule = False - incfile, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) - condstack[-1].append(parserdata.Include(incfile, required, deps)) - - continue - - if kword == 'vpath': - currule = False - e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) - condstack[-1].append(parserdata.VPathDirective(e)) - continue - - if kword == 'override': - currule = False - vname, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars) - vname.lstrip() - vname.rstrip() - - if token is None: - raise SyntaxError("Malformed override directive, need =", d.getloc(d.lstart)) - - value = flattenmakesyntax(d, offset).lstrip() - - condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=d.getloc(offset), token=token, targetexp=None, source=data.Variables.SOURCE_OVERRIDE)) - continue - - if kword == 'export': - currule = False - e, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars) - e.lstrip() - e.rstrip() - - if token is None: - condstack[-1].append(parserdata.ExportDirective(e, concurrent_set=False)) - else: - condstack[-1].append(parserdata.ExportDirective(e, concurrent_set=True)) - - value = flattenmakesyntax(d, offset).lstrip() - condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None)) - - continue - - if kword == 'unexport': - e, token, offset = parsemakesyntax(d, offset, (), itermakefilechars) - condstack[-1].append(parserdata.UnexportDirective(e)) - continue - - e, token, offset = parsemakesyntax(d, offset, _varsettokens + ('::', ':'), itermakefilechars) - if token is None: - e.rstrip() - e.lstrip() - if not e.isempty(): - condstack[-1].append(parserdata.EmptyDirective(e)) - continue - - # if we encountered real makefile syntax, the current rule is over - currule = False - - if token in _varsettokens: - e.lstrip() - e.rstrip() - - value = flattenmakesyntax(d, offset).lstrip() - - condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None)) - else: - doublecolon = token == '::' - - # `e` is targets or target patterns, which can end up as - # * a rule - # * an implicit rule - # * a static pattern rule - # * a target-specific variable definition - # * a pattern-specific variable definition - # any of the rules may have order-only prerequisites - # delimited by |, and a command delimited by ; - targets = e - - e, token, offset = parsemakesyntax(d, offset, - _varsettokens + (':', '|', ';'), - itermakefilechars) - if token in (None, ';'): - condstack[-1].append(parserdata.Rule(targets, e, doublecolon)) - currule = True - - if token == ';': - offset = d.skipwhitespace(offset) - e, t, offset = parsemakesyntax(d, offset, (), itercommandchars) - condstack[-1].append(parserdata.Command(e)) - - elif token in _varsettokens: - e.lstrip() - e.rstrip() - - value = flattenmakesyntax(d, offset).lstrip() - condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=targets)) - elif token == '|': - raise SyntaxError('order-only prerequisites not implemented', d.getloc(offset)) - else: - assert token == ':' - # static pattern rule - - pattern = e - - deps, token, offset = parsemakesyntax(d, offset, (';',), itermakefilechars) - - condstack[-1].append(parserdata.StaticPatternRule(targets, pattern, deps, doublecolon)) - currule = True - - if token == ';': - offset = d.skipwhitespace(offset) - e, token, offset = parsemakesyntax(d, offset, (), itercommandchars) - condstack[-1].append(parserdata.Command(e)) - - if len(condstack) != 1: - raise SyntaxError("Condition never terminated with endif", condstack[-1].loc) - - return condstack[0] - -_PARSESTATE_TOPLEVEL = 0 # at the top level -_PARSESTATE_FUNCTION = 1 # expanding a function call -_PARSESTATE_VARNAME = 2 # expanding a variable expansion. -_PARSESTATE_SUBSTFROM = 3 # expanding a variable expansion substitution "from" value -_PARSESTATE_SUBSTTO = 4 # expanding a variable expansion substitution "to" value -_PARSESTATE_PARENMATCH = 5 # inside nested parentheses/braces that must be matched - -class ParseStackFrame(object): - __slots__ = ('parsestate', 'parent', 'expansion', 'tokenlist', 'openbrace', 'closebrace', 'function', 'loc', 'varname', 'substfrom') - - def __init__(self, parsestate, parent, expansion, tokenlist, openbrace, closebrace, function=None, loc=None): - self.parsestate = parsestate - self.parent = parent - self.expansion = expansion - self.tokenlist = tokenlist - self.openbrace = openbrace - self.closebrace = closebrace - self.function = function - self.loc = loc - - def __str__(self): - return "" % (self.parsestate, self.expansion, self.tokenlist, self.openbrace, self.closebrace) - -_matchingbrace = { - '(': ')', - '{': '}', - } - -def parsemakesyntax(d, offset, stopon, iterfunc): - """ - Given Data, parse it into a data.Expansion. - - @param stopon (sequence) - Indicate characters where toplevel parsing should stop. - - @param iterfunc (generator function) - A function which is used to iterate over d, yielding (char, offset, loc) - @see iterdata - @see itermakefilechars - @see itercommandchars - - @return a tuple (expansion, token, offset). If all the data is consumed, - token and offset will be None - """ - - assert callable(iterfunc) - - stacktop = ParseStackFrame(_PARSESTATE_TOPLEVEL, None, data.Expansion(loc=d.getloc(d.lstart)), - tokenlist=stopon + ('$',), - openbrace=None, closebrace=None) - - tokeniterator = _alltokens.finditer(d.s, offset, d.lend) - - di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator) - while True: # this is not a for loop because `di` changes during the function - assert stacktop is not None - try: - s, token, tokenoffset, offset = di.next() - except StopIteration: - break - - stacktop.expansion.appendstr(s) - if token is None: - continue - - parsestate = stacktop.parsestate - - if token[0] == '$': - if tokenoffset + 1 == d.lend: - # an unterminated $ expands to nothing - break - - loc = d.getloc(tokenoffset) - c = token[1] - if c == '$': - assert len(token) == 2 - stacktop.expansion.appendstr('$') - elif c in ('(', '{'): - closebrace = _matchingbrace[c] - - if len(token) > 2: - fname = token[2:].rstrip() - fn = functions.functionmap[fname](loc) - e = data.Expansion() - if len(fn) + 1 == fn.maxargs: - tokenlist = (c, closebrace, '$') - else: - tokenlist = (',', c, closebrace, '$') - - stacktop = ParseStackFrame(_PARSESTATE_FUNCTION, stacktop, - e, tokenlist, function=fn, - openbrace=c, closebrace=closebrace) - else: - e = data.Expansion() - tokenlist = (':', c, closebrace, '$') - stacktop = ParseStackFrame(_PARSESTATE_VARNAME, stacktop, - e, tokenlist, - openbrace=c, closebrace=closebrace, loc=loc) - else: - assert len(token) == 2 - e = data.Expansion.fromstring(c, loc) - stacktop.expansion.appendfunc(functions.VariableRef(loc, e)) - elif token in ('(', '{'): - assert token == stacktop.openbrace - - stacktop.expansion.appendstr(token) - stacktop = ParseStackFrame(_PARSESTATE_PARENMATCH, stacktop, - stacktop.expansion, - (token, stacktop.closebrace, '$'), - openbrace=token, closebrace=stacktop.closebrace, loc=d.getloc(tokenoffset)) - elif parsestate == _PARSESTATE_PARENMATCH: - assert token == stacktop.closebrace - stacktop.expansion.appendstr(token) - stacktop = stacktop.parent - elif parsestate == _PARSESTATE_TOPLEVEL: - assert stacktop.parent is None - return stacktop.expansion.finish(), token, offset - elif parsestate == _PARSESTATE_FUNCTION: - if token == ',': - stacktop.function.append(stacktop.expansion.finish()) - - stacktop.expansion = data.Expansion() - if len(stacktop.function) + 1 == stacktop.function.maxargs: - tokenlist = (stacktop.openbrace, stacktop.closebrace, '$') - stacktop.tokenlist = tokenlist - elif token in (')', '}'): - fn = stacktop.function - fn.append(stacktop.expansion.finish()) - fn.setup() - - stacktop = stacktop.parent - stacktop.expansion.appendfunc(fn) - else: - assert False, "Not reached, _PARSESTATE_FUNCTION" - elif parsestate == _PARSESTATE_VARNAME: - if token == ':': - stacktop.varname = stacktop.expansion - stacktop.parsestate = _PARSESTATE_SUBSTFROM - stacktop.expansion = data.Expansion() - stacktop.tokenlist = ('=', stacktop.openbrace, stacktop.closebrace, '$') - elif token in (')', '}'): - fn = functions.VariableRef(stacktop.loc, stacktop.expansion.finish()) - stacktop = stacktop.parent - stacktop.expansion.appendfunc(fn) - else: - assert False, "Not reached, _PARSESTATE_VARNAME" - elif parsestate == _PARSESTATE_SUBSTFROM: - if token == '=': - stacktop.substfrom = stacktop.expansion - stacktop.parsestate = _PARSESTATE_SUBSTTO - stacktop.expansion = data.Expansion() - stacktop.tokenlist = (stacktop.openbrace, stacktop.closebrace, '$') - elif token in (')', '}'): - # A substitution of the form $(VARNAME:.ee) is probably a mistake, but make - # parses it. Issue a warning. Combine the varname and substfrom expansions to - # make the compatible varname. See tests/var-substitutions.mk SIMPLE3SUBSTNAME - _log.warning("%s: Variable reference looks like substitution without =", stacktop.loc) - stacktop.varname.appendstr(':') - stacktop.varname.concat(stacktop.expansion) - fn = functions.VariableRef(stacktop.loc, stacktop.varname.finish()) - stacktop = stacktop.parent - stacktop.expansion.appendfunc(fn) - else: - assert False, "Not reached, _PARSESTATE_SUBSTFROM" - elif parsestate == _PARSESTATE_SUBSTTO: - assert token in (')','}'), "Not reached, _PARSESTATE_SUBSTTO" - - fn = functions.SubstitutionRef(stacktop.loc, stacktop.varname.finish(), - stacktop.substfrom.finish(), stacktop.expansion.finish()) - stacktop = stacktop.parent - stacktop.expansion.appendfunc(fn) - else: - assert False, "Unexpected parse state %s" % stacktop.parsestate - - if stacktop.parent is not None and iterfunc == itercommandchars: - di = itermakefilechars(d, offset, stacktop.tokenlist, tokeniterator, - ignorecomments=True) - else: - di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator) - - if stacktop.parent is not None: - raise SyntaxError("Unterminated function call", d.getloc(offset)) - - assert stacktop.parsestate == _PARSESTATE_TOPLEVEL - - return stacktop.expansion.finish(), None, None diff --git a/build/pymake/pymake/parserdata.py b/build/pymake/pymake/parserdata.py deleted file mode 100644 index 7b2e5443d..000000000 --- a/build/pymake/pymake/parserdata.py +++ /dev/null @@ -1,1006 +0,0 @@ -import logging, re, os -import data, parser, functions, util -from cStringIO import StringIO -from pymake.globrelative import hasglob, glob - -_log = logging.getLogger('pymake.data') -_tabwidth = 4 - -class Location(object): - """ - A location within a makefile. - - For the moment, locations are just path/line/column, but in the future - they may reference parent locations for more accurate "included from" - or "evaled at" error reporting. - """ - __slots__ = ('path', 'line', 'column') - - def __init__(self, path, line, column): - self.path = path - self.line = line - self.column = column - - def offset(self, s, start, end): - """ - Returns a new location offset by - the specified string. - """ - - if start == end: - return self - - skiplines = s.count('\n', start, end) - line = self.line + skiplines - if skiplines: - lastnl = s.rfind('\n', start, end) - assert lastnl != -1 - start = lastnl + 1 - column = 0 - else: - column = self.column - - while True: - j = s.find('\t', start, end) - if j == -1: - column += end - start - break - - column += j - start - column += _tabwidth - column -= column % _tabwidth - start = j + 1 - - return Location(self.path, line, column) - - def __str__(self): - return "%s:%s:%s" % (self.path, self.line, self.column) - -def _expandwildcards(makefile, tlist): - for t in tlist: - if not hasglob(t): - yield t - else: - l = glob(makefile.workdir, t) - for r in l: - yield r - -_flagescape = re.compile(r'([\s\\])') - -def parsecommandlineargs(args): - """ - Given a set of arguments from a command-line invocation of make, - parse out the variable definitions and return (stmts, arglist, overridestr) - """ - - overrides = [] - stmts = StatementList() - r = [] - for i in xrange(0, len(args)): - a = args[i] - - vname, t, val = util.strpartition(a, ':=') - if t == '': - vname, t, val = util.strpartition(a, '=') - if t != '': - overrides.append(_flagescape.sub(r'\\\1', a)) - - vname = vname.strip() - vnameexp = data.Expansion.fromstring(vname, "Command-line argument") - - stmts.append(ExportDirective(vnameexp, concurrent_set=True)) - stmts.append(SetVariable(vnameexp, token=t, - value=val, valueloc=Location('', i, len(vname) + len(t)), - targetexp=None, source=data.Variables.SOURCE_COMMANDLINE)) - else: - r.append(data.stripdotslash(a)) - - return stmts, r, ' '.join(overrides) - -class Statement(object): - """ - Represents parsed make file syntax. - - This is an abstract base class. Child classes are expected to implement - basic methods defined below. - """ - - def execute(self, makefile, context): - """Executes this Statement within a make file execution context.""" - raise Exception("%s must implement execute()." % self.__class__) - - def to_source(self): - """Obtain the make file "source" representation of the Statement. - - This converts an individual Statement back to a string that can again - be parsed into this Statement. - """ - raise Exception("%s must implement to_source()." % self.__class__) - - def __eq__(self, other): - raise Exception("%s must implement __eq__." % self.__class__) - - def __ne__(self, other): - return self.__eq__(other) - -class DummyRule(object): - __slots__ = () - - def addcommand(self, r): - pass - -class Rule(Statement): - """ - Rules represent how to make specific targets. - - See https://www.gnu.org/software/make/manual/make.html#Rules. - - An individual rule is composed of a target, dependencies, and a recipe. - This class only contains references to the first 2. The recipe will be - contained in Command classes which follow this one in a stream of Statement - instances. - - Instances also contain a boolean property `doublecolon` which says whether - this is a doublecolon rule. Doublecolon rules are rules that are always - executed, if they are evaluated. Normally, rules are only executed if their - target is out of date. - """ - __slots__ = ('targetexp', 'depexp', 'doublecolon') - - def __init__(self, targetexp, depexp, doublecolon): - assert isinstance(targetexp, (data.Expansion, data.StringExpansion)) - assert isinstance(depexp, (data.Expansion, data.StringExpansion)) - - self.targetexp = targetexp - self.depexp = depexp - self.doublecolon = doublecolon - - def execute(self, makefile, context): - if context.weak: - self._executeweak(makefile, context) - else: - self._execute(makefile, context) - - def _executeweak(self, makefile, context): - """ - If the context is weak (we're just handling dependencies) we can make a number of assumptions here. - This lets us go really fast and is generally good. - """ - assert context.weak - deps = self.depexp.resolvesplit(makefile, makefile.variables) - # Skip targets with no rules and no dependencies - if not deps: - return - targets = data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables)) - rule = data.Rule(list(data.stripdotslashes(deps)), self.doublecolon, loc=self.targetexp.loc, weakdeps=True) - for target in targets: - makefile.gettarget(target).addrule(rule) - makefile.foundtarget(target) - context.currule = rule - - def _execute(self, makefile, context): - assert not context.weak - - atargets = data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables)) - targets = [data.Pattern(p) for p in _expandwildcards(makefile, atargets)] - - if not len(targets): - context.currule = DummyRule() - return - - ispatterns = set((t.ispattern() for t in targets)) - if len(ispatterns) == 2: - raise data.DataError("Mixed implicit and normal rule", self.targetexp.loc) - ispattern, = ispatterns - - deps = list(_expandwildcards(makefile, data.stripdotslashes(self.depexp.resolvesplit(makefile, makefile.variables)))) - if ispattern: - rule = data.PatternRule(targets, map(data.Pattern, deps), self.doublecolon, loc=self.targetexp.loc) - makefile.appendimplicitrule(rule) - else: - rule = data.Rule(deps, self.doublecolon, loc=self.targetexp.loc, weakdeps=False) - for t in targets: - makefile.gettarget(t.gettarget()).addrule(rule) - - makefile.foundtarget(targets[0].gettarget()) - - context.currule = rule - - def dump(self, fd, indent): - print >>fd, "%sRule %s: %s" % (indent, self.targetexp, self.depexp) - - def to_source(self): - sep = ':' - - if self.doublecolon: - sep = '::' - - deps = self.depexp.to_source() - if len(deps) > 0 and not deps[0].isspace(): - sep += ' ' - - return '\n%s%s%s' % ( - self.targetexp.to_source(escape_variables=True), - sep, - deps) - - def __eq__(self, other): - if not isinstance(other, Rule): - return False - - return self.targetexp == other.targetexp \ - and self.depexp == other.depexp \ - and self.doublecolon == other.doublecolon - -class StaticPatternRule(Statement): - """ - Static pattern rules are rules which specify multiple targets based on a - string pattern. - - See https://www.gnu.org/software/make/manual/make.html#Static-Pattern - - They are like `Rule` instances except an added property, `patternexp` is - present. It contains the Expansion which represents the rule pattern. - """ - __slots__ = ('targetexp', 'patternexp', 'depexp', 'doublecolon') - - def __init__(self, targetexp, patternexp, depexp, doublecolon): - assert isinstance(targetexp, (data.Expansion, data.StringExpansion)) - assert isinstance(patternexp, (data.Expansion, data.StringExpansion)) - assert isinstance(depexp, (data.Expansion, data.StringExpansion)) - - self.targetexp = targetexp - self.patternexp = patternexp - self.depexp = depexp - self.doublecolon = doublecolon - - def execute(self, makefile, context): - if context.weak: - raise data.DataError("Static pattern rules not allowed in includedeps", self.targetexp.loc) - - targets = list(_expandwildcards(makefile, data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables)))) - - if not len(targets): - context.currule = DummyRule() - return - - patterns = list(data.stripdotslashes(self.patternexp.resolvesplit(makefile, makefile.variables))) - if len(patterns) != 1: - raise data.DataError("Static pattern rules must have a single pattern", self.patternexp.loc) - pattern = data.Pattern(patterns[0]) - - deps = [data.Pattern(p) for p in _expandwildcards(makefile, data.stripdotslashes(self.depexp.resolvesplit(makefile, makefile.variables)))] - - rule = data.PatternRule([pattern], deps, self.doublecolon, loc=self.targetexp.loc) - - for t in targets: - if data.Pattern(t).ispattern(): - raise data.DataError("Target '%s' of a static pattern rule must not be a pattern" % (t,), self.targetexp.loc) - stem = pattern.match(t) - if stem is None: - raise data.DataError("Target '%s' does not match the static pattern '%s'" % (t, pattern), self.targetexp.loc) - makefile.gettarget(t).addrule(data.PatternRuleInstance(rule, '', stem, pattern.ismatchany())) - - makefile.foundtarget(targets[0]) - context.currule = rule - - def dump(self, fd, indent): - print >>fd, "%sStaticPatternRule %s: %s: %s" % (indent, self.targetexp, self.patternexp, self.depexp) - - def to_source(self): - sep = ':' - - if self.doublecolon: - sep = '::' - - pattern = self.patternexp.to_source() - deps = self.depexp.to_source() - - if len(pattern) > 0 and pattern[0] not in (' ', '\t'): - sep += ' ' - - return '\n%s%s%s:%s' % ( - self.targetexp.to_source(escape_variables=True), - sep, - pattern, - deps) - - def __eq__(self, other): - if not isinstance(other, StaticPatternRule): - return False - - return self.targetexp == other.targetexp \ - and self.patternexp == other.patternexp \ - and self.depexp == other.depexp \ - and self.doublecolon == other.doublecolon - -class Command(Statement): - """ - Commands are things that get executed by a rule. - - A rule's recipe is composed of 0 or more Commands. - - A command is simply an expansion. Commands typically represent strings to - be executed in a shell (e.g. via system()). Although, since make files - allow arbitrary shells to be used for command execution, this isn't a - guarantee. - """ - __slots__ = ('exp',) - - def __init__(self, exp): - assert isinstance(exp, (data.Expansion, data.StringExpansion)) - self.exp = exp - - def execute(self, makefile, context): - assert context.currule is not None - if context.weak: - raise data.DataError("rules not allowed in includedeps", self.exp.loc) - - context.currule.addcommand(self.exp) - - def dump(self, fd, indent): - print >>fd, "%sCommand %s" % (indent, self.exp,) - - def to_source(self): - # Commands have some interesting quirks when it comes to source - # formatting. First, they can be multi-line. Second, a tab needs to be - # inserted at the beginning of every line. Finally, there might be - # variable references inside the command. This means we need to escape - # variable references inside command strings. Luckily, this is handled - # by the Expansion. - s = self.exp.to_source(escape_variables=True) - - return '\n'.join(['\t%s' % line for line in s.split('\n')]) - - def __eq__(self, other): - if not isinstance(other, Command): - return False - - return self.exp == other.exp - -class SetVariable(Statement): - """ - Represents a variable assignment. - - Variable assignment comes in two different flavors. - - Simple assignment has the form: - - - - e.g. FOO := bar - - These correspond to the fields `vnameexp`, `token`, and `value`. In - addition, `valueloc` will be a Location and `source` will be a - pymake.data.Variables.SOURCE_* constant. - - There are also target-specific variables. These are variables that only - apply in the context of a specific target. They are like the aforementioned - assignment except the `targetexp` field is set to an Expansion representing - the target they apply to. - """ - __slots__ = ('vnameexp', 'token', 'value', 'valueloc', 'targetexp', 'source') - - def __init__(self, vnameexp, token, value, valueloc, targetexp, source=None): - assert isinstance(vnameexp, (data.Expansion, data.StringExpansion)) - assert isinstance(value, str) - assert targetexp is None or isinstance(targetexp, (data.Expansion, data.StringExpansion)) - - if source is None: - source = data.Variables.SOURCE_MAKEFILE - - self.vnameexp = vnameexp - self.token = token - self.value = value - self.valueloc = valueloc - self.targetexp = targetexp - self.source = source - - def execute(self, makefile, context): - vname = self.vnameexp.resolvestr(makefile, makefile.variables) - if len(vname) == 0: - raise data.DataError("Empty variable name", self.vnameexp.loc) - - if self.targetexp is None: - setvariables = [makefile.variables] - else: - setvariables = [] - - targets = [data.Pattern(t) for t in data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables))] - for t in targets: - if t.ispattern(): - setvariables.append(makefile.getpatternvariables(t)) - else: - setvariables.append(makefile.gettarget(t.gettarget()).variables) - - for v in setvariables: - if self.token == '+=': - v.append(vname, self.source, self.value, makefile.variables, makefile) - continue - - if self.token == '?=': - flavor = data.Variables.FLAVOR_RECURSIVE - oldflavor, oldsource, oldval = v.get(vname, expand=False) - if oldval is not None: - continue - value = self.value - elif self.token == '=': - flavor = data.Variables.FLAVOR_RECURSIVE - value = self.value - else: - assert self.token == ':=' - - flavor = data.Variables.FLAVOR_SIMPLE - d = parser.Data.fromstring(self.value, self.valueloc) - e, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata) - value = e.resolvestr(makefile, makefile.variables) - - v.set(vname, flavor, self.source, value) - - def dump(self, fd, indent): - print >>fd, "%sSetVariable<%s> %s %s\n%s %r" % (indent, self.valueloc, self.vnameexp, self.token, indent, self.value) - - def __eq__(self, other): - if not isinstance(other, SetVariable): - return False - - return self.vnameexp == other.vnameexp \ - and self.token == other.token \ - and self.value == other.value \ - and self.targetexp == other.targetexp \ - and self.source == other.source - - def to_source(self): - chars = [] - for i in xrange(0, len(self.value)): - c = self.value[i] - - # Literal # is escaped in variable assignment otherwise it would be - # a comment. - if c == '#': - # If a backslash precedes this, we need to escape it as well. - if i > 0 and self.value[i-1] == '\\': - chars.append('\\') - - chars.append('\\#') - continue - - chars.append(c) - - value = ''.join(chars) - - prefix = '' - if self.source == data.Variables.SOURCE_OVERRIDE: - prefix = 'override ' - - # SetVariable come in two flavors: simple and target-specific. - - # We handle the target-specific syntax first. - if self.targetexp is not None: - return '%s: %s %s %s' % ( - self.targetexp.to_source(), - self.vnameexp.to_source(), - self.token, - value) - - # The variable could be multi-line or have leading whitespace. For - # regular variable assignment, whitespace after the token but before - # the value is ignored. If we see leading whitespace in the value here, - # the variable must have come from a define. - if value.count('\n') > 0 or (len(value) and value[0].isspace()): - # The parser holds the token in vnameexp for whatever reason. - return '%sdefine %s\n%s\nendef' % ( - prefix, - self.vnameexp.to_source(), - value) - - return '%s%s %s %s' % ( - prefix, - self.vnameexp.to_source(), - self.token, - value) - -class Condition(object): - """ - An abstract "condition", either ifeq or ifdef, perhaps negated. - - See https://www.gnu.org/software/make/manual/make.html#Conditional-Syntax - - Subclasses must implement: - - def evaluate(self, makefile) - """ - - def __eq__(self, other): - raise Exception("%s must implement __eq__." % __class__) - - def __ne__(self, other): - return not self.__eq__(other) - -class EqCondition(Condition): - """ - Represents an ifeq or ifneq conditional directive. - - This directive consists of two Expansions which are compared for equality. - - The `expected` field is a bool indicating what the condition must evaluate - to in order for its body to be executed. If True, this is an "ifeq" - conditional directive. If False, an "ifneq." - """ - __slots__ = ('exp1', 'exp2', 'expected') - - def __init__(self, exp1, exp2): - assert isinstance(exp1, (data.Expansion, data.StringExpansion)) - assert isinstance(exp2, (data.Expansion, data.StringExpansion)) - - self.expected = True - self.exp1 = exp1 - self.exp2 = exp2 - - def evaluate(self, makefile): - r1 = self.exp1.resolvestr(makefile, makefile.variables) - r2 = self.exp2.resolvestr(makefile, makefile.variables) - return (r1 == r2) == self.expected - - def __str__(self): - return "ifeq (expected=%s) %s %s" % (self.expected, self.exp1, self.exp2) - - def __eq__(self, other): - if not isinstance(other, EqCondition): - return False - - return self.exp1 == other.exp1 \ - and self.exp2 == other.exp2 \ - and self.expected == other.expected - -class IfdefCondition(Condition): - """ - Represents an ifdef or ifndef conditional directive. - - This directive consists of a single expansion which represents the name of - a variable (without the leading '$') which will be checked for definition. - - The `expected` field is a bool and has the same behavior as EqCondition. - If it is True, this represents a "ifdef" conditional. If False, "ifndef." - """ - __slots__ = ('exp', 'expected') - - def __init__(self, exp): - assert isinstance(exp, (data.Expansion, data.StringExpansion)) - self.exp = exp - self.expected = True - - def evaluate(self, makefile): - vname = self.exp.resolvestr(makefile, makefile.variables) - flavor, source, value = makefile.variables.get(vname, expand=False) - - if value is None: - return not self.expected - - return (len(value) > 0) == self.expected - - def __str__(self): - return "ifdef (expected=%s) %s" % (self.expected, self.exp) - - def __eq__(self, other): - if not isinstance(other, IfdefCondition): - return False - - return self.exp == other.exp and self.expected == other.expected - -class ElseCondition(Condition): - """ - Represents the transition between branches in a ConditionBlock. - """ - __slots__ = () - - def evaluate(self, makefile): - return True - - def __str__(self): - return "else" - - def __eq__(self, other): - return isinstance(other, ElseCondition) - -class ConditionBlock(Statement): - """ - A set of related Conditions. - - This is essentially a list of 2-tuples of (Condition, list(Statement)). - - The parser creates a ConditionBlock for all statements related to the same - conditional group. If iterating over the parser's output, where you think - you would see an ifeq, you will see a ConditionBlock containing an IfEq. In - other words, the parser collapses separate statements into this container - class. - - ConditionBlock instances may exist within other ConditionBlock if the - conditional logic is multiple levels deep. - """ - __slots__ = ('loc', '_groups') - - def __init__(self, loc, condition): - self.loc = loc - self._groups = [] - self.addcondition(loc, condition) - - def getloc(self): - return self.loc - - def addcondition(self, loc, condition): - assert isinstance(condition, Condition) - condition.loc = loc - - if len(self._groups) and isinstance(self._groups[-1][0], ElseCondition): - raise parser.SyntaxError("Multiple else conditions for block starting at %s" % self.loc, loc) - - self._groups.append((condition, StatementList())) - - def append(self, statement): - self._groups[-1][1].append(statement) - - def execute(self, makefile, context): - i = 0 - for c, statements in self._groups: - if c.evaluate(makefile): - _log.debug("Condition at %s met by clause #%i", self.loc, i) - statements.execute(makefile, context) - return - - i += 1 - - def dump(self, fd, indent): - print >>fd, "%sConditionBlock" % (indent,) - - indent2 = indent + ' ' - for c, statements in self._groups: - print >>fd, "%s Condition %s" % (indent, c) - statements.dump(fd, indent2) - print >>fd, "%s ~Condition" % (indent,) - print >>fd, "%s~ConditionBlock" % (indent,) - - def to_source(self): - lines = [] - index = 0 - for condition, statements in self: - lines.append(ConditionBlock.condition_source(condition, index)) - index += 1 - - for statement in statements: - lines.append(statement.to_source()) - - lines.append('endif') - - return '\n'.join(lines) - - def __eq__(self, other): - if not isinstance(other, ConditionBlock): - return False - - if len(self) != len(other): - return False - - for i in xrange(0, len(self)): - our_condition, our_statements = self[i] - other_condition, other_statements = other[i] - - if our_condition != other_condition: - return False - - if our_statements != other_statements: - return False - - return True - - @staticmethod - def condition_source(statement, index): - """Convert a condition to its source representation. - - The index argument defines the index of this condition inside a - ConditionBlock. If it is greater than 0, an "else" will be prepended - to the result, if necessary. - """ - prefix = '' - if isinstance(statement, (EqCondition, IfdefCondition)) and index > 0: - prefix = 'else ' - - if isinstance(statement, IfdefCondition): - s = statement.exp.s - - if statement.expected: - return '%sifdef %s' % (prefix, s) - - return '%sifndef %s' % (prefix, s) - - if isinstance(statement, EqCondition): - args = [ - statement.exp1.to_source(escape_comments=True), - statement.exp2.to_source(escape_comments=True)] - - use_quotes = False - single_quote_present = False - double_quote_present = False - for i, arg in enumerate(args): - if len(arg) > 0 and (arg[0].isspace() or arg[-1].isspace()): - use_quotes = True - - if "'" in arg: - single_quote_present = True - - if '"' in arg: - double_quote_present = True - - # Quote everything if needed. - if single_quote_present and double_quote_present: - raise Exception('Cannot format condition with multiple quotes.') - - if use_quotes: - for i, arg in enumerate(args): - # Double to single quotes. - if single_quote_present: - args[i] = '"' + arg + '"' - else: - args[i] = "'" + arg + "'" - - body = None - if use_quotes: - body = ' '.join(args) - else: - body = '(%s)' % ','.join(args) - - if statement.expected: - return '%sifeq %s' % (prefix, body) - - return '%sifneq %s' % (prefix, body) - - if isinstance(statement, ElseCondition): - return 'else' - - raise Exception('Unhandled Condition statement: %s' % - statement.__class__) - - def __iter__(self): - return iter(self._groups) - - def __len__(self): - return len(self._groups) - - def __getitem__(self, i): - return self._groups[i] - -class Include(Statement): - """ - Represents the include directive. - - See https://www.gnu.org/software/make/manual/make.html#Include - - The file to be included is represented by the Expansion defined in the - field `exp`. `required` is a bool indicating whether execution should fail - if the specified file could not be processed. - """ - __slots__ = ('exp', 'required', 'deps') - - def __init__(self, exp, required, weak): - assert isinstance(exp, (data.Expansion, data.StringExpansion)) - self.exp = exp - self.required = required - self.weak = weak - - def execute(self, makefile, context): - files = self.exp.resolvesplit(makefile, makefile.variables) - for f in files: - makefile.include(f, self.required, loc=self.exp.loc, weak=self.weak) - - def dump(self, fd, indent): - print >>fd, "%sInclude %s" % (indent, self.exp) - - def to_source(self): - prefix = '' - - if not self.required: - prefix = '-' - - return '%sinclude %s' % (prefix, self.exp.to_source()) - - def __eq__(self, other): - if not isinstance(other, Include): - return False - - return self.exp == other.exp and self.required == other.required - -class VPathDirective(Statement): - """ - Represents the vpath directive. - - See https://www.gnu.org/software/make/manual/make.html#Selective-Search - """ - __slots__ = ('exp',) - - def __init__(self, exp): - assert isinstance(exp, (data.Expansion, data.StringExpansion)) - self.exp = exp - - def execute(self, makefile, context): - words = list(data.stripdotslashes(self.exp.resolvesplit(makefile, makefile.variables))) - if len(words) == 0: - makefile.clearallvpaths() - else: - pattern = data.Pattern(words[0]) - mpaths = words[1:] - - if len(mpaths) == 0: - makefile.clearvpath(pattern) - else: - dirs = [] - for mpath in mpaths: - dirs.extend((dir for dir in mpath.split(os.pathsep) - if dir != '')) - if len(dirs): - makefile.addvpath(pattern, dirs) - - def dump(self, fd, indent): - print >>fd, "%sVPath %s" % (indent, self.exp) - - def to_source(self): - return 'vpath %s' % self.exp.to_source() - - def __eq__(self, other): - if not isinstance(other, VPathDirective): - return False - - return self.exp == other.exp - -class ExportDirective(Statement): - """ - Represents the "export" directive. - - This is used to control exporting variables to sub makes. - - See https://www.gnu.org/software/make/manual/make.html#Variables_002fRecursion - - The `concurrent_set` field defines whether this statement occurred with or - without a variable assignment. If False, no variable assignment was - present. If True, the SetVariable immediately following this statement - originally came from this export directive (the parser splits it into - multiple statements). - """ - - __slots__ = ('exp', 'concurrent_set') - - def __init__(self, exp, concurrent_set): - assert isinstance(exp, (data.Expansion, data.StringExpansion)) - self.exp = exp - self.concurrent_set = concurrent_set - - def execute(self, makefile, context): - if self.concurrent_set: - vlist = [self.exp.resolvestr(makefile, makefile.variables)] - else: - vlist = list(self.exp.resolvesplit(makefile, makefile.variables)) - if not len(vlist): - raise data.DataError("Exporting all variables is not supported", self.exp.loc) - - for v in vlist: - makefile.exportedvars[v] = True - - def dump(self, fd, indent): - print >>fd, "%sExport (single=%s) %s" % (indent, self.single, self.exp) - - def to_source(self): - return ('export %s' % self.exp.to_source()).rstrip() - - def __eq__(self, other): - if not isinstance(other, ExportDirective): - return False - - # single is irrelevant because it just says whether the next Statement - # contains a variable definition. - return self.exp == other.exp - -class UnexportDirective(Statement): - """ - Represents the "unexport" directive. - - This is the opposite of ExportDirective. - """ - __slots__ = ('exp',) - - def __init__(self, exp): - self.exp = exp - - def execute(self, makefile, context): - vlist = list(self.exp.resolvesplit(makefile, makefile.variables)) - for v in vlist: - makefile.exportedvars[v] = False - - def dump(self, fd, indent): - print >>fd, "%sUnexport %s" % (indent, self.exp) - - def to_source(self): - return 'unexport %s' % self.exp.to_source() - - def __eq__(self, other): - if not isinstance(other, UnexportDirective): - return False - - return self.exp == other.exp - -class EmptyDirective(Statement): - """ - Represents a standalone statement, usually an Expansion. - - You will encounter EmptyDirective instances if there is a function - or similar at the top-level of a make file (e.g. outside of a rule or - variable assignment). You can also find them as the bodies of - ConditionBlock branches. - """ - __slots__ = ('exp',) - - def __init__(self, exp): - assert isinstance(exp, (data.Expansion, data.StringExpansion)) - self.exp = exp - - def execute(self, makefile, context): - v = self.exp.resolvestr(makefile, makefile.variables) - if v.strip() != '': - raise data.DataError("Line expands to non-empty value", self.exp.loc) - - def dump(self, fd, indent): - print >>fd, "%sEmptyDirective: %s" % (indent, self.exp) - - def to_source(self): - return self.exp.to_source() - - def __eq__(self, other): - if not isinstance(other, EmptyDirective): - return False - - return self.exp == other.exp - -class _EvalContext(object): - __slots__ = ('currule', 'weak') - - def __init__(self, weak): - self.weak = weak - -class StatementList(list): - """ - A list of Statement instances. - - This is what is generated by the parser when a make file is parsed. - - Consumers can iterate over all Statement instances in this collection to - statically inspect (and even modify) make files before they are executed. - """ - __slots__ = ('mtime',) - - def append(self, statement): - assert isinstance(statement, Statement) - list.append(self, statement) - - def execute(self, makefile, context=None, weak=False): - if context is None: - context = _EvalContext(weak=weak) - - for s in self: - s.execute(makefile, context) - - def dump(self, fd, indent): - for s in self: - s.dump(fd, indent) - - def __str__(self): - fd = StringIO() - self.dump(fd, '') - return fd.getvalue() - - def to_source(self): - return '\n'.join([s.to_source() for s in self]) - -def iterstatements(stmts): - for s in stmts: - yield s - if isinstance(s, ConditionBlock): - for c, sl in s: - for s2 in iterstatments(sl): yield s2 diff --git a/build/pymake/pymake/process.py b/build/pymake/pymake/process.py deleted file mode 100644 index 01cadf5a9..000000000 --- a/build/pymake/pymake/process.py +++ /dev/null @@ -1,556 +0,0 @@ -""" -Skipping shell invocations is good, when possible. This wrapper around subprocess does dirty work of -parsing command lines into argv and making sure that no shell magic is being used. -""" - -#TODO: ship pyprocessing? -import multiprocessing -import subprocess, shlex, re, logging, sys, traceback, os, imp, glob -import site -from collections import deque -# XXXkhuey Work around http://bugs.python.org/issue1731717 -subprocess._cleanup = lambda: None -import command, util -if sys.platform=='win32': - import win32process - -_log = logging.getLogger('pymake.process') - -_escapednewlines = re.compile(r'\\\n') - -def tokens2re(tokens): - # Create a pattern for non-escaped tokens, in the form: - # (?pattern) - # which matches the pattern and captures it in a named match group. - # The group names and patterns come are given as a dict in the function - # argument. - nonescaped = r'(?%s)' % (name, value) for name, value in tokens.iteritems()) - # The final pattern matches either the above pattern, or an escaped - # backslash, captured in the "escape" match group. - return re.compile('(?:%s|%s)' % (nonescaped, r'(?P\\\\)')) - -_unquoted_tokens = tokens2re({ - 'whitespace': r'[\t\r\n ]+', - 'quote': r'[\'"]', - 'comment': '#', - 'special': r'[<>&|`~(){}$;]', - 'backslashed': r'\\[^\\]', - 'glob': r'[\*\?]', -}) - -_doubly_quoted_tokens = tokens2re({ - 'quote': '"', - 'backslashedquote': r'\\"', - 'special': '\$', - 'backslashed': r'\\[^\\"]', -}) - -class MetaCharacterException(Exception): - def __init__(self, char): - self.char = char - -class ClineSplitter(list): - """ - Parses a given command line string and creates a list of command - and arguments, with wildcard expansion. - """ - def __init__(self, cline, cwd): - self.cwd = cwd - self.arg = None - self.cline = cline - self.glob = False - self._parse_unquoted() - - def _push(self, str): - """ - Push the given string as part of the current argument - """ - if self.arg is None: - self.arg = '' - self.arg += str - - def _next(self): - """ - Finalize current argument, effectively adding it to the list. - Perform globbing if needed. - """ - if self.arg is None: - return - if self.glob: - if os.path.isabs(self.arg): - path = self.arg - else: - path = os.path.join(self.cwd, self.arg) - globbed = glob.glob(path) - if not globbed: - # If globbing doesn't find anything, the literal string is - # used. - self.append(self.arg) - else: - self.extend(f[len(path)-len(self.arg):] for f in globbed) - self.glob = False - else: - self.append(self.arg) - self.arg = None - - def _parse_unquoted(self): - """ - Parse command line remainder in the context of an unquoted string. - """ - while self.cline: - # Find the next token - m = _unquoted_tokens.search(self.cline) - # If we find none, the remainder of the string can be pushed to - # the current argument and the argument finalized - if not m: - self._push(self.cline) - break - # The beginning of the string, up to the found token, is part of - # the current argument - if m.start(): - self._push(self.cline[:m.start()]) - self.cline = self.cline[m.end():] - - match = dict([(name, value) for name, value in m.groupdict().items() if value]) - if 'quote' in match: - # " or ' start a quoted string - if match['quote'] == '"': - self._parse_doubly_quoted() - else: - self._parse_quoted() - elif 'comment' in match: - # Comments are ignored. The current argument can be finalized, - # and parsing stopped. - break - elif 'special' in match: - # Unquoted, non-escaped special characters need to be sent to a - # shell. - raise MetaCharacterException, match['special'] - elif 'whitespace' in match: - # Whitespaces terminate current argument. - self._next() - elif 'escape' in match: - # Escaped backslashes turn into a single backslash - self._push('\\') - elif 'backslashed' in match: - # Backslashed characters are unbackslashed - # e.g. echo \a -> a - self._push(match['backslashed'][1]) - elif 'glob' in match: - # ? or * will need globbing - self.glob = True - self._push(m.group(0)) - else: - raise Exception, "Shouldn't reach here" - if self.arg: - self._next() - - def _parse_quoted(self): - # Single quoted strings are preserved, except for the final quote - index = self.cline.find("'") - if index == -1: - raise Exception, 'Unterminated quoted string in command' - self._push(self.cline[:index]) - self.cline = self.cline[index+1:] - - def _parse_doubly_quoted(self): - if not self.cline: - raise Exception, 'Unterminated quoted string in command' - while self.cline: - m = _doubly_quoted_tokens.search(self.cline) - if not m: - raise Exception, 'Unterminated quoted string in command' - self._push(self.cline[:m.start()]) - self.cline = self.cline[m.end():] - match = dict([(name, value) for name, value in m.groupdict().items() if value]) - if 'quote' in match: - # a double quote ends the quoted string, so go back to - # unquoted parsing - return - elif 'special' in match: - # Unquoted, non-escaped special characters in a doubly quoted - # string still have a special meaning and need to be sent to a - # shell. - raise MetaCharacterException, match['special'] - elif 'escape' in match: - # Escaped backslashes turn into a single backslash - self._push('\\') - elif 'backslashedquote' in match: - # Backslashed double quotes are un-backslashed - self._push('"') - elif 'backslashed' in match: - # Backslashed characters are kept backslashed - self._push(match['backslashed']) - -def clinetoargv(cline, cwd): - """ - If this command line can safely skip the shell, return an argv array. - @returns argv, badchar - """ - str = _escapednewlines.sub('', cline) - try: - args = ClineSplitter(str, cwd) - except MetaCharacterException, e: - return None, e.char - - if len(args) and args[0].find('=') != -1: - return None, '=' - - return args, None - -# shellwords contains a set of shell builtin commands that need to be -# executed within a shell. It also contains a set of commands that are known -# to be giving problems when run directly instead of through the msys shell. -shellwords = (':', '.', 'break', 'cd', 'continue', 'exec', 'exit', 'export', - 'getopts', 'hash', 'pwd', 'readonly', 'return', 'shift', - 'test', 'times', 'trap', 'umask', 'unset', 'alias', - 'set', 'bind', 'builtin', 'caller', 'command', 'declare', - 'echo', 'enable', 'help', 'let', 'local', 'logout', - 'printf', 'read', 'shopt', 'source', 'type', 'typeset', - 'ulimit', 'unalias', 'set', 'find') - -def prepare_command(cline, cwd, loc): - """ - Returns a list of command and arguments for the given command line string. - If the command needs to be run through a shell for some reason, the - returned list contains the shell invocation. - """ - - #TODO: call this once up-front somewhere and save the result? - shell, msys = util.checkmsyscompat() - - shellreason = None - executable = None - if msys and cline.startswith('/'): - shellreason = "command starts with /" - else: - argv, badchar = clinetoargv(cline, cwd) - if argv is None: - shellreason = "command contains shell-special character '%s'" % (badchar,) - elif len(argv) and argv[0] in shellwords: - shellreason = "command starts with shell primitive '%s'" % (argv[0],) - elif argv and (os.sep in argv[0] or os.altsep and os.altsep in argv[0]): - executable = util.normaljoin(cwd, argv[0]) - # Avoid "%1 is not a valid Win32 application" errors, assuming - # that if the executable path is to be resolved with PATH, it will - # be a Win32 executable. - if sys.platform == 'win32' and os.path.isfile(executable) and open(executable, 'rb').read(2) == "#!": - shellreason = "command executable starts with a hashbang" - - if shellreason is not None: - _log.debug("%s: using shell: %s: '%s'", loc, shellreason, cline) - if msys: - if len(cline) > 3 and cline[1] == ':' and cline[2] == '/': - cline = '/' + cline[0] + cline[2:] - argv = [shell, "-c", cline] - executable = None - - return executable, argv - -def call(cline, env, cwd, loc, cb, context, echo, justprint=False): - executable, argv = prepare_command(cline, cwd, loc) - - if not len(argv): - cb(res=0) - return - - if argv[0] == command.makepypath: - command.main(argv[1:], env, cwd, cb) - return - - if argv[0:2] == [sys.executable.replace('\\', '/'), - command.makepypath.replace('\\', '/')]: - command.main(argv[2:], env, cwd, cb) - return - - context.call(argv, executable=executable, shell=False, env=env, cwd=cwd, cb=cb, - echo=echo, justprint=justprint) - -def call_native(module, method, argv, env, cwd, loc, cb, context, echo, justprint=False, - pycommandpath=None): - context.call_native(module, method, argv, env=env, cwd=cwd, cb=cb, - echo=echo, justprint=justprint, pycommandpath=pycommandpath) - -def statustoresult(status): - """ - Convert the status returned from waitpid into a prettier numeric result. - """ - sig = status & 0xFF - if sig: - return -sig - - return status >>8 - -class Job(object): - """ - A single job to be executed on the process pool. - """ - done = False # set to true when the job completes - - def __init__(self): - self.exitcode = -127 - - def notify(self, condition, result): - condition.acquire() - self.done = True - self.exitcode = result - condition.notify() - condition.release() - - def get_callback(self, condition): - return lambda result: self.notify(condition, result) - -class PopenJob(Job): - """ - A job that executes a command using subprocess.Popen. - """ - def __init__(self, argv, executable, shell, env, cwd): - Job.__init__(self) - self.argv = argv - self.executable = executable - self.shell = shell - self.env = env - self.cwd = cwd - self.parentpid = os.getpid() - - def run(self): - assert os.getpid() != self.parentpid - # subprocess.Popen doesn't use the PATH set in the env argument for - # finding the executable on some platforms (but strangely it does on - # others!), so set os.environ['PATH'] explicitly. This is parallel- - # safe because pymake uses separate processes for parallelism, and - # each process is serial. See http://bugs.python.org/issue8557 for a - # general overview of "subprocess PATH semantics and portability". - oldpath = os.environ['PATH'] - try: - if self.env is not None and self.env.has_key('PATH'): - os.environ['PATH'] = self.env['PATH'] - p = subprocess.Popen(self.argv, executable=self.executable, shell=self.shell, env=self.env, cwd=self.cwd) - return p.wait() - except OSError, e: - print >>sys.stderr, e - return -127 - finally: - os.environ['PATH'] = oldpath - -class PythonException(Exception): - def __init__(self, message, exitcode): - Exception.__init__(self) - self.message = message - self.exitcode = exitcode - - def __str__(self): - return self.message - - -class PythonJob(Job): - """ - A job that calls a Python method. - """ - def __init__(self, module, method, argv, env, cwd, pycommandpath=None): - self.module = module - self.method = method - self.argv = argv - self.env = env - self.cwd = cwd - self.pycommandpath = pycommandpath or [] - self.parentpid = os.getpid() - - def run(self): - assert os.getpid() != self.parentpid - # os.environ is a magic dictionary. Setting it to something else - # doesn't affect the environment of subprocesses, so use clear/update - oldenv = dict(os.environ) - - # sys.path is adjusted for the entire lifetime of the command - # execution. This ensures any delayed imports will still work. - oldsyspath = list(sys.path) - try: - os.chdir(self.cwd) - os.environ.clear() - os.environ.update(self.env) - - sys.path = [] - for p in sys.path + self.pycommandpath: - site.addsitedir(p) - sys.path.extend(oldsyspath) - - if self.module not in sys.modules: - try: - __import__(self.module) - except Exception as e: - print >>sys.stderr, 'Error importing %s: %s' % ( - self.module, e) - return -127 - - m = sys.modules[self.module] - if self.method not in m.__dict__: - print >>sys.stderr, "No method named '%s' in module %s" % (self.method, self.module) - return -127 - rv = m.__dict__[self.method](self.argv) - if rv != 0 and rv is not None: - print >>sys.stderr, ( - "Native command '%s %s' returned value '%s'" % - (self.module, self.method, rv)) - return (rv if isinstance(rv, int) else 1) - - except PythonException, e: - print >>sys.stderr, e - return e.exitcode - except: - e = sys.exc_info()[1] - if isinstance(e, SystemExit) and (e.code == 0 or e.code is None): - pass # sys.exit(0) is not a failure - else: - print >>sys.stderr, e - traceback.print_exc() - return -127 - finally: - os.environ.clear() - os.environ.update(oldenv) - sys.path = oldsyspath - # multiprocessing exits via os._exit, make sure that all output - # from command gets written out before that happens. - sys.stdout.flush() - sys.stderr.flush() - - return 0 - -def job_runner(job): - """ - Run a job. Called in a Process pool. - """ - return job.run() - -class ParallelContext(object): - """ - Manages the parallel execution of processes. - """ - - _allcontexts = set() - _condition = multiprocessing.Condition() - - def __init__(self, jcount): - self.jcount = jcount - self.exit = False - - self.processpool = multiprocessing.Pool(processes=jcount) - self.pending = deque() # deque of (cb, args, kwargs) - self.running = [] # list of (subprocess, cb) - - self._allcontexts.add(self) - - def finish(self): - assert len(self.pending) == 0 and len(self.running) == 0, "pending: %i running: %i" % (len(self.pending), len(self.running)) - self.processpool.close() - self.processpool.join() - self._allcontexts.remove(self) - - def run(self): - while len(self.pending) and len(self.running) < self.jcount: - cb, args, kwargs = self.pending.popleft() - cb(*args, **kwargs) - - def defer(self, cb, *args, **kwargs): - assert self.jcount > 1 or not len(self.pending), "Serial execution error defering %r %r %r: currently pending %r" % (cb, args, kwargs, self.pending) - self.pending.append((cb, args, kwargs)) - - def _docall_generic(self, pool, job, cb, echo, justprint): - if echo is not None: - print echo - processcb = job.get_callback(ParallelContext._condition) - if justprint: - processcb(0) - else: - pool.apply_async(job_runner, args=(job,), callback=processcb) - self.running.append((job, cb)) - - def call(self, argv, shell, env, cwd, cb, echo, justprint=False, executable=None): - """ - Asynchronously call the process - """ - - job = PopenJob(argv, executable=executable, shell=shell, env=env, cwd=cwd) - self.defer(self._docall_generic, self.processpool, job, cb, echo, justprint) - - def call_native(self, module, method, argv, env, cwd, cb, - echo, justprint=False, pycommandpath=None): - """ - Asynchronously call the native function - """ - - job = PythonJob(module, method, argv, env, cwd, pycommandpath) - self.defer(self._docall_generic, self.processpool, job, cb, echo, justprint) - - @staticmethod - def _waitany(condition): - def _checkdone(): - jobs = [] - for c in ParallelContext._allcontexts: - for i in xrange(0, len(c.running)): - if c.running[i][0].done: - jobs.append(c.running[i]) - for j in jobs: - if j in c.running: - c.running.remove(j) - return jobs - - # We must acquire the lock, and then check to see if any jobs have - # finished. If we don't check after acquiring the lock it's possible - # that all outstanding jobs will have completed before we wait and we'll - # wait for notifications that have already occurred. - condition.acquire() - jobs = _checkdone() - - if jobs == []: - condition.wait() - jobs = _checkdone() - - condition.release() - - return jobs - - @staticmethod - def spin(): - """ - Spin the 'event loop', and never return. - """ - - while True: - clist = list(ParallelContext._allcontexts) - for c in clist: - c.run() - - dowait = util.any((len(c.running) for c in ParallelContext._allcontexts)) - if dowait: - # Wait on local jobs first for perf - for job, cb in ParallelContext._waitany(ParallelContext._condition): - cb(job.exitcode) - else: - assert any(len(c.pending) for c in ParallelContext._allcontexts) - -def makedeferrable(usercb, **userkwargs): - def cb(*args, **kwargs): - kwargs.update(userkwargs) - return usercb(*args, **kwargs) - - return cb - -_serialContext = None -_parallelContext = None - -def getcontext(jcount): - global _serialContext, _parallelContext - if jcount == 1: - if _serialContext is None: - _serialContext = ParallelContext(1) - return _serialContext - else: - if _parallelContext is None: - _parallelContext = ParallelContext(jcount) - return _parallelContext - diff --git a/build/pymake/pymake/util.py b/build/pymake/pymake/util.py deleted file mode 100644 index c63f930cc..000000000 --- a/build/pymake/pymake/util.py +++ /dev/null @@ -1,150 +0,0 @@ -import os - -class MakeError(Exception): - def __init__(self, message, loc=None): - self.msg = message - self.loc = loc - - def __str__(self): - locstr = '' - if self.loc is not None: - locstr = "%s:" % (self.loc,) - - return "%s%s" % (locstr, self.msg) - -def normaljoin(path, suffix): - """ - Combine the given path with the suffix, and normalize if necessary to shrink the path to avoid hitting path length limits - """ - result = os.path.join(path, suffix) - if len(result) > 255: - result = os.path.normpath(result) - return result - -def joiniter(fd, it): - """ - Given an iterator that returns strings, write the words with a space in between each. - """ - - it = iter(it) - for i in it: - fd.write(i) - break - - for i in it: - fd.write(' ') - fd.write(i) - -def checkmsyscompat(): - """For msys compatibility on windows, honor the SHELL environment variable, - and if $MSYSTEM == MINGW32, run commands through $SHELL -c instead of - letting Python use the system shell.""" - if 'SHELL' in os.environ: - shell = os.environ['SHELL'] - elif 'MOZILLABUILD' in os.environ: - shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe' - elif 'COMSPEC' in os.environ: - shell = os.environ['COMSPEC'] - else: - raise DataError("Can't find a suitable shell!") - - msys = False - if 'MSYSTEM' in os.environ and os.environ['MSYSTEM'] == 'MINGW32': - msys = True - if not shell.lower().endswith(".exe"): - shell += ".exe" - return (shell, msys) - -if hasattr(str, 'partition'): - def strpartition(str, token): - return str.partition(token) - - def strrpartition(str, token): - return str.rpartition(token) - -else: - def strpartition(str, token): - """Python 2.4 compatible str.partition""" - - offset = str.find(token) - if offset == -1: - return str, '', '' - - return str[:offset], token, str[offset + len(token):] - - def strrpartition(str, token): - """Python 2.4 compatible str.rpartition""" - - offset = str.rfind(token) - if offset == -1: - return '', '', str - - return str[:offset], token, str[offset + len(token):] - -try: - from __builtin__ import any -except ImportError: - def any(it): - for i in it: - if i: - return True - return False - -class _MostUsedItem(object): - __slots__ = ('key', 'o', 'count') - - def __init__(self, key): - self.key = key - self.o = None - self.count = 1 - - def __repr__(self): - return "MostUsedItem(key=%r, count=%i, o=%r)" % (self.key, self.count, self.o) - -class MostUsedCache(object): - def __init__(self, capacity, creationfunc, verifyfunc): - self.capacity = capacity - self.cfunc = creationfunc - self.vfunc = verifyfunc - - self.d = {} - self.active = [] # lazily sorted! - - def setactive(self, item): - if item in self.active: - return - - if len(self.active) == self.capacity: - self.active.sort(key=lambda i: i.count) - old = self.active.pop(0) - old.o = None - # print "Evicting %s" % old.key - - self.active.append(item) - - def get(self, key): - item = self.d.get(key, None) - if item is None: - item = _MostUsedItem(key) - self.d[key] = item - else: - item.count += 1 - - if item.o is not None and self.vfunc(key, item.o): - return item.o - - item.o = self.cfunc(key) - self.setactive(item) - return item.o - - def verify(self): - for k, v in self.d.iteritems(): - if v.o: - assert v in self.active - else: - assert v not in self.active - - def debugitems(self): - l = [i.key for i in self.active] - l.sort() - return l diff --git a/build/pymake/pymake/win32process.py b/build/pymake/pymake/win32process.py deleted file mode 100644 index 880a26a5b..000000000 --- a/build/pymake/pymake/win32process.py +++ /dev/null @@ -1,28 +0,0 @@ -from ctypes import windll, POINTER, byref, WinError -from ctypes.wintypes import WINFUNCTYPE, HANDLE, DWORD, BOOL - -INFINITE = -1 -WAIT_FAILED = 0xFFFFFFFF - -LPDWORD = POINTER(DWORD) -_GetExitCodeProcessProto = WINFUNCTYPE(BOOL, HANDLE, LPDWORD) -_GetExitCodeProcess = _GetExitCodeProcessProto(("GetExitCodeProcess", windll.kernel32)) -def GetExitCodeProcess(h): - exitcode = DWORD() - r = _GetExitCodeProcess(h, byref(exitcode)) - if r is 0: - raise WinError() - return exitcode.value - -_WaitForMultipleObjectsProto = WINFUNCTYPE(DWORD, DWORD, POINTER(HANDLE), BOOL, DWORD) -_WaitForMultipleObjects = _WaitForMultipleObjectsProto(("WaitForMultipleObjects", windll.kernel32)) - -def WaitForAnyProcess(processes): - arrtype = HANDLE * len(processes) - harray = arrtype(*(int(p._handle) for p in processes)) - - r = _WaitForMultipleObjects(len(processes), harray, False, INFINITE) - if r == WAIT_FAILED: - raise WinError() - - return processes[r], GetExitCodeProcess(int(processes[r]._handle)) <<8 diff --git a/build/pymake/tests/automatic-variables.mk b/build/pymake/tests/automatic-variables.mk deleted file mode 100644 index 5302c08ea..000000000 --- a/build/pymake/tests/automatic-variables.mk +++ /dev/null @@ -1,79 +0,0 @@ -$(shell \ -mkdir -p src/subd; \ -mkdir subd; \ -touch dummy; \ -sleep 2; \ -touch subd/test.out src/subd/test.in2; \ -sleep 2; \ -touch subd/test.out2 src/subd/test.in; \ -sleep 2; \ -touch subd/host_test.out subd/host_test.out2; \ -sleep 2; \ -touch host_prog; \ -) - -VPATH = src - -all: prog host_prog prog dir/ - test "$@" = "all" - test "$<" = "prog" - test "$^" = "prog host_prog dir" - test "$?" = "prog host_prog dir" - test "$+" = "prog host_prog prog dir" - test "$(@D)" = "." - test "$(@F)" = "all" - test "$($@ - -%.out2: %.in2 dummy - @echo TEST_FAIL No need to remake - -.PHONY: all diff --git a/build/pymake/tests/bad-command-continuation.mk b/build/pymake/tests/bad-command-continuation.mk deleted file mode 100644 index d9ceccfc2..000000000 --- a/build/pymake/tests/bad-command-continuation.mk +++ /dev/null @@ -1,3 +0,0 @@ -all: - echo 'hello'\ -TEST-PASS diff --git a/build/pymake/tests/call.mk b/build/pymake/tests/call.mk deleted file mode 100644 index 9eeb7e00c..000000000 --- a/build/pymake/tests/call.mk +++ /dev/null @@ -1,12 +0,0 @@ -test = $0 -reverse = $2 $1 -twice = $1$1 -sideeffect = $(shell echo "called$1:" >>dummyfile) - -all: - test "$(call test)" = "test" - test "$(call reverse,1,2)" = "2 1" -# expansion happens *before* substitution, thank sanity - test "$(call twice,$(sideeffect))" = "" - test `cat dummyfile` = "called:" - @echo TEST-PASS diff --git a/build/pymake/tests/cmd-stripdotslash.mk b/build/pymake/tests/cmd-stripdotslash.mk deleted file mode 100644 index ce5ed4244..000000000 --- a/build/pymake/tests/cmd-stripdotslash.mk +++ /dev/null @@ -1,5 +0,0 @@ -all: - $(MAKE) -f $(TESTPATH)/cmd-stripdotslash.mk ./foo - -./foo: - @echo TEST-PASS diff --git a/build/pymake/tests/cmdgoals.mk b/build/pymake/tests/cmdgoals.mk deleted file mode 100644 index a3b25e751..000000000 --- a/build/pymake/tests/cmdgoals.mk +++ /dev/null @@ -1,9 +0,0 @@ -default: - test "$(MAKECMDGOALS)" = "" - $(MAKE) -f $(TESTPATH)/cmdgoals.mk t1 t2 - @echo TEST-PASS - -t1: - test "$(MAKECMDGOALS)" = "t1 t2" - -t2: diff --git a/build/pymake/tests/commandmodifiers.mk b/build/pymake/tests/commandmodifiers.mk deleted file mode 100644 index 8440462f3..000000000 --- a/build/pymake/tests/commandmodifiers.mk +++ /dev/null @@ -1,21 +0,0 @@ -define COMMAND -$(1) - $(1) - -endef - -all: - $(call COMMAND,@true #TEST-FAIL) - $(call COMMAND,-exit 4) - $(call COMMAND,@-exit 1 # TEST-FAIL) - $(call COMMAND,-@exit 1 # TEST-FAIL) - $(call COMMAND,+exit 0) - $(call COMMAND,+-exit 1) - $(call COMMAND,@+exit 0 # TEST-FAIL) - $(call COMMAND,+@exit 0 # TEST-FAIL) - $(call COMMAND,-+@exit 1 # TEST-FAIL) - $(call COMMAND,+-@exit 1 # TEST-FAIL) - $(call COMMAND,@+-exit 1 # TEST-FAIL) - $(call COMMAND,@+-@+-exit 1 # TEST-FAIL) - $(call COMMAND,@@++exit 0 # TEST-FAIL) - @echo TEST-PASS diff --git a/build/pymake/tests/comment-parsing.mk b/build/pymake/tests/comment-parsing.mk deleted file mode 100644 index d469e1aea..000000000 --- a/build/pymake/tests/comment-parsing.mk +++ /dev/null @@ -1,29 +0,0 @@ -# where do comments take effect? - -VAR = val1 # comment -VAR2 = lit2\#hash -VAR2_1 = lit2.1\\\#hash -VAR3 = val3 -VAR4 = lit4\\#backslash -VAR4_1 = lit4\\\\#backslash -VAR5 = lit5\char -VAR6 = lit6\\char -VAR7 = lit7\\ -VAR8 = lit8\\\\ -VAR9 = lit9\\\\extra -# This comment extends to the next line \ -VAR3 = ignored - -all: - test "$(VAR)" = "val1 " - test "$(VAR2)" = "lit2#hash" - test '$(VAR2_1)' = 'lit2.1\#hash' - test "$(VAR3)" = "val3" - test '$(VAR4)' = 'lit4\' - test '$(VAR4_1)' = 'lit4\\' - test '$(VAR5)' = 'lit5\char' - test '$(VAR6)' = 'lit6\\char' - test '$(VAR7)' = 'lit7\\' - test '$(VAR8)' = 'lit8\\\\' - test '$(VAR9)' = 'lit9\\\\extra' - @echo "TEST-PASS" diff --git a/build/pymake/tests/continuations-in-functions.mk b/build/pymake/tests/continuations-in-functions.mk deleted file mode 100644 index 533df6176..000000000 --- a/build/pymake/tests/continuations-in-functions.mk +++ /dev/null @@ -1,6 +0,0 @@ -all: - test 'Hello world.' = '$(if 1,Hello \ - world.)' - test '(Hello world.)' != '(Hello \ - world.)' - @echo TEST-PASS diff --git a/build/pymake/tests/datatests.py b/build/pymake/tests/datatests.py deleted file mode 100644 index 513028b0b..000000000 --- a/build/pymake/tests/datatests.py +++ /dev/null @@ -1,237 +0,0 @@ -import pymake.data, pymake.functions, pymake.util -import unittest -import re -from cStringIO import StringIO - -def multitest(cls): - for name in cls.testdata.iterkeys(): - def m(self, name=name): - return self.runSingle(*self.testdata[name]) - - setattr(cls, 'test_%s' % name, m) - return cls - -class SplitWordsTest(unittest.TestCase): - testdata = ( - (' test test.c test.o ', ['test', 'test.c', 'test.o']), - ('\ttest\t test.c \ntest.o', ['test', 'test.c', 'test.o']), - ) - - def runTest(self): - for s, e in self.testdata: - w = s.split() - self.assertEqual(w, e, 'splitwords(%r)' % (s,)) - -class GetPatSubstTest(unittest.TestCase): - testdata = ( - ('%.c', '%.o', ' test test.c test.o ', 'test test.o test.o'), - ('%', '%.o', ' test.c test.o ', 'test.c.o test.o.o'), - ('foo', 'bar', 'test foo bar', 'test bar bar'), - ('foo', '%bar', 'test foo bar', 'test %bar bar'), - ('%', 'perc_%', 'path', 'perc_path'), - ('\\%', 'sub%', 'p %', 'p sub%'), - ('%.c', '\\%%.o', 'foo.c bar.o baz.cpp', '%foo.o bar.o baz.cpp'), - ) - - def runTest(self): - for s, r, d, e in self.testdata: - words = d.split() - p = pymake.data.Pattern(s) - a = ' '.join((p.subst(r, word, False) - for word in words)) - self.assertEqual(a, e, 'Pattern(%r).subst(%r, %r)' % (s, r, d)) - -class LRUTest(unittest.TestCase): - # getkey, expected, funccount, debugitems - expected = ( - (0, '', 1, (0,)), - (0, '', 2, (0,)), - (1, ' ', 3, (1, 0)), - (1, ' ', 3, (1, 0)), - (0, '', 4, (0, 1)), - (2, ' ', 5, (2, 0, 1)), - (1, ' ', 5, (1, 2, 0)), - (3, ' ', 6, (3, 1, 2)), - ) - - def spaceFunc(self, l): - self.funccount += 1 - return ''.ljust(l) - - def runTest(self): - self.funccount = 0 - c = pymake.util.LRUCache(3, self.spaceFunc, lambda k, v: k % 2) - self.assertEqual(tuple(c.debugitems()), ()) - - for i in xrange(0, len(self.expected)): - k, e, fc, di = self.expected[i] - - v = c.get(k) - self.assertEqual(v, e) - self.assertEqual(self.funccount, fc, - "funccount, iteration %i, got %i expected %i" % (i, self.funccount, fc)) - goti = tuple(c.debugitems()) - self.assertEqual(goti, di, - "debugitems, iteration %i, got %r expected %r" % (i, goti, di)) - -class EqualityTest(unittest.TestCase): - def test_string_expansion(self): - s1 = pymake.data.StringExpansion('foo bar', None) - s2 = pymake.data.StringExpansion('foo bar', None) - - self.assertEqual(s1, s2) - - def test_expansion_simple(self): - s1 = pymake.data.Expansion(None) - s2 = pymake.data.Expansion(None) - - self.assertEqual(s1, s2) - - s1.appendstr('foo') - s2.appendstr('foo') - self.assertEqual(s1, s2) - - def test_expansion_string_finish(self): - """Adjacent strings should normalize to same value.""" - s1 = pymake.data.Expansion(None) - s2 = pymake.data.Expansion(None) - - s1.appendstr('foo') - s2.appendstr('foo') - - s1.appendstr(' bar') - s1.appendstr(' baz') - s2.appendstr(' bar baz') - - self.assertEqual(s1, s2) - - def test_function(self): - s1 = pymake.data.Expansion(None) - s2 = pymake.data.Expansion(None) - - n1 = pymake.data.StringExpansion('FOO', None) - n2 = pymake.data.StringExpansion('FOO', None) - - v1 = pymake.functions.VariableRef(None, n1) - v2 = pymake.functions.VariableRef(None, n2) - - s1.appendfunc(v1) - s2.appendfunc(v2) - - self.assertEqual(s1, s2) - - -class StringExpansionTest(unittest.TestCase): - def test_base_expansion_interface(self): - s1 = pymake.data.StringExpansion('FOO', None) - - self.assertTrue(s1.is_static_string) - - funcs = list(s1.functions()) - self.assertEqual(len(funcs), 0) - - funcs = list(s1.functions(True)) - self.assertEqual(len(funcs), 0) - - refs = list(s1.variable_references()) - self.assertEqual(len(refs), 0) - - -class ExpansionTest(unittest.TestCase): - def test_is_static_string(self): - e1 = pymake.data.Expansion() - e1.appendstr('foo') - - self.assertTrue(e1.is_static_string) - - e1.appendstr('bar') - self.assertTrue(e1.is_static_string) - - vname = pymake.data.StringExpansion('FOO', None) - func = pymake.functions.VariableRef(None, vname) - - e1.appendfunc(func) - - self.assertFalse(e1.is_static_string) - - def test_get_functions(self): - e1 = pymake.data.Expansion() - e1.appendstr('foo') - - vname1 = pymake.data.StringExpansion('FOO', None) - vname2 = pymake.data.StringExpansion('BAR', None) - - func1 = pymake.functions.VariableRef(None, vname1) - func2 = pymake.functions.VariableRef(None, vname2) - - e1.appendfunc(func1) - e1.appendfunc(func2) - - funcs = list(e1.functions()) - self.assertEqual(len(funcs), 2) - - func3 = pymake.functions.SortFunction(None) - func3.append(vname1) - - e1.appendfunc(func3) - - funcs = list(e1.functions()) - self.assertEqual(len(funcs), 3) - - refs = list(e1.variable_references()) - self.assertEqual(len(refs), 2) - - def test_get_functions_descend(self): - e1 = pymake.data.Expansion() - vname1 = pymake.data.StringExpansion('FOO', None) - func1 = pymake.functions.VariableRef(None, vname1) - e2 = pymake.data.Expansion() - e2.appendfunc(func1) - - func2 = pymake.functions.SortFunction(None) - func2.append(e2) - - e1.appendfunc(func2) - - funcs = list(e1.functions()) - self.assertEqual(len(funcs), 1) - - funcs = list(e1.functions(True)) - self.assertEqual(len(funcs), 2) - - self.assertTrue(isinstance(funcs[0], pymake.functions.SortFunction)) - - def test_is_filesystem_dependent(self): - e = pymake.data.Expansion() - vname1 = pymake.data.StringExpansion('FOO', None) - func1 = pymake.functions.VariableRef(None, vname1) - e.appendfunc(func1) - - self.assertFalse(e.is_filesystem_dependent) - - func2 = pymake.functions.WildcardFunction(None) - func2.append(vname1) - e.appendfunc(func2) - - self.assertTrue(e.is_filesystem_dependent) - - def test_is_filesystem_dependent_descend(self): - sort = pymake.functions.SortFunction(None) - wildcard = pymake.functions.WildcardFunction(None) - - e = pymake.data.StringExpansion('foo/*', None) - wildcard.append(e) - - e = pymake.data.Expansion(None) - e.appendfunc(wildcard) - - sort.append(e) - - e = pymake.data.Expansion(None) - e.appendfunc(sort) - - self.assertTrue(e.is_filesystem_dependent) - - -if __name__ == '__main__': - unittest.main() diff --git a/build/pymake/tests/default-goal-set-first.mk b/build/pymake/tests/default-goal-set-first.mk deleted file mode 100644 index 00a5b53a2..000000000 --- a/build/pymake/tests/default-goal-set-first.mk +++ /dev/null @@ -1,7 +0,0 @@ -.DEFAULT_GOAL := default - -not-default: - @echo TEST-FAIL did not run default rule - -default: - @echo TEST-PASS diff --git a/build/pymake/tests/default-goal.mk b/build/pymake/tests/default-goal.mk deleted file mode 100644 index 699d6c0cd..000000000 --- a/build/pymake/tests/default-goal.mk +++ /dev/null @@ -1,8 +0,0 @@ -not-default: - @echo TEST-FAIL did not run default rule - -default: - @echo $(if $(filter not-default,$(INTERMEDIATE_DEFAULT_GOAL)),TEST-PASS,TEST-FAIL .DEFAULT_GOAL not set by $(MAKE)) - -INTERMEDIATE_DEFAULT_GOAL := $(.DEFAULT_GOAL) -.DEFAULT_GOAL := default diff --git a/build/pymake/tests/default-target.mk b/build/pymake/tests/default-target.mk deleted file mode 100644 index 701ac6916..000000000 --- a/build/pymake/tests/default-target.mk +++ /dev/null @@ -1,14 +0,0 @@ -test: VAR = value - -%.do: - @echo TEST-FAIL: ran target "$@", should have run "all" - -.PHONY: test - -all: - @echo TEST-PASS: the default target is all - -test: - @echo TEST-FAIL: ran target "$@", should have run "all" - -test.do: diff --git a/build/pymake/tests/default-target2.mk b/build/pymake/tests/default-target2.mk deleted file mode 100644 index b5a4b1bbf..000000000 --- a/build/pymake/tests/default-target2.mk +++ /dev/null @@ -1,6 +0,0 @@ -test.foo: %.foo: - test "$@" = "test.foo" - @echo TEST-PASS made test.foo by default - -all: - @echo TEST-FAIL made $@, should have made test.foo diff --git a/build/pymake/tests/define-directive.mk b/build/pymake/tests/define-directive.mk deleted file mode 100644 index 789988666..000000000 --- a/build/pymake/tests/define-directive.mk +++ /dev/null @@ -1,69 +0,0 @@ -define COMMANDS -shellvar=hello -test "$$shellvar" != "hello" -endef - -define COMMANDS2 -shellvar=hello; \ - test "$$shellvar" = "hello" -endef - -define VARWITHCOMMENT # comment -value -endef - -define TEST3 - whitespace -endef - -define TEST4 -define TEST5 -random -endef - endef - -ifdef TEST5 -$(error TEST5 should not be set) -endif - -define TEST6 - define TEST7 -random -endef -endef - -ifdef TEST7 -$(error TEST7 should not be set) -endif - -define TEST8 -is this # a comment? -endef - -ifneq ($(TEST8),is this \# a comment?) -$(error TEST8 value not expected: $(TEST8)) -endif - -# A backslash continuation "hides" the endef -define TEST9 -value \ -endef -endef - -# Test ridiculous spacing - define TEST10 - define TEST11 - baz -endef -define TEST12 - foo - endef - endef - -all: - $(COMMANDS) - $(COMMANDS2) - test '$(VARWITHCOMMENT)' = 'value' - test '$(COMMANDS2)' = 'shellvar=hello; test "$$shellvar" = "hello"' - test "$(TEST3)" = " whitespace" - @echo TEST-PASS diff --git a/build/pymake/tests/depfailed.mk b/build/pymake/tests/depfailed.mk deleted file mode 100644 index ce4137c38..000000000 --- a/build/pymake/tests/depfailed.mk +++ /dev/null @@ -1,4 +0,0 @@ -#T returncode: 2 - -all: foo.out foo.in - @echo TEST-PASS diff --git a/build/pymake/tests/depfailedj.mk b/build/pymake/tests/depfailedj.mk deleted file mode 100644 index a94c74f6f..000000000 --- a/build/pymake/tests/depfailedj.mk +++ /dev/null @@ -1,10 +0,0 @@ -#T returncode: 2 -#T commandline: ['-j4'] - -$(shell touch foo.in) - -all: foo.in foo.out missing - @echo TEST-PASS - -%.out: %.in - cp $< $@ diff --git a/build/pymake/tests/diamond-deps.mk b/build/pymake/tests/diamond-deps.mk deleted file mode 100644 index 40a4176d9..000000000 --- a/build/pymake/tests/diamond-deps.mk +++ /dev/null @@ -1,13 +0,0 @@ -# If the dependency graph includes a diamond dependency, we should only remake -# once! - -all: depA depB - cat testfile - test `cat testfile` = "data"; - @echo TEST-PASS - -depA: testfile -depB: testfile - -testfile: - printf "data" >>$@ diff --git a/build/pymake/tests/dotslash-dir.mk b/build/pymake/tests/dotslash-dir.mk deleted file mode 100644 index 8b30d1e3c..000000000 --- a/build/pymake/tests/dotslash-dir.mk +++ /dev/null @@ -1,8 +0,0 @@ -#T grep-for: "dotslash-built" -.PHONY: $(dir foo) - -all: $(dir foo) - @echo TEST-PASS - -$(dir foo): - @echo dotslash-built diff --git a/build/pymake/tests/dotslash-parse.mk b/build/pymake/tests/dotslash-parse.mk deleted file mode 100644 index 91461bedb..000000000 --- a/build/pymake/tests/dotslash-parse.mk +++ /dev/null @@ -1,4 +0,0 @@ -./: - -# This is merely a test to see that pymake doesn't choke on parsing ./ -$(info TEST-PASS) diff --git a/build/pymake/tests/dotslash-phony.mk b/build/pymake/tests/dotslash-phony.mk deleted file mode 100644 index 06b6ae78d..000000000 --- a/build/pymake/tests/dotslash-phony.mk +++ /dev/null @@ -1,3 +0,0 @@ -.PHONY: ./ -./: - @echo TEST-PASS diff --git a/build/pymake/tests/dotslash.mk b/build/pymake/tests/dotslash.mk deleted file mode 100644 index 585db96b7..000000000 --- a/build/pymake/tests/dotslash.mk +++ /dev/null @@ -1,9 +0,0 @@ -$(shell touch foo.in) - -all: foo.out - test "$(wildcard ./*.in)" = "./foo.in" - @echo TEST-PASS - -./%.out: %.in - test "$@" = "foo.out" - cp $< $@ diff --git a/build/pymake/tests/doublecolon-exists.mk b/build/pymake/tests/doublecolon-exists.mk deleted file mode 100644 index 5d99a1f6b..000000000 --- a/build/pymake/tests/doublecolon-exists.mk +++ /dev/null @@ -1,16 +0,0 @@ -$(shell touch foo.testfile1 foo.testfile2) - -# when a rule has commands and no prerequisites, should it be executed? -# double-colon: yes -# single-colon: no - -all: foo.testfile1 foo.testfile2 - test "$$(cat foo.testfile1)" = "" - test "$$(cat foo.testfile2)" = "remade:foo.testfile2" - @echo TEST-PASS - -foo.testfile1: - @echo TEST-FAIL - -foo.testfile2:: - printf "remade:$@"> $@ diff --git a/build/pymake/tests/doublecolon-priordeps.mk b/build/pymake/tests/doublecolon-priordeps.mk deleted file mode 100644 index 6cdf3a8e7..000000000 --- a/build/pymake/tests/doublecolon-priordeps.mk +++ /dev/null @@ -1,19 +0,0 @@ -#T commandline: ['-j3'] - -# All *prior* dependencies of a doublecolon rule must be satisfied before -# subsequent commands are run. - -all:: target1 - -all:: target2 - test -f target1 - @echo TEST-PASS - -target1: - touch starting-$@ - sleep 1 - touch $@ - -target2: - sleep 0.1 - test -f starting-target1 diff --git a/build/pymake/tests/doublecolon-remake.mk b/build/pymake/tests/doublecolon-remake.mk deleted file mode 100644 index 52aa9265c..000000000 --- a/build/pymake/tests/doublecolon-remake.mk +++ /dev/null @@ -1,4 +0,0 @@ -$(shell touch somefile) - -all:: somefile - @echo TEST-PASS diff --git a/build/pymake/tests/dynamic-var.mk b/build/pymake/tests/dynamic-var.mk deleted file mode 100644 index 0993b9ccf..000000000 --- a/build/pymake/tests/dynamic-var.mk +++ /dev/null @@ -1,18 +0,0 @@ -# The *name* of variables can be constructed dynamically. - -VARNAME = FOOBAR - -$(VARNAME) = foovalue -$(VARNAME)2 = foo2value - -$(VARNAME:%BAR=%BAM) = foobam - -all: - test "$(FOOBAR)" = "foovalue" - test "$(flavor FOOBAZ)" = "undefined" - test "$(FOOBAR2)" = "bazvalue" - test "$(FOOBAM)" = "foobam" - @echo TEST-PASS - -VARNAME = FOOBAZ -FOOBAR2 = bazvalue diff --git a/build/pymake/tests/empty-arg.mk b/build/pymake/tests/empty-arg.mk deleted file mode 100644 index 616e5b694..000000000 --- a/build/pymake/tests/empty-arg.mk +++ /dev/null @@ -1,2 +0,0 @@ -all: - @ sh -c 'if [ $$# = 3 ] ; then echo TEST-PASS; else echo TEST-FAIL; fi' -- a "" b diff --git a/build/pymake/tests/empty-command-semicolon.mk b/build/pymake/tests/empty-command-semicolon.mk deleted file mode 100644 index 07789f3f1..000000000 --- a/build/pymake/tests/empty-command-semicolon.mk +++ /dev/null @@ -1,5 +0,0 @@ -all: - @echo TEST-PASS - -foo: ; - diff --git a/build/pymake/tests/empty-with-deps.mk b/build/pymake/tests/empty-with-deps.mk deleted file mode 100644 index 284e5a113..000000000 --- a/build/pymake/tests/empty-with-deps.mk +++ /dev/null @@ -1,4 +0,0 @@ -default.test: default.c - -default.c: - @echo TEST-PASS diff --git a/build/pymake/tests/env-var-append.mk b/build/pymake/tests/env-var-append.mk deleted file mode 100644 index 4db39c45f..000000000 --- a/build/pymake/tests/env-var-append.mk +++ /dev/null @@ -1,7 +0,0 @@ -#T environment: {'FOO': 'TEST'} - -FOO += $(BAR) -BAR := PASS - -all: - @echo $(subst $(NULL) ,-,$(FOO)) diff --git a/build/pymake/tests/env-var-append2.mk b/build/pymake/tests/env-var-append2.mk deleted file mode 100644 index fc0735d88..000000000 --- a/build/pymake/tests/env-var-append2.mk +++ /dev/null @@ -1,8 +0,0 @@ -#T environment: {'FOO': '$(BAZ)'} - -FOO += $(BAR) -BAR := PASS -BAZ := TEST - -all: - @echo $(subst $(NULL) ,-,$(FOO)) diff --git a/build/pymake/tests/eof-continuation.mk b/build/pymake/tests/eof-continuation.mk deleted file mode 100644 index daeaabc3e..000000000 --- a/build/pymake/tests/eof-continuation.mk +++ /dev/null @@ -1,5 +0,0 @@ -all: - test '$(TESTVAR)' = 'testval\' - @echo TEST-PASS - -TESTVAR = testval\ \ No newline at end of file diff --git a/build/pymake/tests/escape-chars.mk b/build/pymake/tests/escape-chars.mk deleted file mode 100644 index ebea33074..000000000 --- a/build/pymake/tests/escape-chars.mk +++ /dev/null @@ -1,26 +0,0 @@ -space = $(NULL) $(NULL) -hello$(space)world$(space) = hellovalue - -A = aval - -VAR = value1\\ -VARAWFUL = value1\\#comment -VAR2 = value2 -VAR3 = test\$A -VAR4 = value4\\value5 - -VAR5 = value1\\ \ \ - value2 - -EPERCENT = \% - -all: - test "$(hello world )" = "hellovalue" - test "$(VAR)" = "value1\\" - test '$(VARAWFUL)' = 'value1\' - test "$(VAR2)" = "value2" - test "$(VAR3)" = "test\aval" - test "$(VAR4)" = "value4\\value5" - test "$(VAR5)" = "value1\\ \ value2" - test "$(EPERCENT)" = "\%" - @echo TEST-PASS diff --git a/build/pymake/tests/escaped-continuation.mk b/build/pymake/tests/escaped-continuation.mk deleted file mode 100644 index 537f7547f..000000000 --- a/build/pymake/tests/escaped-continuation.mk +++ /dev/null @@ -1,6 +0,0 @@ -#T returncode: 2 - -all: - echo "Hello" \\ - test "world" = "not!" - @echo TEST-PASS diff --git a/build/pymake/tests/eval-duringexecute.mk b/build/pymake/tests/eval-duringexecute.mk deleted file mode 100644 index dff848032..000000000 --- a/build/pymake/tests/eval-duringexecute.mk +++ /dev/null @@ -1,12 +0,0 @@ -#T returncode: 2 - -# Once parsing is finished, recursive expansion in commands are not allowed to create any new rules (it may only set variables) - -define MORERULE -all: - @echo TEST-FAIL -endef - -all: - $(eval $(MORERULE)) - @echo done diff --git a/build/pymake/tests/eval.mk b/build/pymake/tests/eval.mk deleted file mode 100644 index de9759f02..000000000 --- a/build/pymake/tests/eval.mk +++ /dev/null @@ -1,7 +0,0 @@ -TESTVAR = val1 - -$(eval TESTVAR = val2) - -all: - test "$(TESTVAR)" = "val2" - @echo TEST-PASS diff --git a/build/pymake/tests/exit-code.mk b/build/pymake/tests/exit-code.mk deleted file mode 100644 index 84dcffcf9..000000000 --- a/build/pymake/tests/exit-code.mk +++ /dev/null @@ -1,5 +0,0 @@ -#T returncode: 2 - -all: - exit 1 - @echo TEST-PASS diff --git a/build/pymake/tests/file-functions-symlinks.mk b/build/pymake/tests/file-functions-symlinks.mk deleted file mode 100644 index dcc0f6eef..000000000 --- a/build/pymake/tests/file-functions-symlinks.mk +++ /dev/null @@ -1,22 +0,0 @@ -#T returncode-on: {'win32': 2} -$(shell \ -touch test.file; \ -ln -s test.file test.symlink; \ -ln -s test.missing missing.symlink; \ -touch .testhidden; \ -mkdir foo; \ -touch foo/testfile; \ -ln -s foo symdir; \ -) - -all: - test "$(abspath test.file test.symlink)" = "$(CURDIR)/test.file $(CURDIR)/test.symlink" - test "$(realpath test.file test.symlink)" = "$(CURDIR)/test.file $(CURDIR)/test.file" - test "$(sort $(wildcard *))" = "foo symdir test.file test.symlink" - test "$(sort $(wildcard .*))" = ". .. .testhidden" - test "$(sort $(wildcard test*))" = "test.file test.symlink" - test "$(sort $(wildcard foo/*))" = "foo/testfile" - test "$(sort $(wildcard ./*))" = "./foo ./symdir ./test.file ./test.symlink" - test "$(sort $(wildcard f?o/*))" = "foo/testfile" - test "$(sort $(wildcard */*))" = "foo/testfile symdir/testfile" - @echo TEST-PASS diff --git a/build/pymake/tests/file-functions.mk b/build/pymake/tests/file-functions.mk deleted file mode 100644 index 7e4c68e85..000000000 --- a/build/pymake/tests/file-functions.mk +++ /dev/null @@ -1,19 +0,0 @@ -$(shell \ -touch test.file; \ -touch .testhidden; \ -mkdir foo; \ -touch foo/testfile; \ -) - -all: - test "$(abspath test.file)" = "$(CURDIR)/test.file" - test "$(realpath test.file)" = "$(CURDIR)/test.file" - test "$(sort $(wildcard *))" = "foo test.file" -# commented out because GNU make matches . and .. while python doesn't, and I don't -# care enough -# test "$(sort $(wildcard .*))" = ". .. .testhidden" - test "$(sort $(wildcard test*))" = "test.file" - test "$(sort $(wildcard foo/*))" = "foo/testfile" - test "$(sort $(wildcard ./*))" = "./foo ./test.file" - test "$(sort $(wildcard f?o/*))" = "foo/testfile" - @echo TEST-PASS diff --git a/build/pymake/tests/foreach-local-variable.mk b/build/pymake/tests/foreach-local-variable.mk deleted file mode 100644 index 2551621eb..000000000 --- a/build/pymake/tests/foreach-local-variable.mk +++ /dev/null @@ -1,8 +0,0 @@ -# This test ensures that a local variable in a $(foreach) is bound to -# the local value, not a global value. -i := dummy - -all: - test "$(foreach i,foo bar,found:$(i))" = "found:foo found:bar" - test "$(i)" = "dummy" - @echo TEST-PASS diff --git a/build/pymake/tests/formattingtests.py b/build/pymake/tests/formattingtests.py deleted file mode 100644 index 7aad6d4cc..000000000 --- a/build/pymake/tests/formattingtests.py +++ /dev/null @@ -1,289 +0,0 @@ -# This file contains test code for the formatting of parsed statements back to -# make file "source." It essentially verifies to to_source() functions -# scattered across the tree. - -import glob -import logging -import os.path -import unittest - -from pymake.data import Expansion -from pymake.data import StringExpansion -from pymake.functions import BasenameFunction -from pymake.functions import SubstitutionRef -from pymake.functions import VariableRef -from pymake.functions import WordlistFunction -from pymake.parserdata import Include -from pymake.parserdata import SetVariable -from pymake.parser import parsestring -from pymake.parser import SyntaxError - -class TestBase(unittest.TestCase): - pass - -class VariableRefTest(TestBase): - def test_string_name(self): - e = StringExpansion('foo', None) - v = VariableRef(None, e) - - self.assertEqual(v.to_source(), '$(foo)') - - def test_special_variable(self): - e = StringExpansion('<', None) - v = VariableRef(None, e) - - self.assertEqual(v.to_source(), '$<') - - def test_expansion_simple(self): - e = Expansion() - e.appendstr('foo') - e.appendstr('bar') - - v = VariableRef(None, e) - - self.assertEqual(v.to_source(), '$(foobar)') - -class StandardFunctionTest(TestBase): - def test_basename(self): - e1 = StringExpansion('foo', None) - v = VariableRef(None, e1) - e2 = Expansion(None) - e2.appendfunc(v) - - b = BasenameFunction(None) - b.append(e2) - - self.assertEqual(b.to_source(), '$(basename $(foo))') - - def test_wordlist(self): - e1 = StringExpansion('foo', None) - e2 = StringExpansion('bar ', None) - e3 = StringExpansion(' baz', None) - - w = WordlistFunction(None) - w.append(e1) - w.append(e2) - w.append(e3) - - self.assertEqual(w.to_source(), '$(wordlist foo,bar , baz)') - - def test_curly_brackets(self): - e1 = Expansion(None) - e1.appendstr('foo') - - e2 = Expansion(None) - e2.appendstr('foo ( bar') - - f = WordlistFunction(None) - f.append(e1) - f.append(e2) - - self.assertEqual(f.to_source(), '${wordlist foo,foo ( bar}') - -class StringExpansionTest(TestBase): - def test_simple(self): - e = StringExpansion('foobar', None) - self.assertEqual(e.to_source(), 'foobar') - - e = StringExpansion('$var', None) - self.assertEqual(e.to_source(), '$var') - - def test_escaping(self): - e = StringExpansion('$var', None) - self.assertEqual(e.to_source(escape_variables=True), '$$var') - - e = StringExpansion('this is # not a comment', None) - self.assertEqual(e.to_source(escape_comments=True), - 'this is \# not a comment') - - def test_empty(self): - e = StringExpansion('', None) - self.assertEqual(e.to_source(), '') - - e = StringExpansion(' ', None) - self.assertEqual(e.to_source(), ' ') - -class ExpansionTest(TestBase): - def test_single_string(self): - e = Expansion() - e.appendstr('foo') - - self.assertEqual(e.to_source(), 'foo') - - def test_multiple_strings(self): - e = Expansion() - e.appendstr('hello') - e.appendstr('world') - - self.assertEqual(e.to_source(), 'helloworld') - - def test_string_escape(self): - e = Expansion() - e.appendstr('$var') - self.assertEqual(e.to_source(), '$var') - self.assertEqual(e.to_source(escape_variables=True), '$$var') - - e = Expansion() - e.appendstr('foo') - e.appendstr(' $bar') - self.assertEqual(e.to_source(escape_variables=True), 'foo $$bar') - -class SubstitutionRefTest(TestBase): - def test_simple(self): - name = StringExpansion('foo', None) - c = StringExpansion('%.c', None) - o = StringExpansion('%.o', None) - s = SubstitutionRef(None, name, c, o) - - self.assertEqual(s.to_source(), '$(foo:%.c=%.o)') - -class SetVariableTest(TestBase): - def test_simple(self): - v = SetVariable(StringExpansion('foo', None), '=', 'bar', None, None) - self.assertEqual(v.to_source(), 'foo = bar') - - def test_multiline(self): - s = 'hello\nworld' - foo = StringExpansion('FOO', None) - - v = SetVariable(foo, '=', s, None, None) - - self.assertEqual(v.to_source(), 'define FOO\nhello\nworld\nendef') - - def test_multiline_immediate(self): - source = 'define FOO :=\nhello\nworld\nendef' - - statements = parsestring(source, 'foo.mk') - self.assertEqual(statements.to_source(), source) - - def test_target_specific(self): - foo = StringExpansion('FOO', None) - bar = StringExpansion('BAR', None) - - v = SetVariable(foo, '+=', 'value', None, bar) - - self.assertEqual(v.to_source(), 'BAR: FOO += value') - -class IncludeTest(TestBase): - def test_include(self): - e = StringExpansion('rules.mk', None) - i = Include(e, True, False) - self.assertEqual(i.to_source(), 'include rules.mk') - - i = Include(e, False, False) - self.assertEqual(i.to_source(), '-include rules.mk') - -class IfdefTest(TestBase): - def test_simple(self): - source = 'ifdef FOO\nbar := $(value)\nendif' - - statements = parsestring(source, 'foo.mk') - self.assertEqual(statements[0].to_source(), source) - - def test_nested(self): - source = 'ifdef FOO\nifdef BAR\nhello = world\nendif\nendif' - - statements = parsestring(source, 'foo.mk') - self.assertEqual(statements[0].to_source(), source) - - def test_negation(self): - source = 'ifndef FOO\nbar += value\nendif' - - statements = parsestring(source, 'foo.mk') - self.assertEqual(statements[0].to_source(), source) - -class IfeqTest(TestBase): - def test_simple(self): - source = 'ifeq ($(foo),bar)\nhello = $(world)\nendif' - - statements = parsestring(source, 'foo.mk') - self.assertEqual(statements[0].to_source(), source) - - def test_negation(self): - source = 'ifneq (foo,bar)\nhello = world\nendif' - - statements = parsestring(source, 'foo.mk') - self.assertEqual(statements.to_source(), source) - -class ConditionBlocksTest(TestBase): - def test_mixed_conditions(self): - source = 'ifdef FOO\nifeq ($(FOO),bar)\nvar += $(value)\nendif\nendif' - - statements = parsestring(source, 'foo.mk') - self.assertEqual(statements.to_source(), source) - - def test_extra_statements(self): - source = 'ifdef FOO\nF := 1\nifdef BAR\nB += 1\nendif\nC = 1\nendif' - - statements = parsestring(source, 'foo.mk') - self.assertEqual(statements.to_source(), source) - - def test_whitespace_preservation(self): - source = "ifeq ' x' 'x '\n$(error stripping)\nendif" - - statements = parsestring(source, 'foo.mk') - self.assertEqual(statements.to_source(), source) - - source = 'ifneq (x , x)\n$(error stripping)\nendif' - statements = parsestring(source, 'foo.mk') - self.assertEqual(statements.to_source(), - 'ifneq (x,x)\n$(error stripping)\nendif') - -class MakefileCorupusTest(TestBase): - """Runs the make files from the pymake corpus through the formatter. - - All the above tests are child's play compared to this. - """ - - # Our reformatting isn't perfect. We ignore files with known failures until - # we make them work. - # TODO Address these formatting corner cases. - _IGNORE_FILES = [ - # We are thrown off by backslashes at end of lines. - 'comment-parsing.mk', - 'escape-chars.mk', - 'include-notfound.mk', - ] - - def _get_test_files(self): - ourdir = os.path.dirname(os.path.abspath(__file__)) - - for makefile in glob.glob(os.path.join(ourdir, '*.mk')): - if os.path.basename(makefile) in self._IGNORE_FILES: - continue - - source = None - with open(makefile, 'rU') as fh: - source = fh.read() - - try: - yield (makefile, source, parsestring(source, makefile)) - except SyntaxError: - continue - - def test_reparse_consistency(self): - for filename, source, statements in self._get_test_files(): - reformatted = statements.to_source() - - # We should be able to parse the reformatted source fine. - new_statements = parsestring(reformatted, filename) - - # If we do the formatting again, the representation shouldn't - # change. i.e. the only lossy change should be the original - # (whitespace and some semantics aren't preserved). - reformatted_again = new_statements.to_source() - self.assertEqual(reformatted, reformatted_again, - '%s has lossless reformat.' % filename) - - self.assertEqual(len(statements), len(new_statements)) - - for i in xrange(0, len(statements)): - original = statements[i] - formatted = new_statements[i] - - self.assertEqual(original, formatted, '%s %d: %s != %s' % (filename, - i, original, formatted)) - -if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) - unittest.main() diff --git a/build/pymake/tests/func-refs.mk b/build/pymake/tests/func-refs.mk deleted file mode 100644 index 82ab17ba8..000000000 --- a/build/pymake/tests/func-refs.mk +++ /dev/null @@ -1,11 +0,0 @@ -unknown var = uval - -all: - test "$(subst a,b,value)" = "vblue" - test "${subst a,b,va)lue}" = "vb)lue" - test "$(subst /,\,ab/c)" = "ab\c" - test '$(subst a,b,\\#)' = '\\#' - test "$( subst a,b,value)" = "" - test "$(Subst a,b,value)" = "" - test "$(unknown var)" = "uval" - @echo TEST-PASS diff --git a/build/pymake/tests/functions.mk b/build/pymake/tests/functions.mk deleted file mode 100644 index 817be07aa..000000000 --- a/build/pymake/tests/functions.mk +++ /dev/null @@ -1,36 +0,0 @@ -all: - test "$(subst e,EE,hello)" = "hEEllo" - test "$(strip $(NULL) test data )" = "test data" - test "$(findstring hell,hello)" = "hell" - test "$(findstring heaven,hello)" = "" - test "$(filter foo/%.c b%,foo/a.c b.c foo/a.o)" = "foo/a.c b.c" - test "$(filter foo,foo bar)" = "foo" - test "$(filter-out foo/%.c b%,foo/a.c b.c foo/a.o)" = "foo/a.o" - test "$(filter-out %.c,foo,bar.c foo,bar.o)" = "foo,bar.o" - test "$(sort .go a b aa A c cc)" = ".go A a aa b c cc" - test "$(word 1, hello )" = "hello" - test "$(word 2, hello )" = "" - test "$(wordlist 1, 2, foo bar baz )" = "foo bar" - test "$(words 1 2 3)" = "3" - test "$(words )" = "0" - test "$(firstword $(NULL) foo bar baz)" = "foo" - test "$(firstword )" = "" - test "$(dir foo.c path/foo.o dir/dir2/)" = "./ path/ dir/dir2/" - test "$(notdir foo.c path/foo.o dir/dir2/)" = "foo.c foo.o " - test "$(suffix src/foo.c dir/my.dir/foo foo.o)" = ".c .o" - test "$(basename src/foo.c dir/my.dir/foo foo.c .c)" = "src/foo dir/my.dir/foo foo " - test "$(addprefix src/,foo bar.c dir/foo)" = "src/foo src/bar.c src/dir/foo" - test "$(addsuffix .c,foo dir/bar)" = "foo.c dir/bar.c" - test "$(join a b c, 1 2 3)" = "a1 b2 c3" - test "$(join a b, 1 2 3)" = "a1 b2 3" - test "$(join a b c, 1 2)" = "a1 b2 c" - test "$(if $(NULL) ,yes)" = "" - test "$(if 1,yes,no)" = "yes" - test "$(if ,yes,no )" = "no " - test "$(if ,$(error Short-circuit problem))" = "" - test "$(or $(NULL),1)" = "1" - test "$(or $(NULL),2,$(warning TEST-FAIL bad or short-circuit))" = "2" - test "$(and ,$(warning TEST-FAIL bad and short-circuit))" = "" - test "$(and 1,2)" = "2" - test "$(foreach i,foo bar,found:$(i))" = "found:foo found:bar" - @echo TEST-PASS diff --git a/build/pymake/tests/functiontests.py b/build/pymake/tests/functiontests.py deleted file mode 100644 index 43a344a05..000000000 --- a/build/pymake/tests/functiontests.py +++ /dev/null @@ -1,54 +0,0 @@ -import unittest - -import pymake.data -import pymake.functions - -class VariableRefTest(unittest.TestCase): - def test_get_expansions(self): - e = pymake.data.StringExpansion('FOO', None) - f = pymake.functions.VariableRef(None, e) - - exps = list(f.expansions()) - self.assertEqual(len(exps), 1) - -class GetExpansionsTest(unittest.TestCase): - def test_get_arguments(self): - f = pymake.functions.SubstFunction(None) - - e1 = pymake.data.StringExpansion('FOO', None) - e2 = pymake.data.StringExpansion('BAR', None) - e3 = pymake.data.StringExpansion('BAZ', None) - - f.append(e1) - f.append(e2) - f.append(e3) - - exps = list(f.expansions()) - self.assertEqual(len(exps), 3) - - def test_descend(self): - f = pymake.functions.StripFunction(None) - - e = pymake.data.Expansion(None) - - e1 = pymake.data.StringExpansion('FOO', None) - f1 = pymake.functions.VariableRef(None, e1) - e.appendfunc(f1) - - f2 = pymake.functions.WildcardFunction(None) - e2 = pymake.data.StringExpansion('foo/*', None) - f2.append(e2) - e.appendfunc(f2) - - f.append(e) - - exps = list(f.expansions()) - self.assertEqual(len(exps), 1) - - exps = list(f.expansions(True)) - self.assertEqual(len(exps), 3) - - self.assertFalse(f.is_filesystem_dependent) - -if __name__ == '__main__': - unittest.main() diff --git a/build/pymake/tests/if-syntaxerr.mk b/build/pymake/tests/if-syntaxerr.mk deleted file mode 100644 index c172492ef..000000000 --- a/build/pymake/tests/if-syntaxerr.mk +++ /dev/null @@ -1,6 +0,0 @@ -#T returncode: 2 - -ifeq ($(FOO,VAR)) -all: - @echo TEST_FAIL -endif diff --git a/build/pymake/tests/ifdefs-nesting.mk b/build/pymake/tests/ifdefs-nesting.mk deleted file mode 100644 index 340530ffa..000000000 --- a/build/pymake/tests/ifdefs-nesting.mk +++ /dev/null @@ -1,13 +0,0 @@ -ifdef RANDOM -ifeq (,$(error Not evaluated!)) -endif -endif - -ifdef RANDOM -ifeq (,) -else ifeq (,$(error Not evaluated!)) -endif -endif - -all: - @echo TEST-PASS diff --git a/build/pymake/tests/ifdefs.mk b/build/pymake/tests/ifdefs.mk deleted file mode 100644 index a779d197b..000000000 --- a/build/pymake/tests/ifdefs.mk +++ /dev/null @@ -1,127 +0,0 @@ -ifdef FOO -$(error FOO is not defined!) -endif - -FOO = foo -FOOFOUND = false -BARFOUND = false -BAZFOUND = false - -ifdef FOO -FOOFOUND = true -else ifdef BAR -BARFOUND = true -else -BAZFOUND = true -endif - -BAR2 = bar2 -FOO2FOUND = false -BAR2FOUND = false -BAZ2FOUND = false - -ifdef FOO2 -FOO2FOUND = true -else ifdef BAR2 -BAR2FOUND = true -else -BAZ2FOUND = true -endif - -FOO3FOUND = false -BAR3FOUND = false -BAZ3FOUND = false - -ifdef FOO3 -FOO3FOUND = true -else ifdef BAR3 -BAR3FOUND = true -else -BAZ3FOUND = true -endif - -ifdef RANDOM -CONTINUATION = \ -else \ -endif -endif - -ifndef ASDFJK -else -$(error ASFDJK was not set) -endif - -TESTSET = - -ifdef TESTSET -$(error TESTSET was not set) -endif - -TESTEMPTY = $(NULL) -ifndef TESTEMPTY -$(error TEST-FAIL TESTEMPTY was probably expanded!) -endif - -# ifneq ( a,a) -# $(error Arguments to ifeq should be stripped before evaluation) -# endif - -XSPACE = x # trick - -ifneq ($(NULL),$(NULL)) -$(error TEST-FAIL ifneq) -endif - -ifneq (x , x) -$(error argument-stripping1) -endif - -ifeq ( x,x ) -$(error argument-stripping2) -endif - -ifneq ($(XSPACE), x ) -$(error argument-stripping3) -endif - -ifeq 'x ' ' x' -$(error TEST-FAIL argument-stripping4) -endif - -all: - test $(FOOFOUND) = true # FOOFOUND - test $(BARFOUND) = false # BARFOUND - test $(BAZFOUND) = false # BAZFOUND - test $(FOO2FOUND) = false # FOO2FOUND - test $(BAR2FOUND) = true # BAR2FOUND - test $(BAZ2FOUND) = false # BAZ2FOUND - test $(FOO3FOUND) = false # FOO3FOUND - test $(BAR3FOUND) = false # BAR3FOUND - test $(BAZ3FOUND) = true # BAZ3FOUND -ifneq ($(FOO),foo) - echo TEST-FAIL 'FOO neq foo: "$(FOO)"' -endif -ifneq ($(FOO), foo) # Whitespace after the comma is stripped - echo TEST-FAIL 'FOO plus whitespace' -endif -ifeq ($(FOO), foo ) # But not trailing whitespace - echo TEST-FAIL 'FOO plus trailing whitespace' -endif -ifeq ( $(FOO),foo) # Not whitespace after the paren - echo TEST-FAIL 'FOO with leading whitespace' -endif -ifeq ($(FOO),$(NULL) foo) # Nor whitespace after expansion - echo TEST-FAIL 'FOO with embedded ws' -endif -ifeq ($(BAR2),bar) - echo TEST-FAIL 'BAR2 eq bar' -endif -ifeq '$(BAR3FOUND)' 'false' - echo BAR3FOUND is ok -else - echo TEST-FAIL BAR3FOUND is not ok -endif -ifndef FOO - echo TEST-FAIL "foo not defined?" -endif - @echo TEST-PASS diff --git a/build/pymake/tests/ignore-error.mk b/build/pymake/tests/ignore-error.mk deleted file mode 100644 index dc8d3a72c..000000000 --- a/build/pymake/tests/ignore-error.mk +++ /dev/null @@ -1,13 +0,0 @@ -all: - -rm foo - +-rm bar - -+rm baz - @-rm bah - -@rm humbug - +-@rm sincere - +@-rm flattery - @+-rm will - @-+rm not - -+@rm save - -@+rm you - @echo TEST-PASS diff --git a/build/pymake/tests/implicit-chain.mk b/build/pymake/tests/implicit-chain.mk deleted file mode 100644 index 16288b3f5..000000000 --- a/build/pymake/tests/implicit-chain.mk +++ /dev/null @@ -1,12 +0,0 @@ -all: test.prog - test "$$(cat $<)" = "Program: Object: Source: test.source" - @echo TEST-PASS - -%.prog: %.object - printf "Program: %s" "$$(cat $<)" > $@ - -%.object: %.source - printf "Object: %s" "$$(cat $<)" > $@ - -%.source: - printf "Source: %s" $@ > $@ diff --git a/build/pymake/tests/implicit-dir.mk b/build/pymake/tests/implicit-dir.mk deleted file mode 100644 index c7f75e8d4..000000000 --- a/build/pymake/tests/implicit-dir.mk +++ /dev/null @@ -1,16 +0,0 @@ -# Implicit rules have special instructions to deal with directories, so that a pattern rule which doesn't directly apply -# may still be used. - -all: dir/host_test.otest - -host_%.otest: %.osource extra.file - @echo making $@ from $< - -test.osource: - @echo TEST-FAIL should have made dir/test.osource - -dir/test.osource: - @echo TEST-PASS made the correct dependency - -extra.file: - @echo building $@ diff --git a/build/pymake/tests/implicit-terminal.mk b/build/pymake/tests/implicit-terminal.mk deleted file mode 100644 index db2e244ed..000000000 --- a/build/pymake/tests/implicit-terminal.mk +++ /dev/null @@ -1,16 +0,0 @@ -#T returncode: 2 - -# the %.object rule is "terminal". This means that additional implicit rules cannot be chained to it. - -all: test.prog - test "$$(cat $<)" = "Program: Object: Source: test.source" - @echo TEST-FAIL - -%.prog: %.object - printf "Program: %s" "$$(cat $<)" > $@ - -%.object:: %.source - printf "Object: %s" "$$(cat $<)" > $@ - -%.source: - printf "Source: %s" $@ > $@ diff --git a/build/pymake/tests/implicitsubdir.mk b/build/pymake/tests/implicitsubdir.mk deleted file mode 100644 index b9d854a2a..000000000 --- a/build/pymake/tests/implicitsubdir.mk +++ /dev/null @@ -1,12 +0,0 @@ -$(shell \ -mkdir foo; \ -touch test.in \ -) - -all: foo/test.out - @echo TEST-PASS - -foo/%.out: %.in - cp $< $@ - - diff --git a/build/pymake/tests/include-dynamic.mk b/build/pymake/tests/include-dynamic.mk deleted file mode 100644 index 571895dc3..000000000 --- a/build/pymake/tests/include-dynamic.mk +++ /dev/null @@ -1,21 +0,0 @@ -$(shell \ -if ! test -f include-dynamic.inc; then \ - echo "TESTVAR = oldval" > include-dynamic.inc; \ - sleep 2; \ - echo "TESTVAR = newval" > include-dynamic.inc.in; \ -fi \ -) - -# before running the 'all' rule, we should be rebuilding include-dynamic.inc, -# because there is a rule to do so - -all: - test $(TESTVAR) = newval - test "$(MAKE_RESTARTS)" = 1 - @echo TEST-PASS - -include-dynamic.inc: include-dynamic.inc.in - test "$(MAKE_RESTARTS)" = "" - cp $< $@ - -include include-dynamic.inc diff --git a/build/pymake/tests/include-file.inc b/build/pymake/tests/include-file.inc deleted file mode 100644 index d5d495dec..000000000 --- a/build/pymake/tests/include-file.inc +++ /dev/null @@ -1 +0,0 @@ -INCLUDED = yes diff --git a/build/pymake/tests/include-missing.mk b/build/pymake/tests/include-missing.mk deleted file mode 100644 index 583d0a065..000000000 --- a/build/pymake/tests/include-missing.mk +++ /dev/null @@ -1,9 +0,0 @@ -#T returncode: 2 - -# If an include file isn't present and doesn't have a rule to remake it, make -# should fail. - -include notfound.mk - -all: - @echo TEST-FAIL diff --git a/build/pymake/tests/include-notfound.mk b/build/pymake/tests/include-notfound.mk deleted file mode 100644 index 1ee7e05b2..000000000 --- a/build/pymake/tests/include-notfound.mk +++ /dev/null @@ -1,19 +0,0 @@ -ifdef __WIN32__ -PS:=\\# -else -PS:=/ -endif - -ifneq ($(strip $(MAKEFILE_LIST)),$(NATIVE_TESTPATH)$(PS)include-notfound.mk) -$(error MAKEFILE_LIST incorrect: '$(MAKEFILE_LIST)' (expected '$(NATIVE_TESTPATH)$(PS)include-notfound.mk')) -endif - --include notfound.inc-dummy - -ifneq ($(strip $(MAKEFILE_LIST)),$(NATIVE_TESTPATH)$(PS)include-notfound.mk) -$(error MAKEFILE_LIST incorrect: '$(MAKEFILE_LIST)' (expected '$(NATIVE_TESTPATH)$(PS)include-notfound.mk')) -endif - -all: - @echo TEST-PASS - diff --git a/build/pymake/tests/include-optional-warning.mk b/build/pymake/tests/include-optional-warning.mk deleted file mode 100644 index 901938dff..000000000 --- a/build/pymake/tests/include-optional-warning.mk +++ /dev/null @@ -1,4 +0,0 @@ --include TEST-FAIL.mk - -all: - @echo TEST-PASS diff --git a/build/pymake/tests/include-regen.mk b/build/pymake/tests/include-regen.mk deleted file mode 100644 index c86e0c78d..000000000 --- a/build/pymake/tests/include-regen.mk +++ /dev/null @@ -1,10 +0,0 @@ -# avoid infinite loops by not remaking makefiles with -# double-colon no-dependency rules -# http://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles --include notfound.mk - -all: - @echo TEST-PASS - -notfound.mk:: - @echo TEST-FAIL diff --git a/build/pymake/tests/include-regen2.mk b/build/pymake/tests/include-regen2.mk deleted file mode 100644 index fc7fef073..000000000 --- a/build/pymake/tests/include-regen2.mk +++ /dev/null @@ -1,10 +0,0 @@ -# make should make makefiles that it has rules for if they are -# included -include test.mk - -all: - test "$(X)" = "1" - @echo "TEST-PASS" - -test.mk: - @echo "X = 1" > $@ diff --git a/build/pymake/tests/include-regen3.mk b/build/pymake/tests/include-regen3.mk deleted file mode 100644 index 878ce0adc..000000000 --- a/build/pymake/tests/include-regen3.mk +++ /dev/null @@ -1,10 +0,0 @@ -# make should make makefiles that it has rules for if they are -# included --include test.mk - -all: - test "$(X)" = "1" - @echo "TEST-PASS" - -test.mk: - @echo "X = 1" > $@ diff --git a/build/pymake/tests/include-test.mk b/build/pymake/tests/include-test.mk deleted file mode 100644 index 3608fc269..000000000 --- a/build/pymake/tests/include-test.mk +++ /dev/null @@ -1,8 +0,0 @@ -$(shell echo "INCLUDED2 = yes" >local-include.inc) - -include $(TESTPATH)/include-file.inc local-include.inc - -all: - test "$(INCLUDED)" = "yes" - test "$(INCLUDED2)" = "yes" - @echo TEST-PASS diff --git a/build/pymake/tests/includedeps-norebuild.mk b/build/pymake/tests/includedeps-norebuild.mk deleted file mode 100644 index e30abd439..000000000 --- a/build/pymake/tests/includedeps-norebuild.mk +++ /dev/null @@ -1,15 +0,0 @@ -#T gmake skip - -$(shell \ -touch filemissing; \ -sleep 2; \ -touch file1; \ -) - -all: file1 - @echo TEST-PASS - -includedeps $(TESTPATH)/includedeps.deps - -file1: - @echo TEST-FAIL diff --git a/build/pymake/tests/includedeps-sideeffects.mk b/build/pymake/tests/includedeps-sideeffects.mk deleted file mode 100644 index 7e4ea30a2..000000000 --- a/build/pymake/tests/includedeps-sideeffects.mk +++ /dev/null @@ -1,10 +0,0 @@ -#T gmake skip -#T returncode: 2 - -all: file1 filemissing - @echo TEST-PASS - -includedeps $(TESTPATH)/includedeps.deps - -file: - touch $@ diff --git a/build/pymake/tests/includedeps-stripdotslash.deps b/build/pymake/tests/includedeps-stripdotslash.deps deleted file mode 100644 index 352fca1bb..000000000 --- a/build/pymake/tests/includedeps-stripdotslash.deps +++ /dev/null @@ -1 +0,0 @@ -./test: TEST-PASS diff --git a/build/pymake/tests/includedeps-stripdotslash.mk b/build/pymake/tests/includedeps-stripdotslash.mk deleted file mode 100644 index ee942e6db..000000000 --- a/build/pymake/tests/includedeps-stripdotslash.mk +++ /dev/null @@ -1,8 +0,0 @@ -#T gmake skip - -test: - @echo $< - -includedeps $(TESTPATH)/includedeps-stripdotslash.deps - -TEST-PASS: diff --git a/build/pymake/tests/includedeps-variables.deps b/build/pymake/tests/includedeps-variables.deps deleted file mode 100644 index ba69e9b6c..000000000 --- a/build/pymake/tests/includedeps-variables.deps +++ /dev/null @@ -1 +0,0 @@ -$(FILE)1: filemissing diff --git a/build/pymake/tests/includedeps-variables.mk b/build/pymake/tests/includedeps-variables.mk deleted file mode 100644 index 314618da4..000000000 --- a/build/pymake/tests/includedeps-variables.mk +++ /dev/null @@ -1,10 +0,0 @@ -#T gmake skip - -FILE = includedeps-variables - -all: $(FILE)1 - -includedeps $(TESTPATH)/includedeps-variables.deps - -filemissing: - @echo TEST-PASS diff --git a/build/pymake/tests/includedeps.deps b/build/pymake/tests/includedeps.deps deleted file mode 100644 index d3017c078..000000000 --- a/build/pymake/tests/includedeps.deps +++ /dev/null @@ -1 +0,0 @@ -file1: filemissing diff --git a/build/pymake/tests/includedeps.mk b/build/pymake/tests/includedeps.mk deleted file mode 100644 index deaa71fe8..000000000 --- a/build/pymake/tests/includedeps.mk +++ /dev/null @@ -1,9 +0,0 @@ -#T gmake skip - -all: file1 - @echo TEST-PASS - -includedeps $(TESTPATH)/includedeps.deps - -file1: - touch $@ diff --git a/build/pymake/tests/info.mk b/build/pymake/tests/info.mk deleted file mode 100644 index 8dddfd815..000000000 --- a/build/pymake/tests/info.mk +++ /dev/null @@ -1,8 +0,0 @@ -#T grep-for: "info-printed\ninfo-nth" -all: - -INFO = info-printed - -$(info $(INFO)) -$(info $(subst second,nth,info-second)) -$(info TEST-PASS) diff --git a/build/pymake/tests/justprint-native.mk b/build/pymake/tests/justprint-native.mk deleted file mode 100644 index 580e402e9..000000000 --- a/build/pymake/tests/justprint-native.mk +++ /dev/null @@ -1,28 +0,0 @@ -## $(TOUCH) and $(RM) are native commands in pymake. -## Test that pymake --just-print just prints them. - -ifndef TOUCH -TOUCH = touch -endif - -all: - $(RM) justprint-native-file1.txt - $(TOUCH) justprint-native-file2.txt - $(MAKE) --just-print -f $(TESTPATH)/justprint-native.mk justprint_target > justprint.log -# make --just-print shouldn't have actually done anything. - test ! -f justprint-native-file1.txt - test -f justprint-native-file2.txt -# but it should have printed each command - grep -q 'touch justprint-native-file1.txt' justprint.log - grep -q 'rm -f justprint-native-file2.txt' justprint.log - grep -q 'this string is "unlikely to appear in the log by chance"' justprint.log -# tidy up - $(RM) justprint-native-file2.txt - @echo TEST-PASS - -justprint_target: - $(TOUCH) justprint-native-file1.txt - $(RM) justprint-native-file2.txt - this string is "unlikely to appear in the log by chance" - -.PHONY: justprint_target diff --git a/build/pymake/tests/justprint.mk b/build/pymake/tests/justprint.mk deleted file mode 100644 index be11ba8de..000000000 --- a/build/pymake/tests/justprint.mk +++ /dev/null @@ -1,5 +0,0 @@ -#T commandline: ['-n'] - -all: - false # without -n, we wouldn't get past this - TEST-PASS # heh diff --git a/build/pymake/tests/keep-going-doublecolon.mk b/build/pymake/tests/keep-going-doublecolon.mk deleted file mode 100644 index fa5b31df8..000000000 --- a/build/pymake/tests/keep-going-doublecolon.mk +++ /dev/null @@ -1,16 +0,0 @@ -#T commandline: ['-k'] -#T returncode: 2 -#T grep-for: "TEST-PASS" - -all:: t1 - @echo TEST-FAIL "(t1)" - -all:: t2 - @echo TEST-PASS - -t1: - @false - -t2: - touch $@ - diff --git a/build/pymake/tests/keep-going-parallel.mk b/build/pymake/tests/keep-going-parallel.mk deleted file mode 100644 index a91d1a6ed..000000000 --- a/build/pymake/tests/keep-going-parallel.mk +++ /dev/null @@ -1,11 +0,0 @@ -#T commandline: ['-k', '-j2'] -#T returncode: 2 -#T grep-for: "TEST-PASS" - -all: t1 slow1 slow2 slow3 t2 - -t2: - @echo TEST-PASS - -slow%: - sleep 1 diff --git a/build/pymake/tests/keep-going.mk b/build/pymake/tests/keep-going.mk deleted file mode 100644 index 4c709288c..000000000 --- a/build/pymake/tests/keep-going.mk +++ /dev/null @@ -1,14 +0,0 @@ -#T commandline: ['-k'] -#T returncode: 2 -#T grep-for: "TEST-PASS" - -all: t2 t3 - -t1: - @false - -t2: t1 - @echo TEST-FAIL - -t3: - @echo TEST-PASS diff --git a/build/pymake/tests/line-continuations.mk b/build/pymake/tests/line-continuations.mk deleted file mode 100644 index 8b44480ea..000000000 --- a/build/pymake/tests/line-continuations.mk +++ /dev/null @@ -1,24 +0,0 @@ -VAR = val1 \ - val2 - -VAR2 = val1space\ -val2 - -VAR3 = val3 \\\ - cont3 - -all: otarget test.target - test "$(VAR)" = "val1 val2 " - test "$(VAR2)" = "val1space val2" - test '$(VAR3)' = 'val3 \ cont3' - test "hello \ - world" = "hello world" - test "hello" = \ -"hello" - @echo TEST-PASS - -otarget: ; test "hello\ - world" = "helloworld" - -test.target: %.target: ; test "hello\ - world" = "helloworld" diff --git a/build/pymake/tests/link-search.mk b/build/pymake/tests/link-search.mk deleted file mode 100644 index ea827f391..000000000 --- a/build/pymake/tests/link-search.mk +++ /dev/null @@ -1,7 +0,0 @@ -$(shell \ -touch libfoo.so \ -) - -all: -lfoo - test "$<" = "libfoo.so" - @echo TEST-PASS diff --git a/build/pymake/tests/makeflags.mk b/build/pymake/tests/makeflags.mk deleted file mode 100644 index 288ff7866..000000000 --- a/build/pymake/tests/makeflags.mk +++ /dev/null @@ -1,7 +0,0 @@ -#T environment: {'MAKEFLAGS': 'OVAR=oval'} - -all: - test "$(OVAR)" = "oval" - test "$$OVAR" = "oval" - @echo TEST-PASS - diff --git a/build/pymake/tests/matchany.mk b/build/pymake/tests/matchany.mk deleted file mode 100644 index 7876c90a3..000000000 --- a/build/pymake/tests/matchany.mk +++ /dev/null @@ -1,14 +0,0 @@ -#T returncode: 2 - -# we should fail to make foo.ooo from foo.ooo.test -all: foo.ooo - @echo TEST-FAIL - -%.ooo: - -# this match-anything pattern should not apply to %.ooo -%: %.test - cp $< $@ - -foo.ooo.test: - touch $@ diff --git a/build/pymake/tests/matchany2.mk b/build/pymake/tests/matchany2.mk deleted file mode 100644 index d21d9702c..000000000 --- a/build/pymake/tests/matchany2.mk +++ /dev/null @@ -1,13 +0,0 @@ -# we should succeed in making foo.ooo from foo.ooo.test -all: foo.ooo - @echo TEST-PASS - -%.ooo: %.ccc - exit 1 - -# this match-anything rule is terminal, and therefore applies -%:: %.test - cp $< $@ - -foo.ooo.test: - touch $@ diff --git a/build/pymake/tests/matchany3.mk b/build/pymake/tests/matchany3.mk deleted file mode 100644 index 83de8af2b..000000000 --- a/build/pymake/tests/matchany3.mk +++ /dev/null @@ -1,10 +0,0 @@ -$(shell \ -echo "target" > target.in; \ -) - -all: target - test "$$(cat $^)" = "target" - @echo TEST-PASS - -%: %.in - cp $< $@ diff --git a/build/pymake/tests/mkdir-fail.mk b/build/pymake/tests/mkdir-fail.mk deleted file mode 100644 index b05734aa9..000000000 --- a/build/pymake/tests/mkdir-fail.mk +++ /dev/null @@ -1,7 +0,0 @@ -#T returncode: 2 -all: - mkdir newdir/subdir - test ! -d newdir/subdir - test ! -d newdir - rm -r newdir - @echo TEST-PASS diff --git a/build/pymake/tests/mkdir.mk b/build/pymake/tests/mkdir.mk deleted file mode 100644 index 413348f77..000000000 --- a/build/pymake/tests/mkdir.mk +++ /dev/null @@ -1,27 +0,0 @@ -MKDIR ?= mkdir - -all: - $(MKDIR) newdir - test -d newdir - # subdir, parent exists - $(MKDIR) newdir/subdir - test -d newdir/subdir - # -p, existing dir - $(MKDIR) -p newdir - # -p, existing subdir - $(MKDIR) -p newdir/subdir - # multiple subdirs, existing parent - $(MKDIR) newdir/subdir1 newdir/subdir2 - test -d newdir/subdir1 -a -d newdir/subdir2 - rm -r newdir - # -p, subdir, no existing parent - $(MKDIR) -p newdir/subdir - test -d newdir/subdir - rm -r newdir - # -p, multiple subdirs, no existing parent - $(MKDIR) -p newdir/subdir1 newdir/subdir2 - test -d newdir/subdir1 -a -d newdir/subdir2 - # -p, multiple existing subdirs - $(MKDIR) -p newdir/subdir1 newdir/subdir2 - rm -r newdir - @echo TEST-PASS diff --git a/build/pymake/tests/multiple-rules-prerequisite-merge.mk b/build/pymake/tests/multiple-rules-prerequisite-merge.mk deleted file mode 100644 index 480d3b58c..000000000 --- a/build/pymake/tests/multiple-rules-prerequisite-merge.mk +++ /dev/null @@ -1,25 +0,0 @@ -# When a target is defined multiple times, the prerequisites should get -# merged. - -default: foo bar baz - -foo: - test "$<" = "foo.in1" - @echo TEST-PASS - -foo: foo.in1 - -bar: bar.in1 - test "$<" = "bar.in1" - test "$^" = "bar.in1 bar.in2" - @echo TEST-PASS - -bar: bar.in2 - -baz: baz.in2 -baz: baz.in1 - test "$<" = "baz.in1" - test "$^" = "baz.in1 baz.in2" - @echo TEST-PASS - -foo.in1 bar.in1 bar.in2 baz.in1 baz.in2: diff --git a/build/pymake/tests/native-command-delay-load.mk b/build/pymake/tests/native-command-delay-load.mk deleted file mode 100644 index a9f3774eb..000000000 --- a/build/pymake/tests/native-command-delay-load.mk +++ /dev/null @@ -1,12 +0,0 @@ -#T gmake skip - -# This test exists to verify that sys.path is adjusted during command -# execution and that delay importing a module will work. - -CMD = %pycmd delayloadfn -PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir - -all: - $(CMD) - @echo TEST-PASS - diff --git a/build/pymake/tests/native-command-raise.mk b/build/pymake/tests/native-command-raise.mk deleted file mode 100644 index d1b28b331..000000000 --- a/build/pymake/tests/native-command-raise.mk +++ /dev/null @@ -1,9 +0,0 @@ -#T gmake skip -#T returncode: 2 -#T grep-for: "Exception: info-exception" - -CMD = %pycmd asplode_raise -PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir - -all: - @$(CMD) info-exception diff --git a/build/pymake/tests/native-command-return-fail1.mk b/build/pymake/tests/native-command-return-fail1.mk deleted file mode 100644 index 0cf085ae2..000000000 --- a/build/pymake/tests/native-command-return-fail1.mk +++ /dev/null @@ -1,8 +0,0 @@ -#T gmake skip -#T returncode: 2 - -CMD = %pycmd asplode_return -PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir - -all: - $(CMD) 1 diff --git a/build/pymake/tests/native-command-return-fail2.mk b/build/pymake/tests/native-command-return-fail2.mk deleted file mode 100644 index c071fc879..000000000 --- a/build/pymake/tests/native-command-return-fail2.mk +++ /dev/null @@ -1,8 +0,0 @@ -#T gmake skip -#T returncode: 2 - -CMD = %pycmd asplode_return -PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir - -all: - $(CMD) not-an-integer diff --git a/build/pymake/tests/native-command-return.mk b/build/pymake/tests/native-command-return.mk deleted file mode 100644 index 3e4d2e0c4..000000000 --- a/build/pymake/tests/native-command-return.mk +++ /dev/null @@ -1,11 +0,0 @@ -#T gmake skip - -CMD = %pycmd asplode_return -PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir - -all: - $(CMD) 0 - -$(CMD) 1 - $(CMD) None - -$(CMD) not-an-integer - @echo TEST-PASS diff --git a/build/pymake/tests/native-command-shell-glob.mk b/build/pymake/tests/native-command-shell-glob.mk deleted file mode 100644 index 4bcdad8b9..000000000 --- a/build/pymake/tests/native-command-shell-glob.mk +++ /dev/null @@ -1,11 +0,0 @@ -#T gmake skip -all: - mkdir shell-glob-test - touch shell-glob-test/foo.txt - touch shell-glob-test/bar.txt - touch shell-glob-test/a.foo - touch shell-glob-test/b.foo - $(RM) shell-glob-test/*.txt - $(RM) shell-glob-test/?.foo - rmdir shell-glob-test - @echo TEST-PASS diff --git a/build/pymake/tests/native-command-sys-exit-fail1.mk b/build/pymake/tests/native-command-sys-exit-fail1.mk deleted file mode 100644 index 8e74800ed..000000000 --- a/build/pymake/tests/native-command-sys-exit-fail1.mk +++ /dev/null @@ -1,8 +0,0 @@ -#T gmake skip -#T returncode: 2 - -CMD = %pycmd asplode -PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir - -all: - $(CMD) 1 diff --git a/build/pymake/tests/native-command-sys-exit-fail2.mk b/build/pymake/tests/native-command-sys-exit-fail2.mk deleted file mode 100644 index 0a04395ad..000000000 --- a/build/pymake/tests/native-command-sys-exit-fail2.mk +++ /dev/null @@ -1,8 +0,0 @@ -#T gmake skip -#T returncode: 2 - -CMD = %pycmd asplode -PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir - -all: - $(CMD) not-an-integer diff --git a/build/pymake/tests/native-command-sys-exit.mk b/build/pymake/tests/native-command-sys-exit.mk deleted file mode 100644 index c04913aca..000000000 --- a/build/pymake/tests/native-command-sys-exit.mk +++ /dev/null @@ -1,11 +0,0 @@ -#T gmake skip - -CMD = %pycmd asplode -PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir - -all: - $(CMD) 0 - -$(CMD) 1 - $(CMD) None - -$(CMD) not-an-integer - @echo TEST-PASS diff --git a/build/pymake/tests/native-environment.mk b/build/pymake/tests/native-environment.mk deleted file mode 100644 index 36bd5894a..000000000 --- a/build/pymake/tests/native-environment.mk +++ /dev/null @@ -1,11 +0,0 @@ -#T gmake skip -export EXPECTED := some data - -PYCOMMANDPATH = $(TESTPATH) - -all: - %pycmd writeenvtofile results EXPECTED - test "$$(cat results)" = "$(EXPECTED)" - %pycmd writesubprocessenvtofile results EXPECTED - test "$$(cat results)" = "$(EXPECTED)" - @echo TEST-PASS diff --git a/build/pymake/tests/native-pycommandpath-sep.mk b/build/pymake/tests/native-pycommandpath-sep.mk deleted file mode 100644 index b1c2c2b97..000000000 --- a/build/pymake/tests/native-pycommandpath-sep.mk +++ /dev/null @@ -1,21 +0,0 @@ -#T gmake skip -EXPECTED := some data - -# verify that we can load native command modules from -# multiple directories in PYCOMMANDPATH separated by the native -# path separator -ifdef __WIN32__ -PS:=; -else -PS:=: -endif -CMD = %pycmd writetofile -CMD2 = %pymod writetofile -PYCOMMANDPATH = $(TESTPATH)$(PS)$(TESTPATH)/subdir - -all: - $(CMD) results $(EXPECTED) - test "$$(cat results)" = "$(EXPECTED)" - $(CMD2) results2 $(EXPECTED) - test "$$(cat results2)" = "$(EXPECTED)" - @echo TEST-PASS diff --git a/build/pymake/tests/native-pycommandpath.mk b/build/pymake/tests/native-pycommandpath.mk deleted file mode 100644 index dd0fbc9f9..000000000 --- a/build/pymake/tests/native-pycommandpath.mk +++ /dev/null @@ -1,15 +0,0 @@ -#T gmake skip -EXPECTED := some data - -# verify that we can load native command modules from -# multiple space-separated directories in PYCOMMANDPATH -CMD = %pycmd writetofile -CMD2 = %pymod writetofile -PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir - -all: - $(CMD) results $(EXPECTED) - test "$$(cat results)" = "$(EXPECTED)" - $(CMD2) results2 $(EXPECTED) - test "$$(cat results2)" = "$(EXPECTED)" - @echo TEST-PASS diff --git a/build/pymake/tests/native-simple.mk b/build/pymake/tests/native-simple.mk deleted file mode 100644 index 626a58670..000000000 --- a/build/pymake/tests/native-simple.mk +++ /dev/null @@ -1,12 +0,0 @@ -ifndef TOUCH -TOUCH = touch -endif - -all: testfile {testfile2} (testfile3) - test -f testfile - test -f {testfile2} - test -f "(testfile3)" - @echo TEST-PASS - -testfile {testfile2} (testfile3): - $(TOUCH) "$@" diff --git a/build/pymake/tests/native-touch.mk b/build/pymake/tests/native-touch.mk deleted file mode 100644 index 811161ece..000000000 --- a/build/pymake/tests/native-touch.mk +++ /dev/null @@ -1,15 +0,0 @@ -TOUCH ?= touch - -foo: - $(TOUCH) bar - $(TOUCH) baz - $(MAKE) -f $(TESTPATH)/native-touch.mk baz - $(TOUCH) -t 198007040802 baz - $(MAKE) -f $(TESTPATH)/native-touch.mk baz - -bar: - $(TOUCH) $@ - -baz: bar - echo TEST-PASS - $(TOUCH) $@ diff --git a/build/pymake/tests/newlines.mk b/build/pymake/tests/newlines.mk deleted file mode 100644 index 5d8195c94..000000000 --- a/build/pymake/tests/newlines.mk +++ /dev/null @@ -1,30 +0,0 @@ -#T gmake skip - -# Test that we handle \\\n properly - -all: dep1 dep2 dep3 - cat testfile - test `cat testfile` = "data"; - test "$$(cat results)" = "$(EXPECTED)"; - @echo TEST-PASS - -# Test that something that still needs to go to the shell works -testfile: - printf "data" \ - >>$@ - -dep1: testfile - -# Test that something that does not need to go to the shell works -dep2: - $(echo foo) \ - $(echo bar) - -export EXPECTED := some data - -CMD = %pycmd writeenvtofile -PYCOMMANDPATH = $(TESTPATH) - -dep3: - $(CMD) \ - results EXPECTED diff --git a/build/pymake/tests/no-remake.mk b/build/pymake/tests/no-remake.mk deleted file mode 100644 index c8df81bc3..000000000 --- a/build/pymake/tests/no-remake.mk +++ /dev/null @@ -1,7 +0,0 @@ -$(shell date >testfile) - -all: testfile - @echo TEST-PASS - -testfile: - @echo TEST-FAIL "We shouldn't have remade this!" diff --git a/build/pymake/tests/nosuchfile.mk b/build/pymake/tests/nosuchfile.mk deleted file mode 100644 index cca9ce1e9..000000000 --- a/build/pymake/tests/nosuchfile.mk +++ /dev/null @@ -1,4 +0,0 @@ -#T returncode: 2 - -all: - reallythereisnosuchcommand diff --git a/build/pymake/tests/notargets.mk b/build/pymake/tests/notargets.mk deleted file mode 100644 index 8e55d944f..000000000 --- a/build/pymake/tests/notargets.mk +++ /dev/null @@ -1,5 +0,0 @@ -$(NULL): foo.c - @echo TEST-FAIL - -all: - @echo TEST-PASS diff --git a/build/pymake/tests/notparallel.mk b/build/pymake/tests/notparallel.mk deleted file mode 100644 index 4fd8b1a8d..000000000 --- a/build/pymake/tests/notparallel.mk +++ /dev/null @@ -1,8 +0,0 @@ -#T commandline: ['-j3'] - -include $(TESTPATH)/serial-rule-execution.mk - -all:: - $(MAKE) -f $(TESTPATH)/parallel-simple.mk - -.NOTPARALLEL: diff --git a/build/pymake/tests/oneline-command-continuations.mk b/build/pymake/tests/oneline-command-continuations.mk deleted file mode 100644 index c11f3df52..000000000 --- a/build/pymake/tests/oneline-command-continuations.mk +++ /dev/null @@ -1,5 +0,0 @@ -all: test - @echo TEST-PASS - -test: ; test "Hello \ - world" = "Hello world" diff --git a/build/pymake/tests/override-propagate.mk b/build/pymake/tests/override-propagate.mk deleted file mode 100644 index a1663ff41..000000000 --- a/build/pymake/tests/override-propagate.mk +++ /dev/null @@ -1,37 +0,0 @@ -#T commandline: ['-w', 'OVAR=oval'] - -OVAR=mval - -all: vartest run-override - $(MAKE) -f $(TESTPATH)/override-propagate.mk vartest - @echo TEST-PASS - -CLINE := OVAR=oval TESTPATH=$(TESTPATH) NATIVE_TESTPATH=$(NATIVE_TESTPATH) -ifdef __WIN32__ -CLINE += __WIN32__=1 -endif - -SORTED_CLINE := $(subst \,\\,$(sort $(CLINE))) - -vartest: - @echo MAKELEVEL: '$(MAKELEVEL)' - test '$(value MAKEFLAGS)' = 'w -- $$(MAKEOVERRIDES)' - test '$(origin MAKEFLAGS)' = 'file' - test '$(value MAKEOVERRIDES)' = '$${-*-command-variables-*-}' - test "$(sort $(MAKEOVERRIDES))" = "$(SORTED_CLINE)" - test '$(origin MAKEOVERRIDES)' = 'environment' - test '$(origin -*-command-variables-*-)' = 'automatic' - test "$(origin OVAR)" = "command line" - test "$(OVAR)" = "oval" - -run-override: MAKEOVERRIDES= -run-override: - test "$(OVAR)" = "oval" - $(MAKE) -f $(TESTPATH)/override-propagate.mk otest - -otest: - test '$(value MAKEFLAGS)' = 'w' - test '$(value MAKEOVERRIDES)' = '$${-*-command-variables-*-}' - test '$(MAKEOVERRIDES)' = '' - test '$(origin -*-command-variables-*-)' = 'undefined' - test "$(OVAR)" = "mval" diff --git a/build/pymake/tests/parallel-dep-resolution.mk b/build/pymake/tests/parallel-dep-resolution.mk deleted file mode 100644 index 7967eba2d..000000000 --- a/build/pymake/tests/parallel-dep-resolution.mk +++ /dev/null @@ -1,8 +0,0 @@ -#T commandline: ['-j3'] -#T returncode: 2 - -all: t1 t2 - -t1: - sleep 1 - touch t1 t2 diff --git a/build/pymake/tests/parallel-dep-resolution2.mk b/build/pymake/tests/parallel-dep-resolution2.mk deleted file mode 100644 index 7d61e6b3e..000000000 --- a/build/pymake/tests/parallel-dep-resolution2.mk +++ /dev/null @@ -1,9 +0,0 @@ -#T commandline: ['-j3'] -#T returncode: 2 - -all:: - sleep 1 - touch somefile - -all:: somefile - @echo TEST-PASS diff --git a/build/pymake/tests/parallel-native.mk b/build/pymake/tests/parallel-native.mk deleted file mode 100644 index d50cfbdbb..000000000 --- a/build/pymake/tests/parallel-native.mk +++ /dev/null @@ -1,21 +0,0 @@ -#T commandline: ['-j2'] - -# ensure that calling python commands doesn't block other targets -ifndef SLEEP -SLEEP := sleep -endif - -PRINTF = printf "$@:0:" >>results -EXPECTED = target2:0:target1:0: - -all:: target1 target2 - cat results - test "$$(cat results)" = "$(EXPECTED)" - @echo TEST-PASS - -target1: - $(SLEEP) 0.1 - $(PRINTF) - -target2: - $(PRINTF) diff --git a/build/pymake/tests/parallel-simple.mk b/build/pymake/tests/parallel-simple.mk deleted file mode 100644 index f1aafc5f1..000000000 --- a/build/pymake/tests/parallel-simple.mk +++ /dev/null @@ -1,27 +0,0 @@ -#T commandline: ['-j2'] - -# CAUTION: this makefile is also used by serial-toparallel.mk - -define SLOWMAKE -printf "$@:0:" >>results -sleep 0.5 -printf "$@:1:" >>results -sleep 0.5 -printf "$@:2:" >>results -endef - -EXPECTED = target1:0:target2:0:target1:1:target2:1:target1:2:target2:2: - -all:: target1 target2 - cat results - test "$$(cat results)" = "$(EXPECTED)" - @echo TEST-PASS - -target1: - $(SLOWMAKE) - -target2: - sleep 0.1 - $(SLOWMAKE) - -.PHONY: all diff --git a/build/pymake/tests/parallel-submake.mk b/build/pymake/tests/parallel-submake.mk deleted file mode 100644 index 65cb2cf7c..000000000 --- a/build/pymake/tests/parallel-submake.mk +++ /dev/null @@ -1,17 +0,0 @@ -#T commandline: ['-j2'] - -# A submake shouldn't return control to the parent until it has actually finished doing everything. - -all: - -$(MAKE) -f $(TESTPATH)/parallel-submake.mk subtarget - cat results - test "$$(cat results)" = "0123" - @echo TEST-PASS - -subtarget: succeed-slowly fail-quickly - -succeed-slowly: - printf 0 >>results; sleep 1; printf 1 >>results; sleep 1; printf 2 >>results; sleep 1; printf 3 >>results - -fail-quickly: - exit 1 diff --git a/build/pymake/tests/parallel-toserial.mk b/build/pymake/tests/parallel-toserial.mk deleted file mode 100644 index 9a355eb33..000000000 --- a/build/pymake/tests/parallel-toserial.mk +++ /dev/null @@ -1,31 +0,0 @@ -#T commandline: ['-j4'] - -# Test that -j1 in a submake has the proper effect. - -define SLOWCMD -printf "$@:0:" >>$(RFILE) -sleep 0.5 -printf "$@:1:" >>$(RFILE) -endef - -all: p1 p2 -subtarget: s1 s2 - -p1 p2: RFILE = presult -s1 s2: RFILE = sresult - -p1 s1: - $(SLOWCMD) - -p2 s2: - sleep 0.1 - $(SLOWCMD) - -all: - $(MAKE) -j1 -f $(TESTPATH)/parallel-toserial.mk subtarget - printf "presult: %s\n" "$$(cat presult)" - test "$$(cat presult)" = "p1:0:p2:0:p1:1:p2:1:" - printf "sresult: %s\n" "$$(cat sresult)" - test "$$(cat sresult)" = "s1:0:s1:1:s2:0:s2:1:" - @echo TEST-PASS - diff --git a/build/pymake/tests/parallel-waiting.mk b/build/pymake/tests/parallel-waiting.mk deleted file mode 100644 index 40a6e0d50..000000000 --- a/build/pymake/tests/parallel-waiting.mk +++ /dev/null @@ -1,21 +0,0 @@ -#T commandline: ['-j2'] - -EXPECTED = target1:before:target2:1:target2:2:target2:3:target1:after - -all:: target1 target2 - cat results - test "$$(cat results)" = "$(EXPECTED)" - @echo TEST-PASS - -target1: - printf "$@:before:" >>results - sleep 4 - printf "$@:after" >>results - -target2: - sleep 0.2 - printf "$@:1:" >>results - sleep 0.1 - printf "$@:2:" >>results - sleep 0.1 - printf "$@:3:" >>results diff --git a/build/pymake/tests/parentheses.mk b/build/pymake/tests/parentheses.mk deleted file mode 100644 index f207234ff..000000000 --- a/build/pymake/tests/parentheses.mk +++ /dev/null @@ -1,2 +0,0 @@ -all: - @(echo TEST-PASS) diff --git a/build/pymake/tests/parsertests.py b/build/pymake/tests/parsertests.py deleted file mode 100644 index ab6406be0..000000000 --- a/build/pymake/tests/parsertests.py +++ /dev/null @@ -1,314 +0,0 @@ -import pymake.data, pymake.parser, pymake.parserdata, pymake.functions -import unittest -import logging - -from cStringIO import StringIO - -def multitest(cls): - for name in cls.testdata.iterkeys(): - def m(self, name=name): - return self.runSingle(*self.testdata[name]) - - setattr(cls, 'test_%s' % name, m) - return cls - -class TestBase(unittest.TestCase): - def assertEqual(self, a, b, msg=""): - """Actually print the values which weren't equal, if things don't work out!""" - unittest.TestCase.assertEqual(self, a, b, "%s got %r expected %r" % (msg, a, b)) - -class DataTest(TestBase): - testdata = { - 'oneline': - ("He\tllo", "f", 1, 0, - ((0, "f", 1, 0), (2, "f", 1, 2), (3, "f", 1, 4))), - 'twoline': - ("line1 \n\tl\tine2", "f", 1, 4, - ((0, "f", 1, 4), (5, "f", 1, 9), (6, "f", 1, 10), (7, "f", 2, 0), (8, "f", 2, 4), (10, "f", 2, 8), (13, "f", 2, 11))), - } - - def runSingle(self, data, filename, line, col, results): - d = pymake.parser.Data(data, 0, len(data), pymake.parserdata.Location(filename, line, col)) - for pos, file, lineno, col in results: - loc = d.getloc(pos) - self.assertEqual(loc.path, file, "data file offset %i" % pos) - self.assertEqual(loc.line, lineno, "data line offset %i" % pos) - self.assertEqual(loc.column, col, "data col offset %i" % pos) -multitest(DataTest) - -class LineEnumeratorTest(TestBase): - testdata = { - 'simple': ( - 'Hello, world', [ - ('Hello, world', 1), - ] - ), - 'multi': ( - 'Hello\nhappy \n\nworld\n', [ - ('Hello', 1), - ('happy ', 2), - ('', 3), - ('world', 4), - ('', 5), - ] - ), - 'continuation': ( - 'Hello, \\\n world\nJellybeans!', [ - ('Hello, \\\n world', 1), - ('Jellybeans!', 3), - ] - ), - 'multislash': ( - 'Hello, \\\\\n world', [ - ('Hello, \\\\', 1), - (' world', 2), - ] - ) - } - - def runSingle(self, s, lines): - gotlines = [(d.s[d.lstart:d.lend], d.loc.line) for d in pymake.parser.enumeratelines(s, 'path')] - self.assertEqual(gotlines, lines) - -multitest(LineEnumeratorTest) - -class IterTest(TestBase): - testdata = { - 'plaindata': ( - pymake.parser.iterdata, - "plaindata # test\n", - "plaindata # test\n" - ), - 'makecomment': ( - pymake.parser.itermakefilechars, - "VAR = val # comment", - "VAR = val " - ), - 'makeescapedcomment': ( - pymake.parser.itermakefilechars, - "VAR = val \# escaped hash", - "VAR = val # escaped hash" - ), - 'makeescapedslash': ( - pymake.parser.itermakefilechars, - "VAR = val\\\\", - "VAR = val\\\\", - ), - 'makecontinuation': ( - pymake.parser.itermakefilechars, - "VAR = VAL \\\n continuation # comment \\\n continuation", - "VAR = VAL continuation " - ), - 'makecontinuation2': ( - pymake.parser.itermakefilechars, - "VAR = VAL \\ \\\n continuation", - "VAR = VAL \\ continuation" - ), - 'makeawful': ( - pymake.parser.itermakefilechars, - "VAR = VAL \\\\# comment\n", - "VAR = VAL \\" - ), - 'command': ( - pymake.parser.itercommandchars, - "echo boo # comment", - "echo boo # comment", - ), - 'commandcomment': ( - pymake.parser.itercommandchars, - "echo boo \# comment", - "echo boo \# comment", - ), - 'commandcontinue': ( - pymake.parser.itercommandchars, - "echo boo # \\\n\t command 2", - "echo boo # \\\n command 2" - ), - } - - def runSingle(self, ifunc, idata, expected): - d = pymake.parser.Data.fromstring(idata, 'IterTest data') - - it = pymake.parser._alltokens.finditer(d.s, 0, d.lend) - actual = ''.join( [c for c, t, o, oo in ifunc(d, 0, ('dummy-token',), it)] ) - self.assertEqual(actual, expected) - - if ifunc == pymake.parser.itermakefilechars: - print "testing %r" % expected - self.assertEqual(pymake.parser.flattenmakesyntax(d, 0), expected) - -multitest(IterTest) - - -# 'define': ( -# pymake.parser.iterdefinechars, -# "endef", -# "" -# ), -# 'definenesting': ( -# pymake.parser.iterdefinechars, -# """define BAR # comment -#random text -#endef not what you think! -#endef # comment is ok\n""", -# """define BAR # comment -#random text -#endef not what you think!""" -# ), -# 'defineescaped': ( -# pymake.parser.iterdefinechars, -# """value \\ -#endef -#endef\n""", -# "value endef" -# ), - -class MakeSyntaxTest(TestBase): - # (string, startat, stopat, stopoffset, expansion - testdata = { - 'text': ('hello world', 0, (), None, ['hello world']), - 'singlechar': ('hello $W', 0, (), None, - ['hello ', - {'type': 'VariableRef', - '.vname': ['W']} - ]), - 'stopat': ('hello: world', 0, (':', '='), 6, ['hello']), - 'funccall': ('h $(flavor FOO)', 0, (), None, - ['h ', - {'type': 'FlavorFunction', - '[0]': ['FOO']} - ]), - 'escapedollar': ('hello$$world', 0, (), None, ['hello$world']), - 'varref': ('echo $(VAR)', 0, (), None, - ['echo ', - {'type': 'VariableRef', - '.vname': ['VAR']} - ]), - 'dynamicvarname': ('echo $($(VARNAME):.c=.o)', 0, (':',), None, - ['echo ', - {'type': 'SubstitutionRef', - '.vname': [{'type': 'VariableRef', - '.vname': ['VARNAME']} - ], - '.substfrom': ['.c'], - '.substto': ['.o']} - ]), - 'substref': (' $(VAR:VAL) := $(VAL)', 0, (':=', '+=', '=', ':'), 15, - [' ', - {'type': 'VariableRef', - '.vname': ['VAR:VAL']}, - ' ']), - 'vadsubstref': (' $(VAR:VAL) = $(VAL)', 15, (), None, - [{'type': 'VariableRef', - '.vname': ['VAL']}, - ]), - } - - def compareRecursive(self, actual, expected, path): - self.assertEqual(len(actual), len(expected), - "compareRecursive: %s %r" % (path, actual)) - for i in xrange(0, len(actual)): - ipath = path + [i] - - a, isfunc = actual[i] - e = expected[i] - if isinstance(e, str): - self.assertEqual(a, e, "compareRecursive: %s" % (ipath,)) - else: - self.assertEqual(type(a), getattr(pymake.functions, e['type']), - "compareRecursive: %s" % (ipath,)) - for k, v in e.iteritems(): - if k == 'type': - pass - elif k[0] == '[': - item = int(k[1:-1]) - proppath = ipath + [item] - self.compareRecursive(a[item], v, proppath) - elif k[0] == '.': - item = k[1:] - proppath = ipath + [item] - self.compareRecursive(getattr(a, item), v, proppath) - else: - raise Exception("Unexpected property at %s: %s" % (ipath, k)) - - def runSingle(self, s, startat, stopat, stopoffset, expansion): - d = pymake.parser.Data.fromstring(s, pymake.parserdata.Location('testdata', 1, 0)) - - a, t, offset = pymake.parser.parsemakesyntax(d, startat, stopat, pymake.parser.itermakefilechars) - self.compareRecursive(a, expansion, []) - self.assertEqual(offset, stopoffset) - -multitest(MakeSyntaxTest) - -class VariableTest(TestBase): - testdata = """ - VAR = value - VARNAME = TESTVAR - $(VARNAME) = testvalue - $(VARNAME:VAR=VAL) = moretesting - IMM := $(VARNAME) # this is a comment - MULTIVAR = val1 \\ - val2 - VARNAME = newname - """ - expected = {'VAR': 'value', - 'VARNAME': 'newname', - 'TESTVAR': 'testvalue', - 'TESTVAL': 'moretesting', - 'IMM': 'TESTVAR ', - 'MULTIVAR': 'val1 val2', - 'UNDEF': None} - - def runTest(self): - stmts = pymake.parser.parsestring(self.testdata, 'VariableTest') - - m = pymake.data.Makefile() - stmts.execute(m) - for k, v in self.expected.iteritems(): - flavor, source, val = m.variables.get(k) - if val is None: - self.assertEqual(val, v, 'variable named %s' % k) - else: - self.assertEqual(val.resolvestr(m, m.variables), v, 'variable named %s' % k) - -class SimpleRuleTest(TestBase): - testdata = """ - VAR = value -TSPEC = dummy -all: TSPEC = myrule -all:: test test2 $(VAR) - echo "Hello, $(TSPEC)" - -%.o: %.c - $(CC) -o $@ $< -""" - - def runTest(self): - stmts = pymake.parser.parsestring(self.testdata, 'SimpleRuleTest') - - m = pymake.data.Makefile() - stmts.execute(m) - self.assertEqual(m.defaulttarget, 'all', "Default target") - - self.assertTrue(m.hastarget('all'), "Has 'all' target") - target = m.gettarget('all') - rules = target.rules - self.assertEqual(len(rules), 1, "Number of rules") - prereqs = rules[0].prerequisites - self.assertEqual(prereqs, ['test', 'test2', 'value'], "Prerequisites") - commands = rules[0].commands - self.assertEqual(len(commands), 1, "Number of commands") - expanded = commands[0].resolvestr(m, target.variables) - self.assertEqual(expanded, 'echo "Hello, myrule"') - - irules = m.implicitrules - self.assertEqual(len(irules), 1, "Number of implicit rules") - - irule = irules[0] - self.assertEqual(len(irule.targetpatterns), 1, "%.o target pattern count") - self.assertEqual(len(irule.prerequisites), 1, "%.o prerequisite count") - self.assertEqual(irule.targetpatterns[0].match('foo.o'), 'foo', "%.o stem") - -if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) - unittest.main() diff --git a/build/pymake/tests/path-length.mk b/build/pymake/tests/path-length.mk deleted file mode 100644 index 10c33b5ed..000000000 --- a/build/pymake/tests/path-length.mk +++ /dev/null @@ -1,9 +0,0 @@ -#T gmake skip - -$(shell \ -mkdir foo; \ -touch tfile; \ -) - -all: foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../tfile - @echo TEST-PASS diff --git a/build/pymake/tests/pathdir/pathtest b/build/pymake/tests/pathdir/pathtest deleted file mode 100755 index 17037159f..000000000 --- a/build/pymake/tests/pathdir/pathtest +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -echo Called shell script: 2f7cdd0b-7277-48c1-beaf-56cb0dbacb24 diff --git a/build/pymake/tests/pathdir/pathtest.exe b/build/pymake/tests/pathdir/pathtest.exe deleted file mode 100644 index 3178db9a9..000000000 Binary files a/build/pymake/tests/pathdir/pathtest.exe and /dev/null differ diff --git a/build/pymake/tests/pathdir/src/Makefile b/build/pymake/tests/pathdir/src/Makefile deleted file mode 100644 index 6c24bd8f9..000000000 --- a/build/pymake/tests/pathdir/src/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -pathtest.exe: pathtest.cpp - cl -EHsc -MT $^ diff --git a/build/pymake/tests/pathdir/src/pathtest.cpp b/build/pymake/tests/pathdir/src/pathtest.cpp deleted file mode 100644 index bef8d8a11..000000000 --- a/build/pymake/tests/pathdir/src/pathtest.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main() { - std::printf("Called Windows executable: 2f7cdd0b-7277-48c1-beaf-56cb0dbacb24\n"); - return 0; -} diff --git a/build/pymake/tests/patsubst.mk b/build/pymake/tests/patsubst.mk deleted file mode 100644 index 0c3efdc4b..000000000 --- a/build/pymake/tests/patsubst.mk +++ /dev/null @@ -1,7 +0,0 @@ -all: - test "$(patsubst foo,%.bar,foo)" = "%.bar" - test "$(patsubst \%word,replace,word %word other)" = "word replace other" - test "$(patsubst %.c,\%%.o,foo.c bar.o baz.cpp)" = "%foo.o bar.o baz.cpp" - test "$(patsubst host_%.c,host_%.o,dir/host_foo.c host_bar.c)" = "dir/host_foo.c host_bar.o" - test "$(patsubst foo,bar,dir/foo foo baz)" = "dir/foo bar baz" - @echo TEST-PASS diff --git a/build/pymake/tests/phony.mk b/build/pymake/tests/phony.mk deleted file mode 100644 index 36db4d121..000000000 --- a/build/pymake/tests/phony.mk +++ /dev/null @@ -1,10 +0,0 @@ -$(shell \ -touch dep; \ -sleep 2; \ -touch all; \ -) - -all:: dep - @echo TEST-PASS - -.PHONY: all diff --git a/build/pymake/tests/pycmd.py b/build/pymake/tests/pycmd.py deleted file mode 100644 index 83b9b966b..000000000 --- a/build/pymake/tests/pycmd.py +++ /dev/null @@ -1,38 +0,0 @@ -import os, sys, subprocess - -def writetofile(args): - with open(args[0], 'w') as f: - f.write(' '.join(args[1:])) - -def writeenvtofile(args): - with open(args[0], 'w') as f: - f.write(os.environ[args[1]]) - -def writesubprocessenvtofile(args): - with open(args[0], 'w') as f: - p = subprocess.Popen([sys.executable, "-c", - "import os; print os.environ['%s']" % args[1]], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() - assert p.returncode == 0 - f.write(stdout) - -def convertasplode(arg): - try: - return int(arg) - except: - return (None if arg == "None" else arg) - -def asplode(args): - arg0 = convertasplode(args[0]) - sys.exit(arg0) - -def asplode_return(args): - arg0 = convertasplode(args[0]) - return arg0 - -def asplode_raise(args): - raise Exception(args[0]) - -def delayloadfn(args): - import delayload diff --git a/build/pymake/tests/recursive-set.mk b/build/pymake/tests/recursive-set.mk deleted file mode 100644 index 853f90463..000000000 --- a/build/pymake/tests/recursive-set.mk +++ /dev/null @@ -1,7 +0,0 @@ -#T returncode: 2 - -FOO = $(FOO) - -all: - echo $(FOO) - @echo TEST-FAIL diff --git a/build/pymake/tests/recursive-set2.mk b/build/pymake/tests/recursive-set2.mk deleted file mode 100644 index b68e34f0d..000000000 --- a/build/pymake/tests/recursive-set2.mk +++ /dev/null @@ -1,8 +0,0 @@ -#T returncode: 2 - -FOO = $(BAR) -BAR = $(FOO) - -all: - echo $(FOO) - @echo TEST-FAIL diff --git a/build/pymake/tests/remake-mtime.mk b/build/pymake/tests/remake-mtime.mk deleted file mode 100644 index 47c775b93..000000000 --- a/build/pymake/tests/remake-mtime.mk +++ /dev/null @@ -1,14 +0,0 @@ -# mtime(dep1) < mtime(target) so the target should not be made -$(shell touch dep1; sleep 1; touch target) - -all: target - echo TEST-PASS - -target: dep1 - echo TEST-FAIL target should not have been made - -dep1: dep2 - @echo "Remaking dep1 (actually not)" - -dep2: - @echo "Making dep2 (actually not)" diff --git a/build/pymake/tests/rm-fail.mk b/build/pymake/tests/rm-fail.mk deleted file mode 100644 index 1a9aefb57..000000000 --- a/build/pymake/tests/rm-fail.mk +++ /dev/null @@ -1,7 +0,0 @@ -#T returncode: 2 -all: - mkdir newdir - test -d newdir - touch newdir/newfile - $(RM) newdir - @echo TEST-PASS diff --git a/build/pymake/tests/rm.mk b/build/pymake/tests/rm.mk deleted file mode 100644 index 6c7140e39..000000000 --- a/build/pymake/tests/rm.mk +++ /dev/null @@ -1,21 +0,0 @@ -all: -# $(RM) defaults to -f - $(RM) nosuchfile - touch newfile - test -f newfile - $(RM) newfile - test ! -f newfile - mkdir newdir - test -d newdir - touch newdir/newfile - mkdir newdir/subdir - $(RM) -r newdir/subdir - test ! -d newdir/subdir - test -d newdir - mkdir newdir/subdir1 newdir/subdir2 - $(RM) -r newdir/subdir1 newdir/subdir2 - test ! -d newdir/subdir1 -a ! -d newdir/subdir2 - test -d newdir - $(RM) -r newdir - test ! -d newdir - @echo TEST-PASS diff --git a/build/pymake/tests/runtests.py b/build/pymake/tests/runtests.py deleted file mode 100644 index ab149ecfb..000000000 --- a/build/pymake/tests/runtests.py +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env python -""" -Run the test(s) listed on the command line. If a directory is listed, the script will recursively -walk the directory for files named .mk and run each. - -For each test, we run gmake -f test.mk. By default, make must exit with an exit code of 0, and must print 'TEST-PASS'. - -Each test is run in an empty directory. - -The test file may contain lines at the beginning to alter the default behavior. These are all evaluated as python: - -#T commandline: ['extra', 'params', 'here'] -#T returncode: 2 -#T returncode-on: {'win32': 2} -#T environment: {'VAR': 'VALUE} -#T grep-for: "text" -""" - -from subprocess import Popen, PIPE, STDOUT -from optparse import OptionParser -import os, re, sys, shutil, glob - -class ParentDict(dict): - def __init__(self, parent, **kwargs): - self.d = dict(kwargs) - self.parent = parent - - def __setitem__(self, k, v): - self.d[k] = v - - def __getitem__(self, k): - if k in self.d: - return self.d[k] - - return self.parent[k] - -thisdir = os.path.dirname(os.path.abspath(__file__)) - -pymake = [sys.executable, os.path.join(os.path.dirname(thisdir), 'make.py')] -manifest = os.path.join(thisdir, 'tests.manifest') - -o = OptionParser() -o.add_option('-g', '--gmake', - dest="gmake", default="gmake") -o.add_option('-d', '--tempdir', - dest="tempdir", default="_mktests") -opts, args = o.parse_args() - -if len(args) == 0: - args = [thisdir] - -makefiles = [] -for a in args: - if os.path.isfile(a): - makefiles.append(a) - elif os.path.isdir(a): - makefiles.extend(sorted(glob.glob(os.path.join(a, '*.mk')))) - -def runTest(makefile, make, logfile, options): - """ - Given a makefile path, test it with a given `make` and return - (pass, message). - """ - - if os.path.exists(opts.tempdir): shutil.rmtree(opts.tempdir) - os.mkdir(opts.tempdir, 0755) - - logfd = open(logfile, 'w') - p = Popen(make + options['commandline'], stdout=logfd, stderr=STDOUT, env=options['env']) - logfd.close() - retcode = p.wait() - - if retcode != options['returncode']: - return False, "FAIL (returncode=%i)" % retcode - - logfd = open(logfile) - stdout = logfd.read() - logfd.close() - - if stdout.find('TEST-FAIL') != -1: - print stdout - return False, "FAIL (TEST-FAIL printed)" - - if options['grepfor'] and stdout.find(options['grepfor']) == -1: - print stdout - return False, "FAIL (%s not in output)" % options['grepfor'] - - if options['returncode'] == 0 and stdout.find('TEST-PASS') == -1: - print stdout - return False, 'FAIL (No TEST-PASS printed)' - - if options['returncode'] != 0: - return True, 'PASS (retcode=%s)' % retcode - - return True, 'PASS' - -print "%-30s%-28s%-28s" % ("Test:", "gmake:", "pymake:") - -gmakefails = 0 -pymakefails = 0 - -tre = re.compile('^#T (gmake |pymake )?([a-z-]+)(?:: (.*))?$') - -for makefile in makefiles: - # For some reason, MAKEFILE_LIST uses native paths in GNU make on Windows - # (even in MSYS!) so we pass both TESTPATH and NATIVE_TESTPATH - cline = ['-C', opts.tempdir, '-f', os.path.abspath(makefile), 'TESTPATH=%s' % thisdir.replace('\\','/'), 'NATIVE_TESTPATH=%s' % thisdir] - if sys.platform == 'win32': - #XXX: hack so we can specialize the separator character on windows. - # we really shouldn't need this, but y'know - cline += ['__WIN32__=1'] - - options = { - 'returncode': 0, - 'grepfor': None, - 'env': dict(os.environ), - 'commandline': cline, - 'pass': True, - 'skip': False, - } - - gmakeoptions = ParentDict(options) - pymakeoptions = ParentDict(options) - - dmap = {None: options, 'gmake ': gmakeoptions, 'pymake ': pymakeoptions} - - mdata = open(makefile) - for line in mdata: - line = line.strip() - m = tre.search(line) - if m is None: - break - - make, key, data = m.group(1, 2, 3) - d = dmap[make] - if data is not None: - data = eval(data) - if key == 'commandline': - assert make is None - d['commandline'].extend(data) - elif key == 'returncode': - d['returncode'] = data - elif key == 'returncode-on': - if sys.platform in data: - d['returncode'] = data[sys.platform] - elif key == 'environment': - for k, v in data.iteritems(): - d['env'][k] = v - elif key == 'grep-for': - d['grepfor'] = data - elif key == 'fail': - d['pass'] = False - elif key == 'skip': - d['skip'] = True - else: - print >>sys.stderr, "%s: Unexpected #T key: %s" % (makefile, key) - sys.exit(1) - - mdata.close() - - if gmakeoptions['skip']: - gmakepass, gmakemsg = True, '' - else: - gmakepass, gmakemsg = runTest(makefile, [opts.gmake], - makefile + '.gmakelog', gmakeoptions) - - if gmakeoptions['pass']: - if not gmakepass: - gmakefails += 1 - else: - if gmakepass: - gmakefails += 1 - gmakemsg = "UNEXPECTED PASS" - else: - gmakemsg = "KNOWN FAIL" - - if pymakeoptions['skip']: - pymakepass, pymakemsg = True, '' - else: - pymakepass, pymakemsg = runTest(makefile, pymake, - makefile + '.pymakelog', pymakeoptions) - - if pymakeoptions['pass']: - if not pymakepass: - pymakefails += 1 - else: - if pymakepass: - pymakefails += 1 - pymakemsg = "UNEXPECTED PASS" - else: - pymakemsg = "OK (known fail)" - - print "%-30.30s%-28.28s%-28.28s" % (os.path.basename(makefile), - gmakemsg, pymakemsg) - -print -print "Summary:" -print "%-30s%-28s%-28s" % ("", "gmake:", "pymake:") - -if gmakefails == 0: - gmakemsg = 'PASS' -else: - gmakemsg = 'FAIL (%i failures)' % gmakefails - -if pymakefails == 0: - pymakemsg = 'PASS' -else: - pymakemsg = 'FAIL (%i failures)' % pymakefails - -print "%-30.30s%-28.28s%-28.28s" % ('', gmakemsg, pymakemsg) - -shutil.rmtree(opts.tempdir) - -if gmakefails or pymakefails: - sys.exit(1) diff --git a/build/pymake/tests/serial-dep-resolution.mk b/build/pymake/tests/serial-dep-resolution.mk deleted file mode 100644 index e65f1ed03..000000000 --- a/build/pymake/tests/serial-dep-resolution.mk +++ /dev/null @@ -1,5 +0,0 @@ -all: t1 t2 - @echo TEST-PASS - -t1: - touch t1 t2 diff --git a/build/pymake/tests/serial-doublecolon-execution.mk b/build/pymake/tests/serial-doublecolon-execution.mk deleted file mode 100644 index 1871cb13a..000000000 --- a/build/pymake/tests/serial-doublecolon-execution.mk +++ /dev/null @@ -1,18 +0,0 @@ -#T commandline: ['-j3'] - -# Commands of double-colon rules are always executed in order. - -all: dc - cat status - test "$$(cat status)" = "all1:all2:" - @echo TEST-PASS - -dc:: slowt - printf "all1:" >> status - -dc:: - sleep 0.2 - printf "all2:" >> status - -slowt: - sleep 1 diff --git a/build/pymake/tests/serial-rule-execution.mk b/build/pymake/tests/serial-rule-execution.mk deleted file mode 100644 index da5b177de..000000000 --- a/build/pymake/tests/serial-rule-execution.mk +++ /dev/null @@ -1,5 +0,0 @@ -all:: - touch somefile - -all:: somefile - @echo TEST-PASS diff --git a/build/pymake/tests/serial-rule-execution2.mk b/build/pymake/tests/serial-rule-execution2.mk deleted file mode 100644 index 252a7df83..000000000 --- a/build/pymake/tests/serial-rule-execution2.mk +++ /dev/null @@ -1,13 +0,0 @@ -#T returncode: 2 - -# The dependencies of the command rule of a single-colon target are resolved before the rules without commands. - -all: export - -export: - sleep 1 - touch somefile - -all: somefile - test -f somefile - @echo TEST-PASS diff --git a/build/pymake/tests/serial-toparallel.mk b/build/pymake/tests/serial-toparallel.mk deleted file mode 100644 index a980badc7..000000000 --- a/build/pymake/tests/serial-toparallel.mk +++ /dev/null @@ -1,5 +0,0 @@ -all:: - $(MAKE) -j2 -f $(TESTPATH)/parallel-simple.mk - -all:: results - @echo TEST-PASS diff --git a/build/pymake/tests/shellfunc.mk b/build/pymake/tests/shellfunc.mk deleted file mode 100644 index 1e408dbac..000000000 --- a/build/pymake/tests/shellfunc.mk +++ /dev/null @@ -1,7 +0,0 @@ -all: testfile - test "$(shell cat $<)" = "Hello world" - test "$(shell printf "\n")" = "" - @echo TEST-PASS - -testfile: - printf "Hello\nworld\n" > $@ diff --git a/build/pymake/tests/simple-makeflags.mk b/build/pymake/tests/simple-makeflags.mk deleted file mode 100644 index c7c92ec9d..000000000 --- a/build/pymake/tests/simple-makeflags.mk +++ /dev/null @@ -1,10 +0,0 @@ -# There once was a time when MAKEFLAGS=w without any following spaces would -# cause us to treat w as a target, not a flag. Silly! - -MAKEFLAGS=w - -all: - $(MAKE) -f $(TESTPATH)/simple-makeflags.mk subt - @echo TEST-PASS - -subt: diff --git a/build/pymake/tests/sort.mk b/build/pymake/tests/sort.mk deleted file mode 100644 index e1313ad5c..000000000 --- a/build/pymake/tests/sort.mk +++ /dev/null @@ -1,4 +0,0 @@ -# sort should remove duplicates -all: - @test "$(sort x a y b z c a z b x c y)" = "a b c x y z" - @echo "TEST-PASS" diff --git a/build/pymake/tests/specified-target.mk b/build/pymake/tests/specified-target.mk deleted file mode 100644 index 3b23fbf69..000000000 --- a/build/pymake/tests/specified-target.mk +++ /dev/null @@ -1,7 +0,0 @@ -#T commandline: ['VAR=all', '$(VAR)'] - -all: - @echo TEST-FAIL: unexpected target 'all' - -$$(VAR): - @echo TEST-PASS: expected target '$$(VAR)' diff --git a/build/pymake/tests/static-pattern.mk b/build/pymake/tests/static-pattern.mk deleted file mode 100644 index f613b8c9a..000000000 --- a/build/pymake/tests/static-pattern.mk +++ /dev/null @@ -1,5 +0,0 @@ -#T returncode: 2 - -out/host_foo.o: host_%.o: host_%.c out - cp $< $@ - @echo TEST-FAIL diff --git a/build/pymake/tests/static-pattern2.mk b/build/pymake/tests/static-pattern2.mk deleted file mode 100644 index 08ed834fd..000000000 --- a/build/pymake/tests/static-pattern2.mk +++ /dev/null @@ -1,10 +0,0 @@ -all: foo.out - test -f $^ - @echo TEST-PASS - -foo.out: %.out: %.in - test "$*" = "foo" - cp $^ $@ - -foo.in: - touch $@ diff --git a/build/pymake/tests/subdir/delayload.py b/build/pymake/tests/subdir/delayload.py deleted file mode 100644 index bdd6669db..000000000 --- a/build/pymake/tests/subdir/delayload.py +++ /dev/null @@ -1 +0,0 @@ -# This module exists to test delay importing of modules at run-time. diff --git a/build/pymake/tests/subdir/pymod.py b/build/pymake/tests/subdir/pymod.py deleted file mode 100644 index 1a47d8af2..000000000 --- a/build/pymake/tests/subdir/pymod.py +++ /dev/null @@ -1,5 +0,0 @@ -import testmodule - -def writetofile(args): - with open(args[0], 'w') as f: - f.write(' '.join(args[1:])) diff --git a/build/pymake/tests/subdir/testmodule.py b/build/pymake/tests/subdir/testmodule.py deleted file mode 100644 index 05b2f821a..000000000 --- a/build/pymake/tests/subdir/testmodule.py +++ /dev/null @@ -1,3 +0,0 @@ -# This is an empty module. It is imported by pymod.py to test that if a module -# is loaded from the PYCOMMANDPATH, it can import other modules from the same -# directory correctly. diff --git a/build/pymake/tests/submake-path.makefile2 b/build/pymake/tests/submake-path.makefile2 deleted file mode 100644 index 1266db7d1..000000000 --- a/build/pymake/tests/submake-path.makefile2 +++ /dev/null @@ -1,11 +0,0 @@ -# -*- Mode: Makefile -*- - -shellresult := $(shell pathtest) -ifneq (2f7cdd0b-7277-48c1-beaf-56cb0dbacb24,$(filter $(shellresult),2f7cdd0b-7277-48c1-beaf-56cb0dbacb24)) -$(error pathtest not found in submake shell function) -endif - -all: - @pathtest - @pathtest | grep -q 2f7cdd0b-7277-48c1-beaf-56cb0dbacb24 - @echo TEST-PASS diff --git a/build/pymake/tests/submake-path.mk b/build/pymake/tests/submake-path.mk deleted file mode 100644 index b6432276d..000000000 --- a/build/pymake/tests/submake-path.mk +++ /dev/null @@ -1,16 +0,0 @@ -#T gmake skip -#T grep-for: "2f7cdd0b-7277-48c1-beaf-56cb0dbacb24" - -ifdef __WIN32__ -PS:=; -else -PS:=: -endif - -export PATH := $(TESTPATH)/pathdir$(PS)$(PATH) - -# This is similar to subprocess-path.mk, except we also check $(shell) -# invocations since they're affected by exported environment variables too, -# but only in submakes! -all: - $(MAKE) -f $(TESTPATH)/submake-path.makefile2 diff --git a/build/pymake/tests/submake.makefile2 b/build/pymake/tests/submake.makefile2 deleted file mode 100644 index 12ce94834..000000000 --- a/build/pymake/tests/submake.makefile2 +++ /dev/null @@ -1,24 +0,0 @@ -# -*- Mode: Makefile -*- - -$(info MAKEFLAGS = '$(MAKEFLAGS)') -$(info MAKE = '$(MAKE)') -$(info value MAKE = "$(value MAKE)") - -shellresult := $(shell echo -n $$EVAR) -ifneq ($(shellresult),eval) -$(error EVAR should be eval, is instead $(shellresult)) -endif - -all: - env - test "$(MAKELEVEL)" = "1" - echo "value(MAKE)" '$(value MAKE)' - echo "value(MAKE_COMMAND)" = '$(value MAKE_COMMAND)' - test "$(origin CVAR)" = "command line" - test "$(CVAR)" = "c val=spac\ed" - test "$(origin EVAR)" = "environment" - test "$(EVAR)" = "eval" - test "$(OVAL)" = "cline" - test "$(OVAL2)" = "cline2" - test "$(ALLVAR)" = "allspecific" - @echo TEST-PASS diff --git a/build/pymake/tests/submake.mk b/build/pymake/tests/submake.mk deleted file mode 100644 index 41e47134b..000000000 --- a/build/pymake/tests/submake.mk +++ /dev/null @@ -1,16 +0,0 @@ -#T commandline: ['CVAR=c val=spac\\ed', 'OVAL=cline', 'OVAL2=cline2'] - -export EVAR = eval -override OVAL = makefile - -# exporting an override variable doesn't mean it's an override variable -override OVAL2 = makefile2 -export OVAL2 - -export ALLVAR -ALLVAR = general -all: ALLVAR = allspecific - -all: - test "$(MAKELEVEL)" = "0" - $(MAKE) -f $(TESTPATH)/submake.makefile2 diff --git a/build/pymake/tests/subprocess-path.mk b/build/pymake/tests/subprocess-path.mk deleted file mode 100644 index f63921414..000000000 --- a/build/pymake/tests/subprocess-path.mk +++ /dev/null @@ -1,32 +0,0 @@ -#T gmake skip -#T grep-for: "2f7cdd0b-7277-48c1-beaf-56cb0dbacb24" - -ifdef __WIN32__ -PS:=; -else -PS:=: -endif - -export PATH := $(TESTPATH)/pathdir$(PS)$(PATH) - -# Test two commands. The first one shouldn't go through the shell and the -# second one should. The pathdir subdirectory has a Windows executable called -# pathtest.exe and a shell script called pathtest. We don't care which one is -# run, just that one of the two is (we use a uuid + grep-for to make sure -# that happens). -# -# FAQ: -# Q. Why skip GNU Make? -# A. Because $(TESTPATH) is a Windows-style path, and MSYS make doesn't take -# too kindly to Windows paths in the PATH environment variable. -# -# Q. Why use an exe and not a batch file? -# A. The use cases here were all exe files without the extension. Batch file -# lookup has broken semantics if the .bat extension isn't passed. -# -# Q. Why are the commands silent? -# A. So that we don't pass the grep-for test by mistake. -all: - @pathtest - @pathtest | grep -q 2f7cdd0b-7277-48c1-beaf-56cb0dbacb24 - @echo TEST-PASS diff --git a/build/pymake/tests/tab-intro.mk b/build/pymake/tests/tab-intro.mk deleted file mode 100644 index 1c25ce747..000000000 --- a/build/pymake/tests/tab-intro.mk +++ /dev/null @@ -1,16 +0,0 @@ -# Initial tab characters should be treated well. - - THIS = a value - - ifdef THIS - VAR = conditional value - endif - -all: - test "$(THIS)" = "another value" - test "$(VAR)" = "conditional value" - @echo TEST-PASS - -THAT = makefile syntax - - THIS = another value diff --git a/build/pymake/tests/target-specific.mk b/build/pymake/tests/target-specific.mk deleted file mode 100644 index 217ed155e..000000000 --- a/build/pymake/tests/target-specific.mk +++ /dev/null @@ -1,30 +0,0 @@ -TESTVAR = anonval - -all: target.suffix target.suffix2 dummy host_test.py my.test1 my.test2 - @echo TEST-PASS - -target.suffix: TESTVAR = testval - -%.suffix: - test "$(TESTVAR)" = "testval" - -%.suffix2: TESTVAR = testval2 - -%.suffix2: - test "$(TESTVAR)" = "testval2" - -%my: TESTVAR = dummyval - -dummy: - test "$(TESTVAR)" = "dummyval" - -%.py: TESTVAR = pyval -host_%.py: TESTVAR = hostval - -host_test.py: - test "$(TESTVAR)" = "hostval" - -%.test1 %.test2: TESTVAR = %val - -my.test1 my.test2: - test "$(TESTVAR)" = "%val" diff --git a/build/pymake/tests/unexport.mk b/build/pymake/tests/unexport.mk deleted file mode 100644 index 424411603..000000000 --- a/build/pymake/tests/unexport.mk +++ /dev/null @@ -1,15 +0,0 @@ -#T environment: {'ENVVAR': 'envval'} - -VAR1 = val1 -VAR2 = val2 -VAR3 = val3 - -unexport VAR3 -export VAR1 VAR2 VAR3 -unexport VAR2 ENVVAR -unexport - -all: - test "$(ENVVAR)" = "envval" # unexport.mk - $(MAKE) -f $(TESTPATH)/unexport.submk - @echo TEST-PASS diff --git a/build/pymake/tests/unexport.submk b/build/pymake/tests/unexport.submk deleted file mode 100644 index 8db6163de..000000000 --- a/build/pymake/tests/unexport.submk +++ /dev/null @@ -1,15 +0,0 @@ -# -@- Mode: Makefile -@- - -unexport VAR1 - -all: - env - test "$(VAR1)" = "val1" - test "$(origin VAR1)" = "environment" - test "$(VAR2)" = "" # VAR2 - test "$(VAR3)" = "val3" - test "$(ENVVAR)" = "" - $(MAKE) -f $(TESTPATH)/unexport.submk subt - -subt: - test "$(VAR1)" = "" diff --git a/build/pymake/tests/unterminated-dollar.mk b/build/pymake/tests/unterminated-dollar.mk deleted file mode 100644 index dee9a207b..000000000 --- a/build/pymake/tests/unterminated-dollar.mk +++ /dev/null @@ -1,6 +0,0 @@ -VAR = value$ -VAR2 = other - -all: - test "$(VAR)" = "value" - @echo TEST-PASS diff --git a/build/pymake/tests/var-change-flavor.mk b/build/pymake/tests/var-change-flavor.mk deleted file mode 100644 index 0cccf0bd6..000000000 --- a/build/pymake/tests/var-change-flavor.mk +++ /dev/null @@ -1,12 +0,0 @@ -VAR = value1 -VAR := value2 - -VAR2 := val1 -VAR2 = val2 - -default: - test "$(flavor VAR)" = "simple" - test "$(VAR)" = "value2" - test "$(flavor VAR2)" = "recursive" - test "$(VAR2)" = "val2" - @echo "TEST-PASS" diff --git a/build/pymake/tests/var-commandline.mk b/build/pymake/tests/var-commandline.mk deleted file mode 100644 index e2cdad457..000000000 --- a/build/pymake/tests/var-commandline.mk +++ /dev/null @@ -1,8 +0,0 @@ -#T commandline: ['TESTVAR=$(MAKEVAL)', 'TESTVAR2:=$(MAKEVAL)'] - -MAKEVAL=testvalue - -all: - test "$(TESTVAR)" = "testvalue" - test "$(TESTVAR2)" = "" - @echo "TEST-PASS" \ No newline at end of file diff --git a/build/pymake/tests/var-overrides.mk b/build/pymake/tests/var-overrides.mk deleted file mode 100644 index bd0765d19..000000000 --- a/build/pymake/tests/var-overrides.mk +++ /dev/null @@ -1,21 +0,0 @@ -#T commandline: ['CLINEVAR=clineval', 'CLINEVAR2=clineval2'] - -# this doesn't actually test overrides yet, because they aren't implemented in pymake, -# but testing origins in general is important - -MVAR = mval -CLINEVAR = deadbeef - -override CLINEVAR2 = mval2 - -all: - test "$(origin NOVAR)" = "undefined" - test "$(CLINEVAR)" = "clineval" - test "$(origin CLINEVAR)" = "command line" - test "$(MVAR)" = "mval" - test "$(origin MVAR)" = "file" - test "$(@)" = "all" - test "$(origin @)" = "automatic" - test "$(origin CLINEVAR2)" = "override" - test "$(CLINEVAR2)" = "mval2" - @echo TEST-PASS diff --git a/build/pymake/tests/var-ref.mk b/build/pymake/tests/var-ref.mk deleted file mode 100644 index 3bc1886f9..000000000 --- a/build/pymake/tests/var-ref.mk +++ /dev/null @@ -1,19 +0,0 @@ -VAR = value -VAR2 == value - -VAR5 = $(NULL) $(NULL) -VARC = value # comment - -$(VAR3) - $(VAR4) -$(VAR5) - -VAR6$(VAR5) = val6 - -all: - test "$( VAR)" = "" - test "$(VAR2)" = "= value" - test "${VAR2}" = "= value" - test "$(VAR6 )" = "val6" - test "$(VARC)" = "value " - @echo TEST-PASS diff --git a/build/pymake/tests/var-set.mk b/build/pymake/tests/var-set.mk deleted file mode 100644 index 1603e7a35..000000000 --- a/build/pymake/tests/var-set.mk +++ /dev/null @@ -1,55 +0,0 @@ -#T commandline: ['OBASIC=oval'] - -BASIC = val - -TEST = $(TEST) - -TEST2 = $(TES -TEST2 += T) - -TES T = val - -RECVAR = foo -RECVAR += var baz - -IMMVAR := bloo -IMMVAR += $(RECVAR) - -BASIC ?= notval - -all: BASIC = valall -all: RECVAR += $(BASIC) -all: IMMVAR += $(BASIC) -all: UNSET += more -all: OBASIC += allmore - -CHECKLIT = $(NULL) check -all: CHECKLIT += appendliteral - -RECVAR = blimey - -TESTEMPTY = \ - $(NULL) - -all: other - test "$(TEST2)" = "val" - test '$(value TEST2)' = '$$(TES T)' - test "$(RECVAR)" = "blimey valall" - test "$(IMMVAR)" = "bloo foo var baz valall" - test "$(UNSET)" = "more" - test "$(OBASIC)" = "oval" - test "$(CHECKLIT)" = " check appendliteral" - test "$(TESTEMPTY)" = "" - @echo TEST-PASS - -OVAR = oval -OVAR ?= onotval - -other: OVAR ?= ooval -other: LATERVAR ?= lateroverride - -LATERVAR = olater - -other: - test "$(OVAR)" = "oval" - test "$(LATERVAR)" = "lateroverride" diff --git a/build/pymake/tests/var-substitutions.mk b/build/pymake/tests/var-substitutions.mk deleted file mode 100644 index d5627d7bd..000000000 --- a/build/pymake/tests/var-substitutions.mk +++ /dev/null @@ -1,49 +0,0 @@ -SIMPLEVAR = aabb.cc -SIMPLEPERCENT = test_value%extra - -SIMPLE3SUBSTNAME = SIMPLEVAR:.dd -$(SIMPLE3SUBSTNAME) = weirdval - -PERCENT = dummy - -SIMPLESUBST = $(SIMPLEVAR:.cc=.dd) -SIMPLE2SUBST = $(SIMPLEVAR:.cc) -SIMPLE3SUBST = $(SIMPLEVAR:.dd) -SIMPLE4SUBST = $(SIMPLEVAR:.cc=.dd=.ee) -SIMPLE5SUBST = $(SIMPLEVAR:.cc=%.dd) -PERCENTSUBST = $(SIMPLEVAR:%.cc=%.ee) -PERCENT2SUBST = $(SIMPLEVAR:aa%.cc=ff%.f) -PERCENT3SUBST = $(SIMPLEVAR:aa%.dd=gg%.gg) -PERCENT4SUBST = $(SIMPLEVAR:aa%.cc=gg) -PERCENT5SUBST = $(SIMPLEVAR:aa) -PERCENT6SUBST = $(SIMPLEVAR:%.cc=%.dd=%.ee) -PERCENT7SUBST = $(SIMPLEVAR:$(PERCENT).cc=%.dd) -PERCENT8SUBST = $(SIMPLEVAR:%.cc=$(PERCENT).dd) -PERCENT9SUBST = $(SIMPLEVAR:$(PERCENT).cc=$(PERCENT).dd) -PERCENT10SUBST = $(SIMPLEVAR:%%.bb.cc=zz.bb.cc) -PERCENT11SUBST = $(SIMPLEPERCENT:test%value%extra=other%value%extra) - -SPACEDVAR = $(NULL) ex1.c ex2.c $(NULL) -SPACEDSUBST = $(SPACEDVAR:.c=.o) - -all: - test "$(SIMPLESUBST)" = "aabb.dd" - test "$(SIMPLE2SUBST)" = "" - test "$(SIMPLE3SUBST)" = "weirdval" - test "$(SIMPLE4SUBST)" = "aabb.dd=.ee" - test "$(SIMPLE5SUBST)" = "aabb%.dd" - test "$(PERCENTSUBST)" = "aabb.ee" - test "$(PERCENT2SUBST)" = "ffbb.f" - test "$(PERCENT3SUBST)" = "aabb.cc" - test "$(PERCENT4SUBST)" = "gg" - test "$(PERCENT5SUBST)" = "" - test "$(PERCENT6SUBST)" = "aabb.dd=%.ee" - test "$(PERCENT7SUBST)" = "aabb.dd" - test "$(PERCENT8SUBST)" = "aabb.dd" - test "$(PERCENT9SUBST)" = "aabb.dd" - test "$(PERCENT10SUBST)" = "aabb.cc" - test "$(PERCENT11SUBST)" = "other_value%extra" - test "$(SPACEDSUBST)" = "ex1.o ex2.o" - @echo TEST-PASS - -PERCENT = % diff --git a/build/pymake/tests/vpath-directive-dynamic.mk b/build/pymake/tests/vpath-directive-dynamic.mk deleted file mode 100644 index 9aa1bf956..000000000 --- a/build/pymake/tests/vpath-directive-dynamic.mk +++ /dev/null @@ -1,12 +0,0 @@ -$(shell \ -mkdir subd1; \ -touch subd1/test.in; \ -) - -VVAR = %.in subd1 - -vpath $(VVAR) - -all: test.in - test "$<" = "subd1/test.in" - @echo TEST-PASS diff --git a/build/pymake/tests/vpath-directive.mk b/build/pymake/tests/vpath-directive.mk deleted file mode 100644 index 4c7d4bf39..000000000 --- a/build/pymake/tests/vpath-directive.mk +++ /dev/null @@ -1,31 +0,0 @@ -# On Windows, MSYS make takes Unix paths but Pymake takes Windows paths -VPSEP := $(if $(and $(__WIN32__),$(.PYMAKE)),;,:) - -$(shell \ -mkdir subd1 subd2 subd3; \ -printf "reallybaddata" >subd1/foo.in; \ -printf "gooddata" >subd2/foo.in; \ -printf "baddata" >subd3/foo.in; \ -touch subd1/foo.in2 subd2/foo.in2 subd3/foo.in2; \ -) - -vpath %.in subd - -vpath -vpath %.in subd2$(VPSEP)subd3 - -vpath %.in2 subd0 -vpath f%.in2 subd1 -vpath %.in2 $(VPSEP)subd2 - -%.out: %.in - test "$<" = "subd2/foo.in" - cp $< $@ - -%.out2: %.in2 - test "$<" = "subd1/foo.in2" - cp $< $@ - -all: foo.out foo.out2 - test "$$(cat foo.out)" = "gooddata" - @echo TEST-PASS diff --git a/build/pymake/tests/vpath.mk b/build/pymake/tests/vpath.mk deleted file mode 100644 index 06f52180c..000000000 --- a/build/pymake/tests/vpath.mk +++ /dev/null @@ -1,18 +0,0 @@ -VPATH = foo bar - -$(shell \ -mkdir foo; touch foo/tfile1; \ -mkdir bar; touch bar/tfile2 bar/tfile3 bar/test.objtest; \ -sleep 2; \ -touch bar/test.source; \ -) - -all: tfile1 tfile2 tfile3 test.objtest test.source - test "$^" = "foo/tfile1 bar/tfile2 tfile3 test.objtest bar/test.source" - @echo TEST-PASS - -tfile3: test.objtest - -%.objtest: %.source - test "$<" = bar/test.source - test "$@" = test.objtest diff --git a/build/pymake/tests/vpath2.mk b/build/pymake/tests/vpath2.mk deleted file mode 100644 index be73ffe5c..000000000 --- a/build/pymake/tests/vpath2.mk +++ /dev/null @@ -1,18 +0,0 @@ -VPATH = foo bar - -$(shell \ -mkdir bar; touch bar/test.source; \ -sleep 2; \ -mkdir foo; touch foo/tfile1; \ -touch bar/tfile2 bar/tfile3 bar/test.objtest; \ -) - -all: tfile1 tfile2 tfile3 test.objtest test.source - test "$^" = "foo/tfile1 bar/tfile2 bar/tfile3 bar/test.objtest bar/test.source" - @echo TEST-PASS - -tfile3: test.objtest - -%.objtest: %.source - test "$<" = bar/test.source - test "$@" = test.objtest diff --git a/build/pymake/tests/wildcards.mk b/build/pymake/tests/wildcards.mk deleted file mode 100644 index 24ff3f14c..000000000 --- a/build/pymake/tests/wildcards.mk +++ /dev/null @@ -1,22 +0,0 @@ -$(shell \ -mkdir foo; \ -touch a.c b.c c.out foo/d.c; \ -sleep 2; \ -touch c.in; \ -) - -VPATH = foo - -all: c.out prog - cat $< - test "$$(cat $<)" = "remadec.out" - @echo TEST-PASS - -*.out: %.out: %.in - test "$@" = c.out - test "$<" = c.in - printf "remade$@" >$@ - -prog: *.c - test "$^" = "a.c b.c" - touch $@ diff --git a/build/pymake/tests/windows-paths.mk b/build/pymake/tests/windows-paths.mk deleted file mode 100644 index 5f33a9050..000000000 --- a/build/pymake/tests/windows-paths.mk +++ /dev/null @@ -1,5 +0,0 @@ -all: - touch file.in - printf "%s: %s\n\ttrue" '$(CURDIR)/file.out' '$(CURDIR)/file.in' >test.mk - $(MAKE) -f test.mk $(CURDIR)/file.out - @echo TEST-PASS diff --git a/build/virtualenv_packages.txt b/build/virtualenv_packages.txt index 047aa4a29..b8e0ecd84 100644 --- a/build/virtualenv_packages.txt +++ b/build/virtualenv_packages.txt @@ -9,7 +9,7 @@ mach.pth:python/mach mozbuild.pth:python/mozbuild mozversioncontrol.pth:python/mozversioncontrol mozlint.pth:python/mozlint -pymake.pth:build/pymake +pymake.pth:python/pymake optional:setup.py:python/psutil:build_ext:--inplace optional:psutil.pth:python/psutil which.pth:python/which diff --git a/build/win32/mozconfig.vs-latest b/build/win32/mozconfig.vs-latest deleted file mode 100644 index 9c8726a8d..000000000 --- a/build/win32/mozconfig.vs-latest +++ /dev/null @@ -1 +0,0 @@ -. $topsrcdir/build/win32/mozconfig.vs2015-win64 diff --git a/build/win32/mozconfig.vs2015-win64 b/build/win32/mozconfig.vs2015-win64 deleted file mode 100644 index b81afa681..000000000 --- a/build/win32/mozconfig.vs2015-win64 +++ /dev/null @@ -1,25 +0,0 @@ -if [ -z "${VSPATH}" ]; then - TOOLTOOL_DIR=${TOOLTOOL_DIR:-$topsrcdir} - VSPATH="$(cd ${TOOLTOOL_DIR} && pwd)/vs2015u3" -fi - -VSWINPATH="$(cd ${VSPATH} && pwd -W)" - -export WINDOWSSDKDIR="${VSWINPATH}/SDK" -export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x86/Microsoft.VC140.CRT" -export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x86" - -export PATH="${VSPATH}/VC/bin/amd64_x86:${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x86:${VSPATH}/SDK/bin/x64:${VSPATH}/DIA SDK/bin:${PATH}" -export PATH="${VSPATH}/VC/redist/x86/Microsoft.VC140.CRT:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x86:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}" - -export INCLUDE="${VSPATH}/VC/include:${VSPATH}/VC/atlmfc/include:${VSPATH}/SDK/Include/10.0.14393.0/ucrt:${VSPATH}/SDK/Include/10.0.14393.0/shared:${VSPATH}/SDK/Include/10.0.14393.0/um:${VSPATH}/SDK/Include/10.0.14393.0/winrt:${VSPATH}/DIA SDK/include" -export LIB="${VSPATH}/VC/lib:${VSPATH}/VC/atlmfc/lib:${VSPATH}/SDK/lib/10.0.14393.0/ucrt/x86:${VSPATH}/SDK/lib/10.0.14393.0/um/x86:${VSPATH}/DIA SDK/lib" - -. $topsrcdir/build/mozconfig.vs-common - -mk_export_correct_style WINDOWSSDKDIR -mk_export_correct_style INCLUDE -mk_export_correct_style LIB -mk_export_correct_style PATH -mk_export_correct_style WIN32_REDIST_DIR -mk_export_correct_style WIN_UCRT_REDIST_DIR diff --git a/build/win64/mozconfig.vs-latest b/build/win64/mozconfig.vs-latest deleted file mode 100644 index 3470d4ace..000000000 --- a/build/win64/mozconfig.vs-latest +++ /dev/null @@ -1 +0,0 @@ -. $topsrcdir/build/win64/mozconfig.vs2015 diff --git a/build/win64/mozconfig.vs2015 b/build/win64/mozconfig.vs2015 deleted file mode 100644 index e81a00064..000000000 --- a/build/win64/mozconfig.vs2015 +++ /dev/null @@ -1,24 +0,0 @@ -if [ -z "${VSPATH}" ]; then - TOOLTOOL_DIR=${TOOLTOOL_DIR:-$topsrcdir} - VSPATH="$(cd ${TOOLTOOL_DIR} && pwd)/vs2015u3" -fi - -VSWINPATH="$(cd ${VSPATH} && pwd -W)" - -export WINDOWSSDKDIR="${VSWINPATH}/SDK" -export WIN32_REDIST_DIR=${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT -export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x64" - -export PATH="${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x64:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${VSPATH}/DIA SDK/bin/amd64:${PATH}" - -export INCLUDE="${VSPATH}/VC/include:${VSPATH}/VC/atlmfc/include:${VSPATH}/SDK/Include/10.0.14393.0/ucrt:${VSPATH}/SDK/Include/10.0.14393.0/shared:${VSPATH}/SDK/Include/10.0.14393.0/um:${VSPATH}/SDK/Include/10.0.14393.0/winrt:${VSPATH}/DIA SDK/include" -export LIB="${VSPATH}/VC/lib/amd64:${VSPATH}/VC/atlmfc/lib/amd64:${VSPATH}/SDK/lib/10.0.14393.0/ucrt/x64:${VSPATH}/SDK/lib/10.0.14393.0/um/x64:${VSPATH}/DIA SDK/lib/amd64" - -. $topsrcdir/build/mozconfig.vs-common - -mk_export_correct_style WINDOWSSDKDIR -mk_export_correct_style INCLUDE -mk_export_correct_style LIB -mk_export_correct_style PATH -mk_export_correct_style WIN32_REDIST_DIR -mk_export_correct_style WIN_UCRT_REDIST_DIR diff --git a/python/pymake/LICENSE b/python/pymake/LICENSE new file mode 100644 index 000000000..04a7d641d --- /dev/null +++ b/python/pymake/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2009 The Mozilla Foundation + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/python/pymake/README b/python/pymake/README new file mode 100644 index 000000000..4f0fdfea4 --- /dev/null +++ b/python/pymake/README @@ -0,0 +1,64 @@ +INTRODUCTION + +make.py (and the pymake modules that support it) are an implementation of the make tool +which are mostly compatible with makefiles written for GNU make. + +PURPOSE + +The Mozilla project inspired this tool with several goals: + +* Improve build speeds, especially on Windows. This can be done by reducing the total number + of processes that are launched, especially MSYS shell processes which are expensive. + +* Allow writing some complicated build logic directly in Python instead of in shell. + +* Allow computing dependencies for special targets, such as members within ZIP files. + +* Enable experiments with build system. By writing a makefile parser, we can experiment + with converting in-tree makefiles to another build system, such as SCons, waf, ant, ...insert + your favorite build tool here. Or we could experiment along the lines of makepp, keeping + our existing makefiles, but change the engine to build a global dependency graph. + +KNOWN INCOMPATIBILITIES + +* Order-only prerequisites are not yet supported + +* Secondary expansion is not yet supported. + +* Target-specific variables behave differently than in GNU make: in pymake, the target-specific + variable only applies to the specific target that is mentioned, and does not apply recursively + to all dependencies which are remade. This is an intentional change: the behavior of GNU make + is neither deterministic nor intuitive. + +* $(eval) is only supported during the parse phase. Any attempt to recursively expand + an $(eval) function during command execution will fail. This is an intentional incompatibility. + +* There is a subtle difference in execution order that can cause unexpected changes in the + following circumstance: +** A file `foo.c` exists on the VPATH +** A rule for `foo.c` exists with a dependency on `tool` and no commands +** `tool` is remade for some other reason earlier in the file + In this case, pymake resets the VPATH of `foo.c`, while GNU make does not. This shouldn't + happen in the real world, since a target found on the VPATH without commands is silly. But + mozilla/js/src happens to have a rule, which I'm patching. + +* pymake does not implement any of the builtin implicit rules or the related variables. Mozilla + only cares because pymake doesn't implicitly define $(RM), which I'm also fixing in the Mozilla + code. + +ISSUES + +* Speed is a problem. + +FUTURE WORK + +* implement a new type of command which is implemented in python. This would allow us +to replace the current `nsinstall` binary (and execution costs for the shell and binary) with an +in-process python solution. + +AUTHOR + +Initial code was written by Benjamin Smedberg . For future releases see +http://benjamin.smedbergs.us/pymake/ + +See the LICENSE file for license information (MIT license) diff --git a/python/pymake/make.py b/python/pymake/make.py new file mode 100644 index 000000000..0857f3f8c --- /dev/null +++ b/python/pymake/make.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +""" +make.py + +A drop-in or mostly drop-in replacement for GNU make. +""" + +import sys, os +import pymake.command, pymake.process + +import gc + +if __name__ == '__main__': + if 'TINDERBOX_OUTPUT' in os.environ: + # When building on mozilla build slaves, execute mozmake instead. Until bug + # 978211, this is the easiest, albeit hackish, way to do this. + import subprocess + mozmake = os.path.join(os.path.dirname(__file__), '..', '..', + 'mozmake.exe') + cmd = [mozmake] + cmd.extend(sys.argv[1:]) + shell = os.environ.get('SHELL') + if shell and not shell.lower().endswith('.exe'): + cmd += ['SHELL=%s.exe' % shell] + sys.exit(subprocess.call(cmd)) + + sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) + sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0) + + gc.disable() + + pymake.command.main(sys.argv[1:], os.environ, os.getcwd(), cb=sys.exit) + pymake.process.ParallelContext.spin() + assert False, "Not reached" diff --git a/python/pymake/mkformat.py b/python/pymake/mkformat.py new file mode 100644 index 000000000..41dd761b2 --- /dev/null +++ b/python/pymake/mkformat.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +import sys +import pymake.parser + +filename = sys.argv[1] +source = None + +with open(filename, 'rU') as fh: + source = fh.read() + +statements = pymake.parser.parsestring(source, filename) +print statements.to_source() diff --git a/python/pymake/mkparse.py b/python/pymake/mkparse.py new file mode 100644 index 000000000..253683948 --- /dev/null +++ b/python/pymake/mkparse.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +import sys +import pymake.parser + +for f in sys.argv[1:]: + print "Parsing %s" % f + fd = open(f, 'rU') + s = fd.read() + fd.close() + stmts = pymake.parser.parsestring(s, f) + print stmts diff --git a/python/pymake/pymake/__init__.py b/python/pymake/pymake/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/pymake/pymake/builtins.py b/python/pymake/pymake/builtins.py new file mode 100644 index 000000000..eb6f2e11b --- /dev/null +++ b/python/pymake/pymake/builtins.py @@ -0,0 +1,120 @@ +# Basic commands implemented in Python +import errno, sys, os, shutil, time +from getopt import getopt, GetoptError + +from process import PythonException + +__all__ = ["mkdir", "rm", "sleep", "touch"] + +def mkdir(args): + """ + Emulate some of the behavior of mkdir(1). + Only supports the -p (--parents) argument. + """ + try: + opts, args = getopt(args, "p", ["parents"]) + except GetoptError, e: + raise PythonException, ("mkdir: %s" % e, 1) + parents = False + for o, a in opts: + if o in ('-p', '--parents'): + parents = True + for f in args: + try: + if parents: + os.makedirs(f) + else: + os.mkdir(f) + except OSError, e: + if e.errno == errno.EEXIST and parents: + pass + else: + raise PythonException, ("mkdir: %s" % e, 1) + +def rm(args): + """ + Emulate most of the behavior of rm(1). + Only supports the -r (--recursive) and -f (--force) arguments. + """ + try: + opts, args = getopt(args, "rRf", ["force", "recursive"]) + except GetoptError, e: + raise PythonException, ("rm: %s" % e, 1) + force = False + recursive = False + for o, a in opts: + if o in ('-f', '--force'): + force = True + elif o in ('-r', '-R', '--recursive'): + recursive = True + for f in args: + if os.path.isdir(f): + if not recursive: + raise PythonException, ("rm: cannot remove '%s': Is a directory" % f, 1) + else: + shutil.rmtree(f, force) + elif os.path.exists(f): + try: + os.unlink(f) + except: + if not force: + raise PythonException, ("rm: failed to remove '%s': %s" % (f, sys.exc_info()[0]), 1) + elif not force: + raise PythonException, ("rm: cannot remove '%s': No such file or directory" % f, 1) + +def sleep(args): + """ + Emulate the behavior of sleep(1). + """ + total = 0 + values = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400} + for a in args: + multiplier = 1 + for k, v in values.iteritems(): + if a.endswith(k): + a = a[:-1] + multiplier = v + break + try: + f = float(a) + total += f * multiplier + except ValueError: + raise PythonException, ("sleep: invalid time interval '%s'" % a, 1) + time.sleep(total) + +def touch(args): + """ + Emulate the behavior of touch(1). + """ + try: + opts, args = getopt(args, "t:") + except GetoptError, e: + raise PythonException, ("touch: %s" % e, 1) + opts = dict(opts) + times = None + if '-t' in opts: + import re + from time import mktime, localtime + m = re.match('^(?P(?:\d\d)?\d\d)?(?P\d\d)(?P\d\d)(?P\d\d)(?P\d\d)(?:\.(?P\d\d))?$', opts['-t']) + if not m: + raise PythonException, ("touch: invalid date format '%s'" % opts['-t'], 1) + def normalized_field(m, f): + if f == 'Y': + if m.group(f) is None: + return localtime()[0] + y = int(m.group(f)) + if y < 69: + y += 2000 + elif y < 100: + y += 1900 + return y + if m.group(f) is None: + return localtime()[0] if f == 'Y' else 0 + return int(m.group(f)) + time = [normalized_field(m, f) for f in ['Y', 'M', 'D', 'h', 'm', 's']] + [0, 0, -1] + time = mktime(time) + times = (time, time) + for f in args: + if not os.path.exists(f): + open(f, 'a').close() + os.utime(f, times) diff --git a/python/pymake/pymake/command.py b/python/pymake/pymake/command.py new file mode 100644 index 000000000..cd68e4fdb --- /dev/null +++ b/python/pymake/pymake/command.py @@ -0,0 +1,278 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Makefile execution. + +Multiple `makes` can be run within the same process. Each one has an entirely data.Makefile and .Target +structure, environment, and working directory. Typically they will all share a parallel execution context, +except when a submake specifies -j1 when the parent make is building in parallel. +""" + +import os, subprocess, sys, logging, time, traceback, re +from optparse import OptionParser +import data, parserdata, process, util + +# TODO: If this ever goes from relocatable package to system-installed, this may need to be +# a configured-in path. + +makepypath = util.normaljoin(os.path.dirname(__file__), '../make.py') + +_simpleopts = re.compile(r'^[a-zA-Z]+(\s|$)') +def parsemakeflags(env): + """ + Parse MAKEFLAGS from the environment into a sequence of command-line arguments. + """ + + makeflags = env.get('MAKEFLAGS', '') + makeflags = makeflags.strip() + + if makeflags == '': + return [] + + if _simpleopts.match(makeflags): + makeflags = '-' + makeflags + + opts = [] + curopt = '' + + i = 0 + while i < len(makeflags): + c = makeflags[i] + if c.isspace(): + opts.append(curopt) + curopt = '' + i += 1 + while i < len(makeflags) and makeflags[i].isspace(): + i += 1 + continue + + if c == '\\': + i += 1 + if i == len(makeflags): + raise data.DataError("MAKEFLAGS has trailing backslash") + c = makeflags[i] + + curopt += c + i += 1 + + if curopt != '': + opts.append(curopt) + + return opts + +def _version(*args): + print """pymake: GNU-compatible make program +Copyright (C) 2009 The Mozilla Foundation +This is free software; see the source for copying conditions. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE.""" + +_log = logging.getLogger('pymake.execution') + +class _MakeContext(object): + def __init__(self, makeflags, makelevel, workdir, context, env, targets, options, ostmts, overrides, cb): + self.makeflags = makeflags + self.makelevel = makelevel + + self.workdir = workdir + self.context = context + self.env = env + self.targets = targets + self.options = options + self.ostmts = ostmts + self.overrides = overrides + self.cb = cb + + self.restarts = 0 + + self.remakecb(True) + + def remakecb(self, remade, error=None): + if error is not None: + print error + self.context.defer(self.cb, 2) + return + + if remade: + if self.restarts > 0: + _log.info("make.py[%i]: Restarting makefile parsing", self.makelevel) + + self.makefile = data.Makefile(restarts=self.restarts, + make='%s %s' % (sys.executable.replace('\\', '/'), makepypath.replace('\\', '/')), + makeflags=self.makeflags, + makeoverrides=self.overrides, + workdir=self.workdir, + context=self.context, + env=self.env, + makelevel=self.makelevel, + targets=self.targets, + keepgoing=self.options.keepgoing, + silent=self.options.silent, + justprint=self.options.justprint) + + self.restarts += 1 + + try: + self.ostmts.execute(self.makefile) + for f in self.options.makefiles: + self.makefile.include(f) + self.makefile.finishparsing() + self.makefile.remakemakefiles(self.remakecb) + except util.MakeError, e: + print e + self.context.defer(self.cb, 2) + + return + + if len(self.targets) == 0: + if self.makefile.defaulttarget is None: + print "No target specified and no default target found." + self.context.defer(self.cb, 2) + return + + _log.info("Making default target %s", self.makefile.defaulttarget) + self.realtargets = [self.makefile.defaulttarget] + self.tstack = [''] + else: + self.realtargets = self.targets + self.tstack = [''] + + self.makefile.gettarget(self.realtargets.pop(0)).make(self.makefile, self.tstack, cb=self.makecb) + + def makecb(self, error, didanything): + assert error in (True, False) + + if error: + self.context.defer(self.cb, 2) + return + + if not len(self.realtargets): + if self.options.printdir: + print "make.py[%i]: Leaving directory '%s'" % (self.makelevel, self.workdir) + sys.stdout.flush() + + self.context.defer(self.cb, 0) + else: + self.makefile.gettarget(self.realtargets.pop(0)).make(self.makefile, self.tstack, self.makecb) + +def main(args, env, cwd, cb): + """ + Start a single makefile execution, given a command line, working directory, and environment. + + @param cb a callback to notify with an exit code when make execution is finished. + """ + + try: + makelevel = int(env.get('MAKELEVEL', '0')) + + op = OptionParser() + op.add_option('-f', '--file', '--makefile', + action='append', + dest='makefiles', + default=[]) + op.add_option('-d', + action="store_true", + dest="verbose", default=False) + op.add_option('-k', '--keep-going', + action="store_true", + dest="keepgoing", default=False) + op.add_option('--debug-log', + dest="debuglog", default=None) + op.add_option('-C', '--directory', + dest="directory", default=None) + op.add_option('-v', '--version', action="store_true", + dest="printversion", default=False) + op.add_option('-j', '--jobs', type="int", + dest="jobcount", default=1) + op.add_option('-w', '--print-directory', action="store_true", + dest="printdir") + op.add_option('--no-print-directory', action="store_false", + dest="printdir", default=True) + op.add_option('-s', '--silent', action="store_true", + dest="silent", default=False) + op.add_option('-n', '--just-print', '--dry-run', '--recon', + action="store_true", + dest="justprint", default=False) + + options, arguments1 = op.parse_args(parsemakeflags(env)) + options, arguments2 = op.parse_args(args, values=options) + + op.destroy() + + arguments = arguments1 + arguments2 + + if options.printversion: + _version() + cb(0) + return + + shortflags = [] + longflags = [] + + if options.keepgoing: + shortflags.append('k') + + if options.printdir: + shortflags.append('w') + + if options.silent: + shortflags.append('s') + options.printdir = False + + if options.justprint: + shortflags.append('n') + + loglevel = logging.WARNING + if options.verbose: + loglevel = logging.DEBUG + shortflags.append('d') + + logkwargs = {} + if options.debuglog: + logkwargs['filename'] = options.debuglog + longflags.append('--debug-log=%s' % options.debuglog) + + if options.directory is None: + workdir = cwd + else: + workdir = util.normaljoin(cwd, options.directory) + + if options.jobcount != 1: + longflags.append('-j%i' % (options.jobcount,)) + + makeflags = ''.join(shortflags) + if len(longflags): + makeflags += ' ' + ' '.join(longflags) + + logging.basicConfig(level=loglevel, **logkwargs) + + context = process.getcontext(options.jobcount) + + if options.printdir: + print "make.py[%i]: Entering directory '%s'" % (makelevel, workdir) + sys.stdout.flush() + + if len(options.makefiles) == 0: + if os.path.exists(util.normaljoin(workdir, 'Makefile')): + options.makefiles.append('Makefile') + else: + print "No makefile found" + cb(2) + return + + ostmts, targets, overrides = parserdata.parsecommandlineargs(arguments) + + _MakeContext(makeflags, makelevel, workdir, context, env, targets, options, ostmts, overrides, cb) + except (util.MakeError), e: + print e + if options.printdir: + print "make.py[%i]: Leaving directory '%s'" % (makelevel, workdir) + sys.stdout.flush() + cb(2) + return diff --git a/python/pymake/pymake/data.py b/python/pymake/pymake/data.py new file mode 100644 index 000000000..dcd7e9225 --- /dev/null +++ b/python/pymake/pymake/data.py @@ -0,0 +1,1842 @@ +""" +A representation of makefile data structures. +""" + +import logging, re, os, sys +import parserdata, parser, functions, process, util, implicit +from cStringIO import StringIO + +if sys.version_info[0] < 3: + str_type = basestring +else: + str_type = str + +_log = logging.getLogger('pymake.data') + +class DataError(util.MakeError): + pass + +class ResolutionError(DataError): + """ + Raised when dependency resolution fails, either due to recursion or to missing + prerequisites.This is separately catchable so that implicit rule search can try things + without having to commit. + """ + pass + +def withoutdups(it): + r = set() + for i in it: + if not i in r: + r.add(i) + yield i + +def mtimeislater(deptime, targettime): + """ + Is the mtime of the dependency later than the target? + """ + + if deptime is None: + return True + if targettime is None: + return False + # int(1000*x) because of http://bugs.python.org/issue10148 + return int(1000 * deptime) > int(1000 * targettime) + +def getmtime(path): + try: + s = os.stat(path) + return s.st_mtime + except OSError: + return None + +def stripdotslash(s): + if s.startswith('./'): + st = s[2:] + return st if st != '' else '.' + return s + +def stripdotslashes(sl): + for s in sl: + yield stripdotslash(s) + +def getindent(stack): + return ''.ljust(len(stack) - 1) + +def _if_else(c, t, f): + if c: + return t() + return f() + + +class BaseExpansion(object): + """Base class for expansions. + + A make expansion is the parsed representation of a string, which may + contain references to other elements. + """ + + @property + def is_static_string(self): + """Returns whether the expansion is composed of static string content. + + This is always True for StringExpansion. It will be True for Expansion + only if all elements of that Expansion are static strings. + """ + raise Exception('Must be implemented in child class.') + + def functions(self, descend=False): + """Obtain all functions inside this expansion. + + This is a generator for pymake.functions.Function instances. + + By default, this only returns functions existing as the primary + elements of this expansion. If `descend` is True, it will descend into + child expansions and extract all functions in the tree. + """ + # An empty generator. Yeah, it's weird. + for x in []: + yield x + + def variable_references(self, descend=False): + """Obtain all variable references in this expansion. + + This is a generator for pymake.functionsVariableRef instances. + + To retrieve the names of variables, simply query the `vname` field on + the returned instances. Most of the time these will be StringExpansion + instances. + """ + for f in self.functions(descend=descend): + if not isinstance(f, functions.VariableRef): + continue + + yield f + + @property + def is_filesystem_dependent(self): + """Whether this expansion may query the filesystem for evaluation. + + This effectively asks "is any function in this expansion dependent on + the filesystem. + """ + for f in self.functions(descend=True): + if f.is_filesystem_dependent: + return True + + return False + + @property + def is_shell_dependent(self): + """Whether this expansion may invoke a shell for evaluation.""" + + for f in self.functions(descend=True): + if isinstance(f, functions.ShellFunction): + return True + + return False + + +class StringExpansion(BaseExpansion): + """An Expansion representing a static string. + + This essentially wraps a single str instance. + """ + + __slots__ = ('loc', 's',) + simple = True + + def __init__(self, s, loc): + assert isinstance(s, str_type) + self.s = s + self.loc = loc + + def lstrip(self): + self.s = self.s.lstrip() + + def rstrip(self): + self.s = self.s.rstrip() + + def isempty(self): + return self.s == '' + + def resolve(self, i, j, fd, k=None): + fd.write(self.s) + + def resolvestr(self, i, j, k=None): + return self.s + + def resolvesplit(self, i, j, k=None): + return self.s.split() + + def clone(self): + e = Expansion(self.loc) + e.appendstr(self.s) + return e + + @property + def is_static_string(self): + return True + + def __len__(self): + return 1 + + def __getitem__(self, i): + assert i == 0 + return self.s, False + + def __repr__(self): + return "Exp<%s>(%r)" % (self.loc, self.s) + + def __eq__(self, other): + """We only compare the string contents.""" + return self.s == other + + def __ne__(self, other): + return not self.__eq__(other) + + def to_source(self, escape_variables=False, escape_comments=False): + s = self.s + + if escape_comments: + s = s.replace('#', '\\#') + + if escape_variables: + return s.replace('$', '$$') + + return s + + +class Expansion(BaseExpansion, list): + """A representation of expanded data. + + This is effectively an ordered list of StringExpansion and + pymake.function.Function instances. Every item in the collection appears in + the same context in a make file. + """ + + __slots__ = ('loc',) + simple = False + + def __init__(self, loc=None): + # A list of (element, isfunc) tuples + # element is either a string or a function + self.loc = loc + + @staticmethod + def fromstring(s, path): + return StringExpansion(s, parserdata.Location(path, 1, 0)) + + def clone(self): + e = Expansion() + e.extend(self) + return e + + def appendstr(self, s): + assert isinstance(s, str_type) + if s == '': + return + + self.append((s, False)) + + def appendfunc(self, func): + assert isinstance(func, functions.Function) + self.append((func, True)) + + def concat(self, o): + """Concatenate the other expansion on to this one.""" + if o.simple: + self.appendstr(o.s) + else: + self.extend(o) + + def isempty(self): + return (not len(self)) or self[0] == ('', False) + + def lstrip(self): + """Strip leading literal whitespace from this expansion.""" + while True: + i, isfunc = self[0] + if isfunc: + return + + i = i.lstrip() + if i != '': + self[0] = i, False + return + + del self[0] + + def rstrip(self): + """Strip trailing literal whitespace from this expansion.""" + while True: + i, isfunc = self[-1] + if isfunc: + return + + i = i.rstrip() + if i != '': + self[-1] = i, False + return + + del self[-1] + + def finish(self): + # Merge any adjacent literal strings: + strings = [] + elements = [] + for (e, isfunc) in self: + if isfunc: + if strings: + s = ''.join(strings) + if s: + elements.append((s, False)) + strings = [] + elements.append((e, True)) + else: + strings.append(e) + + if not elements: + # This can only happen if there were no function elements. + return StringExpansion(''.join(strings), self.loc) + + if strings: + s = ''.join(strings) + if s: + elements.append((s, False)) + + if len(elements) < len(self): + self[:] = elements + + return self + + def resolve(self, makefile, variables, fd, setting=[]): + """ + Resolve this variable into a value, by interpolating the value + of other variables. + + @param setting (Variable instance) the variable currently + being set, if any. Setting variables must avoid self-referential + loops. + """ + assert isinstance(makefile, Makefile) + assert isinstance(variables, Variables) + assert isinstance(setting, list) + + for e, isfunc in self: + if isfunc: + e.resolve(makefile, variables, fd, setting) + else: + assert isinstance(e, str_type) + fd.write(e) + + def resolvestr(self, makefile, variables, setting=[]): + fd = StringIO() + self.resolve(makefile, variables, fd, setting) + return fd.getvalue() + + def resolvesplit(self, makefile, variables, setting=[]): + return self.resolvestr(makefile, variables, setting).split() + + @property + def is_static_string(self): + """An Expansion is static if all its components are strings, not + functions.""" + for e, is_func in self: + if is_func: + return False + + return True + + def functions(self, descend=False): + for e, is_func in self: + if is_func: + yield e + + if descend: + for exp in e.expansions(descend=True): + for f in exp.functions(descend=True): + yield f + + def __repr__(self): + return "" % ([e for e, isfunc in self],) + + def to_source(self, escape_variables=False, escape_comments=False): + parts = [] + for e, is_func in self: + if is_func: + parts.append(e.to_source()) + continue + + if escape_variables: + parts.append(e.replace('$', '$$')) + continue + + parts.append(e) + + return ''.join(parts) + + def __eq__(self, other): + if not isinstance(other, (Expansion, StringExpansion)): + return False + + # Expansions are equivalent if adjacent string literals normalize to + # the same value. So, we must normalize before any comparisons are + # made. + a = self.clone().finish() + + if isinstance(other, StringExpansion): + if isinstance(a, StringExpansion): + return a == other + + # A normalized Expansion != StringExpansion. + return False + + b = other.clone().finish() + + # b could be a StringExpansion now. + if isinstance(b, StringExpansion): + if isinstance(a, StringExpansion): + return a == b + + # Our normalized Expansion != normalized StringExpansion. + return False + + if len(a) != len(b): + return False + + for i in xrange(len(self)): + e1, is_func1 = a[i] + e2, is_func2 = b[i] + + if is_func1 != is_func2: + return False + + if type(e1) != type(e2): + return False + + if e1 != e2: + return False + + return True + + def __ne__(self, other): + return not self.__eq__(other) + +class Variables(object): + """ + A mapping from variable names to variables. Variables have flavor, source, and value. The value is an + expansion object. + """ + + __slots__ = ('parent', '_map') + + FLAVOR_RECURSIVE = 0 + FLAVOR_SIMPLE = 1 + FLAVOR_APPEND = 2 + + SOURCE_OVERRIDE = 0 + SOURCE_COMMANDLINE = 1 + SOURCE_MAKEFILE = 2 + SOURCE_ENVIRONMENT = 3 + SOURCE_AUTOMATIC = 4 + SOURCE_IMPLICIT = 5 + + def __init__(self, parent=None): + self._map = {} # vname -> flavor, source, valuestr, valueexp + self.parent = parent + + def readfromenvironment(self, env): + for k, v in env.iteritems(): + self.set(k, self.FLAVOR_RECURSIVE, self.SOURCE_ENVIRONMENT, v) + + def get(self, name, expand=True): + """ + Get the value of a named variable. Returns a tuple (flavor, source, value) + + If the variable is not present, returns (None, None, None) + + @param expand If true, the value will be returned as an expansion. If false, + it will be returned as an unexpanded string. + """ + flavor, source, valuestr, valueexp = self._map.get(name, (None, None, None, None)) + if flavor is not None: + if expand and flavor != self.FLAVOR_SIMPLE and valueexp is None: + d = parser.Data.fromstring(valuestr, parserdata.Location("Expansion of variables '%s'" % (name,), 1, 0)) + valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata) + self._map[name] = flavor, source, valuestr, valueexp + + if flavor == self.FLAVOR_APPEND: + if self.parent: + pflavor, psource, pvalue = self.parent.get(name, expand) + else: + pflavor, psource, pvalue = None, None, None + + if pvalue is None: + flavor = self.FLAVOR_RECURSIVE + # fall through + else: + if source > psource: + # TODO: log a warning? + return pflavor, psource, pvalue + + if not expand: + return pflavor, psource, pvalue + ' ' + valuestr + + pvalue = pvalue.clone() + pvalue.appendstr(' ') + pvalue.concat(valueexp) + + return pflavor, psource, pvalue + + if not expand: + return flavor, source, valuestr + + if flavor == self.FLAVOR_RECURSIVE: + val = valueexp + else: + val = Expansion.fromstring(valuestr, "Expansion of variable '%s'" % (name,)) + + return flavor, source, val + + if self.parent is not None: + return self.parent.get(name, expand) + + return (None, None, None) + + def set(self, name, flavor, source, value, force=False): + assert flavor in (self.FLAVOR_RECURSIVE, self.FLAVOR_SIMPLE) + assert source in (self.SOURCE_OVERRIDE, self.SOURCE_COMMANDLINE, self.SOURCE_MAKEFILE, self.SOURCE_ENVIRONMENT, self.SOURCE_AUTOMATIC, self.SOURCE_IMPLICIT) + assert isinstance(value, str_type), "expected str, got %s" % type(value) + + prevflavor, prevsource, prevvalue = self.get(name) + if prevsource is not None and source > prevsource and not force: + # TODO: give a location for this warning + _log.info("not setting variable '%s', set by higher-priority source to value '%s'" % (name, prevvalue)) + return + + self._map[name] = flavor, source, value, None + + def append(self, name, source, value, variables, makefile): + assert source in (self.SOURCE_OVERRIDE, self.SOURCE_MAKEFILE, self.SOURCE_AUTOMATIC) + assert isinstance(value, str_type) + + if name not in self._map: + self._map[name] = self.FLAVOR_APPEND, source, value, None + return + + prevflavor, prevsource, prevvalue, valueexp = self._map[name] + if source > prevsource: + # TODO: log a warning? + return + + if prevflavor == self.FLAVOR_SIMPLE: + d = parser.Data.fromstring(value, parserdata.Location("Expansion of variables '%s'" % (name,), 1, 0)) + valueexp, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata) + + val = valueexp.resolvestr(makefile, variables, [name]) + self._map[name] = prevflavor, prevsource, prevvalue + ' ' + val, None + return + + newvalue = prevvalue + ' ' + value + self._map[name] = prevflavor, prevsource, newvalue, None + + def merge(self, other): + assert isinstance(other, Variables) + for k, flavor, source, value in other: + self.set(k, flavor, source, value) + + def __iter__(self): + for k, (flavor, source, value, valueexp) in self._map.iteritems(): + yield k, flavor, source, value + + def __contains__(self, item): + return item in self._map + +class Pattern(object): + """ + A pattern is a string, possibly with a % substitution character. From the GNU make manual: + + '%' characters in pattern rules can be quoted with precending backslashes ('\'). Backslashes that + would otherwise quote '%' charcters can be quoted with more backslashes. Backslashes that + quote '%' characters or other backslashes are removed from the pattern before it is compared t + file names or has a stem substituted into it. Backslashes that are not in danger of quoting '%' + characters go unmolested. For example, the pattern the\%weird\\%pattern\\ has `the%weird\' preceding + the operative '%' character, and 'pattern\\' following it. The final two backslashes are left alone + because they cannot affect any '%' character. + + This insane behavior probably doesn't matter, but we're compatible just for shits and giggles. + """ + + __slots__ = ('data') + + def __init__(self, s): + r = [] + i = 0 + slen = len(s) + while i < slen: + c = s[i] + if c == '\\': + nc = s[i + 1] + if nc == '%': + r.append('%') + i += 1 + elif nc == '\\': + r.append('\\') + i += 1 + else: + r.append(c) + elif c == '%': + self.data = (''.join(r), s[i+1:]) + return + else: + r.append(c) + i += 1 + + # This is different than (s,) because \% and \\ have been unescaped. Parsing patterns is + # context-sensitive! + self.data = (''.join(r),) + + def ismatchany(self): + return self.data == ('','') + + def ispattern(self): + return len(self.data) == 2 + + def __hash__(self): + return self.data.__hash__() + + def __eq__(self, o): + assert isinstance(o, Pattern) + return self.data == o.data + + def gettarget(self): + assert not self.ispattern() + return self.data[0] + + def hasslash(self): + return self.data[0].find('/') != -1 or self.data[1].find('/') != -1 + + def match(self, word): + """ + Match this search pattern against a word (string). + + @returns None if the word doesn't match, or the matching stem. + If this is a %-less pattern, the stem will always be '' + """ + d = self.data + if len(d) == 1: + if word == d[0]: + return word + return None + + d0, d1 = d + l1 = len(d0) + l2 = len(d1) + if len(word) >= l1 + l2 and word.startswith(d0) and word.endswith(d1): + if l2 == 0: + return word[l1:] + return word[l1:-l2] + + return None + + def resolve(self, dir, stem): + if self.ispattern(): + return dir + self.data[0] + stem + self.data[1] + + return self.data[0] + + def subst(self, replacement, word, mustmatch): + """ + Given a word, replace the current pattern with the replacement pattern, a la 'patsubst' + + @param mustmatch If true and this pattern doesn't match the word, throw a DataError. Otherwise + return word unchanged. + """ + assert isinstance(replacement, str_type) + + stem = self.match(word) + if stem is None: + if mustmatch: + raise DataError("target '%s' doesn't match pattern" % (word,)) + return word + + if not self.ispattern(): + # if we're not a pattern, the replacement is not parsed as a pattern either + return replacement + + return Pattern(replacement).resolve('', stem) + + def __repr__(self): + return "" % (self.data,) + + _backre = re.compile(r'[%\\]') + def __str__(self): + if not self.ispattern(): + return self._backre.sub(r'\\\1', self.data[0]) + + return self._backre.sub(r'\\\1', self.data[0]) + '%' + self.data[1] + +class RemakeTargetSerially(object): + __slots__ = ('target', 'makefile', 'indent', 'rlist') + + def __init__(self, target, makefile, indent, rlist): + self.target = target + self.makefile = makefile + self.indent = indent + self.rlist = rlist + self.commandscb(False) + + def resolvecb(self, error, didanything): + assert error in (True, False) + + if didanything: + self.target.didanything = True + + if error: + self.target.error = True + self.makefile.error = True + if not self.makefile.keepgoing: + self.target.notifydone(self.makefile) + return + else: + # don't run the commands! + del self.rlist[0] + self.commandscb(error=False) + else: + self.rlist.pop(0).runcommands(self.indent, self.commandscb) + + def commandscb(self, error): + assert error in (True, False) + + if error: + self.target.error = True + self.makefile.error = True + + if self.target.error and not self.makefile.keepgoing: + self.target.notifydone(self.makefile) + return + + if not len(self.rlist): + self.target.notifydone(self.makefile) + else: + self.rlist[0].resolvedeps(True, self.resolvecb) + +class RemakeTargetParallel(object): + __slots__ = ('target', 'makefile', 'indent', 'rlist', 'rulesremaining', 'currunning') + + def __init__(self, target, makefile, indent, rlist): + self.target = target + self.makefile = makefile + self.indent = indent + self.rlist = rlist + + self.rulesremaining = len(rlist) + self.currunning = False + + for r in rlist: + makefile.context.defer(self.doresolve, r) + + def doresolve(self, r): + if self.makefile.error and not self.makefile.keepgoing: + r.error = True + self.resolvecb(True, False) + else: + r.resolvedeps(False, self.resolvecb) + + def resolvecb(self, error, didanything): + assert error in (True, False) + + if error: + self.target.error = True + + if didanything: + self.target.didanything = True + + self.rulesremaining -= 1 + + # commandscb takes care of the details if we're currently building + # something + if self.currunning: + return + + self.runnext() + + def runnext(self): + assert not self.currunning + + if self.makefile.error and not self.makefile.keepgoing: + self.rlist = [] + else: + while len(self.rlist) and self.rlist[0].error: + del self.rlist[0] + + if not len(self.rlist): + if not self.rulesremaining: + self.target.notifydone(self.makefile) + return + + if self.rlist[0].depsremaining != 0: + return + + self.currunning = True + rule = self.rlist.pop(0) + self.makefile.context.defer(rule.runcommands, self.indent, self.commandscb) + + def commandscb(self, error): + assert error in (True, False) + if error: + self.target.error = True + self.makefile.error = True + + assert self.currunning + self.currunning = False + self.runnext() + +class RemakeRuleContext(object): + def __init__(self, target, makefile, rule, deps, + targetstack, avoidremakeloop): + self.target = target + self.makefile = makefile + self.rule = rule + self.deps = deps + self.targetstack = targetstack + self.avoidremakeloop = avoidremakeloop + + self.running = False + self.error = False + self.depsremaining = len(deps) + 1 + self.remake = False + + def resolvedeps(self, serial, cb): + self.resolvecb = cb + self.didanything = False + if serial: + self._resolvedepsserial() + else: + self._resolvedepsparallel() + + def _weakdepfinishedserial(self, error, didanything): + if error: + self.remake = True + self._depfinishedserial(False, didanything) + + def _depfinishedserial(self, error, didanything): + assert error in (True, False) + + if didanything: + self.didanything = True + + if error: + self.error = True + if not self.makefile.keepgoing: + self.resolvecb(error=True, didanything=self.didanything) + return + + if len(self.resolvelist): + dep, weak = self.resolvelist.pop(0) + self.makefile.context.defer(dep.make, + self.makefile, self.targetstack, weak and self._weakdepfinishedserial or self._depfinishedserial) + else: + self.resolvecb(error=self.error, didanything=self.didanything) + + def _resolvedepsserial(self): + self.resolvelist = list(self.deps) + self._depfinishedserial(False, False) + + def _startdepparallel(self, d): + dep, weak = d + if weak: + depfinished = self._weakdepfinishedparallel + else: + depfinished = self._depfinishedparallel + if self.makefile.error: + depfinished(True, False) + else: + dep.make(self.makefile, self.targetstack, depfinished) + + def _weakdepfinishedparallel(self, error, didanything): + if error: + self.remake = True + self._depfinishedparallel(False, didanything) + + def _depfinishedparallel(self, error, didanything): + assert error in (True, False) + + if error: + print "<%s>: Found error" % self.target.target + self.error = True + if didanything: + self.didanything = True + + self.depsremaining -= 1 + if self.depsremaining == 0: + self.resolvecb(error=self.error, didanything=self.didanything) + + def _resolvedepsparallel(self): + self.depsremaining -= 1 + if self.depsremaining == 0: + self.resolvecb(error=self.error, didanything=self.didanything) + return + + self.didanything = False + + for d in self.deps: + self.makefile.context.defer(self._startdepparallel, d) + + def _commandcb(self, error): + assert error in (True, False) + + if error: + self.runcb(error=True) + return + + if len(self.commands): + self.commands.pop(0)(self._commandcb) + else: + self.runcb(error=False) + + def runcommands(self, indent, cb): + assert not self.running + self.running = True + + self.runcb = cb + + if self.rule is None or not len(self.rule.commands): + if self.target.mtime is None: + self.target.beingremade() + else: + for d, weak in self.deps: + if mtimeislater(d.mtime, self.target.mtime): + if d.mtime is None: + self.target.beingremade() + else: + _log.info("%sNot remaking %s ubecause it would have no effect, even though %s is newer.", indent, self.target.target, d.target) + break + cb(error=False) + return + + if self.rule.doublecolon: + if len(self.deps) == 0: + if self.avoidremakeloop: + _log.info("%sNot remaking %s using rule at %s because it would introduce an infinite loop.", indent, self.target.target, self.rule.loc) + cb(error=False) + return + + remake = self.remake + if remake: + _log.info("%sRemaking %s using rule at %s: weak dependency was not found.", indent, self.target.target, self.rule.loc) + else: + if self.target.mtime is None: + remake = True + _log.info("%sRemaking %s using rule at %s: target doesn't exist or is a forced target", indent, self.target.target, self.rule.loc) + + if not remake: + if self.rule.doublecolon: + if len(self.deps) == 0: + _log.info("%sRemaking %s using rule at %s because there are no prerequisites listed for a double-colon rule.", indent, self.target.target, self.rule.loc) + remake = True + + if not remake: + for d, weak in self.deps: + if mtimeislater(d.mtime, self.target.mtime): + _log.info("%sRemaking %s using rule at %s because %s is newer.", indent, self.target.target, self.rule.loc, d.target) + remake = True + break + + if remake: + self.target.beingremade() + self.target.didanything = True + try: + self.commands = [c for c in self.rule.getcommands(self.target, self.makefile)] + except util.MakeError, e: + print e + sys.stdout.flush() + cb(error=True) + return + + self._commandcb(False) + else: + cb(error=False) + +MAKESTATE_NONE = 0 +MAKESTATE_FINISHED = 1 +MAKESTATE_WORKING = 2 + +class Target(object): + """ + An actual (non-pattern) target. + + It holds target-specific variables and a list of rules. It may also point to a parent + PatternTarget, if this target is being created by an implicit rule. + + The rules associated with this target may be Rule instances or, in the case of static pattern + rules, PatternRule instances. + """ + + wasremade = False + + def __init__(self, target, makefile): + assert isinstance(target, str_type) + self.target = target + self.vpathtarget = None + self.rules = [] + self.variables = Variables(makefile.variables) + self.explicit = False + self._state = MAKESTATE_NONE + + def addrule(self, rule): + assert isinstance(rule, (Rule, PatternRuleInstance)) + if len(self.rules) and rule.doublecolon != self.rules[0].doublecolon: + raise DataError("Cannot have single- and double-colon rules for the same target. Prior rule location: %s" % self.rules[0].loc, rule.loc) + + if isinstance(rule, PatternRuleInstance): + if len(rule.prule.targetpatterns) != 1: + raise DataError("Static pattern rules must only have one target pattern", rule.prule.loc) + if rule.prule.targetpatterns[0].match(self.target) is None: + raise DataError("Static pattern rule doesn't match target '%s'" % self.target, rule.loc) + + self.rules.append(rule) + + def isdoublecolon(self): + return self.rules[0].doublecolon + + def isphony(self, makefile): + """Is this a phony target? We don't check for existence of phony targets.""" + return makefile.gettarget('.PHONY').hasdependency(self.target) + + def hasdependency(self, t): + for rule in self.rules: + if t in rule.prerequisites: + return True + + return False + + def resolveimplicitrule(self, makefile, targetstack, rulestack): + """ + Try to resolve an implicit rule to build this target. + """ + # The steps in the GNU make manual Implicit-Rule-Search.html are very detailed. I hope they can be trusted. + + indent = getindent(targetstack) + + _log.info("%sSearching for implicit rule to make '%s'", indent, self.target) + + dir, s, file = util.strrpartition(self.target, '/') + dir = dir + s + + candidates = [] # list of PatternRuleInstance + + hasmatch = util.any((r.hasspecificmatch(file) for r in makefile.implicitrules)) + + for r in makefile.implicitrules: + if r in rulestack: + _log.info("%s %s: Avoiding implicit rule recursion", indent, r.loc) + continue + + if not len(r.commands): + continue + + for ri in r.matchesfor(dir, file, hasmatch): + candidates.append(ri) + + newcandidates = [] + + for r in candidates: + depfailed = None + for p in r.prerequisites: + t = makefile.gettarget(p) + t.resolvevpath(makefile) + if not t.explicit and t.mtime is None: + depfailed = p + break + + if depfailed is not None: + if r.doublecolon: + _log.info("%s Terminal rule at %s doesn't match: prerequisite '%s' not mentioned and doesn't exist.", indent, r.loc, depfailed) + else: + newcandidates.append(r) + continue + + _log.info("%sFound implicit rule at %s for target '%s'", indent, r.loc, self.target) + self.rules.append(r) + return + + # Try again, but this time with chaining and without terminal (double-colon) rules + + for r in newcandidates: + newrulestack = rulestack + [r.prule] + + depfailed = None + for p in r.prerequisites: + t = makefile.gettarget(p) + try: + t.resolvedeps(makefile, targetstack, newrulestack, True) + except ResolutionError: + depfailed = p + break + + if depfailed is not None: + _log.info("%s Rule at %s doesn't match: prerequisite '%s' could not be made.", indent, r.loc, depfailed) + continue + + _log.info("%sFound implicit rule at %s for target '%s'", indent, r.loc, self.target) + self.rules.append(r) + return + + _log.info("%sCouldn't find implicit rule to remake '%s'", indent, self.target) + + def ruleswithcommands(self): + "The number of rules with commands" + return reduce(lambda i, rule: i + (len(rule.commands) > 0), self.rules, 0) + + def resolvedeps(self, makefile, targetstack, rulestack, recursive): + """ + Resolve the actual path of this target, using vpath if necessary. + + Recursively resolve dependencies of this target. This means finding implicit + rules which match the target, if appropriate. + + Figure out whether this target needs to be rebuild, and set self.outofdate + appropriately. + + @param targetstack is the current stack of dependencies being resolved. If + this target is already in targetstack, bail to prevent infinite + recursion. + @param rulestack is the current stack of implicit rules being used to resolve + dependencies. A rule chain cannot use the same implicit rule twice. + """ + assert makefile.parsingfinished + + if self.target in targetstack: + raise ResolutionError("Recursive dependency: %s -> %s" % ( + " -> ".join(targetstack), self.target)) + + targetstack = targetstack + [self.target] + + indent = getindent(targetstack) + + _log.info("%sConsidering target '%s'", indent, self.target) + + self.resolvevpath(makefile) + + # Sanity-check our rules. If we're single-colon, only one rule should have commands + ruleswithcommands = self.ruleswithcommands() + if len(self.rules) and not self.isdoublecolon(): + if ruleswithcommands > 1: + # In GNU make this is a warning, not an error. I'm going to be stricter. + # TODO: provide locations + raise DataError("Target '%s' has multiple rules with commands." % self.target) + + if ruleswithcommands == 0: + self.resolveimplicitrule(makefile, targetstack, rulestack) + + # If a target is mentioned, but doesn't exist, has no commands and no + # prerequisites, it is special and exists just to say that targets which + # depend on it are always out of date. This is like .FORCE but more + # compatible with other makes. + # Otherwise, we don't know how to make it. + if not len(self.rules) and self.mtime is None and not util.any((len(rule.prerequisites) > 0 + for rule in self.rules)): + raise ResolutionError("No rule to make target '%s' needed by %r" % (self.target, + targetstack)) + + if recursive: + for r in self.rules: + newrulestack = rulestack + [r] + for d in r.prerequisites: + dt = makefile.gettarget(d) + if dt.explicit: + continue + + dt.resolvedeps(makefile, targetstack, newrulestack, True) + + for v in makefile.getpatternvariablesfor(self.target): + self.variables.merge(v) + + def resolvevpath(self, makefile): + if self.vpathtarget is not None: + return + + if self.isphony(makefile): + self.vpathtarget = self.target + self.mtime = None + return + + if self.target.startswith('-l'): + stem = self.target[2:] + f, s, e = makefile.variables.get('.LIBPATTERNS') + if e is not None: + libpatterns = [Pattern(stripdotslash(s)) for s in e.resolvesplit(makefile, makefile.variables)] + if len(libpatterns): + searchdirs = [''] + searchdirs.extend(makefile.getvpath(self.target)) + + for lp in libpatterns: + if not lp.ispattern(): + raise DataError('.LIBPATTERNS contains a non-pattern') + + libname = lp.resolve('', stem) + + for dir in searchdirs: + libpath = util.normaljoin(dir, libname).replace('\\', '/') + fspath = util.normaljoin(makefile.workdir, libpath) + mtime = getmtime(fspath) + if mtime is not None: + self.vpathtarget = libpath + self.mtime = mtime + return + + self.vpathtarget = self.target + self.mtime = None + return + + search = [self.target] + if not os.path.isabs(self.target): + search += [util.normaljoin(dir, self.target).replace('\\', '/') + for dir in makefile.getvpath(self.target)] + + targetandtime = self.searchinlocs(makefile, search) + if targetandtime is not None: + (self.vpathtarget, self.mtime) = targetandtime + return + + self.vpathtarget = self.target + self.mtime = None + + def searchinlocs(self, makefile, locs): + """ + Look in the given locations relative to the makefile working directory + for a file. Return a pair of the target and the mtime if found, None + if not. + """ + for t in locs: + fspath = util.normaljoin(makefile.workdir, t).replace('\\', '/') + mtime = getmtime(fspath) +# _log.info("Searching %s ... checking %s ... mtime %r" % (t, fspath, mtime)) + if mtime is not None: + return (t, mtime) + + return None + + def beingremade(self): + """ + When we remake ourself, we have to drop any vpath prefixes. + """ + self.vpathtarget = self.target + self.wasremade = True + + def notifydone(self, makefile): + assert self._state == MAKESTATE_WORKING, "State was %s" % self._state + # If we were remade then resolve mtime again + if self.wasremade: + targetandtime = self.searchinlocs(makefile, [self.target]) + if targetandtime is not None: + (_, self.mtime) = targetandtime + else: + self.mtime = None + + self._state = MAKESTATE_FINISHED + for cb in self._callbacks: + makefile.context.defer(cb, error=self.error, didanything=self.didanything) + del self._callbacks + + def make(self, makefile, targetstack, cb, avoidremakeloop=False, printerror=True): + """ + If we are out of date, asynchronously make ourself. This is a multi-stage process, mostly handled + by the helper objects RemakeTargetSerially, RemakeTargetParallel, + RemakeRuleContext. These helper objects should keep us from developing + any cyclical dependencies. + + * resolve dependencies (synchronous) + * gather a list of rules to execute and related dependencies (synchronous) + * for each rule (in parallel) + ** remake dependencies (asynchronous) + ** build list of commands to execute (synchronous) + ** execute each command (asynchronous) + * asynchronously notify when all rules are complete + + @param cb A callback function to notify when remaking is finished. It is called + thusly: callback(error=True/False, didanything=True/False) + If there is no asynchronous activity to perform, the callback may be called directly. + """ + + serial = makefile.context.jcount == 1 + + if self._state == MAKESTATE_FINISHED: + cb(error=self.error, didanything=self.didanything) + return + + if self._state == MAKESTATE_WORKING: + assert not serial + self._callbacks.append(cb) + return + + assert self._state == MAKESTATE_NONE + + self._state = MAKESTATE_WORKING + self._callbacks = [cb] + self.error = False + self.didanything = False + + indent = getindent(targetstack) + + try: + self.resolvedeps(makefile, targetstack, [], False) + except util.MakeError, e: + if printerror: + print e + self.error = True + self.notifydone(makefile) + return + + assert self.vpathtarget is not None, "Target was never resolved!" + if not len(self.rules): + self.notifydone(makefile) + return + + if self.isdoublecolon(): + rulelist = [RemakeRuleContext(self, makefile, r, [(makefile.gettarget(p), False) for p in r.prerequisites], targetstack, avoidremakeloop) for r in self.rules] + else: + alldeps = [] + + commandrule = None + for r in self.rules: + rdeps = [(makefile.gettarget(p), r.weakdeps) for p in r.prerequisites] + if len(r.commands): + assert commandrule is None + commandrule = r + # The dependencies of the command rule are resolved before other dependencies, + # no matter the ordering of the other no-command rules + alldeps[0:0] = rdeps + else: + alldeps.extend(rdeps) + + rulelist = [RemakeRuleContext(self, makefile, commandrule, alldeps, targetstack, avoidremakeloop)] + + targetstack = targetstack + [self.target] + + if serial: + RemakeTargetSerially(self, makefile, indent, rulelist) + else: + RemakeTargetParallel(self, makefile, indent, rulelist) + +def dirpart(p): + d, s, f = util.strrpartition(p, '/') + if d == '': + return '.' + + return d + +def filepart(p): + d, s, f = util.strrpartition(p, '/') + return f + +def setautomatic(v, name, plist): + v.set(name, Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, ' '.join(plist)) + v.set(name + 'D', Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, ' '.join((dirpart(p) for p in plist))) + v.set(name + 'F', Variables.FLAVOR_SIMPLE, Variables.SOURCE_AUTOMATIC, ' '.join((filepart(p) for p in plist))) + +def setautomaticvariables(v, makefile, target, prerequisites): + prtargets = [makefile.gettarget(p) for p in prerequisites] + prall = [pt.vpathtarget for pt in prtargets] + proutofdate = [pt.vpathtarget for pt in withoutdups(prtargets) + if target.mtime is None or mtimeislater(pt.mtime, target.mtime)] + + setautomatic(v, '@', [target.vpathtarget]) + if len(prall): + setautomatic(v, '<', [prall[0]]) + + setautomatic(v, '?', proutofdate) + setautomatic(v, '^', list(withoutdups(prall))) + setautomatic(v, '+', prall) + +def splitcommand(command): + """ + Using the esoteric rules, split command lines by unescaped newlines. + """ + start = 0 + i = 0 + while i < len(command): + c = command[i] + if c == '\\': + i += 1 + elif c == '\n': + yield command[start:i] + i += 1 + start = i + continue + + i += 1 + + if i > start: + yield command[start:i] + +def findmodifiers(command): + """ + Find any of +-@% prefixed on the command. + @returns (command, isHidden, isRecursive, ignoreErrors, isNative) + """ + + isHidden = False + isRecursive = False + ignoreErrors = False + isNative = False + + realcommand = command.lstrip(' \t\n@+-%') + modset = set(command[:-len(realcommand)]) + return realcommand, '@' in modset, '+' in modset, '-' in modset, '%' in modset + +class _CommandWrapper(object): + def __init__(self, cline, ignoreErrors, loc, context, **kwargs): + self.ignoreErrors = ignoreErrors + self.loc = loc + self.cline = cline + self.kwargs = kwargs + self.context = context + + def _cb(self, res): + if res != 0 and not self.ignoreErrors: + print "%s: command '%s' failed, return code %i" % (self.loc, self.cline, res) + self.usercb(error=True) + else: + self.usercb(error=False) + + def __call__(self, cb): + self.usercb = cb + process.call(self.cline, loc=self.loc, cb=self._cb, context=self.context, **self.kwargs) + +class _NativeWrapper(_CommandWrapper): + def __init__(self, cline, ignoreErrors, loc, context, + pycommandpath, **kwargs): + _CommandWrapper.__init__(self, cline, ignoreErrors, loc, context, + **kwargs) + if pycommandpath: + self.pycommandpath = re.split('[%s\s]+' % os.pathsep, + pycommandpath) + else: + self.pycommandpath = None + + def __call__(self, cb): + # get the module and method to call + parts, badchar = process.clinetoargv(self.cline, self.kwargs['cwd']) + if parts is None: + raise DataError("native command '%s': shell metacharacter '%s' in command line" % (self.cline, badchar), self.loc) + if len(parts) < 2: + raise DataError("native command '%s': no method name specified" % self.cline, self.loc) + module = parts[0] + method = parts[1] + cline_list = parts[2:] + self.usercb = cb + process.call_native(module, method, cline_list, + loc=self.loc, cb=self._cb, context=self.context, + pycommandpath=self.pycommandpath, **self.kwargs) + +def getcommandsforrule(rule, target, makefile, prerequisites, stem): + v = Variables(parent=target.variables) + setautomaticvariables(v, makefile, target, prerequisites) + if stem is not None: + setautomatic(v, '*', [stem]) + + env = makefile.getsubenvironment(v) + + for c in rule.commands: + cstring = c.resolvestr(makefile, v) + for cline in splitcommand(cstring): + cline, isHidden, isRecursive, ignoreErrors, isNative = findmodifiers(cline) + if (isHidden or makefile.silent) and not makefile.justprint: + echo = None + else: + echo = "%s$ %s" % (c.loc, cline) + if not isNative: + yield _CommandWrapper(cline, ignoreErrors=ignoreErrors, env=env, cwd=makefile.workdir, loc=c.loc, context=makefile.context, + echo=echo, justprint=makefile.justprint) + else: + f, s, e = v.get("PYCOMMANDPATH", True) + if e: + e = e.resolvestr(makefile, v, ["PYCOMMANDPATH"]) + yield _NativeWrapper(cline, ignoreErrors=ignoreErrors, + env=env, cwd=makefile.workdir, + loc=c.loc, context=makefile.context, + echo=echo, justprint=makefile.justprint, + pycommandpath=e) + +class Rule(object): + """ + A rule contains a list of prerequisites and a list of commands. It may also + contain rule-specific variables. This rule may be associated with multiple targets. + """ + + def __init__(self, prereqs, doublecolon, loc, weakdeps): + self.prerequisites = prereqs + self.doublecolon = doublecolon + self.commands = [] + self.loc = loc + self.weakdeps = weakdeps + + def addcommand(self, c): + assert isinstance(c, (Expansion, StringExpansion)) + self.commands.append(c) + + def getcommands(self, target, makefile): + assert isinstance(target, Target) + # Prerequisites are merged if the target contains multiple rules and is + # not a terminal (double colon) rule. See + # https://www.gnu.org/software/make/manual/make.html#Multiple-Targets. + prereqs = [] + prereqs.extend(self.prerequisites) + + if not self.doublecolon: + for rule in target.rules: + # The current rule comes first, which is already in prereqs so + # we don't need to add it again. + if rule != self: + prereqs.extend(rule.prerequisites) + + return getcommandsforrule(self, target, makefile, prereqs, stem=None) + # TODO: $* in non-pattern rules? + +class PatternRuleInstance(object): + weakdeps = False + + """ + A pattern rule instantiated for a particular target. It has the same API as Rule, but + different internals, forwarding most information on to the PatternRule. + """ + def __init__(self, prule, dir, stem, ismatchany): + assert isinstance(prule, PatternRule) + + self.dir = dir + self.stem = stem + self.prule = prule + self.prerequisites = prule.prerequisitesforstem(dir, stem) + self.doublecolon = prule.doublecolon + self.loc = prule.loc + self.ismatchany = ismatchany + self.commands = prule.commands + + def getcommands(self, target, makefile): + assert isinstance(target, Target) + return getcommandsforrule(self, target, makefile, self.prerequisites, stem=self.dir + self.stem) + + def __str__(self): + return "Pattern rule at %s with stem '%s', matchany: %s doublecolon: %s" % (self.loc, + self.dir + self.stem, + self.ismatchany, + self.doublecolon) + +class PatternRule(object): + """ + An implicit rule or static pattern rule containing target patterns, prerequisite patterns, + and a list of commands. + """ + + def __init__(self, targetpatterns, prerequisites, doublecolon, loc): + self.targetpatterns = targetpatterns + self.prerequisites = prerequisites + self.doublecolon = doublecolon + self.loc = loc + self.commands = [] + + def addcommand(self, c): + assert isinstance(c, (Expansion, StringExpansion)) + self.commands.append(c) + + def ismatchany(self): + return util.any((t.ismatchany() for t in self.targetpatterns)) + + def hasspecificmatch(self, file): + for p in self.targetpatterns: + if not p.ismatchany() and p.match(file) is not None: + return True + + return False + + def matchesfor(self, dir, file, skipsinglecolonmatchany): + """ + Determine all the target patterns of this rule that might match target t. + @yields a PatternRuleInstance for each. + """ + + for p in self.targetpatterns: + matchany = p.ismatchany() + if matchany: + if skipsinglecolonmatchany and not self.doublecolon: + continue + + yield PatternRuleInstance(self, dir, file, True) + else: + stem = p.match(dir + file) + if stem is not None: + yield PatternRuleInstance(self, '', stem, False) + else: + stem = p.match(file) + if stem is not None: + yield PatternRuleInstance(self, dir, stem, False) + + def prerequisitesforstem(self, dir, stem): + return [p.resolve(dir, stem) for p in self.prerequisites] + +class _RemakeContext(object): + def __init__(self, makefile, cb): + self.makefile = makefile + self.included = [(makefile.gettarget(f), required) + for f, required in makefile.included] + self.toremake = list(self.included) + self.cb = cb + + self.remakecb(error=False, didanything=False) + + def remakecb(self, error, didanything): + assert error in (True, False) + + if error and self.required: + print "Error remaking makefiles (ignored)" + + if len(self.toremake): + target, self.required = self.toremake.pop(0) + target.make(self.makefile, [], avoidremakeloop=True, cb=self.remakecb, printerror=False) + else: + for t, required in self.included: + if t.wasremade: + _log.info("Included file %s was remade, restarting make", t.target) + self.cb(remade=True) + return + elif required and t.mtime is None: + self.cb(remade=False, error=DataError("No rule to remake missing include file %s" % t.target)) + return + + self.cb(remade=False) + +class Makefile(object): + """ + The top-level data structure for makefile execution. It holds Targets, implicit rules, and other + state data. + """ + + def __init__(self, workdir=None, env=None, restarts=0, make=None, + makeflags='', makeoverrides='', + makelevel=0, context=None, targets=(), keepgoing=False, + silent=False, justprint=False): + self.defaulttarget = None + + if env is None: + env = os.environ + self.env = env + + self.variables = Variables() + self.variables.readfromenvironment(env) + + self.context = context + self.exportedvars = {} + self._targets = {} + self.keepgoing = keepgoing + self.silent = silent + self.justprint = justprint + self._patternvariables = [] # of (pattern, variables) + self.implicitrules = [] + self.parsingfinished = False + + self._patternvpaths = [] # of (pattern, [dir, ...]) + + if workdir is None: + workdir = os.getcwd() + workdir = os.path.realpath(workdir) + self.workdir = workdir + self.variables.set('CURDIR', Variables.FLAVOR_SIMPLE, + Variables.SOURCE_AUTOMATIC, workdir.replace('\\','/')) + + # the list of included makefiles, whether or not they existed + self.included = [] + + self.variables.set('MAKE_RESTARTS', Variables.FLAVOR_SIMPLE, + Variables.SOURCE_AUTOMATIC, restarts > 0 and str(restarts) or '') + + self.variables.set('.PYMAKE', Variables.FLAVOR_SIMPLE, + Variables.SOURCE_MAKEFILE, "1") + if make is not None: + self.variables.set('MAKE', Variables.FLAVOR_SIMPLE, + Variables.SOURCE_MAKEFILE, make) + + if makeoverrides != '': + self.variables.set('-*-command-variables-*-', Variables.FLAVOR_SIMPLE, + Variables.SOURCE_AUTOMATIC, makeoverrides) + makeflags += ' -- $(MAKEOVERRIDES)' + + self.variables.set('MAKEOVERRIDES', Variables.FLAVOR_RECURSIVE, + Variables.SOURCE_ENVIRONMENT, + '${-*-command-variables-*-}') + + self.variables.set('MAKEFLAGS', Variables.FLAVOR_RECURSIVE, + Variables.SOURCE_MAKEFILE, makeflags) + self.exportedvars['MAKEFLAGS'] = True + + self.makelevel = makelevel + self.variables.set('MAKELEVEL', Variables.FLAVOR_SIMPLE, + Variables.SOURCE_MAKEFILE, str(makelevel)) + + self.variables.set('MAKECMDGOALS', Variables.FLAVOR_SIMPLE, + Variables.SOURCE_AUTOMATIC, ' '.join(targets)) + + for vname, val in implicit.variables.iteritems(): + self.variables.set(vname, + Variables.FLAVOR_SIMPLE, + Variables.SOURCE_IMPLICIT, val) + + def foundtarget(self, t): + """ + Inform the makefile of a target which is a candidate for being the default target, + if there isn't already a default target. + """ + flavor, source, value = self.variables.get('.DEFAULT_GOAL') + if self.defaulttarget is None and t != '.PHONY' and value is None: + self.defaulttarget = t + self.variables.set('.DEFAULT_GOAL', Variables.FLAVOR_SIMPLE, + Variables.SOURCE_AUTOMATIC, t) + + def getpatternvariables(self, pattern): + assert isinstance(pattern, Pattern) + + for p, v in self._patternvariables: + if p == pattern: + return v + + v = Variables() + self._patternvariables.append( (pattern, v) ) + return v + + def getpatternvariablesfor(self, target): + for p, v in self._patternvariables: + if p.match(target): + yield v + + def hastarget(self, target): + return target in self._targets + + _globcheck = re.compile('[[*?]') + def gettarget(self, target): + assert isinstance(target, str_type) + + target = target.rstrip('/') + + assert target != '', "empty target?" + + assert not self._globcheck.match(target) + + t = self._targets.get(target, None) + if t is None: + t = Target(target, self) + self._targets[target] = t + return t + + def appendimplicitrule(self, rule): + assert isinstance(rule, PatternRule) + self.implicitrules.append(rule) + + def finishparsing(self): + """ + Various activities, such as "eval", are not allowed after parsing is + finished. In addition, various warnings and errors can only be issued + after the parsing data model is complete. All dependency resolution + and rule execution requires that parsing be finished. + """ + self.parsingfinished = True + + flavor, source, value = self.variables.get('GPATH') + if value is not None and value.resolvestr(self, self.variables, ['GPATH']).strip() != '': + raise DataError('GPATH was set: pymake does not support GPATH semantics') + + flavor, source, value = self.variables.get('VPATH') + if value is None: + self._vpath = [] + else: + self._vpath = filter(lambda e: e != '', + re.split('[%s\s]+' % os.pathsep, + value.resolvestr(self, self.variables, ['VPATH']))) + + targets = list(self._targets.itervalues()) + for t in targets: + t.explicit = True + for r in t.rules: + for p in r.prerequisites: + self.gettarget(p).explicit = True + + np = self.gettarget('.NOTPARALLEL') + if len(np.rules): + self.context = process.getcontext(1) + + flavor, source, value = self.variables.get('.DEFAULT_GOAL') + if value is not None: + self.defaulttarget = value.resolvestr(self, self.variables, ['.DEFAULT_GOAL']).strip() + + self.error = False + + def include(self, path, required=True, weak=False, loc=None): + """ + Include the makefile at `path`. + """ + self.included.append((path, required)) + fspath = util.normaljoin(self.workdir, path) + if os.path.exists(fspath): + if weak: + stmts = parser.parsedepfile(fspath) + else: + stmts = parser.parsefile(fspath) + self.variables.append('MAKEFILE_LIST', Variables.SOURCE_AUTOMATIC, path, None, self) + stmts.execute(self, weak=weak) + self.gettarget(path).explicit = True + + def addvpath(self, pattern, dirs): + """ + Add a directory to the vpath search for the given pattern. + """ + self._patternvpaths.append((pattern, dirs)) + + def clearvpath(self, pattern): + """ + Clear vpaths for the given pattern. + """ + self._patternvpaths = [(p, dirs) + for p, dirs in self._patternvpaths + if not p.match(pattern)] + + def clearallvpaths(self): + self._patternvpaths = [] + + def getvpath(self, target): + vp = list(self._vpath) + for p, dirs in self._patternvpaths: + if p.match(target): + vp.extend(dirs) + + return withoutdups(vp) + + def remakemakefiles(self, cb): + mlist = [] + for f, required in self.included: + t = self.gettarget(f) + t.explicit = True + t.resolvevpath(self) + oldmtime = t.mtime + + mlist.append((t, oldmtime)) + + _RemakeContext(self, cb) + + def getsubenvironment(self, variables): + env = dict(self.env) + for vname, v in self.exportedvars.iteritems(): + if v: + flavor, source, val = variables.get(vname) + if val is None: + strval = '' + else: + strval = val.resolvestr(self, variables, [vname]) + env[vname] = strval + else: + env.pop(vname, None) + + makeflags = '' + + env['MAKELEVEL'] = str(self.makelevel + 1) + return env diff --git a/python/pymake/pymake/functions.py b/python/pymake/pymake/functions.py new file mode 100644 index 000000000..e53fb5472 --- /dev/null +++ b/python/pymake/pymake/functions.py @@ -0,0 +1,873 @@ +""" +Makefile functions. +""" + +import parser, util +import subprocess, os, logging, sys +from globrelative import glob +from cStringIO import StringIO + +log = logging.getLogger('pymake.data') + +def emit_expansions(descend, *expansions): + """Helper function to emit all expansions within an input set.""" + for expansion in expansions: + yield expansion + + if not descend or not isinstance(expansion, list): + continue + + for e, is_func in expansion: + if is_func: + for exp in e.expansions(True): + yield exp + else: + yield e + +class Function(object): + """ + An object that represents a function call. This class is always subclassed + with the following methods and attributes: + + minargs = minimum # of arguments + maxargs = maximum # of arguments (0 means unlimited) + + def resolve(self, makefile, variables, fd, setting) + Calls the function + calls fd.write() with strings + """ + + __slots__ = ('_arguments', 'loc') + + def __init__(self, loc): + self._arguments = [] + self.loc = loc + assert self.minargs > 0 + + def __getitem__(self, key): + return self._arguments[key] + + def setup(self): + argc = len(self._arguments) + + if argc < self.minargs: + raise data.DataError("Not enough arguments to function %s, requires %s" % (self.name, self.minargs), self.loc) + + assert self.maxargs == 0 or argc <= self.maxargs, "Parser screwed up, gave us too many args" + + def append(self, arg): + assert isinstance(arg, (data.Expansion, data.StringExpansion)) + self._arguments.append(arg) + + def to_source(self): + """Convert the function back to make file "source" code.""" + if not hasattr(self, 'name'): + raise Exception("%s must implement to_source()." % self.__class__) + + # The default implementation simply prints the function name and all + # the arguments joined by a comma. + # According to the GNU make manual Section 8.1, whitespace around + # arguments is *not* part of the argument's value. So, we trim excess + # white space so we have consistent behavior. + args = [] + curly = False + for i, arg in enumerate(self._arguments): + arg = arg.to_source() + + if i == 0: + arg = arg.lstrip() + + # Are balanced parens even OK? + if arg.count('(') != arg.count(')'): + curly = True + + args.append(arg) + + if curly: + return '${%s %s}' % (self.name, ','.join(args)) + + return '$(%s %s)' % (self.name, ','.join(args)) + + def expansions(self, descend=False): + """Obtain all expansions contained within this function. + + By default, only expansions directly part of this function are + returned. If descend is True, we will descend into child expansions and + return all of the composite parts. + + This is a generator for pymake.data.BaseExpansion instances. + """ + # Our default implementation simply returns arguments. More advanced + # functions like variable references may need their own implementation. + return emit_expansions(descend, *self._arguments) + + @property + def is_filesystem_dependent(self): + """Exposes whether this function depends on the filesystem for results. + + If True, the function touches the filesystem as part of evaluation. + + This only tests whether the function itself uses the filesystem. If + this function has arguments that are functions that touch the + filesystem, this will return False. + """ + return False + + def __len__(self): + return len(self._arguments) + + def __repr__(self): + return "%s<%s>(%r)" % ( + self.__class__.__name__, self.loc, + ','.join([repr(a) for a in self._arguments]), + ) + + def __eq__(self, other): + if not hasattr(self, 'name'): + raise Exception("%s must implement __eq__." % self.__class__) + + if type(self) != type(other): + return False + + if self.name != other.name: + return False + + if len(self._arguments) != len(other._arguments): + return False + + for i in xrange(len(self._arguments)): + # According to the GNU make manual Section 8.1, whitespace around + # arguments is *not* part of the argument's value. So, we do a + # whitespace-agnostic comparison. + if i == 0: + a = self._arguments[i] + a.lstrip() + + b = other._arguments[i] + b.lstrip() + + if a != b: + return False + + continue + + if self._arguments[i] != other._arguments[i]: + return False + + return True + + def __ne__(self, other): + return not self.__eq__(other) + +class VariableRef(Function): + AUTOMATIC_VARIABLES = set(['@', '%', '<', '?', '^', '+', '|', '*']) + + __slots__ = ('vname', 'loc') + + def __init__(self, loc, vname): + self.loc = loc + assert isinstance(vname, (data.Expansion, data.StringExpansion)) + self.vname = vname + + def setup(self): + assert False, "Shouldn't get here" + + def resolve(self, makefile, variables, fd, setting): + vname = self.vname.resolvestr(makefile, variables, setting) + if vname in setting: + raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc) + + flavor, source, value = variables.get(vname) + if value is None: + log.debug("%s: variable '%s' was not set" % (self.loc, vname)) + return + + value.resolve(makefile, variables, fd, setting + [vname]) + + def to_source(self): + if isinstance(self.vname, data.StringExpansion): + if self.vname.s in self.AUTOMATIC_VARIABLES: + return '$%s' % self.vname.s + + return '$(%s)' % self.vname.s + + return '$(%s)' % self.vname.to_source() + + def expansions(self, descend=False): + return emit_expansions(descend, self.vname) + + def __repr__(self): + return "VariableRef<%s>(%r)" % (self.loc, self.vname) + + def __eq__(self, other): + if not isinstance(other, VariableRef): + return False + + return self.vname == other.vname + +class SubstitutionRef(Function): + """$(VARNAME:.c=.o) and $(VARNAME:%.c=%.o)""" + + __slots__ = ('loc', 'vname', 'substfrom', 'substto') + + def __init__(self, loc, varname, substfrom, substto): + self.loc = loc + self.vname = varname + self.substfrom = substfrom + self.substto = substto + + def setup(self): + assert False, "Shouldn't get here" + + def resolve(self, makefile, variables, fd, setting): + vname = self.vname.resolvestr(makefile, variables, setting) + if vname in setting: + raise data.DataError("Setting variable '%s' recursively references itself." % (vname,), self.loc) + + substfrom = self.substfrom.resolvestr(makefile, variables, setting) + substto = self.substto.resolvestr(makefile, variables, setting) + + flavor, source, value = variables.get(vname) + if value is None: + log.debug("%s: variable '%s' was not set" % (self.loc, vname)) + return + + f = data.Pattern(substfrom) + if not f.ispattern(): + f = data.Pattern('%' + substfrom) + substto = '%' + substto + + fd.write(' '.join([f.subst(substto, word, False) + for word in value.resolvesplit(makefile, variables, setting + [vname])])) + + def to_source(self): + return '$(%s:%s=%s)' % ( + self.vname.to_source(), + self.substfrom.to_source(), + self.substto.to_source()) + + def expansions(self, descend=False): + return emit_expansions(descend, self.vname, self.substfrom, + self.substto) + + def __repr__(self): + return "SubstitutionRef<%s>(%r:%r=%r)" % ( + self.loc, self.vname, self.substfrom, self.substto,) + + def __eq__(self, other): + if not isinstance(other, SubstitutionRef): + return False + + return self.vname == other.vname and self.substfrom == other.substfrom \ + and self.substto == other.substto + +class SubstFunction(Function): + name = 'subst' + minargs = 3 + maxargs = 3 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + s = self._arguments[0].resolvestr(makefile, variables, setting) + r = self._arguments[1].resolvestr(makefile, variables, setting) + d = self._arguments[2].resolvestr(makefile, variables, setting) + fd.write(d.replace(s, r)) + +class PatSubstFunction(Function): + name = 'patsubst' + minargs = 3 + maxargs = 3 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + s = self._arguments[0].resolvestr(makefile, variables, setting) + r = self._arguments[1].resolvestr(makefile, variables, setting) + + p = data.Pattern(s) + fd.write(' '.join([p.subst(r, word, False) + for word in self._arguments[2].resolvesplit(makefile, variables, setting)])) + +class StripFunction(Function): + name = 'strip' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + util.joiniter(fd, self._arguments[0].resolvesplit(makefile, variables, setting)) + +class FindstringFunction(Function): + name = 'findstring' + minargs = 2 + maxargs = 2 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + s = self._arguments[0].resolvestr(makefile, variables, setting) + r = self._arguments[1].resolvestr(makefile, variables, setting) + if r.find(s) == -1: + return + fd.write(s) + +class FilterFunction(Function): + name = 'filter' + minargs = 2 + maxargs = 2 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + plist = [data.Pattern(p) + for p in self._arguments[0].resolvesplit(makefile, variables, setting)] + + fd.write(' '.join([w for w in self._arguments[1].resolvesplit(makefile, variables, setting) + if util.any((p.match(w) for p in plist))])) + +class FilteroutFunction(Function): + name = 'filter-out' + minargs = 2 + maxargs = 2 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + plist = [data.Pattern(p) + for p in self._arguments[0].resolvesplit(makefile, variables, setting)] + + fd.write(' '.join([w for w in self._arguments[1].resolvesplit(makefile, variables, setting) + if not util.any((p.match(w) for p in plist))])) + +class SortFunction(Function): + name = 'sort' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + d = set(self._arguments[0].resolvesplit(makefile, variables, setting)) + util.joiniter(fd, sorted(d)) + +class WordFunction(Function): + name = 'word' + minargs = 2 + maxargs = 2 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + n = self._arguments[0].resolvestr(makefile, variables, setting) + # TODO: provide better error if this doesn't convert + n = int(n) + words = list(self._arguments[1].resolvesplit(makefile, variables, setting)) + if n < 1 or n > len(words): + return + fd.write(words[n - 1]) + +class WordlistFunction(Function): + name = 'wordlist' + minargs = 3 + maxargs = 3 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + nfrom = self._arguments[0].resolvestr(makefile, variables, setting) + nto = self._arguments[1].resolvestr(makefile, variables, setting) + # TODO: provide better errors if this doesn't convert + nfrom = int(nfrom) + nto = int(nto) + + words = list(self._arguments[2].resolvesplit(makefile, variables, setting)) + + if nfrom < 1: + nfrom = 1 + if nto < 1: + nto = 1 + + util.joiniter(fd, words[nfrom - 1:nto]) + +class WordsFunction(Function): + name = 'words' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + fd.write(str(len(self._arguments[0].resolvesplit(makefile, variables, setting)))) + +class FirstWordFunction(Function): + name = 'firstword' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + l = self._arguments[0].resolvesplit(makefile, variables, setting) + if len(l): + fd.write(l[0]) + +class LastWordFunction(Function): + name = 'lastword' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + l = self._arguments[0].resolvesplit(makefile, variables, setting) + if len(l): + fd.write(l[-1]) + +def pathsplit(path, default='./'): + """ + Splits a path into dirpart, filepart on the last slash. If there is no slash, dirpart + is ./ + """ + dir, slash, file = util.strrpartition(path, '/') + if dir == '': + return default, file + + return dir + slash, file + +class DirFunction(Function): + name = 'dir' + minargs = 1 + maxargs = 1 + + def resolve(self, makefile, variables, fd, setting): + fd.write(' '.join([pathsplit(path)[0] + for path in self._arguments[0].resolvesplit(makefile, variables, setting)])) + +class NotDirFunction(Function): + name = 'notdir' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + fd.write(' '.join([pathsplit(path)[1] + for path in self._arguments[0].resolvesplit(makefile, variables, setting)])) + +class SuffixFunction(Function): + name = 'suffix' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + @staticmethod + def suffixes(words): + for w in words: + dir, file = pathsplit(w) + base, dot, suffix = util.strrpartition(file, '.') + if base != '': + yield dot + suffix + + def resolve(self, makefile, variables, fd, setting): + util.joiniter(fd, self.suffixes(self._arguments[0].resolvesplit(makefile, variables, setting))) + +class BasenameFunction(Function): + name = 'basename' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + @staticmethod + def basenames(words): + for w in words: + dir, file = pathsplit(w, '') + base, dot, suffix = util.strrpartition(file, '.') + if dot == '': + base = suffix + + yield dir + base + + def resolve(self, makefile, variables, fd, setting): + util.joiniter(fd, self.basenames(self._arguments[0].resolvesplit(makefile, variables, setting))) + +class AddSuffixFunction(Function): + name = 'addsuffix' + minargs = 2 + maxargs = 2 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + suffix = self._arguments[0].resolvestr(makefile, variables, setting) + + fd.write(' '.join([w + suffix for w in self._arguments[1].resolvesplit(makefile, variables, setting)])) + +class AddPrefixFunction(Function): + name = 'addprefix' + minargs = 2 + maxargs = 2 + + def resolve(self, makefile, variables, fd, setting): + prefix = self._arguments[0].resolvestr(makefile, variables, setting) + + fd.write(' '.join([prefix + w for w in self._arguments[1].resolvesplit(makefile, variables, setting)])) + +class JoinFunction(Function): + name = 'join' + minargs = 2 + maxargs = 2 + + __slots__ = Function.__slots__ + + @staticmethod + def iterjoin(l1, l2): + for i in xrange(0, max(len(l1), len(l2))): + i1 = i < len(l1) and l1[i] or '' + i2 = i < len(l2) and l2[i] or '' + yield i1 + i2 + + def resolve(self, makefile, variables, fd, setting): + list1 = list(self._arguments[0].resolvesplit(makefile, variables, setting)) + list2 = list(self._arguments[1].resolvesplit(makefile, variables, setting)) + + util.joiniter(fd, self.iterjoin(list1, list2)) + +class WildcardFunction(Function): + name = 'wildcard' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + patterns = self._arguments[0].resolvesplit(makefile, variables, setting) + + fd.write(' '.join([x.replace('\\','/') + for p in patterns + for x in glob(makefile.workdir, p)])) + + @property + def is_filesystem_dependent(self): + return True + +class RealpathFunction(Function): + name = 'realpath' + minargs = 1 + maxargs = 1 + + def resolve(self, makefile, variables, fd, setting): + fd.write(' '.join([os.path.realpath(os.path.join(makefile.workdir, path)).replace('\\', '/') + for path in self._arguments[0].resolvesplit(makefile, variables, setting)])) + + def is_filesystem_dependent(self): + return True + +class AbspathFunction(Function): + name = 'abspath' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + assert os.path.isabs(makefile.workdir) + fd.write(' '.join([util.normaljoin(makefile.workdir, path).replace('\\', '/') + for path in self._arguments[0].resolvesplit(makefile, variables, setting)])) + +class IfFunction(Function): + name = 'if' + minargs = 1 + maxargs = 3 + + __slots__ = Function.__slots__ + + def setup(self): + Function.setup(self) + self._arguments[0].lstrip() + self._arguments[0].rstrip() + + def resolve(self, makefile, variables, fd, setting): + condition = self._arguments[0].resolvestr(makefile, variables, setting) + + if len(condition): + self._arguments[1].resolve(makefile, variables, fd, setting) + elif len(self._arguments) > 2: + return self._arguments[2].resolve(makefile, variables, fd, setting) + +class OrFunction(Function): + name = 'or' + minargs = 1 + maxargs = 0 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + for arg in self._arguments: + r = arg.resolvestr(makefile, variables, setting) + if r != '': + fd.write(r) + return + +class AndFunction(Function): + name = 'and' + minargs = 1 + maxargs = 0 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + r = '' + + for arg in self._arguments: + r = arg.resolvestr(makefile, variables, setting) + if r == '': + return + + fd.write(r) + +class ForEachFunction(Function): + name = 'foreach' + minargs = 3 + maxargs = 3 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + vname = self._arguments[0].resolvestr(makefile, variables, setting) + e = self._arguments[2] + + v = data.Variables(parent=variables) + firstword = True + + for w in self._arguments[1].resolvesplit(makefile, variables, setting): + if firstword: + firstword = False + else: + fd.write(' ') + + # The $(origin) of the local variable must be "automatic" to + # conform with GNU make. However, automatic variables have low + # priority. So, we must force its assignment to occur. + v.set(vname, data.Variables.FLAVOR_SIMPLE, + data.Variables.SOURCE_AUTOMATIC, w, force=True) + e.resolve(makefile, v, fd, setting) + +class CallFunction(Function): + name = 'call' + minargs = 1 + maxargs = 0 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + vname = self._arguments[0].resolvestr(makefile, variables, setting) + if vname in setting: + raise data.DataError("Recursively setting variable '%s'" % (vname,)) + + v = data.Variables(parent=variables) + v.set('0', data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, vname) + for i in xrange(1, len(self._arguments)): + param = self._arguments[i].resolvestr(makefile, variables, setting) + v.set(str(i), data.Variables.FLAVOR_SIMPLE, data.Variables.SOURCE_AUTOMATIC, param) + + flavor, source, e = variables.get(vname) + + if e is None: + return + + if flavor == data.Variables.FLAVOR_SIMPLE: + log.warning("%s: calling variable '%s' which is simply-expanded" % (self.loc, vname)) + + # but we'll do it anyway + e.resolve(makefile, v, fd, setting + [vname]) + +class ValueFunction(Function): + name = 'value' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + varname = self._arguments[0].resolvestr(makefile, variables, setting) + + flavor, source, value = variables.get(varname, expand=False) + if value is not None: + fd.write(value) + +class EvalFunction(Function): + name = 'eval' + minargs = 1 + maxargs = 1 + + def resolve(self, makefile, variables, fd, setting): + if makefile.parsingfinished: + # GNU make allows variables to be set by recursive expansion during + # command execution. This seems really dumb to me, so I don't! + raise data.DataError("$(eval) not allowed via recursive expansion after parsing is finished", self.loc) + + stmts = parser.parsestring(self._arguments[0].resolvestr(makefile, variables, setting), + 'evaluation from %s' % self.loc) + stmts.execute(makefile) + +class OriginFunction(Function): + name = 'origin' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + vname = self._arguments[0].resolvestr(makefile, variables, setting) + + flavor, source, value = variables.get(vname) + if source is None: + r = 'undefined' + elif source == data.Variables.SOURCE_OVERRIDE: + r = 'override' + + elif source == data.Variables.SOURCE_MAKEFILE: + r = 'file' + elif source == data.Variables.SOURCE_ENVIRONMENT: + r = 'environment' + elif source == data.Variables.SOURCE_COMMANDLINE: + r = 'command line' + elif source == data.Variables.SOURCE_AUTOMATIC: + r = 'automatic' + elif source == data.Variables.SOURCE_IMPLICIT: + r = 'default' + + fd.write(r) + +class FlavorFunction(Function): + name = 'flavor' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + varname = self._arguments[0].resolvestr(makefile, variables, setting) + + flavor, source, value = variables.get(varname) + if flavor is None: + r = 'undefined' + elif flavor == data.Variables.FLAVOR_RECURSIVE: + r = 'recursive' + elif flavor == data.Variables.FLAVOR_SIMPLE: + r = 'simple' + fd.write(r) + +class ShellFunction(Function): + name = 'shell' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + from process import prepare_command + cline = self._arguments[0].resolvestr(makefile, variables, setting) + executable, cline = prepare_command(cline, makefile.workdir, self.loc) + + # subprocess.Popen doesn't use the PATH set in the env argument for + # finding the executable on some platforms (but strangely it does on + # others!), so set os.environ['PATH'] explicitly. + oldpath = os.environ['PATH'] + if makefile.env is not None and 'PATH' in makefile.env: + os.environ['PATH'] = makefile.env['PATH'] + + log.debug("%s: running command '%s'" % (self.loc, ' '.join(cline))) + try: + p = subprocess.Popen(cline, executable=executable, env=makefile.env, shell=False, + stdout=subprocess.PIPE, cwd=makefile.workdir) + except OSError, e: + print >>sys.stderr, "Error executing command %s" % cline[0], e + return + finally: + os.environ['PATH'] = oldpath + + stdout, stderr = p.communicate() + stdout = stdout.replace('\r\n', '\n') + if stdout.endswith('\n'): + stdout = stdout[:-1] + stdout = stdout.replace('\n', ' ') + + fd.write(stdout) + +class ErrorFunction(Function): + name = 'error' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + v = self._arguments[0].resolvestr(makefile, variables, setting) + raise data.DataError(v, self.loc) + +class WarningFunction(Function): + name = 'warning' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + v = self._arguments[0].resolvestr(makefile, variables, setting) + log.warning(v) + +class InfoFunction(Function): + name = 'info' + minargs = 1 + maxargs = 1 + + __slots__ = Function.__slots__ + + def resolve(self, makefile, variables, fd, setting): + v = self._arguments[0].resolvestr(makefile, variables, setting) + print v + +functionmap = { + 'subst': SubstFunction, + 'patsubst': PatSubstFunction, + 'strip': StripFunction, + 'findstring': FindstringFunction, + 'filter': FilterFunction, + 'filter-out': FilteroutFunction, + 'sort': SortFunction, + 'word': WordFunction, + 'wordlist': WordlistFunction, + 'words': WordsFunction, + 'firstword': FirstWordFunction, + 'lastword': LastWordFunction, + 'dir': DirFunction, + 'notdir': NotDirFunction, + 'suffix': SuffixFunction, + 'basename': BasenameFunction, + 'addsuffix': AddSuffixFunction, + 'addprefix': AddPrefixFunction, + 'join': JoinFunction, + 'wildcard': WildcardFunction, + 'realpath': RealpathFunction, + 'abspath': AbspathFunction, + 'if': IfFunction, + 'or': OrFunction, + 'and': AndFunction, + 'foreach': ForEachFunction, + 'call': CallFunction, + 'value': ValueFunction, + 'eval': EvalFunction, + 'origin': OriginFunction, + 'flavor': FlavorFunction, + 'shell': ShellFunction, + 'error': ErrorFunction, + 'warning': WarningFunction, + 'info': InfoFunction, +} + +import data diff --git a/python/pymake/pymake/globrelative.py b/python/pymake/pymake/globrelative.py new file mode 100644 index 000000000..37ca28e06 --- /dev/null +++ b/python/pymake/pymake/globrelative.py @@ -0,0 +1,68 @@ +""" +Filename globbing like the python glob module with minor differences: + +* glob relative to an arbitrary directory +* include . and .. +* check that link targets exist, not just links +""" + +import os, re, fnmatch +import util + +_globcheck = re.compile('[[*?]') + +def hasglob(p): + return _globcheck.search(p) is not None + +def glob(fsdir, path): + """ + Yield paths matching the path glob. Sorts as a bonus. Excludes '.' and '..' + """ + + dir, leaf = os.path.split(path) + if dir == '': + return globpattern(fsdir, leaf) + + if hasglob(dir): + dirsfound = glob(fsdir, dir) + else: + dirsfound = [dir] + + r = [] + + for dir in dirsfound: + fspath = util.normaljoin(fsdir, dir) + if not os.path.isdir(fspath): + continue + + r.extend((util.normaljoin(dir, found) for found in globpattern(fspath, leaf))) + + return r + +def globpattern(dir, pattern): + """ + Return leaf names in the specified directory which match the pattern. + """ + + if not hasglob(pattern): + if pattern == '': + if os.path.isdir(dir): + return [''] + return [] + + if os.path.exists(util.normaljoin(dir, pattern)): + return [pattern] + return [] + + leaves = os.listdir(dir) + ['.', '..'] + + # "hidden" filenames are a bit special + if not pattern.startswith('.'): + leaves = [leaf for leaf in leaves + if not leaf.startswith('.')] + + leaves = fnmatch.filter(leaves, pattern) + leaves = filter(lambda l: os.path.exists(util.normaljoin(dir, l)), leaves) + + leaves.sort() + return leaves diff --git a/python/pymake/pymake/implicit.py b/python/pymake/pymake/implicit.py new file mode 100644 index 000000000..d73895cab --- /dev/null +++ b/python/pymake/pymake/implicit.py @@ -0,0 +1,14 @@ +""" +Implicit variables; perhaps in the future this will also include some implicit +rules, at least match-anything cancellation rules. +""" + +variables = { + 'MKDIR': '%pymake.builtins mkdir', + 'RM': '%pymake.builtins rm -f', + 'SLEEP': '%pymake.builtins sleep', + 'TOUCH': '%pymake.builtins touch', + '.LIBPATTERNS': 'lib%.so lib%.a', + '.PYMAKE': '1', + } + diff --git a/python/pymake/pymake/parser.py b/python/pymake/pymake/parser.py new file mode 100644 index 000000000..4bff53368 --- /dev/null +++ b/python/pymake/pymake/parser.py @@ -0,0 +1,822 @@ +""" +Module for parsing Makefile syntax. + +Makefiles use a line-based parsing system. Continuations and substitutions are handled differently based on the +type of line being parsed: + +Lines with makefile syntax condense continuations to a single space, no matter the actual trailing whitespace +of the first line or the leading whitespace of the continuation. In other situations, trailing whitespace is +relevant. + +Lines with command syntax do not condense continuations: the backslash and newline are part of the command. +(GNU Make is buggy in this regard, at least on mac). + +Lines with an initial tab are commands if they can be (there is a rule or a command immediately preceding). +Otherwise, they are parsed as makefile syntax. + +This file parses into the data structures defined in the parserdata module. Those classes are what actually +do the dirty work of "executing" the parsed data into a data.Makefile. + +Four iterator functions are available: +* iterdata +* itermakefilechars +* itercommandchars + +The iterators handle line continuations and comments in different ways, but share a common calling +convention: + +Called with (data, startoffset, tokenlist, finditer) + +yield 4-tuples (flatstr, token, tokenoffset, afteroffset) +flatstr is data, guaranteed to have no tokens (may be '') +token, tokenoffset, afteroffset *may be None*. That means there is more text +coming. +""" + +import logging, re, os, sys +import data, functions, util, parserdata + +_log = logging.getLogger('pymake.parser') + +class SyntaxError(util.MakeError): + pass + +_skipws = re.compile('\S') +class Data(object): + """ + A single virtual "line", which can be multiple source lines joined with + continuations. + """ + + __slots__ = ('s', 'lstart', 'lend', 'loc') + + def __init__(self, s, lstart, lend, loc): + self.s = s + self.lstart = lstart + self.lend = lend + self.loc = loc + + @staticmethod + def fromstring(s, path): + return Data(s, 0, len(s), parserdata.Location(path, 1, 0)) + + def getloc(self, offset): + assert offset >= self.lstart and offset <= self.lend + return self.loc.offset(self.s, self.lstart, offset) + + def skipwhitespace(self, offset): + """ + Return the offset of the first non-whitespace character in data starting at offset, or None if there are + only whitespace characters remaining. + """ + m = _skipws.search(self.s, offset, self.lend) + if m is None: + return self.lend + + return m.start(0) + +_linere = re.compile(r'\\*\n') +def enumeratelines(s, filename): + """ + Enumerate lines in a string as Data objects, joining line + continuations. + """ + + off = 0 + lineno = 1 + curlines = 0 + for m in _linere.finditer(s): + curlines += 1 + start, end = m.span(0) + + if (start - end) % 2 == 0: + # odd number of backslashes is a continuation + continue + + yield Data(s, off, end - 1, parserdata.Location(filename, lineno, 0)) + + lineno += curlines + curlines = 0 + off = end + + yield Data(s, off, len(s), parserdata.Location(filename, lineno, 0)) + +_alltokens = re.compile(r'''\\*\# | # hash mark preceeded by any number of backslashes + := | + \+= | + \?= | + :: | + (?:\$(?:$|[\(\{](?:%s)\s+|.)) | # dollar sign followed by EOF, a function keyword with whitespace, or any character + :(?![\\/]) | # colon followed by anything except a slash (Windows path detection) + [=#{}();,|'"]''' % '|'.join(functions.functionmap.iterkeys()), re.VERBOSE) + +def iterdata(d, offset, tokenlist, it): + """ + Iterate over flat data without line continuations, comments, or any special escaped characters. + + Typically used to parse recursively-expanded variables. + """ + + assert len(tokenlist), "Empty tokenlist passed to iterdata is meaningless!" + assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) + + if offset == d.lend: + return + + s = d.s + for m in it: + mstart, mend = m.span(0) + token = s[mstart:mend] + if token in tokenlist or (token[0] == '$' and '$' in tokenlist): + yield s[offset:mstart], token, mstart, mend + else: + yield s[offset:mend], None, None, mend + offset = mend + + yield s[offset:d.lend], None, None, None + +# multiple backslashes before a newline are unescaped, halving their total number +_makecontinuations = re.compile(r'(?:\s*|((?:\\\\)+))\\\n\s*') +def _replacemakecontinuations(m): + start, end = m.span(1) + if start == -1: + return ' ' + return ' '.rjust((end - start) / 2 + 1, '\\') + +def itermakefilechars(d, offset, tokenlist, it, ignorecomments=False): + """ + Iterate over data in makefile syntax. Comments are found at unescaped # characters, and escaped newlines + are converted to single-space continuations. + """ + + assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) + + if offset == d.lend: + return + + s = d.s + for m in it: + mstart, mend = m.span(0) + token = s[mstart:mend] + + starttext = _makecontinuations.sub(_replacemakecontinuations, s[offset:mstart]) + + if token[-1] == '#' and not ignorecomments: + l = mend - mstart + # multiple backslashes before a hash are unescaped, halving their total number + if l % 2: + # found a comment + yield starttext + token[:(l - 1) / 2], None, None, None + return + else: + yield starttext + token[-l / 2:], None, None, mend + elif token in tokenlist or (token[0] == '$' and '$' in tokenlist): + yield starttext, token, mstart, mend + else: + yield starttext + token, None, None, mend + offset = mend + + yield _makecontinuations.sub(_replacemakecontinuations, s[offset:d.lend]), None, None, None + +_findcomment = re.compile(r'\\*\#') +def flattenmakesyntax(d, offset): + """ + A shortcut method for flattening line continuations and comments in makefile syntax without + looking for other tokens. + """ + + assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) + if offset == d.lend: + return '' + + s = _makecontinuations.sub(_replacemakecontinuations, d.s[offset:d.lend]) + + elements = [] + offset = 0 + for m in _findcomment.finditer(s): + mstart, mend = m.span(0) + elements.append(s[offset:mstart]) + if (mend - mstart) % 2: + # even number of backslashes... it's a comment + elements.append(''.ljust((mend - mstart - 1) / 2, '\\')) + return ''.join(elements) + + # odd number of backslashes + elements.append(''.ljust((mend - mstart - 2) / 2, '\\') + '#') + offset = mend + + elements.append(s[offset:]) + return ''.join(elements) + +def itercommandchars(d, offset, tokenlist, it): + """ + Iterate over command syntax. # comment markers are not special, and escaped newlines are included + in the output text. + """ + + assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend) + + if offset == d.lend: + return + + s = d.s + for m in it: + mstart, mend = m.span(0) + token = s[mstart:mend] + starttext = s[offset:mstart].replace('\n\t', '\n') + + if token in tokenlist or (token[0] == '$' and '$' in tokenlist): + yield starttext, token, mstart, mend + else: + yield starttext + token, None, None, mend + offset = mend + + yield s[offset:d.lend].replace('\n\t', '\n'), None, None, None + +_redefines = re.compile('\s*define|\s*endef') +def iterdefinelines(it, startloc): + """ + Process the insides of a define. Most characters are included literally. Escaped newlines are treated + as they would be in makefile syntax. Internal define/endef pairs are ignored. + """ + + results = [] + + definecount = 1 + for d in it: + m = _redefines.match(d.s, d.lstart, d.lend) + if m is not None: + directive = m.group(0).strip() + if directive == 'endef': + definecount -= 1 + if definecount == 0: + return _makecontinuations.sub(_replacemakecontinuations, '\n'.join(results)) + else: + definecount += 1 + + results.append(d.s[d.lstart:d.lend]) + + # Falling off the end is an unterminated define! + raise SyntaxError("define without matching endef", startloc) + +def _ensureend(d, offset, msg): + """ + Ensure that only whitespace remains in this data. + """ + + s = flattenmakesyntax(d, offset) + if s != '' and not s.isspace(): + raise SyntaxError(msg, d.getloc(offset)) + +_eqargstokenlist = ('(', "'", '"') + +def ifeq(d, offset): + if offset > d.lend - 1: + raise SyntaxError("No arguments after conditional", d.getloc(offset)) + + # the variety of formats for this directive is rather maddening + token = d.s[offset] + if token not in _eqargstokenlist: + raise SyntaxError("No arguments after conditional", d.getloc(offset)) + + offset += 1 + + if token == '(': + arg1, t, offset = parsemakesyntax(d, offset, (',',), itermakefilechars) + if t is None: + raise SyntaxError("Expected two arguments in conditional", d.getloc(d.lend)) + + arg1.rstrip() + + offset = d.skipwhitespace(offset) + arg2, t, offset = parsemakesyntax(d, offset, (')',), itermakefilechars) + if t is None: + raise SyntaxError("Unexpected text in conditional", d.getloc(offset)) + + _ensureend(d, offset, "Unexpected text after conditional") + else: + arg1, t, offset = parsemakesyntax(d, offset, (token,), itermakefilechars) + if t is None: + raise SyntaxError("Unexpected text in conditional", d.getloc(d.lend)) + + offset = d.skipwhitespace(offset) + if offset == d.lend: + raise SyntaxError("Expected two arguments in conditional", d.getloc(offset)) + + token = d.s[offset] + if token not in '\'"': + raise SyntaxError("Unexpected text in conditional", d.getloc(offset)) + + arg2, t, offset = parsemakesyntax(d, offset + 1, (token,), itermakefilechars) + + _ensureend(d, offset, "Unexpected text after conditional") + + return parserdata.EqCondition(arg1, arg2) + +def ifneq(d, offset): + c = ifeq(d, offset) + c.expected = False + return c + +def ifdef(d, offset): + e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) + e.rstrip() + + return parserdata.IfdefCondition(e) + +def ifndef(d, offset): + c = ifdef(d, offset) + c.expected = False + return c + +_conditionkeywords = { + 'ifeq': ifeq, + 'ifneq': ifneq, + 'ifdef': ifdef, + 'ifndef': ifndef + } + +_conditiontokens = tuple(_conditionkeywords.iterkeys()) +_conditionre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_conditiontokens)) + +_directivestokenlist = _conditiontokens + \ + ('else', 'endif', 'define', 'endef', 'override', 'include', '-include', 'includedeps', '-includedeps', 'vpath', 'export', 'unexport') + +_directivesre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_directivestokenlist)) + +_varsettokens = (':=', '+=', '?=', '=') + +def _parsefile(pathname): + fd = open(pathname, "rU") + stmts = parsestring(fd.read(), pathname) + stmts.mtime = os.fstat(fd.fileno()).st_mtime + fd.close() + return stmts + +def _checktime(path, stmts): + mtime = os.path.getmtime(path) + if mtime != stmts.mtime: + _log.debug("Re-parsing makefile '%s': mtimes differ", path) + return False + + return True + +_parsecache = util.MostUsedCache(50, _parsefile, _checktime) + +def parsefile(pathname): + """ + Parse a filename into a parserdata.StatementList. A cache is used to avoid re-parsing + makefiles that have already been parsed and have not changed. + """ + + pathname = os.path.realpath(pathname) + return _parsecache.get(pathname) + +# colon followed by anything except a slash (Windows path detection) +_depfilesplitter = re.compile(r':(?![\\/])') +# simple variable references +_vars = re.compile('\$\((\w+)\)') + +def parsedepfile(pathname): + """ + Parse a filename listing only depencencies into a parserdata.StatementList. + Simple variable references are allowed in such files. + """ + def continuation_iter(lines): + current_line = [] + for line in lines: + line = line.rstrip() + if line.endswith("\\"): + current_line.append(line.rstrip("\\")) + continue + if not len(line): + continue + current_line.append(line) + yield ''.join(current_line) + current_line = [] + if current_line: + yield ''.join(current_line) + + def get_expansion(s): + if '$' in s: + expansion = data.Expansion() + # for an input like e.g. "foo $(bar) baz", + # _vars.split returns ["foo", "bar", "baz"] + # every other element is a variable name. + for i, element in enumerate(_vars.split(s)): + if i % 2: + expansion.appendfunc(functions.VariableRef(None, + data.StringExpansion(element, None))) + elif element: + expansion.appendstr(element) + + return expansion + + return data.StringExpansion(s, None) + + pathname = os.path.realpath(pathname) + stmts = parserdata.StatementList() + for line in continuation_iter(open(pathname).readlines()): + target, deps = _depfilesplitter.split(line, 1) + stmts.append(parserdata.Rule(get_expansion(target), + get_expansion(deps), False)) + return stmts + +def parsestring(s, filename): + """ + Parse a string containing makefile data into a parserdata.StatementList. + """ + + currule = False + condstack = [parserdata.StatementList()] + + fdlines = enumeratelines(s, filename) + for d in fdlines: + assert len(condstack) > 0 + + offset = d.lstart + + if currule and offset < d.lend and d.s[offset] == '\t': + e, token, offset = parsemakesyntax(d, offset + 1, (), itercommandchars) + assert token is None + assert offset is None + condstack[-1].append(parserdata.Command(e)) + continue + + # To parse Makefile syntax, we first strip leading whitespace and + # look for initial keywords. If there are no keywords, it's either + # setting a variable or writing a rule. + + offset = d.skipwhitespace(offset) + if offset is None: + continue + + m = _directivesre.match(d.s, offset, d.lend) + if m is not None: + kword = m.group(1) + offset = m.end(0) + + if kword == 'endif': + _ensureend(d, offset, "Unexpected data after 'endif' directive") + if len(condstack) == 1: + raise SyntaxError("unmatched 'endif' directive", + d.getloc(offset)) + + condstack.pop().endloc = d.getloc(offset) + continue + + if kword == 'else': + if len(condstack) == 1: + raise SyntaxError("unmatched 'else' directive", + d.getloc(offset)) + + m = _conditionre.match(d.s, offset, d.lend) + if m is None: + _ensureend(d, offset, "Unexpected data after 'else' directive.") + condstack[-1].addcondition(d.getloc(offset), parserdata.ElseCondition()) + else: + kword = m.group(1) + if kword not in _conditionkeywords: + raise SyntaxError("Unexpected condition after 'else' directive.", + d.getloc(offset)) + + startoffset = offset + offset = d.skipwhitespace(m.end(1)) + c = _conditionkeywords[kword](d, offset) + condstack[-1].addcondition(d.getloc(startoffset), c) + continue + + if kword in _conditionkeywords: + c = _conditionkeywords[kword](d, offset) + cb = parserdata.ConditionBlock(d.getloc(d.lstart), c) + condstack[-1].append(cb) + condstack.append(cb) + continue + + if kword == 'endef': + raise SyntaxError("endef without matching define", d.getloc(offset)) + + if kword == 'define': + currule = False + vname, t, i = parsemakesyntax(d, offset, (), itermakefilechars) + vname.rstrip() + + startloc = d.getloc(d.lstart) + value = iterdefinelines(fdlines, startloc) + condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=startloc, token='=', targetexp=None)) + continue + + if kword in ('include', '-include', 'includedeps', '-includedeps'): + if kword.startswith('-'): + required = False + kword = kword[1:] + else: + required = True + + deps = kword == 'includedeps' + + currule = False + incfile, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) + condstack[-1].append(parserdata.Include(incfile, required, deps)) + + continue + + if kword == 'vpath': + currule = False + e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars) + condstack[-1].append(parserdata.VPathDirective(e)) + continue + + if kword == 'override': + currule = False + vname, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars) + vname.lstrip() + vname.rstrip() + + if token is None: + raise SyntaxError("Malformed override directive, need =", d.getloc(d.lstart)) + + value = flattenmakesyntax(d, offset).lstrip() + + condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=d.getloc(offset), token=token, targetexp=None, source=data.Variables.SOURCE_OVERRIDE)) + continue + + if kword == 'export': + currule = False + e, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars) + e.lstrip() + e.rstrip() + + if token is None: + condstack[-1].append(parserdata.ExportDirective(e, concurrent_set=False)) + else: + condstack[-1].append(parserdata.ExportDirective(e, concurrent_set=True)) + + value = flattenmakesyntax(d, offset).lstrip() + condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None)) + + continue + + if kword == 'unexport': + e, token, offset = parsemakesyntax(d, offset, (), itermakefilechars) + condstack[-1].append(parserdata.UnexportDirective(e)) + continue + + e, token, offset = parsemakesyntax(d, offset, _varsettokens + ('::', ':'), itermakefilechars) + if token is None: + e.rstrip() + e.lstrip() + if not e.isempty(): + condstack[-1].append(parserdata.EmptyDirective(e)) + continue + + # if we encountered real makefile syntax, the current rule is over + currule = False + + if token in _varsettokens: + e.lstrip() + e.rstrip() + + value = flattenmakesyntax(d, offset).lstrip() + + condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None)) + else: + doublecolon = token == '::' + + # `e` is targets or target patterns, which can end up as + # * a rule + # * an implicit rule + # * a static pattern rule + # * a target-specific variable definition + # * a pattern-specific variable definition + # any of the rules may have order-only prerequisites + # delimited by |, and a command delimited by ; + targets = e + + e, token, offset = parsemakesyntax(d, offset, + _varsettokens + (':', '|', ';'), + itermakefilechars) + if token in (None, ';'): + condstack[-1].append(parserdata.Rule(targets, e, doublecolon)) + currule = True + + if token == ';': + offset = d.skipwhitespace(offset) + e, t, offset = parsemakesyntax(d, offset, (), itercommandchars) + condstack[-1].append(parserdata.Command(e)) + + elif token in _varsettokens: + e.lstrip() + e.rstrip() + + value = flattenmakesyntax(d, offset).lstrip() + condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=targets)) + elif token == '|': + raise SyntaxError('order-only prerequisites not implemented', d.getloc(offset)) + else: + assert token == ':' + # static pattern rule + + pattern = e + + deps, token, offset = parsemakesyntax(d, offset, (';',), itermakefilechars) + + condstack[-1].append(parserdata.StaticPatternRule(targets, pattern, deps, doublecolon)) + currule = True + + if token == ';': + offset = d.skipwhitespace(offset) + e, token, offset = parsemakesyntax(d, offset, (), itercommandchars) + condstack[-1].append(parserdata.Command(e)) + + if len(condstack) != 1: + raise SyntaxError("Condition never terminated with endif", condstack[-1].loc) + + return condstack[0] + +_PARSESTATE_TOPLEVEL = 0 # at the top level +_PARSESTATE_FUNCTION = 1 # expanding a function call +_PARSESTATE_VARNAME = 2 # expanding a variable expansion. +_PARSESTATE_SUBSTFROM = 3 # expanding a variable expansion substitution "from" value +_PARSESTATE_SUBSTTO = 4 # expanding a variable expansion substitution "to" value +_PARSESTATE_PARENMATCH = 5 # inside nested parentheses/braces that must be matched + +class ParseStackFrame(object): + __slots__ = ('parsestate', 'parent', 'expansion', 'tokenlist', 'openbrace', 'closebrace', 'function', 'loc', 'varname', 'substfrom') + + def __init__(self, parsestate, parent, expansion, tokenlist, openbrace, closebrace, function=None, loc=None): + self.parsestate = parsestate + self.parent = parent + self.expansion = expansion + self.tokenlist = tokenlist + self.openbrace = openbrace + self.closebrace = closebrace + self.function = function + self.loc = loc + + def __str__(self): + return "" % (self.parsestate, self.expansion, self.tokenlist, self.openbrace, self.closebrace) + +_matchingbrace = { + '(': ')', + '{': '}', + } + +def parsemakesyntax(d, offset, stopon, iterfunc): + """ + Given Data, parse it into a data.Expansion. + + @param stopon (sequence) + Indicate characters where toplevel parsing should stop. + + @param iterfunc (generator function) + A function which is used to iterate over d, yielding (char, offset, loc) + @see iterdata + @see itermakefilechars + @see itercommandchars + + @return a tuple (expansion, token, offset). If all the data is consumed, + token and offset will be None + """ + + assert callable(iterfunc) + + stacktop = ParseStackFrame(_PARSESTATE_TOPLEVEL, None, data.Expansion(loc=d.getloc(d.lstart)), + tokenlist=stopon + ('$',), + openbrace=None, closebrace=None) + + tokeniterator = _alltokens.finditer(d.s, offset, d.lend) + + di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator) + while True: # this is not a for loop because `di` changes during the function + assert stacktop is not None + try: + s, token, tokenoffset, offset = di.next() + except StopIteration: + break + + stacktop.expansion.appendstr(s) + if token is None: + continue + + parsestate = stacktop.parsestate + + if token[0] == '$': + if tokenoffset + 1 == d.lend: + # an unterminated $ expands to nothing + break + + loc = d.getloc(tokenoffset) + c = token[1] + if c == '$': + assert len(token) == 2 + stacktop.expansion.appendstr('$') + elif c in ('(', '{'): + closebrace = _matchingbrace[c] + + if len(token) > 2: + fname = token[2:].rstrip() + fn = functions.functionmap[fname](loc) + e = data.Expansion() + if len(fn) + 1 == fn.maxargs: + tokenlist = (c, closebrace, '$') + else: + tokenlist = (',', c, closebrace, '$') + + stacktop = ParseStackFrame(_PARSESTATE_FUNCTION, stacktop, + e, tokenlist, function=fn, + openbrace=c, closebrace=closebrace) + else: + e = data.Expansion() + tokenlist = (':', c, closebrace, '$') + stacktop = ParseStackFrame(_PARSESTATE_VARNAME, stacktop, + e, tokenlist, + openbrace=c, closebrace=closebrace, loc=loc) + else: + assert len(token) == 2 + e = data.Expansion.fromstring(c, loc) + stacktop.expansion.appendfunc(functions.VariableRef(loc, e)) + elif token in ('(', '{'): + assert token == stacktop.openbrace + + stacktop.expansion.appendstr(token) + stacktop = ParseStackFrame(_PARSESTATE_PARENMATCH, stacktop, + stacktop.expansion, + (token, stacktop.closebrace, '$'), + openbrace=token, closebrace=stacktop.closebrace, loc=d.getloc(tokenoffset)) + elif parsestate == _PARSESTATE_PARENMATCH: + assert token == stacktop.closebrace + stacktop.expansion.appendstr(token) + stacktop = stacktop.parent + elif parsestate == _PARSESTATE_TOPLEVEL: + assert stacktop.parent is None + return stacktop.expansion.finish(), token, offset + elif parsestate == _PARSESTATE_FUNCTION: + if token == ',': + stacktop.function.append(stacktop.expansion.finish()) + + stacktop.expansion = data.Expansion() + if len(stacktop.function) + 1 == stacktop.function.maxargs: + tokenlist = (stacktop.openbrace, stacktop.closebrace, '$') + stacktop.tokenlist = tokenlist + elif token in (')', '}'): + fn = stacktop.function + fn.append(stacktop.expansion.finish()) + fn.setup() + + stacktop = stacktop.parent + stacktop.expansion.appendfunc(fn) + else: + assert False, "Not reached, _PARSESTATE_FUNCTION" + elif parsestate == _PARSESTATE_VARNAME: + if token == ':': + stacktop.varname = stacktop.expansion + stacktop.parsestate = _PARSESTATE_SUBSTFROM + stacktop.expansion = data.Expansion() + stacktop.tokenlist = ('=', stacktop.openbrace, stacktop.closebrace, '$') + elif token in (')', '}'): + fn = functions.VariableRef(stacktop.loc, stacktop.expansion.finish()) + stacktop = stacktop.parent + stacktop.expansion.appendfunc(fn) + else: + assert False, "Not reached, _PARSESTATE_VARNAME" + elif parsestate == _PARSESTATE_SUBSTFROM: + if token == '=': + stacktop.substfrom = stacktop.expansion + stacktop.parsestate = _PARSESTATE_SUBSTTO + stacktop.expansion = data.Expansion() + stacktop.tokenlist = (stacktop.openbrace, stacktop.closebrace, '$') + elif token in (')', '}'): + # A substitution of the form $(VARNAME:.ee) is probably a mistake, but make + # parses it. Issue a warning. Combine the varname and substfrom expansions to + # make the compatible varname. See tests/var-substitutions.mk SIMPLE3SUBSTNAME + _log.warning("%s: Variable reference looks like substitution without =", stacktop.loc) + stacktop.varname.appendstr(':') + stacktop.varname.concat(stacktop.expansion) + fn = functions.VariableRef(stacktop.loc, stacktop.varname.finish()) + stacktop = stacktop.parent + stacktop.expansion.appendfunc(fn) + else: + assert False, "Not reached, _PARSESTATE_SUBSTFROM" + elif parsestate == _PARSESTATE_SUBSTTO: + assert token in (')','}'), "Not reached, _PARSESTATE_SUBSTTO" + + fn = functions.SubstitutionRef(stacktop.loc, stacktop.varname.finish(), + stacktop.substfrom.finish(), stacktop.expansion.finish()) + stacktop = stacktop.parent + stacktop.expansion.appendfunc(fn) + else: + assert False, "Unexpected parse state %s" % stacktop.parsestate + + if stacktop.parent is not None and iterfunc == itercommandchars: + di = itermakefilechars(d, offset, stacktop.tokenlist, tokeniterator, + ignorecomments=True) + else: + di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator) + + if stacktop.parent is not None: + raise SyntaxError("Unterminated function call", d.getloc(offset)) + + assert stacktop.parsestate == _PARSESTATE_TOPLEVEL + + return stacktop.expansion.finish(), None, None diff --git a/python/pymake/pymake/parserdata.py b/python/pymake/pymake/parserdata.py new file mode 100644 index 000000000..7b2e5443d --- /dev/null +++ b/python/pymake/pymake/parserdata.py @@ -0,0 +1,1006 @@ +import logging, re, os +import data, parser, functions, util +from cStringIO import StringIO +from pymake.globrelative import hasglob, glob + +_log = logging.getLogger('pymake.data') +_tabwidth = 4 + +class Location(object): + """ + A location within a makefile. + + For the moment, locations are just path/line/column, but in the future + they may reference parent locations for more accurate "included from" + or "evaled at" error reporting. + """ + __slots__ = ('path', 'line', 'column') + + def __init__(self, path, line, column): + self.path = path + self.line = line + self.column = column + + def offset(self, s, start, end): + """ + Returns a new location offset by + the specified string. + """ + + if start == end: + return self + + skiplines = s.count('\n', start, end) + line = self.line + skiplines + if skiplines: + lastnl = s.rfind('\n', start, end) + assert lastnl != -1 + start = lastnl + 1 + column = 0 + else: + column = self.column + + while True: + j = s.find('\t', start, end) + if j == -1: + column += end - start + break + + column += j - start + column += _tabwidth + column -= column % _tabwidth + start = j + 1 + + return Location(self.path, line, column) + + def __str__(self): + return "%s:%s:%s" % (self.path, self.line, self.column) + +def _expandwildcards(makefile, tlist): + for t in tlist: + if not hasglob(t): + yield t + else: + l = glob(makefile.workdir, t) + for r in l: + yield r + +_flagescape = re.compile(r'([\s\\])') + +def parsecommandlineargs(args): + """ + Given a set of arguments from a command-line invocation of make, + parse out the variable definitions and return (stmts, arglist, overridestr) + """ + + overrides = [] + stmts = StatementList() + r = [] + for i in xrange(0, len(args)): + a = args[i] + + vname, t, val = util.strpartition(a, ':=') + if t == '': + vname, t, val = util.strpartition(a, '=') + if t != '': + overrides.append(_flagescape.sub(r'\\\1', a)) + + vname = vname.strip() + vnameexp = data.Expansion.fromstring(vname, "Command-line argument") + + stmts.append(ExportDirective(vnameexp, concurrent_set=True)) + stmts.append(SetVariable(vnameexp, token=t, + value=val, valueloc=Location('', i, len(vname) + len(t)), + targetexp=None, source=data.Variables.SOURCE_COMMANDLINE)) + else: + r.append(data.stripdotslash(a)) + + return stmts, r, ' '.join(overrides) + +class Statement(object): + """ + Represents parsed make file syntax. + + This is an abstract base class. Child classes are expected to implement + basic methods defined below. + """ + + def execute(self, makefile, context): + """Executes this Statement within a make file execution context.""" + raise Exception("%s must implement execute()." % self.__class__) + + def to_source(self): + """Obtain the make file "source" representation of the Statement. + + This converts an individual Statement back to a string that can again + be parsed into this Statement. + """ + raise Exception("%s must implement to_source()." % self.__class__) + + def __eq__(self, other): + raise Exception("%s must implement __eq__." % self.__class__) + + def __ne__(self, other): + return self.__eq__(other) + +class DummyRule(object): + __slots__ = () + + def addcommand(self, r): + pass + +class Rule(Statement): + """ + Rules represent how to make specific targets. + + See https://www.gnu.org/software/make/manual/make.html#Rules. + + An individual rule is composed of a target, dependencies, and a recipe. + This class only contains references to the first 2. The recipe will be + contained in Command classes which follow this one in a stream of Statement + instances. + + Instances also contain a boolean property `doublecolon` which says whether + this is a doublecolon rule. Doublecolon rules are rules that are always + executed, if they are evaluated. Normally, rules are only executed if their + target is out of date. + """ + __slots__ = ('targetexp', 'depexp', 'doublecolon') + + def __init__(self, targetexp, depexp, doublecolon): + assert isinstance(targetexp, (data.Expansion, data.StringExpansion)) + assert isinstance(depexp, (data.Expansion, data.StringExpansion)) + + self.targetexp = targetexp + self.depexp = depexp + self.doublecolon = doublecolon + + def execute(self, makefile, context): + if context.weak: + self._executeweak(makefile, context) + else: + self._execute(makefile, context) + + def _executeweak(self, makefile, context): + """ + If the context is weak (we're just handling dependencies) we can make a number of assumptions here. + This lets us go really fast and is generally good. + """ + assert context.weak + deps = self.depexp.resolvesplit(makefile, makefile.variables) + # Skip targets with no rules and no dependencies + if not deps: + return + targets = data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables)) + rule = data.Rule(list(data.stripdotslashes(deps)), self.doublecolon, loc=self.targetexp.loc, weakdeps=True) + for target in targets: + makefile.gettarget(target).addrule(rule) + makefile.foundtarget(target) + context.currule = rule + + def _execute(self, makefile, context): + assert not context.weak + + atargets = data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables)) + targets = [data.Pattern(p) for p in _expandwildcards(makefile, atargets)] + + if not len(targets): + context.currule = DummyRule() + return + + ispatterns = set((t.ispattern() for t in targets)) + if len(ispatterns) == 2: + raise data.DataError("Mixed implicit and normal rule", self.targetexp.loc) + ispattern, = ispatterns + + deps = list(_expandwildcards(makefile, data.stripdotslashes(self.depexp.resolvesplit(makefile, makefile.variables)))) + if ispattern: + rule = data.PatternRule(targets, map(data.Pattern, deps), self.doublecolon, loc=self.targetexp.loc) + makefile.appendimplicitrule(rule) + else: + rule = data.Rule(deps, self.doublecolon, loc=self.targetexp.loc, weakdeps=False) + for t in targets: + makefile.gettarget(t.gettarget()).addrule(rule) + + makefile.foundtarget(targets[0].gettarget()) + + context.currule = rule + + def dump(self, fd, indent): + print >>fd, "%sRule %s: %s" % (indent, self.targetexp, self.depexp) + + def to_source(self): + sep = ':' + + if self.doublecolon: + sep = '::' + + deps = self.depexp.to_source() + if len(deps) > 0 and not deps[0].isspace(): + sep += ' ' + + return '\n%s%s%s' % ( + self.targetexp.to_source(escape_variables=True), + sep, + deps) + + def __eq__(self, other): + if not isinstance(other, Rule): + return False + + return self.targetexp == other.targetexp \ + and self.depexp == other.depexp \ + and self.doublecolon == other.doublecolon + +class StaticPatternRule(Statement): + """ + Static pattern rules are rules which specify multiple targets based on a + string pattern. + + See https://www.gnu.org/software/make/manual/make.html#Static-Pattern + + They are like `Rule` instances except an added property, `patternexp` is + present. It contains the Expansion which represents the rule pattern. + """ + __slots__ = ('targetexp', 'patternexp', 'depexp', 'doublecolon') + + def __init__(self, targetexp, patternexp, depexp, doublecolon): + assert isinstance(targetexp, (data.Expansion, data.StringExpansion)) + assert isinstance(patternexp, (data.Expansion, data.StringExpansion)) + assert isinstance(depexp, (data.Expansion, data.StringExpansion)) + + self.targetexp = targetexp + self.patternexp = patternexp + self.depexp = depexp + self.doublecolon = doublecolon + + def execute(self, makefile, context): + if context.weak: + raise data.DataError("Static pattern rules not allowed in includedeps", self.targetexp.loc) + + targets = list(_expandwildcards(makefile, data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables)))) + + if not len(targets): + context.currule = DummyRule() + return + + patterns = list(data.stripdotslashes(self.patternexp.resolvesplit(makefile, makefile.variables))) + if len(patterns) != 1: + raise data.DataError("Static pattern rules must have a single pattern", self.patternexp.loc) + pattern = data.Pattern(patterns[0]) + + deps = [data.Pattern(p) for p in _expandwildcards(makefile, data.stripdotslashes(self.depexp.resolvesplit(makefile, makefile.variables)))] + + rule = data.PatternRule([pattern], deps, self.doublecolon, loc=self.targetexp.loc) + + for t in targets: + if data.Pattern(t).ispattern(): + raise data.DataError("Target '%s' of a static pattern rule must not be a pattern" % (t,), self.targetexp.loc) + stem = pattern.match(t) + if stem is None: + raise data.DataError("Target '%s' does not match the static pattern '%s'" % (t, pattern), self.targetexp.loc) + makefile.gettarget(t).addrule(data.PatternRuleInstance(rule, '', stem, pattern.ismatchany())) + + makefile.foundtarget(targets[0]) + context.currule = rule + + def dump(self, fd, indent): + print >>fd, "%sStaticPatternRule %s: %s: %s" % (indent, self.targetexp, self.patternexp, self.depexp) + + def to_source(self): + sep = ':' + + if self.doublecolon: + sep = '::' + + pattern = self.patternexp.to_source() + deps = self.depexp.to_source() + + if len(pattern) > 0 and pattern[0] not in (' ', '\t'): + sep += ' ' + + return '\n%s%s%s:%s' % ( + self.targetexp.to_source(escape_variables=True), + sep, + pattern, + deps) + + def __eq__(self, other): + if not isinstance(other, StaticPatternRule): + return False + + return self.targetexp == other.targetexp \ + and self.patternexp == other.patternexp \ + and self.depexp == other.depexp \ + and self.doublecolon == other.doublecolon + +class Command(Statement): + """ + Commands are things that get executed by a rule. + + A rule's recipe is composed of 0 or more Commands. + + A command is simply an expansion. Commands typically represent strings to + be executed in a shell (e.g. via system()). Although, since make files + allow arbitrary shells to be used for command execution, this isn't a + guarantee. + """ + __slots__ = ('exp',) + + def __init__(self, exp): + assert isinstance(exp, (data.Expansion, data.StringExpansion)) + self.exp = exp + + def execute(self, makefile, context): + assert context.currule is not None + if context.weak: + raise data.DataError("rules not allowed in includedeps", self.exp.loc) + + context.currule.addcommand(self.exp) + + def dump(self, fd, indent): + print >>fd, "%sCommand %s" % (indent, self.exp,) + + def to_source(self): + # Commands have some interesting quirks when it comes to source + # formatting. First, they can be multi-line. Second, a tab needs to be + # inserted at the beginning of every line. Finally, there might be + # variable references inside the command. This means we need to escape + # variable references inside command strings. Luckily, this is handled + # by the Expansion. + s = self.exp.to_source(escape_variables=True) + + return '\n'.join(['\t%s' % line for line in s.split('\n')]) + + def __eq__(self, other): + if not isinstance(other, Command): + return False + + return self.exp == other.exp + +class SetVariable(Statement): + """ + Represents a variable assignment. + + Variable assignment comes in two different flavors. + + Simple assignment has the form: + + + + e.g. FOO := bar + + These correspond to the fields `vnameexp`, `token`, and `value`. In + addition, `valueloc` will be a Location and `source` will be a + pymake.data.Variables.SOURCE_* constant. + + There are also target-specific variables. These are variables that only + apply in the context of a specific target. They are like the aforementioned + assignment except the `targetexp` field is set to an Expansion representing + the target they apply to. + """ + __slots__ = ('vnameexp', 'token', 'value', 'valueloc', 'targetexp', 'source') + + def __init__(self, vnameexp, token, value, valueloc, targetexp, source=None): + assert isinstance(vnameexp, (data.Expansion, data.StringExpansion)) + assert isinstance(value, str) + assert targetexp is None or isinstance(targetexp, (data.Expansion, data.StringExpansion)) + + if source is None: + source = data.Variables.SOURCE_MAKEFILE + + self.vnameexp = vnameexp + self.token = token + self.value = value + self.valueloc = valueloc + self.targetexp = targetexp + self.source = source + + def execute(self, makefile, context): + vname = self.vnameexp.resolvestr(makefile, makefile.variables) + if len(vname) == 0: + raise data.DataError("Empty variable name", self.vnameexp.loc) + + if self.targetexp is None: + setvariables = [makefile.variables] + else: + setvariables = [] + + targets = [data.Pattern(t) for t in data.stripdotslashes(self.targetexp.resolvesplit(makefile, makefile.variables))] + for t in targets: + if t.ispattern(): + setvariables.append(makefile.getpatternvariables(t)) + else: + setvariables.append(makefile.gettarget(t.gettarget()).variables) + + for v in setvariables: + if self.token == '+=': + v.append(vname, self.source, self.value, makefile.variables, makefile) + continue + + if self.token == '?=': + flavor = data.Variables.FLAVOR_RECURSIVE + oldflavor, oldsource, oldval = v.get(vname, expand=False) + if oldval is not None: + continue + value = self.value + elif self.token == '=': + flavor = data.Variables.FLAVOR_RECURSIVE + value = self.value + else: + assert self.token == ':=' + + flavor = data.Variables.FLAVOR_SIMPLE + d = parser.Data.fromstring(self.value, self.valueloc) + e, t, o = parser.parsemakesyntax(d, 0, (), parser.iterdata) + value = e.resolvestr(makefile, makefile.variables) + + v.set(vname, flavor, self.source, value) + + def dump(self, fd, indent): + print >>fd, "%sSetVariable<%s> %s %s\n%s %r" % (indent, self.valueloc, self.vnameexp, self.token, indent, self.value) + + def __eq__(self, other): + if not isinstance(other, SetVariable): + return False + + return self.vnameexp == other.vnameexp \ + and self.token == other.token \ + and self.value == other.value \ + and self.targetexp == other.targetexp \ + and self.source == other.source + + def to_source(self): + chars = [] + for i in xrange(0, len(self.value)): + c = self.value[i] + + # Literal # is escaped in variable assignment otherwise it would be + # a comment. + if c == '#': + # If a backslash precedes this, we need to escape it as well. + if i > 0 and self.value[i-1] == '\\': + chars.append('\\') + + chars.append('\\#') + continue + + chars.append(c) + + value = ''.join(chars) + + prefix = '' + if self.source == data.Variables.SOURCE_OVERRIDE: + prefix = 'override ' + + # SetVariable come in two flavors: simple and target-specific. + + # We handle the target-specific syntax first. + if self.targetexp is not None: + return '%s: %s %s %s' % ( + self.targetexp.to_source(), + self.vnameexp.to_source(), + self.token, + value) + + # The variable could be multi-line or have leading whitespace. For + # regular variable assignment, whitespace after the token but before + # the value is ignored. If we see leading whitespace in the value here, + # the variable must have come from a define. + if value.count('\n') > 0 or (len(value) and value[0].isspace()): + # The parser holds the token in vnameexp for whatever reason. + return '%sdefine %s\n%s\nendef' % ( + prefix, + self.vnameexp.to_source(), + value) + + return '%s%s %s %s' % ( + prefix, + self.vnameexp.to_source(), + self.token, + value) + +class Condition(object): + """ + An abstract "condition", either ifeq or ifdef, perhaps negated. + + See https://www.gnu.org/software/make/manual/make.html#Conditional-Syntax + + Subclasses must implement: + + def evaluate(self, makefile) + """ + + def __eq__(self, other): + raise Exception("%s must implement __eq__." % __class__) + + def __ne__(self, other): + return not self.__eq__(other) + +class EqCondition(Condition): + """ + Represents an ifeq or ifneq conditional directive. + + This directive consists of two Expansions which are compared for equality. + + The `expected` field is a bool indicating what the condition must evaluate + to in order for its body to be executed. If True, this is an "ifeq" + conditional directive. If False, an "ifneq." + """ + __slots__ = ('exp1', 'exp2', 'expected') + + def __init__(self, exp1, exp2): + assert isinstance(exp1, (data.Expansion, data.StringExpansion)) + assert isinstance(exp2, (data.Expansion, data.StringExpansion)) + + self.expected = True + self.exp1 = exp1 + self.exp2 = exp2 + + def evaluate(self, makefile): + r1 = self.exp1.resolvestr(makefile, makefile.variables) + r2 = self.exp2.resolvestr(makefile, makefile.variables) + return (r1 == r2) == self.expected + + def __str__(self): + return "ifeq (expected=%s) %s %s" % (self.expected, self.exp1, self.exp2) + + def __eq__(self, other): + if not isinstance(other, EqCondition): + return False + + return self.exp1 == other.exp1 \ + and self.exp2 == other.exp2 \ + and self.expected == other.expected + +class IfdefCondition(Condition): + """ + Represents an ifdef or ifndef conditional directive. + + This directive consists of a single expansion which represents the name of + a variable (without the leading '$') which will be checked for definition. + + The `expected` field is a bool and has the same behavior as EqCondition. + If it is True, this represents a "ifdef" conditional. If False, "ifndef." + """ + __slots__ = ('exp', 'expected') + + def __init__(self, exp): + assert isinstance(exp, (data.Expansion, data.StringExpansion)) + self.exp = exp + self.expected = True + + def evaluate(self, makefile): + vname = self.exp.resolvestr(makefile, makefile.variables) + flavor, source, value = makefile.variables.get(vname, expand=False) + + if value is None: + return not self.expected + + return (len(value) > 0) == self.expected + + def __str__(self): + return "ifdef (expected=%s) %s" % (self.expected, self.exp) + + def __eq__(self, other): + if not isinstance(other, IfdefCondition): + return False + + return self.exp == other.exp and self.expected == other.expected + +class ElseCondition(Condition): + """ + Represents the transition between branches in a ConditionBlock. + """ + __slots__ = () + + def evaluate(self, makefile): + return True + + def __str__(self): + return "else" + + def __eq__(self, other): + return isinstance(other, ElseCondition) + +class ConditionBlock(Statement): + """ + A set of related Conditions. + + This is essentially a list of 2-tuples of (Condition, list(Statement)). + + The parser creates a ConditionBlock for all statements related to the same + conditional group. If iterating over the parser's output, where you think + you would see an ifeq, you will see a ConditionBlock containing an IfEq. In + other words, the parser collapses separate statements into this container + class. + + ConditionBlock instances may exist within other ConditionBlock if the + conditional logic is multiple levels deep. + """ + __slots__ = ('loc', '_groups') + + def __init__(self, loc, condition): + self.loc = loc + self._groups = [] + self.addcondition(loc, condition) + + def getloc(self): + return self.loc + + def addcondition(self, loc, condition): + assert isinstance(condition, Condition) + condition.loc = loc + + if len(self._groups) and isinstance(self._groups[-1][0], ElseCondition): + raise parser.SyntaxError("Multiple else conditions for block starting at %s" % self.loc, loc) + + self._groups.append((condition, StatementList())) + + def append(self, statement): + self._groups[-1][1].append(statement) + + def execute(self, makefile, context): + i = 0 + for c, statements in self._groups: + if c.evaluate(makefile): + _log.debug("Condition at %s met by clause #%i", self.loc, i) + statements.execute(makefile, context) + return + + i += 1 + + def dump(self, fd, indent): + print >>fd, "%sConditionBlock" % (indent,) + + indent2 = indent + ' ' + for c, statements in self._groups: + print >>fd, "%s Condition %s" % (indent, c) + statements.dump(fd, indent2) + print >>fd, "%s ~Condition" % (indent,) + print >>fd, "%s~ConditionBlock" % (indent,) + + def to_source(self): + lines = [] + index = 0 + for condition, statements in self: + lines.append(ConditionBlock.condition_source(condition, index)) + index += 1 + + for statement in statements: + lines.append(statement.to_source()) + + lines.append('endif') + + return '\n'.join(lines) + + def __eq__(self, other): + if not isinstance(other, ConditionBlock): + return False + + if len(self) != len(other): + return False + + for i in xrange(0, len(self)): + our_condition, our_statements = self[i] + other_condition, other_statements = other[i] + + if our_condition != other_condition: + return False + + if our_statements != other_statements: + return False + + return True + + @staticmethod + def condition_source(statement, index): + """Convert a condition to its source representation. + + The index argument defines the index of this condition inside a + ConditionBlock. If it is greater than 0, an "else" will be prepended + to the result, if necessary. + """ + prefix = '' + if isinstance(statement, (EqCondition, IfdefCondition)) and index > 0: + prefix = 'else ' + + if isinstance(statement, IfdefCondition): + s = statement.exp.s + + if statement.expected: + return '%sifdef %s' % (prefix, s) + + return '%sifndef %s' % (prefix, s) + + if isinstance(statement, EqCondition): + args = [ + statement.exp1.to_source(escape_comments=True), + statement.exp2.to_source(escape_comments=True)] + + use_quotes = False + single_quote_present = False + double_quote_present = False + for i, arg in enumerate(args): + if len(arg) > 0 and (arg[0].isspace() or arg[-1].isspace()): + use_quotes = True + + if "'" in arg: + single_quote_present = True + + if '"' in arg: + double_quote_present = True + + # Quote everything if needed. + if single_quote_present and double_quote_present: + raise Exception('Cannot format condition with multiple quotes.') + + if use_quotes: + for i, arg in enumerate(args): + # Double to single quotes. + if single_quote_present: + args[i] = '"' + arg + '"' + else: + args[i] = "'" + arg + "'" + + body = None + if use_quotes: + body = ' '.join(args) + else: + body = '(%s)' % ','.join(args) + + if statement.expected: + return '%sifeq %s' % (prefix, body) + + return '%sifneq %s' % (prefix, body) + + if isinstance(statement, ElseCondition): + return 'else' + + raise Exception('Unhandled Condition statement: %s' % + statement.__class__) + + def __iter__(self): + return iter(self._groups) + + def __len__(self): + return len(self._groups) + + def __getitem__(self, i): + return self._groups[i] + +class Include(Statement): + """ + Represents the include directive. + + See https://www.gnu.org/software/make/manual/make.html#Include + + The file to be included is represented by the Expansion defined in the + field `exp`. `required` is a bool indicating whether execution should fail + if the specified file could not be processed. + """ + __slots__ = ('exp', 'required', 'deps') + + def __init__(self, exp, required, weak): + assert isinstance(exp, (data.Expansion, data.StringExpansion)) + self.exp = exp + self.required = required + self.weak = weak + + def execute(self, makefile, context): + files = self.exp.resolvesplit(makefile, makefile.variables) + for f in files: + makefile.include(f, self.required, loc=self.exp.loc, weak=self.weak) + + def dump(self, fd, indent): + print >>fd, "%sInclude %s" % (indent, self.exp) + + def to_source(self): + prefix = '' + + if not self.required: + prefix = '-' + + return '%sinclude %s' % (prefix, self.exp.to_source()) + + def __eq__(self, other): + if not isinstance(other, Include): + return False + + return self.exp == other.exp and self.required == other.required + +class VPathDirective(Statement): + """ + Represents the vpath directive. + + See https://www.gnu.org/software/make/manual/make.html#Selective-Search + """ + __slots__ = ('exp',) + + def __init__(self, exp): + assert isinstance(exp, (data.Expansion, data.StringExpansion)) + self.exp = exp + + def execute(self, makefile, context): + words = list(data.stripdotslashes(self.exp.resolvesplit(makefile, makefile.variables))) + if len(words) == 0: + makefile.clearallvpaths() + else: + pattern = data.Pattern(words[0]) + mpaths = words[1:] + + if len(mpaths) == 0: + makefile.clearvpath(pattern) + else: + dirs = [] + for mpath in mpaths: + dirs.extend((dir for dir in mpath.split(os.pathsep) + if dir != '')) + if len(dirs): + makefile.addvpath(pattern, dirs) + + def dump(self, fd, indent): + print >>fd, "%sVPath %s" % (indent, self.exp) + + def to_source(self): + return 'vpath %s' % self.exp.to_source() + + def __eq__(self, other): + if not isinstance(other, VPathDirective): + return False + + return self.exp == other.exp + +class ExportDirective(Statement): + """ + Represents the "export" directive. + + This is used to control exporting variables to sub makes. + + See https://www.gnu.org/software/make/manual/make.html#Variables_002fRecursion + + The `concurrent_set` field defines whether this statement occurred with or + without a variable assignment. If False, no variable assignment was + present. If True, the SetVariable immediately following this statement + originally came from this export directive (the parser splits it into + multiple statements). + """ + + __slots__ = ('exp', 'concurrent_set') + + def __init__(self, exp, concurrent_set): + assert isinstance(exp, (data.Expansion, data.StringExpansion)) + self.exp = exp + self.concurrent_set = concurrent_set + + def execute(self, makefile, context): + if self.concurrent_set: + vlist = [self.exp.resolvestr(makefile, makefile.variables)] + else: + vlist = list(self.exp.resolvesplit(makefile, makefile.variables)) + if not len(vlist): + raise data.DataError("Exporting all variables is not supported", self.exp.loc) + + for v in vlist: + makefile.exportedvars[v] = True + + def dump(self, fd, indent): + print >>fd, "%sExport (single=%s) %s" % (indent, self.single, self.exp) + + def to_source(self): + return ('export %s' % self.exp.to_source()).rstrip() + + def __eq__(self, other): + if not isinstance(other, ExportDirective): + return False + + # single is irrelevant because it just says whether the next Statement + # contains a variable definition. + return self.exp == other.exp + +class UnexportDirective(Statement): + """ + Represents the "unexport" directive. + + This is the opposite of ExportDirective. + """ + __slots__ = ('exp',) + + def __init__(self, exp): + self.exp = exp + + def execute(self, makefile, context): + vlist = list(self.exp.resolvesplit(makefile, makefile.variables)) + for v in vlist: + makefile.exportedvars[v] = False + + def dump(self, fd, indent): + print >>fd, "%sUnexport %s" % (indent, self.exp) + + def to_source(self): + return 'unexport %s' % self.exp.to_source() + + def __eq__(self, other): + if not isinstance(other, UnexportDirective): + return False + + return self.exp == other.exp + +class EmptyDirective(Statement): + """ + Represents a standalone statement, usually an Expansion. + + You will encounter EmptyDirective instances if there is a function + or similar at the top-level of a make file (e.g. outside of a rule or + variable assignment). You can also find them as the bodies of + ConditionBlock branches. + """ + __slots__ = ('exp',) + + def __init__(self, exp): + assert isinstance(exp, (data.Expansion, data.StringExpansion)) + self.exp = exp + + def execute(self, makefile, context): + v = self.exp.resolvestr(makefile, makefile.variables) + if v.strip() != '': + raise data.DataError("Line expands to non-empty value", self.exp.loc) + + def dump(self, fd, indent): + print >>fd, "%sEmptyDirective: %s" % (indent, self.exp) + + def to_source(self): + return self.exp.to_source() + + def __eq__(self, other): + if not isinstance(other, EmptyDirective): + return False + + return self.exp == other.exp + +class _EvalContext(object): + __slots__ = ('currule', 'weak') + + def __init__(self, weak): + self.weak = weak + +class StatementList(list): + """ + A list of Statement instances. + + This is what is generated by the parser when a make file is parsed. + + Consumers can iterate over all Statement instances in this collection to + statically inspect (and even modify) make files before they are executed. + """ + __slots__ = ('mtime',) + + def append(self, statement): + assert isinstance(statement, Statement) + list.append(self, statement) + + def execute(self, makefile, context=None, weak=False): + if context is None: + context = _EvalContext(weak=weak) + + for s in self: + s.execute(makefile, context) + + def dump(self, fd, indent): + for s in self: + s.dump(fd, indent) + + def __str__(self): + fd = StringIO() + self.dump(fd, '') + return fd.getvalue() + + def to_source(self): + return '\n'.join([s.to_source() for s in self]) + +def iterstatements(stmts): + for s in stmts: + yield s + if isinstance(s, ConditionBlock): + for c, sl in s: + for s2 in iterstatments(sl): yield s2 diff --git a/python/pymake/pymake/process.py b/python/pymake/pymake/process.py new file mode 100644 index 000000000..01cadf5a9 --- /dev/null +++ b/python/pymake/pymake/process.py @@ -0,0 +1,556 @@ +""" +Skipping shell invocations is good, when possible. This wrapper around subprocess does dirty work of +parsing command lines into argv and making sure that no shell magic is being used. +""" + +#TODO: ship pyprocessing? +import multiprocessing +import subprocess, shlex, re, logging, sys, traceback, os, imp, glob +import site +from collections import deque +# XXXkhuey Work around http://bugs.python.org/issue1731717 +subprocess._cleanup = lambda: None +import command, util +if sys.platform=='win32': + import win32process + +_log = logging.getLogger('pymake.process') + +_escapednewlines = re.compile(r'\\\n') + +def tokens2re(tokens): + # Create a pattern for non-escaped tokens, in the form: + # (?pattern) + # which matches the pattern and captures it in a named match group. + # The group names and patterns come are given as a dict in the function + # argument. + nonescaped = r'(?%s)' % (name, value) for name, value in tokens.iteritems()) + # The final pattern matches either the above pattern, or an escaped + # backslash, captured in the "escape" match group. + return re.compile('(?:%s|%s)' % (nonescaped, r'(?P\\\\)')) + +_unquoted_tokens = tokens2re({ + 'whitespace': r'[\t\r\n ]+', + 'quote': r'[\'"]', + 'comment': '#', + 'special': r'[<>&|`~(){}$;]', + 'backslashed': r'\\[^\\]', + 'glob': r'[\*\?]', +}) + +_doubly_quoted_tokens = tokens2re({ + 'quote': '"', + 'backslashedquote': r'\\"', + 'special': '\$', + 'backslashed': r'\\[^\\"]', +}) + +class MetaCharacterException(Exception): + def __init__(self, char): + self.char = char + +class ClineSplitter(list): + """ + Parses a given command line string and creates a list of command + and arguments, with wildcard expansion. + """ + def __init__(self, cline, cwd): + self.cwd = cwd + self.arg = None + self.cline = cline + self.glob = False + self._parse_unquoted() + + def _push(self, str): + """ + Push the given string as part of the current argument + """ + if self.arg is None: + self.arg = '' + self.arg += str + + def _next(self): + """ + Finalize current argument, effectively adding it to the list. + Perform globbing if needed. + """ + if self.arg is None: + return + if self.glob: + if os.path.isabs(self.arg): + path = self.arg + else: + path = os.path.join(self.cwd, self.arg) + globbed = glob.glob(path) + if not globbed: + # If globbing doesn't find anything, the literal string is + # used. + self.append(self.arg) + else: + self.extend(f[len(path)-len(self.arg):] for f in globbed) + self.glob = False + else: + self.append(self.arg) + self.arg = None + + def _parse_unquoted(self): + """ + Parse command line remainder in the context of an unquoted string. + """ + while self.cline: + # Find the next token + m = _unquoted_tokens.search(self.cline) + # If we find none, the remainder of the string can be pushed to + # the current argument and the argument finalized + if not m: + self._push(self.cline) + break + # The beginning of the string, up to the found token, is part of + # the current argument + if m.start(): + self._push(self.cline[:m.start()]) + self.cline = self.cline[m.end():] + + match = dict([(name, value) for name, value in m.groupdict().items() if value]) + if 'quote' in match: + # " or ' start a quoted string + if match['quote'] == '"': + self._parse_doubly_quoted() + else: + self._parse_quoted() + elif 'comment' in match: + # Comments are ignored. The current argument can be finalized, + # and parsing stopped. + break + elif 'special' in match: + # Unquoted, non-escaped special characters need to be sent to a + # shell. + raise MetaCharacterException, match['special'] + elif 'whitespace' in match: + # Whitespaces terminate current argument. + self._next() + elif 'escape' in match: + # Escaped backslashes turn into a single backslash + self._push('\\') + elif 'backslashed' in match: + # Backslashed characters are unbackslashed + # e.g. echo \a -> a + self._push(match['backslashed'][1]) + elif 'glob' in match: + # ? or * will need globbing + self.glob = True + self._push(m.group(0)) + else: + raise Exception, "Shouldn't reach here" + if self.arg: + self._next() + + def _parse_quoted(self): + # Single quoted strings are preserved, except for the final quote + index = self.cline.find("'") + if index == -1: + raise Exception, 'Unterminated quoted string in command' + self._push(self.cline[:index]) + self.cline = self.cline[index+1:] + + def _parse_doubly_quoted(self): + if not self.cline: + raise Exception, 'Unterminated quoted string in command' + while self.cline: + m = _doubly_quoted_tokens.search(self.cline) + if not m: + raise Exception, 'Unterminated quoted string in command' + self._push(self.cline[:m.start()]) + self.cline = self.cline[m.end():] + match = dict([(name, value) for name, value in m.groupdict().items() if value]) + if 'quote' in match: + # a double quote ends the quoted string, so go back to + # unquoted parsing + return + elif 'special' in match: + # Unquoted, non-escaped special characters in a doubly quoted + # string still have a special meaning and need to be sent to a + # shell. + raise MetaCharacterException, match['special'] + elif 'escape' in match: + # Escaped backslashes turn into a single backslash + self._push('\\') + elif 'backslashedquote' in match: + # Backslashed double quotes are un-backslashed + self._push('"') + elif 'backslashed' in match: + # Backslashed characters are kept backslashed + self._push(match['backslashed']) + +def clinetoargv(cline, cwd): + """ + If this command line can safely skip the shell, return an argv array. + @returns argv, badchar + """ + str = _escapednewlines.sub('', cline) + try: + args = ClineSplitter(str, cwd) + except MetaCharacterException, e: + return None, e.char + + if len(args) and args[0].find('=') != -1: + return None, '=' + + return args, None + +# shellwords contains a set of shell builtin commands that need to be +# executed within a shell. It also contains a set of commands that are known +# to be giving problems when run directly instead of through the msys shell. +shellwords = (':', '.', 'break', 'cd', 'continue', 'exec', 'exit', 'export', + 'getopts', 'hash', 'pwd', 'readonly', 'return', 'shift', + 'test', 'times', 'trap', 'umask', 'unset', 'alias', + 'set', 'bind', 'builtin', 'caller', 'command', 'declare', + 'echo', 'enable', 'help', 'let', 'local', 'logout', + 'printf', 'read', 'shopt', 'source', 'type', 'typeset', + 'ulimit', 'unalias', 'set', 'find') + +def prepare_command(cline, cwd, loc): + """ + Returns a list of command and arguments for the given command line string. + If the command needs to be run through a shell for some reason, the + returned list contains the shell invocation. + """ + + #TODO: call this once up-front somewhere and save the result? + shell, msys = util.checkmsyscompat() + + shellreason = None + executable = None + if msys and cline.startswith('/'): + shellreason = "command starts with /" + else: + argv, badchar = clinetoargv(cline, cwd) + if argv is None: + shellreason = "command contains shell-special character '%s'" % (badchar,) + elif len(argv) and argv[0] in shellwords: + shellreason = "command starts with shell primitive '%s'" % (argv[0],) + elif argv and (os.sep in argv[0] or os.altsep and os.altsep in argv[0]): + executable = util.normaljoin(cwd, argv[0]) + # Avoid "%1 is not a valid Win32 application" errors, assuming + # that if the executable path is to be resolved with PATH, it will + # be a Win32 executable. + if sys.platform == 'win32' and os.path.isfile(executable) and open(executable, 'rb').read(2) == "#!": + shellreason = "command executable starts with a hashbang" + + if shellreason is not None: + _log.debug("%s: using shell: %s: '%s'", loc, shellreason, cline) + if msys: + if len(cline) > 3 and cline[1] == ':' and cline[2] == '/': + cline = '/' + cline[0] + cline[2:] + argv = [shell, "-c", cline] + executable = None + + return executable, argv + +def call(cline, env, cwd, loc, cb, context, echo, justprint=False): + executable, argv = prepare_command(cline, cwd, loc) + + if not len(argv): + cb(res=0) + return + + if argv[0] == command.makepypath: + command.main(argv[1:], env, cwd, cb) + return + + if argv[0:2] == [sys.executable.replace('\\', '/'), + command.makepypath.replace('\\', '/')]: + command.main(argv[2:], env, cwd, cb) + return + + context.call(argv, executable=executable, shell=False, env=env, cwd=cwd, cb=cb, + echo=echo, justprint=justprint) + +def call_native(module, method, argv, env, cwd, loc, cb, context, echo, justprint=False, + pycommandpath=None): + context.call_native(module, method, argv, env=env, cwd=cwd, cb=cb, + echo=echo, justprint=justprint, pycommandpath=pycommandpath) + +def statustoresult(status): + """ + Convert the status returned from waitpid into a prettier numeric result. + """ + sig = status & 0xFF + if sig: + return -sig + + return status >>8 + +class Job(object): + """ + A single job to be executed on the process pool. + """ + done = False # set to true when the job completes + + def __init__(self): + self.exitcode = -127 + + def notify(self, condition, result): + condition.acquire() + self.done = True + self.exitcode = result + condition.notify() + condition.release() + + def get_callback(self, condition): + return lambda result: self.notify(condition, result) + +class PopenJob(Job): + """ + A job that executes a command using subprocess.Popen. + """ + def __init__(self, argv, executable, shell, env, cwd): + Job.__init__(self) + self.argv = argv + self.executable = executable + self.shell = shell + self.env = env + self.cwd = cwd + self.parentpid = os.getpid() + + def run(self): + assert os.getpid() != self.parentpid + # subprocess.Popen doesn't use the PATH set in the env argument for + # finding the executable on some platforms (but strangely it does on + # others!), so set os.environ['PATH'] explicitly. This is parallel- + # safe because pymake uses separate processes for parallelism, and + # each process is serial. See http://bugs.python.org/issue8557 for a + # general overview of "subprocess PATH semantics and portability". + oldpath = os.environ['PATH'] + try: + if self.env is not None and self.env.has_key('PATH'): + os.environ['PATH'] = self.env['PATH'] + p = subprocess.Popen(self.argv, executable=self.executable, shell=self.shell, env=self.env, cwd=self.cwd) + return p.wait() + except OSError, e: + print >>sys.stderr, e + return -127 + finally: + os.environ['PATH'] = oldpath + +class PythonException(Exception): + def __init__(self, message, exitcode): + Exception.__init__(self) + self.message = message + self.exitcode = exitcode + + def __str__(self): + return self.message + + +class PythonJob(Job): + """ + A job that calls a Python method. + """ + def __init__(self, module, method, argv, env, cwd, pycommandpath=None): + self.module = module + self.method = method + self.argv = argv + self.env = env + self.cwd = cwd + self.pycommandpath = pycommandpath or [] + self.parentpid = os.getpid() + + def run(self): + assert os.getpid() != self.parentpid + # os.environ is a magic dictionary. Setting it to something else + # doesn't affect the environment of subprocesses, so use clear/update + oldenv = dict(os.environ) + + # sys.path is adjusted for the entire lifetime of the command + # execution. This ensures any delayed imports will still work. + oldsyspath = list(sys.path) + try: + os.chdir(self.cwd) + os.environ.clear() + os.environ.update(self.env) + + sys.path = [] + for p in sys.path + self.pycommandpath: + site.addsitedir(p) + sys.path.extend(oldsyspath) + + if self.module not in sys.modules: + try: + __import__(self.module) + except Exception as e: + print >>sys.stderr, 'Error importing %s: %s' % ( + self.module, e) + return -127 + + m = sys.modules[self.module] + if self.method not in m.__dict__: + print >>sys.stderr, "No method named '%s' in module %s" % (self.method, self.module) + return -127 + rv = m.__dict__[self.method](self.argv) + if rv != 0 and rv is not None: + print >>sys.stderr, ( + "Native command '%s %s' returned value '%s'" % + (self.module, self.method, rv)) + return (rv if isinstance(rv, int) else 1) + + except PythonException, e: + print >>sys.stderr, e + return e.exitcode + except: + e = sys.exc_info()[1] + if isinstance(e, SystemExit) and (e.code == 0 or e.code is None): + pass # sys.exit(0) is not a failure + else: + print >>sys.stderr, e + traceback.print_exc() + return -127 + finally: + os.environ.clear() + os.environ.update(oldenv) + sys.path = oldsyspath + # multiprocessing exits via os._exit, make sure that all output + # from command gets written out before that happens. + sys.stdout.flush() + sys.stderr.flush() + + return 0 + +def job_runner(job): + """ + Run a job. Called in a Process pool. + """ + return job.run() + +class ParallelContext(object): + """ + Manages the parallel execution of processes. + """ + + _allcontexts = set() + _condition = multiprocessing.Condition() + + def __init__(self, jcount): + self.jcount = jcount + self.exit = False + + self.processpool = multiprocessing.Pool(processes=jcount) + self.pending = deque() # deque of (cb, args, kwargs) + self.running = [] # list of (subprocess, cb) + + self._allcontexts.add(self) + + def finish(self): + assert len(self.pending) == 0 and len(self.running) == 0, "pending: %i running: %i" % (len(self.pending), len(self.running)) + self.processpool.close() + self.processpool.join() + self._allcontexts.remove(self) + + def run(self): + while len(self.pending) and len(self.running) < self.jcount: + cb, args, kwargs = self.pending.popleft() + cb(*args, **kwargs) + + def defer(self, cb, *args, **kwargs): + assert self.jcount > 1 or not len(self.pending), "Serial execution error defering %r %r %r: currently pending %r" % (cb, args, kwargs, self.pending) + self.pending.append((cb, args, kwargs)) + + def _docall_generic(self, pool, job, cb, echo, justprint): + if echo is not None: + print echo + processcb = job.get_callback(ParallelContext._condition) + if justprint: + processcb(0) + else: + pool.apply_async(job_runner, args=(job,), callback=processcb) + self.running.append((job, cb)) + + def call(self, argv, shell, env, cwd, cb, echo, justprint=False, executable=None): + """ + Asynchronously call the process + """ + + job = PopenJob(argv, executable=executable, shell=shell, env=env, cwd=cwd) + self.defer(self._docall_generic, self.processpool, job, cb, echo, justprint) + + def call_native(self, module, method, argv, env, cwd, cb, + echo, justprint=False, pycommandpath=None): + """ + Asynchronously call the native function + """ + + job = PythonJob(module, method, argv, env, cwd, pycommandpath) + self.defer(self._docall_generic, self.processpool, job, cb, echo, justprint) + + @staticmethod + def _waitany(condition): + def _checkdone(): + jobs = [] + for c in ParallelContext._allcontexts: + for i in xrange(0, len(c.running)): + if c.running[i][0].done: + jobs.append(c.running[i]) + for j in jobs: + if j in c.running: + c.running.remove(j) + return jobs + + # We must acquire the lock, and then check to see if any jobs have + # finished. If we don't check after acquiring the lock it's possible + # that all outstanding jobs will have completed before we wait and we'll + # wait for notifications that have already occurred. + condition.acquire() + jobs = _checkdone() + + if jobs == []: + condition.wait() + jobs = _checkdone() + + condition.release() + + return jobs + + @staticmethod + def spin(): + """ + Spin the 'event loop', and never return. + """ + + while True: + clist = list(ParallelContext._allcontexts) + for c in clist: + c.run() + + dowait = util.any((len(c.running) for c in ParallelContext._allcontexts)) + if dowait: + # Wait on local jobs first for perf + for job, cb in ParallelContext._waitany(ParallelContext._condition): + cb(job.exitcode) + else: + assert any(len(c.pending) for c in ParallelContext._allcontexts) + +def makedeferrable(usercb, **userkwargs): + def cb(*args, **kwargs): + kwargs.update(userkwargs) + return usercb(*args, **kwargs) + + return cb + +_serialContext = None +_parallelContext = None + +def getcontext(jcount): + global _serialContext, _parallelContext + if jcount == 1: + if _serialContext is None: + _serialContext = ParallelContext(1) + return _serialContext + else: + if _parallelContext is None: + _parallelContext = ParallelContext(jcount) + return _parallelContext + diff --git a/python/pymake/pymake/util.py b/python/pymake/pymake/util.py new file mode 100644 index 000000000..c63f930cc --- /dev/null +++ b/python/pymake/pymake/util.py @@ -0,0 +1,150 @@ +import os + +class MakeError(Exception): + def __init__(self, message, loc=None): + self.msg = message + self.loc = loc + + def __str__(self): + locstr = '' + if self.loc is not None: + locstr = "%s:" % (self.loc,) + + return "%s%s" % (locstr, self.msg) + +def normaljoin(path, suffix): + """ + Combine the given path with the suffix, and normalize if necessary to shrink the path to avoid hitting path length limits + """ + result = os.path.join(path, suffix) + if len(result) > 255: + result = os.path.normpath(result) + return result + +def joiniter(fd, it): + """ + Given an iterator that returns strings, write the words with a space in between each. + """ + + it = iter(it) + for i in it: + fd.write(i) + break + + for i in it: + fd.write(' ') + fd.write(i) + +def checkmsyscompat(): + """For msys compatibility on windows, honor the SHELL environment variable, + and if $MSYSTEM == MINGW32, run commands through $SHELL -c instead of + letting Python use the system shell.""" + if 'SHELL' in os.environ: + shell = os.environ['SHELL'] + elif 'MOZILLABUILD' in os.environ: + shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe' + elif 'COMSPEC' in os.environ: + shell = os.environ['COMSPEC'] + else: + raise DataError("Can't find a suitable shell!") + + msys = False + if 'MSYSTEM' in os.environ and os.environ['MSYSTEM'] == 'MINGW32': + msys = True + if not shell.lower().endswith(".exe"): + shell += ".exe" + return (shell, msys) + +if hasattr(str, 'partition'): + def strpartition(str, token): + return str.partition(token) + + def strrpartition(str, token): + return str.rpartition(token) + +else: + def strpartition(str, token): + """Python 2.4 compatible str.partition""" + + offset = str.find(token) + if offset == -1: + return str, '', '' + + return str[:offset], token, str[offset + len(token):] + + def strrpartition(str, token): + """Python 2.4 compatible str.rpartition""" + + offset = str.rfind(token) + if offset == -1: + return '', '', str + + return str[:offset], token, str[offset + len(token):] + +try: + from __builtin__ import any +except ImportError: + def any(it): + for i in it: + if i: + return True + return False + +class _MostUsedItem(object): + __slots__ = ('key', 'o', 'count') + + def __init__(self, key): + self.key = key + self.o = None + self.count = 1 + + def __repr__(self): + return "MostUsedItem(key=%r, count=%i, o=%r)" % (self.key, self.count, self.o) + +class MostUsedCache(object): + def __init__(self, capacity, creationfunc, verifyfunc): + self.capacity = capacity + self.cfunc = creationfunc + self.vfunc = verifyfunc + + self.d = {} + self.active = [] # lazily sorted! + + def setactive(self, item): + if item in self.active: + return + + if len(self.active) == self.capacity: + self.active.sort(key=lambda i: i.count) + old = self.active.pop(0) + old.o = None + # print "Evicting %s" % old.key + + self.active.append(item) + + def get(self, key): + item = self.d.get(key, None) + if item is None: + item = _MostUsedItem(key) + self.d[key] = item + else: + item.count += 1 + + if item.o is not None and self.vfunc(key, item.o): + return item.o + + item.o = self.cfunc(key) + self.setactive(item) + return item.o + + def verify(self): + for k, v in self.d.iteritems(): + if v.o: + assert v in self.active + else: + assert v not in self.active + + def debugitems(self): + l = [i.key for i in self.active] + l.sort() + return l diff --git a/python/pymake/pymake/win32process.py b/python/pymake/pymake/win32process.py new file mode 100644 index 000000000..880a26a5b --- /dev/null +++ b/python/pymake/pymake/win32process.py @@ -0,0 +1,28 @@ +from ctypes import windll, POINTER, byref, WinError +from ctypes.wintypes import WINFUNCTYPE, HANDLE, DWORD, BOOL + +INFINITE = -1 +WAIT_FAILED = 0xFFFFFFFF + +LPDWORD = POINTER(DWORD) +_GetExitCodeProcessProto = WINFUNCTYPE(BOOL, HANDLE, LPDWORD) +_GetExitCodeProcess = _GetExitCodeProcessProto(("GetExitCodeProcess", windll.kernel32)) +def GetExitCodeProcess(h): + exitcode = DWORD() + r = _GetExitCodeProcess(h, byref(exitcode)) + if r is 0: + raise WinError() + return exitcode.value + +_WaitForMultipleObjectsProto = WINFUNCTYPE(DWORD, DWORD, POINTER(HANDLE), BOOL, DWORD) +_WaitForMultipleObjects = _WaitForMultipleObjectsProto(("WaitForMultipleObjects", windll.kernel32)) + +def WaitForAnyProcess(processes): + arrtype = HANDLE * len(processes) + harray = arrtype(*(int(p._handle) for p in processes)) + + r = _WaitForMultipleObjects(len(processes), harray, False, INFINITE) + if r == WAIT_FAILED: + raise WinError() + + return processes[r], GetExitCodeProcess(int(processes[r]._handle)) <<8 diff --git a/python/pymake/tests/automatic-variables.mk b/python/pymake/tests/automatic-variables.mk new file mode 100644 index 000000000..5302c08ea --- /dev/null +++ b/python/pymake/tests/automatic-variables.mk @@ -0,0 +1,79 @@ +$(shell \ +mkdir -p src/subd; \ +mkdir subd; \ +touch dummy; \ +sleep 2; \ +touch subd/test.out src/subd/test.in2; \ +sleep 2; \ +touch subd/test.out2 src/subd/test.in; \ +sleep 2; \ +touch subd/host_test.out subd/host_test.out2; \ +sleep 2; \ +touch host_prog; \ +) + +VPATH = src + +all: prog host_prog prog dir/ + test "$@" = "all" + test "$<" = "prog" + test "$^" = "prog host_prog dir" + test "$?" = "prog host_prog dir" + test "$+" = "prog host_prog prog dir" + test "$(@D)" = "." + test "$(@F)" = "all" + test "$($@ + +%.out2: %.in2 dummy + @echo TEST_FAIL No need to remake + +.PHONY: all diff --git a/python/pymake/tests/bad-command-continuation.mk b/python/pymake/tests/bad-command-continuation.mk new file mode 100644 index 000000000..d9ceccfc2 --- /dev/null +++ b/python/pymake/tests/bad-command-continuation.mk @@ -0,0 +1,3 @@ +all: + echo 'hello'\ +TEST-PASS diff --git a/python/pymake/tests/call.mk b/python/pymake/tests/call.mk new file mode 100644 index 000000000..9eeb7e00c --- /dev/null +++ b/python/pymake/tests/call.mk @@ -0,0 +1,12 @@ +test = $0 +reverse = $2 $1 +twice = $1$1 +sideeffect = $(shell echo "called$1:" >>dummyfile) + +all: + test "$(call test)" = "test" + test "$(call reverse,1,2)" = "2 1" +# expansion happens *before* substitution, thank sanity + test "$(call twice,$(sideeffect))" = "" + test `cat dummyfile` = "called:" + @echo TEST-PASS diff --git a/python/pymake/tests/cmd-stripdotslash.mk b/python/pymake/tests/cmd-stripdotslash.mk new file mode 100644 index 000000000..ce5ed4244 --- /dev/null +++ b/python/pymake/tests/cmd-stripdotslash.mk @@ -0,0 +1,5 @@ +all: + $(MAKE) -f $(TESTPATH)/cmd-stripdotslash.mk ./foo + +./foo: + @echo TEST-PASS diff --git a/python/pymake/tests/cmdgoals.mk b/python/pymake/tests/cmdgoals.mk new file mode 100644 index 000000000..a3b25e751 --- /dev/null +++ b/python/pymake/tests/cmdgoals.mk @@ -0,0 +1,9 @@ +default: + test "$(MAKECMDGOALS)" = "" + $(MAKE) -f $(TESTPATH)/cmdgoals.mk t1 t2 + @echo TEST-PASS + +t1: + test "$(MAKECMDGOALS)" = "t1 t2" + +t2: diff --git a/python/pymake/tests/commandmodifiers.mk b/python/pymake/tests/commandmodifiers.mk new file mode 100644 index 000000000..8440462f3 --- /dev/null +++ b/python/pymake/tests/commandmodifiers.mk @@ -0,0 +1,21 @@ +define COMMAND +$(1) + $(1) + +endef + +all: + $(call COMMAND,@true #TEST-FAIL) + $(call COMMAND,-exit 4) + $(call COMMAND,@-exit 1 # TEST-FAIL) + $(call COMMAND,-@exit 1 # TEST-FAIL) + $(call COMMAND,+exit 0) + $(call COMMAND,+-exit 1) + $(call COMMAND,@+exit 0 # TEST-FAIL) + $(call COMMAND,+@exit 0 # TEST-FAIL) + $(call COMMAND,-+@exit 1 # TEST-FAIL) + $(call COMMAND,+-@exit 1 # TEST-FAIL) + $(call COMMAND,@+-exit 1 # TEST-FAIL) + $(call COMMAND,@+-@+-exit 1 # TEST-FAIL) + $(call COMMAND,@@++exit 0 # TEST-FAIL) + @echo TEST-PASS diff --git a/python/pymake/tests/comment-parsing.mk b/python/pymake/tests/comment-parsing.mk new file mode 100644 index 000000000..d469e1aea --- /dev/null +++ b/python/pymake/tests/comment-parsing.mk @@ -0,0 +1,29 @@ +# where do comments take effect? + +VAR = val1 # comment +VAR2 = lit2\#hash +VAR2_1 = lit2.1\\\#hash +VAR3 = val3 +VAR4 = lit4\\#backslash +VAR4_1 = lit4\\\\#backslash +VAR5 = lit5\char +VAR6 = lit6\\char +VAR7 = lit7\\ +VAR8 = lit8\\\\ +VAR9 = lit9\\\\extra +# This comment extends to the next line \ +VAR3 = ignored + +all: + test "$(VAR)" = "val1 " + test "$(VAR2)" = "lit2#hash" + test '$(VAR2_1)' = 'lit2.1\#hash' + test "$(VAR3)" = "val3" + test '$(VAR4)' = 'lit4\' + test '$(VAR4_1)' = 'lit4\\' + test '$(VAR5)' = 'lit5\char' + test '$(VAR6)' = 'lit6\\char' + test '$(VAR7)' = 'lit7\\' + test '$(VAR8)' = 'lit8\\\\' + test '$(VAR9)' = 'lit9\\\\extra' + @echo "TEST-PASS" diff --git a/python/pymake/tests/continuations-in-functions.mk b/python/pymake/tests/continuations-in-functions.mk new file mode 100644 index 000000000..533df6176 --- /dev/null +++ b/python/pymake/tests/continuations-in-functions.mk @@ -0,0 +1,6 @@ +all: + test 'Hello world.' = '$(if 1,Hello \ + world.)' + test '(Hello world.)' != '(Hello \ + world.)' + @echo TEST-PASS diff --git a/python/pymake/tests/datatests.py b/python/pymake/tests/datatests.py new file mode 100644 index 000000000..513028b0b --- /dev/null +++ b/python/pymake/tests/datatests.py @@ -0,0 +1,237 @@ +import pymake.data, pymake.functions, pymake.util +import unittest +import re +from cStringIO import StringIO + +def multitest(cls): + for name in cls.testdata.iterkeys(): + def m(self, name=name): + return self.runSingle(*self.testdata[name]) + + setattr(cls, 'test_%s' % name, m) + return cls + +class SplitWordsTest(unittest.TestCase): + testdata = ( + (' test test.c test.o ', ['test', 'test.c', 'test.o']), + ('\ttest\t test.c \ntest.o', ['test', 'test.c', 'test.o']), + ) + + def runTest(self): + for s, e in self.testdata: + w = s.split() + self.assertEqual(w, e, 'splitwords(%r)' % (s,)) + +class GetPatSubstTest(unittest.TestCase): + testdata = ( + ('%.c', '%.o', ' test test.c test.o ', 'test test.o test.o'), + ('%', '%.o', ' test.c test.o ', 'test.c.o test.o.o'), + ('foo', 'bar', 'test foo bar', 'test bar bar'), + ('foo', '%bar', 'test foo bar', 'test %bar bar'), + ('%', 'perc_%', 'path', 'perc_path'), + ('\\%', 'sub%', 'p %', 'p sub%'), + ('%.c', '\\%%.o', 'foo.c bar.o baz.cpp', '%foo.o bar.o baz.cpp'), + ) + + def runTest(self): + for s, r, d, e in self.testdata: + words = d.split() + p = pymake.data.Pattern(s) + a = ' '.join((p.subst(r, word, False) + for word in words)) + self.assertEqual(a, e, 'Pattern(%r).subst(%r, %r)' % (s, r, d)) + +class LRUTest(unittest.TestCase): + # getkey, expected, funccount, debugitems + expected = ( + (0, '', 1, (0,)), + (0, '', 2, (0,)), + (1, ' ', 3, (1, 0)), + (1, ' ', 3, (1, 0)), + (0, '', 4, (0, 1)), + (2, ' ', 5, (2, 0, 1)), + (1, ' ', 5, (1, 2, 0)), + (3, ' ', 6, (3, 1, 2)), + ) + + def spaceFunc(self, l): + self.funccount += 1 + return ''.ljust(l) + + def runTest(self): + self.funccount = 0 + c = pymake.util.LRUCache(3, self.spaceFunc, lambda k, v: k % 2) + self.assertEqual(tuple(c.debugitems()), ()) + + for i in xrange(0, len(self.expected)): + k, e, fc, di = self.expected[i] + + v = c.get(k) + self.assertEqual(v, e) + self.assertEqual(self.funccount, fc, + "funccount, iteration %i, got %i expected %i" % (i, self.funccount, fc)) + goti = tuple(c.debugitems()) + self.assertEqual(goti, di, + "debugitems, iteration %i, got %r expected %r" % (i, goti, di)) + +class EqualityTest(unittest.TestCase): + def test_string_expansion(self): + s1 = pymake.data.StringExpansion('foo bar', None) + s2 = pymake.data.StringExpansion('foo bar', None) + + self.assertEqual(s1, s2) + + def test_expansion_simple(self): + s1 = pymake.data.Expansion(None) + s2 = pymake.data.Expansion(None) + + self.assertEqual(s1, s2) + + s1.appendstr('foo') + s2.appendstr('foo') + self.assertEqual(s1, s2) + + def test_expansion_string_finish(self): + """Adjacent strings should normalize to same value.""" + s1 = pymake.data.Expansion(None) + s2 = pymake.data.Expansion(None) + + s1.appendstr('foo') + s2.appendstr('foo') + + s1.appendstr(' bar') + s1.appendstr(' baz') + s2.appendstr(' bar baz') + + self.assertEqual(s1, s2) + + def test_function(self): + s1 = pymake.data.Expansion(None) + s2 = pymake.data.Expansion(None) + + n1 = pymake.data.StringExpansion('FOO', None) + n2 = pymake.data.StringExpansion('FOO', None) + + v1 = pymake.functions.VariableRef(None, n1) + v2 = pymake.functions.VariableRef(None, n2) + + s1.appendfunc(v1) + s2.appendfunc(v2) + + self.assertEqual(s1, s2) + + +class StringExpansionTest(unittest.TestCase): + def test_base_expansion_interface(self): + s1 = pymake.data.StringExpansion('FOO', None) + + self.assertTrue(s1.is_static_string) + + funcs = list(s1.functions()) + self.assertEqual(len(funcs), 0) + + funcs = list(s1.functions(True)) + self.assertEqual(len(funcs), 0) + + refs = list(s1.variable_references()) + self.assertEqual(len(refs), 0) + + +class ExpansionTest(unittest.TestCase): + def test_is_static_string(self): + e1 = pymake.data.Expansion() + e1.appendstr('foo') + + self.assertTrue(e1.is_static_string) + + e1.appendstr('bar') + self.assertTrue(e1.is_static_string) + + vname = pymake.data.StringExpansion('FOO', None) + func = pymake.functions.VariableRef(None, vname) + + e1.appendfunc(func) + + self.assertFalse(e1.is_static_string) + + def test_get_functions(self): + e1 = pymake.data.Expansion() + e1.appendstr('foo') + + vname1 = pymake.data.StringExpansion('FOO', None) + vname2 = pymake.data.StringExpansion('BAR', None) + + func1 = pymake.functions.VariableRef(None, vname1) + func2 = pymake.functions.VariableRef(None, vname2) + + e1.appendfunc(func1) + e1.appendfunc(func2) + + funcs = list(e1.functions()) + self.assertEqual(len(funcs), 2) + + func3 = pymake.functions.SortFunction(None) + func3.append(vname1) + + e1.appendfunc(func3) + + funcs = list(e1.functions()) + self.assertEqual(len(funcs), 3) + + refs = list(e1.variable_references()) + self.assertEqual(len(refs), 2) + + def test_get_functions_descend(self): + e1 = pymake.data.Expansion() + vname1 = pymake.data.StringExpansion('FOO', None) + func1 = pymake.functions.VariableRef(None, vname1) + e2 = pymake.data.Expansion() + e2.appendfunc(func1) + + func2 = pymake.functions.SortFunction(None) + func2.append(e2) + + e1.appendfunc(func2) + + funcs = list(e1.functions()) + self.assertEqual(len(funcs), 1) + + funcs = list(e1.functions(True)) + self.assertEqual(len(funcs), 2) + + self.assertTrue(isinstance(funcs[0], pymake.functions.SortFunction)) + + def test_is_filesystem_dependent(self): + e = pymake.data.Expansion() + vname1 = pymake.data.StringExpansion('FOO', None) + func1 = pymake.functions.VariableRef(None, vname1) + e.appendfunc(func1) + + self.assertFalse(e.is_filesystem_dependent) + + func2 = pymake.functions.WildcardFunction(None) + func2.append(vname1) + e.appendfunc(func2) + + self.assertTrue(e.is_filesystem_dependent) + + def test_is_filesystem_dependent_descend(self): + sort = pymake.functions.SortFunction(None) + wildcard = pymake.functions.WildcardFunction(None) + + e = pymake.data.StringExpansion('foo/*', None) + wildcard.append(e) + + e = pymake.data.Expansion(None) + e.appendfunc(wildcard) + + sort.append(e) + + e = pymake.data.Expansion(None) + e.appendfunc(sort) + + self.assertTrue(e.is_filesystem_dependent) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/pymake/tests/default-goal-set-first.mk b/python/pymake/tests/default-goal-set-first.mk new file mode 100644 index 000000000..00a5b53a2 --- /dev/null +++ b/python/pymake/tests/default-goal-set-first.mk @@ -0,0 +1,7 @@ +.DEFAULT_GOAL := default + +not-default: + @echo TEST-FAIL did not run default rule + +default: + @echo TEST-PASS diff --git a/python/pymake/tests/default-goal.mk b/python/pymake/tests/default-goal.mk new file mode 100644 index 000000000..699d6c0cd --- /dev/null +++ b/python/pymake/tests/default-goal.mk @@ -0,0 +1,8 @@ +not-default: + @echo TEST-FAIL did not run default rule + +default: + @echo $(if $(filter not-default,$(INTERMEDIATE_DEFAULT_GOAL)),TEST-PASS,TEST-FAIL .DEFAULT_GOAL not set by $(MAKE)) + +INTERMEDIATE_DEFAULT_GOAL := $(.DEFAULT_GOAL) +.DEFAULT_GOAL := default diff --git a/python/pymake/tests/default-target.mk b/python/pymake/tests/default-target.mk new file mode 100644 index 000000000..701ac6916 --- /dev/null +++ b/python/pymake/tests/default-target.mk @@ -0,0 +1,14 @@ +test: VAR = value + +%.do: + @echo TEST-FAIL: ran target "$@", should have run "all" + +.PHONY: test + +all: + @echo TEST-PASS: the default target is all + +test: + @echo TEST-FAIL: ran target "$@", should have run "all" + +test.do: diff --git a/python/pymake/tests/default-target2.mk b/python/pymake/tests/default-target2.mk new file mode 100644 index 000000000..b5a4b1bbf --- /dev/null +++ b/python/pymake/tests/default-target2.mk @@ -0,0 +1,6 @@ +test.foo: %.foo: + test "$@" = "test.foo" + @echo TEST-PASS made test.foo by default + +all: + @echo TEST-FAIL made $@, should have made test.foo diff --git a/python/pymake/tests/define-directive.mk b/python/pymake/tests/define-directive.mk new file mode 100644 index 000000000..789988666 --- /dev/null +++ b/python/pymake/tests/define-directive.mk @@ -0,0 +1,69 @@ +define COMMANDS +shellvar=hello +test "$$shellvar" != "hello" +endef + +define COMMANDS2 +shellvar=hello; \ + test "$$shellvar" = "hello" +endef + +define VARWITHCOMMENT # comment +value +endef + +define TEST3 + whitespace +endef + +define TEST4 +define TEST5 +random +endef + endef + +ifdef TEST5 +$(error TEST5 should not be set) +endif + +define TEST6 + define TEST7 +random +endef +endef + +ifdef TEST7 +$(error TEST7 should not be set) +endif + +define TEST8 +is this # a comment? +endef + +ifneq ($(TEST8),is this \# a comment?) +$(error TEST8 value not expected: $(TEST8)) +endif + +# A backslash continuation "hides" the endef +define TEST9 +value \ +endef +endef + +# Test ridiculous spacing + define TEST10 + define TEST11 + baz +endef +define TEST12 + foo + endef + endef + +all: + $(COMMANDS) + $(COMMANDS2) + test '$(VARWITHCOMMENT)' = 'value' + test '$(COMMANDS2)' = 'shellvar=hello; test "$$shellvar" = "hello"' + test "$(TEST3)" = " whitespace" + @echo TEST-PASS diff --git a/python/pymake/tests/depfailed.mk b/python/pymake/tests/depfailed.mk new file mode 100644 index 000000000..ce4137c38 --- /dev/null +++ b/python/pymake/tests/depfailed.mk @@ -0,0 +1,4 @@ +#T returncode: 2 + +all: foo.out foo.in + @echo TEST-PASS diff --git a/python/pymake/tests/depfailedj.mk b/python/pymake/tests/depfailedj.mk new file mode 100644 index 000000000..a94c74f6f --- /dev/null +++ b/python/pymake/tests/depfailedj.mk @@ -0,0 +1,10 @@ +#T returncode: 2 +#T commandline: ['-j4'] + +$(shell touch foo.in) + +all: foo.in foo.out missing + @echo TEST-PASS + +%.out: %.in + cp $< $@ diff --git a/python/pymake/tests/diamond-deps.mk b/python/pymake/tests/diamond-deps.mk new file mode 100644 index 000000000..40a4176d9 --- /dev/null +++ b/python/pymake/tests/diamond-deps.mk @@ -0,0 +1,13 @@ +# If the dependency graph includes a diamond dependency, we should only remake +# once! + +all: depA depB + cat testfile + test `cat testfile` = "data"; + @echo TEST-PASS + +depA: testfile +depB: testfile + +testfile: + printf "data" >>$@ diff --git a/python/pymake/tests/dotslash-dir.mk b/python/pymake/tests/dotslash-dir.mk new file mode 100644 index 000000000..8b30d1e3c --- /dev/null +++ b/python/pymake/tests/dotslash-dir.mk @@ -0,0 +1,8 @@ +#T grep-for: "dotslash-built" +.PHONY: $(dir foo) + +all: $(dir foo) + @echo TEST-PASS + +$(dir foo): + @echo dotslash-built diff --git a/python/pymake/tests/dotslash-parse.mk b/python/pymake/tests/dotslash-parse.mk new file mode 100644 index 000000000..91461bedb --- /dev/null +++ b/python/pymake/tests/dotslash-parse.mk @@ -0,0 +1,4 @@ +./: + +# This is merely a test to see that pymake doesn't choke on parsing ./ +$(info TEST-PASS) diff --git a/python/pymake/tests/dotslash-phony.mk b/python/pymake/tests/dotslash-phony.mk new file mode 100644 index 000000000..06b6ae78d --- /dev/null +++ b/python/pymake/tests/dotslash-phony.mk @@ -0,0 +1,3 @@ +.PHONY: ./ +./: + @echo TEST-PASS diff --git a/python/pymake/tests/dotslash.mk b/python/pymake/tests/dotslash.mk new file mode 100644 index 000000000..585db96b7 --- /dev/null +++ b/python/pymake/tests/dotslash.mk @@ -0,0 +1,9 @@ +$(shell touch foo.in) + +all: foo.out + test "$(wildcard ./*.in)" = "./foo.in" + @echo TEST-PASS + +./%.out: %.in + test "$@" = "foo.out" + cp $< $@ diff --git a/python/pymake/tests/doublecolon-exists.mk b/python/pymake/tests/doublecolon-exists.mk new file mode 100644 index 000000000..5d99a1f6b --- /dev/null +++ b/python/pymake/tests/doublecolon-exists.mk @@ -0,0 +1,16 @@ +$(shell touch foo.testfile1 foo.testfile2) + +# when a rule has commands and no prerequisites, should it be executed? +# double-colon: yes +# single-colon: no + +all: foo.testfile1 foo.testfile2 + test "$$(cat foo.testfile1)" = "" + test "$$(cat foo.testfile2)" = "remade:foo.testfile2" + @echo TEST-PASS + +foo.testfile1: + @echo TEST-FAIL + +foo.testfile2:: + printf "remade:$@"> $@ diff --git a/python/pymake/tests/doublecolon-priordeps.mk b/python/pymake/tests/doublecolon-priordeps.mk new file mode 100644 index 000000000..6cdf3a8e7 --- /dev/null +++ b/python/pymake/tests/doublecolon-priordeps.mk @@ -0,0 +1,19 @@ +#T commandline: ['-j3'] + +# All *prior* dependencies of a doublecolon rule must be satisfied before +# subsequent commands are run. + +all:: target1 + +all:: target2 + test -f target1 + @echo TEST-PASS + +target1: + touch starting-$@ + sleep 1 + touch $@ + +target2: + sleep 0.1 + test -f starting-target1 diff --git a/python/pymake/tests/doublecolon-remake.mk b/python/pymake/tests/doublecolon-remake.mk new file mode 100644 index 000000000..52aa9265c --- /dev/null +++ b/python/pymake/tests/doublecolon-remake.mk @@ -0,0 +1,4 @@ +$(shell touch somefile) + +all:: somefile + @echo TEST-PASS diff --git a/python/pymake/tests/dynamic-var.mk b/python/pymake/tests/dynamic-var.mk new file mode 100644 index 000000000..0993b9ccf --- /dev/null +++ b/python/pymake/tests/dynamic-var.mk @@ -0,0 +1,18 @@ +# The *name* of variables can be constructed dynamically. + +VARNAME = FOOBAR + +$(VARNAME) = foovalue +$(VARNAME)2 = foo2value + +$(VARNAME:%BAR=%BAM) = foobam + +all: + test "$(FOOBAR)" = "foovalue" + test "$(flavor FOOBAZ)" = "undefined" + test "$(FOOBAR2)" = "bazvalue" + test "$(FOOBAM)" = "foobam" + @echo TEST-PASS + +VARNAME = FOOBAZ +FOOBAR2 = bazvalue diff --git a/python/pymake/tests/empty-arg.mk b/python/pymake/tests/empty-arg.mk new file mode 100644 index 000000000..616e5b694 --- /dev/null +++ b/python/pymake/tests/empty-arg.mk @@ -0,0 +1,2 @@ +all: + @ sh -c 'if [ $$# = 3 ] ; then echo TEST-PASS; else echo TEST-FAIL; fi' -- a "" b diff --git a/python/pymake/tests/empty-command-semicolon.mk b/python/pymake/tests/empty-command-semicolon.mk new file mode 100644 index 000000000..07789f3f1 --- /dev/null +++ b/python/pymake/tests/empty-command-semicolon.mk @@ -0,0 +1,5 @@ +all: + @echo TEST-PASS + +foo: ; + diff --git a/python/pymake/tests/empty-with-deps.mk b/python/pymake/tests/empty-with-deps.mk new file mode 100644 index 000000000..284e5a113 --- /dev/null +++ b/python/pymake/tests/empty-with-deps.mk @@ -0,0 +1,4 @@ +default.test: default.c + +default.c: + @echo TEST-PASS diff --git a/python/pymake/tests/env-var-append.mk b/python/pymake/tests/env-var-append.mk new file mode 100644 index 000000000..4db39c45f --- /dev/null +++ b/python/pymake/tests/env-var-append.mk @@ -0,0 +1,7 @@ +#T environment: {'FOO': 'TEST'} + +FOO += $(BAR) +BAR := PASS + +all: + @echo $(subst $(NULL) ,-,$(FOO)) diff --git a/python/pymake/tests/env-var-append2.mk b/python/pymake/tests/env-var-append2.mk new file mode 100644 index 000000000..fc0735d88 --- /dev/null +++ b/python/pymake/tests/env-var-append2.mk @@ -0,0 +1,8 @@ +#T environment: {'FOO': '$(BAZ)'} + +FOO += $(BAR) +BAR := PASS +BAZ := TEST + +all: + @echo $(subst $(NULL) ,-,$(FOO)) diff --git a/python/pymake/tests/eof-continuation.mk b/python/pymake/tests/eof-continuation.mk new file mode 100644 index 000000000..daeaabc3e --- /dev/null +++ b/python/pymake/tests/eof-continuation.mk @@ -0,0 +1,5 @@ +all: + test '$(TESTVAR)' = 'testval\' + @echo TEST-PASS + +TESTVAR = testval\ \ No newline at end of file diff --git a/python/pymake/tests/escape-chars.mk b/python/pymake/tests/escape-chars.mk new file mode 100644 index 000000000..ebea33074 --- /dev/null +++ b/python/pymake/tests/escape-chars.mk @@ -0,0 +1,26 @@ +space = $(NULL) $(NULL) +hello$(space)world$(space) = hellovalue + +A = aval + +VAR = value1\\ +VARAWFUL = value1\\#comment +VAR2 = value2 +VAR3 = test\$A +VAR4 = value4\\value5 + +VAR5 = value1\\ \ \ + value2 + +EPERCENT = \% + +all: + test "$(hello world )" = "hellovalue" + test "$(VAR)" = "value1\\" + test '$(VARAWFUL)' = 'value1\' + test "$(VAR2)" = "value2" + test "$(VAR3)" = "test\aval" + test "$(VAR4)" = "value4\\value5" + test "$(VAR5)" = "value1\\ \ value2" + test "$(EPERCENT)" = "\%" + @echo TEST-PASS diff --git a/python/pymake/tests/escaped-continuation.mk b/python/pymake/tests/escaped-continuation.mk new file mode 100644 index 000000000..537f7547f --- /dev/null +++ b/python/pymake/tests/escaped-continuation.mk @@ -0,0 +1,6 @@ +#T returncode: 2 + +all: + echo "Hello" \\ + test "world" = "not!" + @echo TEST-PASS diff --git a/python/pymake/tests/eval-duringexecute.mk b/python/pymake/tests/eval-duringexecute.mk new file mode 100644 index 000000000..dff848032 --- /dev/null +++ b/python/pymake/tests/eval-duringexecute.mk @@ -0,0 +1,12 @@ +#T returncode: 2 + +# Once parsing is finished, recursive expansion in commands are not allowed to create any new rules (it may only set variables) + +define MORERULE +all: + @echo TEST-FAIL +endef + +all: + $(eval $(MORERULE)) + @echo done diff --git a/python/pymake/tests/eval.mk b/python/pymake/tests/eval.mk new file mode 100644 index 000000000..de9759f02 --- /dev/null +++ b/python/pymake/tests/eval.mk @@ -0,0 +1,7 @@ +TESTVAR = val1 + +$(eval TESTVAR = val2) + +all: + test "$(TESTVAR)" = "val2" + @echo TEST-PASS diff --git a/python/pymake/tests/exit-code.mk b/python/pymake/tests/exit-code.mk new file mode 100644 index 000000000..84dcffcf9 --- /dev/null +++ b/python/pymake/tests/exit-code.mk @@ -0,0 +1,5 @@ +#T returncode: 2 + +all: + exit 1 + @echo TEST-PASS diff --git a/python/pymake/tests/file-functions-symlinks.mk b/python/pymake/tests/file-functions-symlinks.mk new file mode 100644 index 000000000..dcc0f6eef --- /dev/null +++ b/python/pymake/tests/file-functions-symlinks.mk @@ -0,0 +1,22 @@ +#T returncode-on: {'win32': 2} +$(shell \ +touch test.file; \ +ln -s test.file test.symlink; \ +ln -s test.missing missing.symlink; \ +touch .testhidden; \ +mkdir foo; \ +touch foo/testfile; \ +ln -s foo symdir; \ +) + +all: + test "$(abspath test.file test.symlink)" = "$(CURDIR)/test.file $(CURDIR)/test.symlink" + test "$(realpath test.file test.symlink)" = "$(CURDIR)/test.file $(CURDIR)/test.file" + test "$(sort $(wildcard *))" = "foo symdir test.file test.symlink" + test "$(sort $(wildcard .*))" = ". .. .testhidden" + test "$(sort $(wildcard test*))" = "test.file test.symlink" + test "$(sort $(wildcard foo/*))" = "foo/testfile" + test "$(sort $(wildcard ./*))" = "./foo ./symdir ./test.file ./test.symlink" + test "$(sort $(wildcard f?o/*))" = "foo/testfile" + test "$(sort $(wildcard */*))" = "foo/testfile symdir/testfile" + @echo TEST-PASS diff --git a/python/pymake/tests/file-functions.mk b/python/pymake/tests/file-functions.mk new file mode 100644 index 000000000..7e4c68e85 --- /dev/null +++ b/python/pymake/tests/file-functions.mk @@ -0,0 +1,19 @@ +$(shell \ +touch test.file; \ +touch .testhidden; \ +mkdir foo; \ +touch foo/testfile; \ +) + +all: + test "$(abspath test.file)" = "$(CURDIR)/test.file" + test "$(realpath test.file)" = "$(CURDIR)/test.file" + test "$(sort $(wildcard *))" = "foo test.file" +# commented out because GNU make matches . and .. while python doesn't, and I don't +# care enough +# test "$(sort $(wildcard .*))" = ". .. .testhidden" + test "$(sort $(wildcard test*))" = "test.file" + test "$(sort $(wildcard foo/*))" = "foo/testfile" + test "$(sort $(wildcard ./*))" = "./foo ./test.file" + test "$(sort $(wildcard f?o/*))" = "foo/testfile" + @echo TEST-PASS diff --git a/python/pymake/tests/foreach-local-variable.mk b/python/pymake/tests/foreach-local-variable.mk new file mode 100644 index 000000000..2551621eb --- /dev/null +++ b/python/pymake/tests/foreach-local-variable.mk @@ -0,0 +1,8 @@ +# This test ensures that a local variable in a $(foreach) is bound to +# the local value, not a global value. +i := dummy + +all: + test "$(foreach i,foo bar,found:$(i))" = "found:foo found:bar" + test "$(i)" = "dummy" + @echo TEST-PASS diff --git a/python/pymake/tests/formattingtests.py b/python/pymake/tests/formattingtests.py new file mode 100644 index 000000000..7aad6d4cc --- /dev/null +++ b/python/pymake/tests/formattingtests.py @@ -0,0 +1,289 @@ +# This file contains test code for the formatting of parsed statements back to +# make file "source." It essentially verifies to to_source() functions +# scattered across the tree. + +import glob +import logging +import os.path +import unittest + +from pymake.data import Expansion +from pymake.data import StringExpansion +from pymake.functions import BasenameFunction +from pymake.functions import SubstitutionRef +from pymake.functions import VariableRef +from pymake.functions import WordlistFunction +from pymake.parserdata import Include +from pymake.parserdata import SetVariable +from pymake.parser import parsestring +from pymake.parser import SyntaxError + +class TestBase(unittest.TestCase): + pass + +class VariableRefTest(TestBase): + def test_string_name(self): + e = StringExpansion('foo', None) + v = VariableRef(None, e) + + self.assertEqual(v.to_source(), '$(foo)') + + def test_special_variable(self): + e = StringExpansion('<', None) + v = VariableRef(None, e) + + self.assertEqual(v.to_source(), '$<') + + def test_expansion_simple(self): + e = Expansion() + e.appendstr('foo') + e.appendstr('bar') + + v = VariableRef(None, e) + + self.assertEqual(v.to_source(), '$(foobar)') + +class StandardFunctionTest(TestBase): + def test_basename(self): + e1 = StringExpansion('foo', None) + v = VariableRef(None, e1) + e2 = Expansion(None) + e2.appendfunc(v) + + b = BasenameFunction(None) + b.append(e2) + + self.assertEqual(b.to_source(), '$(basename $(foo))') + + def test_wordlist(self): + e1 = StringExpansion('foo', None) + e2 = StringExpansion('bar ', None) + e3 = StringExpansion(' baz', None) + + w = WordlistFunction(None) + w.append(e1) + w.append(e2) + w.append(e3) + + self.assertEqual(w.to_source(), '$(wordlist foo,bar , baz)') + + def test_curly_brackets(self): + e1 = Expansion(None) + e1.appendstr('foo') + + e2 = Expansion(None) + e2.appendstr('foo ( bar') + + f = WordlistFunction(None) + f.append(e1) + f.append(e2) + + self.assertEqual(f.to_source(), '${wordlist foo,foo ( bar}') + +class StringExpansionTest(TestBase): + def test_simple(self): + e = StringExpansion('foobar', None) + self.assertEqual(e.to_source(), 'foobar') + + e = StringExpansion('$var', None) + self.assertEqual(e.to_source(), '$var') + + def test_escaping(self): + e = StringExpansion('$var', None) + self.assertEqual(e.to_source(escape_variables=True), '$$var') + + e = StringExpansion('this is # not a comment', None) + self.assertEqual(e.to_source(escape_comments=True), + 'this is \# not a comment') + + def test_empty(self): + e = StringExpansion('', None) + self.assertEqual(e.to_source(), '') + + e = StringExpansion(' ', None) + self.assertEqual(e.to_source(), ' ') + +class ExpansionTest(TestBase): + def test_single_string(self): + e = Expansion() + e.appendstr('foo') + + self.assertEqual(e.to_source(), 'foo') + + def test_multiple_strings(self): + e = Expansion() + e.appendstr('hello') + e.appendstr('world') + + self.assertEqual(e.to_source(), 'helloworld') + + def test_string_escape(self): + e = Expansion() + e.appendstr('$var') + self.assertEqual(e.to_source(), '$var') + self.assertEqual(e.to_source(escape_variables=True), '$$var') + + e = Expansion() + e.appendstr('foo') + e.appendstr(' $bar') + self.assertEqual(e.to_source(escape_variables=True), 'foo $$bar') + +class SubstitutionRefTest(TestBase): + def test_simple(self): + name = StringExpansion('foo', None) + c = StringExpansion('%.c', None) + o = StringExpansion('%.o', None) + s = SubstitutionRef(None, name, c, o) + + self.assertEqual(s.to_source(), '$(foo:%.c=%.o)') + +class SetVariableTest(TestBase): + def test_simple(self): + v = SetVariable(StringExpansion('foo', None), '=', 'bar', None, None) + self.assertEqual(v.to_source(), 'foo = bar') + + def test_multiline(self): + s = 'hello\nworld' + foo = StringExpansion('FOO', None) + + v = SetVariable(foo, '=', s, None, None) + + self.assertEqual(v.to_source(), 'define FOO\nhello\nworld\nendef') + + def test_multiline_immediate(self): + source = 'define FOO :=\nhello\nworld\nendef' + + statements = parsestring(source, 'foo.mk') + self.assertEqual(statements.to_source(), source) + + def test_target_specific(self): + foo = StringExpansion('FOO', None) + bar = StringExpansion('BAR', None) + + v = SetVariable(foo, '+=', 'value', None, bar) + + self.assertEqual(v.to_source(), 'BAR: FOO += value') + +class IncludeTest(TestBase): + def test_include(self): + e = StringExpansion('rules.mk', None) + i = Include(e, True, False) + self.assertEqual(i.to_source(), 'include rules.mk') + + i = Include(e, False, False) + self.assertEqual(i.to_source(), '-include rules.mk') + +class IfdefTest(TestBase): + def test_simple(self): + source = 'ifdef FOO\nbar := $(value)\nendif' + + statements = parsestring(source, 'foo.mk') + self.assertEqual(statements[0].to_source(), source) + + def test_nested(self): + source = 'ifdef FOO\nifdef BAR\nhello = world\nendif\nendif' + + statements = parsestring(source, 'foo.mk') + self.assertEqual(statements[0].to_source(), source) + + def test_negation(self): + source = 'ifndef FOO\nbar += value\nendif' + + statements = parsestring(source, 'foo.mk') + self.assertEqual(statements[0].to_source(), source) + +class IfeqTest(TestBase): + def test_simple(self): + source = 'ifeq ($(foo),bar)\nhello = $(world)\nendif' + + statements = parsestring(source, 'foo.mk') + self.assertEqual(statements[0].to_source(), source) + + def test_negation(self): + source = 'ifneq (foo,bar)\nhello = world\nendif' + + statements = parsestring(source, 'foo.mk') + self.assertEqual(statements.to_source(), source) + +class ConditionBlocksTest(TestBase): + def test_mixed_conditions(self): + source = 'ifdef FOO\nifeq ($(FOO),bar)\nvar += $(value)\nendif\nendif' + + statements = parsestring(source, 'foo.mk') + self.assertEqual(statements.to_source(), source) + + def test_extra_statements(self): + source = 'ifdef FOO\nF := 1\nifdef BAR\nB += 1\nendif\nC = 1\nendif' + + statements = parsestring(source, 'foo.mk') + self.assertEqual(statements.to_source(), source) + + def test_whitespace_preservation(self): + source = "ifeq ' x' 'x '\n$(error stripping)\nendif" + + statements = parsestring(source, 'foo.mk') + self.assertEqual(statements.to_source(), source) + + source = 'ifneq (x , x)\n$(error stripping)\nendif' + statements = parsestring(source, 'foo.mk') + self.assertEqual(statements.to_source(), + 'ifneq (x,x)\n$(error stripping)\nendif') + +class MakefileCorupusTest(TestBase): + """Runs the make files from the pymake corpus through the formatter. + + All the above tests are child's play compared to this. + """ + + # Our reformatting isn't perfect. We ignore files with known failures until + # we make them work. + # TODO Address these formatting corner cases. + _IGNORE_FILES = [ + # We are thrown off by backslashes at end of lines. + 'comment-parsing.mk', + 'escape-chars.mk', + 'include-notfound.mk', + ] + + def _get_test_files(self): + ourdir = os.path.dirname(os.path.abspath(__file__)) + + for makefile in glob.glob(os.path.join(ourdir, '*.mk')): + if os.path.basename(makefile) in self._IGNORE_FILES: + continue + + source = None + with open(makefile, 'rU') as fh: + source = fh.read() + + try: + yield (makefile, source, parsestring(source, makefile)) + except SyntaxError: + continue + + def test_reparse_consistency(self): + for filename, source, statements in self._get_test_files(): + reformatted = statements.to_source() + + # We should be able to parse the reformatted source fine. + new_statements = parsestring(reformatted, filename) + + # If we do the formatting again, the representation shouldn't + # change. i.e. the only lossy change should be the original + # (whitespace and some semantics aren't preserved). + reformatted_again = new_statements.to_source() + self.assertEqual(reformatted, reformatted_again, + '%s has lossless reformat.' % filename) + + self.assertEqual(len(statements), len(new_statements)) + + for i in xrange(0, len(statements)): + original = statements[i] + formatted = new_statements[i] + + self.assertEqual(original, formatted, '%s %d: %s != %s' % (filename, + i, original, formatted)) + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + unittest.main() diff --git a/python/pymake/tests/func-refs.mk b/python/pymake/tests/func-refs.mk new file mode 100644 index 000000000..82ab17ba8 --- /dev/null +++ b/python/pymake/tests/func-refs.mk @@ -0,0 +1,11 @@ +unknown var = uval + +all: + test "$(subst a,b,value)" = "vblue" + test "${subst a,b,va)lue}" = "vb)lue" + test "$(subst /,\,ab/c)" = "ab\c" + test '$(subst a,b,\\#)' = '\\#' + test "$( subst a,b,value)" = "" + test "$(Subst a,b,value)" = "" + test "$(unknown var)" = "uval" + @echo TEST-PASS diff --git a/python/pymake/tests/functions.mk b/python/pymake/tests/functions.mk new file mode 100644 index 000000000..817be07aa --- /dev/null +++ b/python/pymake/tests/functions.mk @@ -0,0 +1,36 @@ +all: + test "$(subst e,EE,hello)" = "hEEllo" + test "$(strip $(NULL) test data )" = "test data" + test "$(findstring hell,hello)" = "hell" + test "$(findstring heaven,hello)" = "" + test "$(filter foo/%.c b%,foo/a.c b.c foo/a.o)" = "foo/a.c b.c" + test "$(filter foo,foo bar)" = "foo" + test "$(filter-out foo/%.c b%,foo/a.c b.c foo/a.o)" = "foo/a.o" + test "$(filter-out %.c,foo,bar.c foo,bar.o)" = "foo,bar.o" + test "$(sort .go a b aa A c cc)" = ".go A a aa b c cc" + test "$(word 1, hello )" = "hello" + test "$(word 2, hello )" = "" + test "$(wordlist 1, 2, foo bar baz )" = "foo bar" + test "$(words 1 2 3)" = "3" + test "$(words )" = "0" + test "$(firstword $(NULL) foo bar baz)" = "foo" + test "$(firstword )" = "" + test "$(dir foo.c path/foo.o dir/dir2/)" = "./ path/ dir/dir2/" + test "$(notdir foo.c path/foo.o dir/dir2/)" = "foo.c foo.o " + test "$(suffix src/foo.c dir/my.dir/foo foo.o)" = ".c .o" + test "$(basename src/foo.c dir/my.dir/foo foo.c .c)" = "src/foo dir/my.dir/foo foo " + test "$(addprefix src/,foo bar.c dir/foo)" = "src/foo src/bar.c src/dir/foo" + test "$(addsuffix .c,foo dir/bar)" = "foo.c dir/bar.c" + test "$(join a b c, 1 2 3)" = "a1 b2 c3" + test "$(join a b, 1 2 3)" = "a1 b2 3" + test "$(join a b c, 1 2)" = "a1 b2 c" + test "$(if $(NULL) ,yes)" = "" + test "$(if 1,yes,no)" = "yes" + test "$(if ,yes,no )" = "no " + test "$(if ,$(error Short-circuit problem))" = "" + test "$(or $(NULL),1)" = "1" + test "$(or $(NULL),2,$(warning TEST-FAIL bad or short-circuit))" = "2" + test "$(and ,$(warning TEST-FAIL bad and short-circuit))" = "" + test "$(and 1,2)" = "2" + test "$(foreach i,foo bar,found:$(i))" = "found:foo found:bar" + @echo TEST-PASS diff --git a/python/pymake/tests/functiontests.py b/python/pymake/tests/functiontests.py new file mode 100644 index 000000000..43a344a05 --- /dev/null +++ b/python/pymake/tests/functiontests.py @@ -0,0 +1,54 @@ +import unittest + +import pymake.data +import pymake.functions + +class VariableRefTest(unittest.TestCase): + def test_get_expansions(self): + e = pymake.data.StringExpansion('FOO', None) + f = pymake.functions.VariableRef(None, e) + + exps = list(f.expansions()) + self.assertEqual(len(exps), 1) + +class GetExpansionsTest(unittest.TestCase): + def test_get_arguments(self): + f = pymake.functions.SubstFunction(None) + + e1 = pymake.data.StringExpansion('FOO', None) + e2 = pymake.data.StringExpansion('BAR', None) + e3 = pymake.data.StringExpansion('BAZ', None) + + f.append(e1) + f.append(e2) + f.append(e3) + + exps = list(f.expansions()) + self.assertEqual(len(exps), 3) + + def test_descend(self): + f = pymake.functions.StripFunction(None) + + e = pymake.data.Expansion(None) + + e1 = pymake.data.StringExpansion('FOO', None) + f1 = pymake.functions.VariableRef(None, e1) + e.appendfunc(f1) + + f2 = pymake.functions.WildcardFunction(None) + e2 = pymake.data.StringExpansion('foo/*', None) + f2.append(e2) + e.appendfunc(f2) + + f.append(e) + + exps = list(f.expansions()) + self.assertEqual(len(exps), 1) + + exps = list(f.expansions(True)) + self.assertEqual(len(exps), 3) + + self.assertFalse(f.is_filesystem_dependent) + +if __name__ == '__main__': + unittest.main() diff --git a/python/pymake/tests/if-syntaxerr.mk b/python/pymake/tests/if-syntaxerr.mk new file mode 100644 index 000000000..c172492ef --- /dev/null +++ b/python/pymake/tests/if-syntaxerr.mk @@ -0,0 +1,6 @@ +#T returncode: 2 + +ifeq ($(FOO,VAR)) +all: + @echo TEST_FAIL +endif diff --git a/python/pymake/tests/ifdefs-nesting.mk b/python/pymake/tests/ifdefs-nesting.mk new file mode 100644 index 000000000..340530ffa --- /dev/null +++ b/python/pymake/tests/ifdefs-nesting.mk @@ -0,0 +1,13 @@ +ifdef RANDOM +ifeq (,$(error Not evaluated!)) +endif +endif + +ifdef RANDOM +ifeq (,) +else ifeq (,$(error Not evaluated!)) +endif +endif + +all: + @echo TEST-PASS diff --git a/python/pymake/tests/ifdefs.mk b/python/pymake/tests/ifdefs.mk new file mode 100644 index 000000000..a779d197b --- /dev/null +++ b/python/pymake/tests/ifdefs.mk @@ -0,0 +1,127 @@ +ifdef FOO +$(error FOO is not defined!) +endif + +FOO = foo +FOOFOUND = false +BARFOUND = false +BAZFOUND = false + +ifdef FOO +FOOFOUND = true +else ifdef BAR +BARFOUND = true +else +BAZFOUND = true +endif + +BAR2 = bar2 +FOO2FOUND = false +BAR2FOUND = false +BAZ2FOUND = false + +ifdef FOO2 +FOO2FOUND = true +else ifdef BAR2 +BAR2FOUND = true +else +BAZ2FOUND = true +endif + +FOO3FOUND = false +BAR3FOUND = false +BAZ3FOUND = false + +ifdef FOO3 +FOO3FOUND = true +else ifdef BAR3 +BAR3FOUND = true +else +BAZ3FOUND = true +endif + +ifdef RANDOM +CONTINUATION = \ +else \ +endif +endif + +ifndef ASDFJK +else +$(error ASFDJK was not set) +endif + +TESTSET = + +ifdef TESTSET +$(error TESTSET was not set) +endif + +TESTEMPTY = $(NULL) +ifndef TESTEMPTY +$(error TEST-FAIL TESTEMPTY was probably expanded!) +endif + +# ifneq ( a,a) +# $(error Arguments to ifeq should be stripped before evaluation) +# endif + +XSPACE = x # trick + +ifneq ($(NULL),$(NULL)) +$(error TEST-FAIL ifneq) +endif + +ifneq (x , x) +$(error argument-stripping1) +endif + +ifeq ( x,x ) +$(error argument-stripping2) +endif + +ifneq ($(XSPACE), x ) +$(error argument-stripping3) +endif + +ifeq 'x ' ' x' +$(error TEST-FAIL argument-stripping4) +endif + +all: + test $(FOOFOUND) = true # FOOFOUND + test $(BARFOUND) = false # BARFOUND + test $(BAZFOUND) = false # BAZFOUND + test $(FOO2FOUND) = false # FOO2FOUND + test $(BAR2FOUND) = true # BAR2FOUND + test $(BAZ2FOUND) = false # BAZ2FOUND + test $(FOO3FOUND) = false # FOO3FOUND + test $(BAR3FOUND) = false # BAR3FOUND + test $(BAZ3FOUND) = true # BAZ3FOUND +ifneq ($(FOO),foo) + echo TEST-FAIL 'FOO neq foo: "$(FOO)"' +endif +ifneq ($(FOO), foo) # Whitespace after the comma is stripped + echo TEST-FAIL 'FOO plus whitespace' +endif +ifeq ($(FOO), foo ) # But not trailing whitespace + echo TEST-FAIL 'FOO plus trailing whitespace' +endif +ifeq ( $(FOO),foo) # Not whitespace after the paren + echo TEST-FAIL 'FOO with leading whitespace' +endif +ifeq ($(FOO),$(NULL) foo) # Nor whitespace after expansion + echo TEST-FAIL 'FOO with embedded ws' +endif +ifeq ($(BAR2),bar) + echo TEST-FAIL 'BAR2 eq bar' +endif +ifeq '$(BAR3FOUND)' 'false' + echo BAR3FOUND is ok +else + echo TEST-FAIL BAR3FOUND is not ok +endif +ifndef FOO + echo TEST-FAIL "foo not defined?" +endif + @echo TEST-PASS diff --git a/python/pymake/tests/ignore-error.mk b/python/pymake/tests/ignore-error.mk new file mode 100644 index 000000000..dc8d3a72c --- /dev/null +++ b/python/pymake/tests/ignore-error.mk @@ -0,0 +1,13 @@ +all: + -rm foo + +-rm bar + -+rm baz + @-rm bah + -@rm humbug + +-@rm sincere + +@-rm flattery + @+-rm will + @-+rm not + -+@rm save + -@+rm you + @echo TEST-PASS diff --git a/python/pymake/tests/implicit-chain.mk b/python/pymake/tests/implicit-chain.mk new file mode 100644 index 000000000..16288b3f5 --- /dev/null +++ b/python/pymake/tests/implicit-chain.mk @@ -0,0 +1,12 @@ +all: test.prog + test "$$(cat $<)" = "Program: Object: Source: test.source" + @echo TEST-PASS + +%.prog: %.object + printf "Program: %s" "$$(cat $<)" > $@ + +%.object: %.source + printf "Object: %s" "$$(cat $<)" > $@ + +%.source: + printf "Source: %s" $@ > $@ diff --git a/python/pymake/tests/implicit-dir.mk b/python/pymake/tests/implicit-dir.mk new file mode 100644 index 000000000..c7f75e8d4 --- /dev/null +++ b/python/pymake/tests/implicit-dir.mk @@ -0,0 +1,16 @@ +# Implicit rules have special instructions to deal with directories, so that a pattern rule which doesn't directly apply +# may still be used. + +all: dir/host_test.otest + +host_%.otest: %.osource extra.file + @echo making $@ from $< + +test.osource: + @echo TEST-FAIL should have made dir/test.osource + +dir/test.osource: + @echo TEST-PASS made the correct dependency + +extra.file: + @echo building $@ diff --git a/python/pymake/tests/implicit-terminal.mk b/python/pymake/tests/implicit-terminal.mk new file mode 100644 index 000000000..db2e244ed --- /dev/null +++ b/python/pymake/tests/implicit-terminal.mk @@ -0,0 +1,16 @@ +#T returncode: 2 + +# the %.object rule is "terminal". This means that additional implicit rules cannot be chained to it. + +all: test.prog + test "$$(cat $<)" = "Program: Object: Source: test.source" + @echo TEST-FAIL + +%.prog: %.object + printf "Program: %s" "$$(cat $<)" > $@ + +%.object:: %.source + printf "Object: %s" "$$(cat $<)" > $@ + +%.source: + printf "Source: %s" $@ > $@ diff --git a/python/pymake/tests/implicitsubdir.mk b/python/pymake/tests/implicitsubdir.mk new file mode 100644 index 000000000..b9d854a2a --- /dev/null +++ b/python/pymake/tests/implicitsubdir.mk @@ -0,0 +1,12 @@ +$(shell \ +mkdir foo; \ +touch test.in \ +) + +all: foo/test.out + @echo TEST-PASS + +foo/%.out: %.in + cp $< $@ + + diff --git a/python/pymake/tests/include-dynamic.mk b/python/pymake/tests/include-dynamic.mk new file mode 100644 index 000000000..571895dc3 --- /dev/null +++ b/python/pymake/tests/include-dynamic.mk @@ -0,0 +1,21 @@ +$(shell \ +if ! test -f include-dynamic.inc; then \ + echo "TESTVAR = oldval" > include-dynamic.inc; \ + sleep 2; \ + echo "TESTVAR = newval" > include-dynamic.inc.in; \ +fi \ +) + +# before running the 'all' rule, we should be rebuilding include-dynamic.inc, +# because there is a rule to do so + +all: + test $(TESTVAR) = newval + test "$(MAKE_RESTARTS)" = 1 + @echo TEST-PASS + +include-dynamic.inc: include-dynamic.inc.in + test "$(MAKE_RESTARTS)" = "" + cp $< $@ + +include include-dynamic.inc diff --git a/python/pymake/tests/include-file.inc b/python/pymake/tests/include-file.inc new file mode 100644 index 000000000..d5d495dec --- /dev/null +++ b/python/pymake/tests/include-file.inc @@ -0,0 +1 @@ +INCLUDED = yes diff --git a/python/pymake/tests/include-missing.mk b/python/pymake/tests/include-missing.mk new file mode 100644 index 000000000..583d0a065 --- /dev/null +++ b/python/pymake/tests/include-missing.mk @@ -0,0 +1,9 @@ +#T returncode: 2 + +# If an include file isn't present and doesn't have a rule to remake it, make +# should fail. + +include notfound.mk + +all: + @echo TEST-FAIL diff --git a/python/pymake/tests/include-notfound.mk b/python/pymake/tests/include-notfound.mk new file mode 100644 index 000000000..1ee7e05b2 --- /dev/null +++ b/python/pymake/tests/include-notfound.mk @@ -0,0 +1,19 @@ +ifdef __WIN32__ +PS:=\\# +else +PS:=/ +endif + +ifneq ($(strip $(MAKEFILE_LIST)),$(NATIVE_TESTPATH)$(PS)include-notfound.mk) +$(error MAKEFILE_LIST incorrect: '$(MAKEFILE_LIST)' (expected '$(NATIVE_TESTPATH)$(PS)include-notfound.mk')) +endif + +-include notfound.inc-dummy + +ifneq ($(strip $(MAKEFILE_LIST)),$(NATIVE_TESTPATH)$(PS)include-notfound.mk) +$(error MAKEFILE_LIST incorrect: '$(MAKEFILE_LIST)' (expected '$(NATIVE_TESTPATH)$(PS)include-notfound.mk')) +endif + +all: + @echo TEST-PASS + diff --git a/python/pymake/tests/include-optional-warning.mk b/python/pymake/tests/include-optional-warning.mk new file mode 100644 index 000000000..901938dff --- /dev/null +++ b/python/pymake/tests/include-optional-warning.mk @@ -0,0 +1,4 @@ +-include TEST-FAIL.mk + +all: + @echo TEST-PASS diff --git a/python/pymake/tests/include-regen.mk b/python/pymake/tests/include-regen.mk new file mode 100644 index 000000000..c86e0c78d --- /dev/null +++ b/python/pymake/tests/include-regen.mk @@ -0,0 +1,10 @@ +# avoid infinite loops by not remaking makefiles with +# double-colon no-dependency rules +# http://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles +-include notfound.mk + +all: + @echo TEST-PASS + +notfound.mk:: + @echo TEST-FAIL diff --git a/python/pymake/tests/include-regen2.mk b/python/pymake/tests/include-regen2.mk new file mode 100644 index 000000000..fc7fef073 --- /dev/null +++ b/python/pymake/tests/include-regen2.mk @@ -0,0 +1,10 @@ +# make should make makefiles that it has rules for if they are +# included +include test.mk + +all: + test "$(X)" = "1" + @echo "TEST-PASS" + +test.mk: + @echo "X = 1" > $@ diff --git a/python/pymake/tests/include-regen3.mk b/python/pymake/tests/include-regen3.mk new file mode 100644 index 000000000..878ce0adc --- /dev/null +++ b/python/pymake/tests/include-regen3.mk @@ -0,0 +1,10 @@ +# make should make makefiles that it has rules for if they are +# included +-include test.mk + +all: + test "$(X)" = "1" + @echo "TEST-PASS" + +test.mk: + @echo "X = 1" > $@ diff --git a/python/pymake/tests/include-test.mk b/python/pymake/tests/include-test.mk new file mode 100644 index 000000000..3608fc269 --- /dev/null +++ b/python/pymake/tests/include-test.mk @@ -0,0 +1,8 @@ +$(shell echo "INCLUDED2 = yes" >local-include.inc) + +include $(TESTPATH)/include-file.inc local-include.inc + +all: + test "$(INCLUDED)" = "yes" + test "$(INCLUDED2)" = "yes" + @echo TEST-PASS diff --git a/python/pymake/tests/includedeps-norebuild.mk b/python/pymake/tests/includedeps-norebuild.mk new file mode 100644 index 000000000..e30abd439 --- /dev/null +++ b/python/pymake/tests/includedeps-norebuild.mk @@ -0,0 +1,15 @@ +#T gmake skip + +$(shell \ +touch filemissing; \ +sleep 2; \ +touch file1; \ +) + +all: file1 + @echo TEST-PASS + +includedeps $(TESTPATH)/includedeps.deps + +file1: + @echo TEST-FAIL diff --git a/python/pymake/tests/includedeps-sideeffects.mk b/python/pymake/tests/includedeps-sideeffects.mk new file mode 100644 index 000000000..7e4ea30a2 --- /dev/null +++ b/python/pymake/tests/includedeps-sideeffects.mk @@ -0,0 +1,10 @@ +#T gmake skip +#T returncode: 2 + +all: file1 filemissing + @echo TEST-PASS + +includedeps $(TESTPATH)/includedeps.deps + +file: + touch $@ diff --git a/python/pymake/tests/includedeps-stripdotslash.deps b/python/pymake/tests/includedeps-stripdotslash.deps new file mode 100644 index 000000000..352fca1bb --- /dev/null +++ b/python/pymake/tests/includedeps-stripdotslash.deps @@ -0,0 +1 @@ +./test: TEST-PASS diff --git a/python/pymake/tests/includedeps-stripdotslash.mk b/python/pymake/tests/includedeps-stripdotslash.mk new file mode 100644 index 000000000..ee942e6db --- /dev/null +++ b/python/pymake/tests/includedeps-stripdotslash.mk @@ -0,0 +1,8 @@ +#T gmake skip + +test: + @echo $< + +includedeps $(TESTPATH)/includedeps-stripdotslash.deps + +TEST-PASS: diff --git a/python/pymake/tests/includedeps-variables.deps b/python/pymake/tests/includedeps-variables.deps new file mode 100644 index 000000000..ba69e9b6c --- /dev/null +++ b/python/pymake/tests/includedeps-variables.deps @@ -0,0 +1 @@ +$(FILE)1: filemissing diff --git a/python/pymake/tests/includedeps-variables.mk b/python/pymake/tests/includedeps-variables.mk new file mode 100644 index 000000000..314618da4 --- /dev/null +++ b/python/pymake/tests/includedeps-variables.mk @@ -0,0 +1,10 @@ +#T gmake skip + +FILE = includedeps-variables + +all: $(FILE)1 + +includedeps $(TESTPATH)/includedeps-variables.deps + +filemissing: + @echo TEST-PASS diff --git a/python/pymake/tests/includedeps.deps b/python/pymake/tests/includedeps.deps new file mode 100644 index 000000000..d3017c078 --- /dev/null +++ b/python/pymake/tests/includedeps.deps @@ -0,0 +1 @@ +file1: filemissing diff --git a/python/pymake/tests/includedeps.mk b/python/pymake/tests/includedeps.mk new file mode 100644 index 000000000..deaa71fe8 --- /dev/null +++ b/python/pymake/tests/includedeps.mk @@ -0,0 +1,9 @@ +#T gmake skip + +all: file1 + @echo TEST-PASS + +includedeps $(TESTPATH)/includedeps.deps + +file1: + touch $@ diff --git a/python/pymake/tests/info.mk b/python/pymake/tests/info.mk new file mode 100644 index 000000000..8dddfd815 --- /dev/null +++ b/python/pymake/tests/info.mk @@ -0,0 +1,8 @@ +#T grep-for: "info-printed\ninfo-nth" +all: + +INFO = info-printed + +$(info $(INFO)) +$(info $(subst second,nth,info-second)) +$(info TEST-PASS) diff --git a/python/pymake/tests/justprint-native.mk b/python/pymake/tests/justprint-native.mk new file mode 100644 index 000000000..580e402e9 --- /dev/null +++ b/python/pymake/tests/justprint-native.mk @@ -0,0 +1,28 @@ +## $(TOUCH) and $(RM) are native commands in pymake. +## Test that pymake --just-print just prints them. + +ifndef TOUCH +TOUCH = touch +endif + +all: + $(RM) justprint-native-file1.txt + $(TOUCH) justprint-native-file2.txt + $(MAKE) --just-print -f $(TESTPATH)/justprint-native.mk justprint_target > justprint.log +# make --just-print shouldn't have actually done anything. + test ! -f justprint-native-file1.txt + test -f justprint-native-file2.txt +# but it should have printed each command + grep -q 'touch justprint-native-file1.txt' justprint.log + grep -q 'rm -f justprint-native-file2.txt' justprint.log + grep -q 'this string is "unlikely to appear in the log by chance"' justprint.log +# tidy up + $(RM) justprint-native-file2.txt + @echo TEST-PASS + +justprint_target: + $(TOUCH) justprint-native-file1.txt + $(RM) justprint-native-file2.txt + this string is "unlikely to appear in the log by chance" + +.PHONY: justprint_target diff --git a/python/pymake/tests/justprint.mk b/python/pymake/tests/justprint.mk new file mode 100644 index 000000000..be11ba8de --- /dev/null +++ b/python/pymake/tests/justprint.mk @@ -0,0 +1,5 @@ +#T commandline: ['-n'] + +all: + false # without -n, we wouldn't get past this + TEST-PASS # heh diff --git a/python/pymake/tests/keep-going-doublecolon.mk b/python/pymake/tests/keep-going-doublecolon.mk new file mode 100644 index 000000000..fa5b31df8 --- /dev/null +++ b/python/pymake/tests/keep-going-doublecolon.mk @@ -0,0 +1,16 @@ +#T commandline: ['-k'] +#T returncode: 2 +#T grep-for: "TEST-PASS" + +all:: t1 + @echo TEST-FAIL "(t1)" + +all:: t2 + @echo TEST-PASS + +t1: + @false + +t2: + touch $@ + diff --git a/python/pymake/tests/keep-going-parallel.mk b/python/pymake/tests/keep-going-parallel.mk new file mode 100644 index 000000000..a91d1a6ed --- /dev/null +++ b/python/pymake/tests/keep-going-parallel.mk @@ -0,0 +1,11 @@ +#T commandline: ['-k', '-j2'] +#T returncode: 2 +#T grep-for: "TEST-PASS" + +all: t1 slow1 slow2 slow3 t2 + +t2: + @echo TEST-PASS + +slow%: + sleep 1 diff --git a/python/pymake/tests/keep-going.mk b/python/pymake/tests/keep-going.mk new file mode 100644 index 000000000..4c709288c --- /dev/null +++ b/python/pymake/tests/keep-going.mk @@ -0,0 +1,14 @@ +#T commandline: ['-k'] +#T returncode: 2 +#T grep-for: "TEST-PASS" + +all: t2 t3 + +t1: + @false + +t2: t1 + @echo TEST-FAIL + +t3: + @echo TEST-PASS diff --git a/python/pymake/tests/line-continuations.mk b/python/pymake/tests/line-continuations.mk new file mode 100644 index 000000000..8b44480ea --- /dev/null +++ b/python/pymake/tests/line-continuations.mk @@ -0,0 +1,24 @@ +VAR = val1 \ + val2 + +VAR2 = val1space\ +val2 + +VAR3 = val3 \\\ + cont3 + +all: otarget test.target + test "$(VAR)" = "val1 val2 " + test "$(VAR2)" = "val1space val2" + test '$(VAR3)' = 'val3 \ cont3' + test "hello \ + world" = "hello world" + test "hello" = \ +"hello" + @echo TEST-PASS + +otarget: ; test "hello\ + world" = "helloworld" + +test.target: %.target: ; test "hello\ + world" = "helloworld" diff --git a/python/pymake/tests/link-search.mk b/python/pymake/tests/link-search.mk new file mode 100644 index 000000000..ea827f391 --- /dev/null +++ b/python/pymake/tests/link-search.mk @@ -0,0 +1,7 @@ +$(shell \ +touch libfoo.so \ +) + +all: -lfoo + test "$<" = "libfoo.so" + @echo TEST-PASS diff --git a/python/pymake/tests/makeflags.mk b/python/pymake/tests/makeflags.mk new file mode 100644 index 000000000..288ff7866 --- /dev/null +++ b/python/pymake/tests/makeflags.mk @@ -0,0 +1,7 @@ +#T environment: {'MAKEFLAGS': 'OVAR=oval'} + +all: + test "$(OVAR)" = "oval" + test "$$OVAR" = "oval" + @echo TEST-PASS + diff --git a/python/pymake/tests/matchany.mk b/python/pymake/tests/matchany.mk new file mode 100644 index 000000000..7876c90a3 --- /dev/null +++ b/python/pymake/tests/matchany.mk @@ -0,0 +1,14 @@ +#T returncode: 2 + +# we should fail to make foo.ooo from foo.ooo.test +all: foo.ooo + @echo TEST-FAIL + +%.ooo: + +# this match-anything pattern should not apply to %.ooo +%: %.test + cp $< $@ + +foo.ooo.test: + touch $@ diff --git a/python/pymake/tests/matchany2.mk b/python/pymake/tests/matchany2.mk new file mode 100644 index 000000000..d21d9702c --- /dev/null +++ b/python/pymake/tests/matchany2.mk @@ -0,0 +1,13 @@ +# we should succeed in making foo.ooo from foo.ooo.test +all: foo.ooo + @echo TEST-PASS + +%.ooo: %.ccc + exit 1 + +# this match-anything rule is terminal, and therefore applies +%:: %.test + cp $< $@ + +foo.ooo.test: + touch $@ diff --git a/python/pymake/tests/matchany3.mk b/python/pymake/tests/matchany3.mk new file mode 100644 index 000000000..83de8af2b --- /dev/null +++ b/python/pymake/tests/matchany3.mk @@ -0,0 +1,10 @@ +$(shell \ +echo "target" > target.in; \ +) + +all: target + test "$$(cat $^)" = "target" + @echo TEST-PASS + +%: %.in + cp $< $@ diff --git a/python/pymake/tests/mkdir-fail.mk b/python/pymake/tests/mkdir-fail.mk new file mode 100644 index 000000000..b05734aa9 --- /dev/null +++ b/python/pymake/tests/mkdir-fail.mk @@ -0,0 +1,7 @@ +#T returncode: 2 +all: + mkdir newdir/subdir + test ! -d newdir/subdir + test ! -d newdir + rm -r newdir + @echo TEST-PASS diff --git a/python/pymake/tests/mkdir.mk b/python/pymake/tests/mkdir.mk new file mode 100644 index 000000000..413348f77 --- /dev/null +++ b/python/pymake/tests/mkdir.mk @@ -0,0 +1,27 @@ +MKDIR ?= mkdir + +all: + $(MKDIR) newdir + test -d newdir + # subdir, parent exists + $(MKDIR) newdir/subdir + test -d newdir/subdir + # -p, existing dir + $(MKDIR) -p newdir + # -p, existing subdir + $(MKDIR) -p newdir/subdir + # multiple subdirs, existing parent + $(MKDIR) newdir/subdir1 newdir/subdir2 + test -d newdir/subdir1 -a -d newdir/subdir2 + rm -r newdir + # -p, subdir, no existing parent + $(MKDIR) -p newdir/subdir + test -d newdir/subdir + rm -r newdir + # -p, multiple subdirs, no existing parent + $(MKDIR) -p newdir/subdir1 newdir/subdir2 + test -d newdir/subdir1 -a -d newdir/subdir2 + # -p, multiple existing subdirs + $(MKDIR) -p newdir/subdir1 newdir/subdir2 + rm -r newdir + @echo TEST-PASS diff --git a/python/pymake/tests/multiple-rules-prerequisite-merge.mk b/python/pymake/tests/multiple-rules-prerequisite-merge.mk new file mode 100644 index 000000000..480d3b58c --- /dev/null +++ b/python/pymake/tests/multiple-rules-prerequisite-merge.mk @@ -0,0 +1,25 @@ +# When a target is defined multiple times, the prerequisites should get +# merged. + +default: foo bar baz + +foo: + test "$<" = "foo.in1" + @echo TEST-PASS + +foo: foo.in1 + +bar: bar.in1 + test "$<" = "bar.in1" + test "$^" = "bar.in1 bar.in2" + @echo TEST-PASS + +bar: bar.in2 + +baz: baz.in2 +baz: baz.in1 + test "$<" = "baz.in1" + test "$^" = "baz.in1 baz.in2" + @echo TEST-PASS + +foo.in1 bar.in1 bar.in2 baz.in1 baz.in2: diff --git a/python/pymake/tests/native-command-delay-load.mk b/python/pymake/tests/native-command-delay-load.mk new file mode 100644 index 000000000..a9f3774eb --- /dev/null +++ b/python/pymake/tests/native-command-delay-load.mk @@ -0,0 +1,12 @@ +#T gmake skip + +# This test exists to verify that sys.path is adjusted during command +# execution and that delay importing a module will work. + +CMD = %pycmd delayloadfn +PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir + +all: + $(CMD) + @echo TEST-PASS + diff --git a/python/pymake/tests/native-command-raise.mk b/python/pymake/tests/native-command-raise.mk new file mode 100644 index 000000000..d1b28b331 --- /dev/null +++ b/python/pymake/tests/native-command-raise.mk @@ -0,0 +1,9 @@ +#T gmake skip +#T returncode: 2 +#T grep-for: "Exception: info-exception" + +CMD = %pycmd asplode_raise +PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir + +all: + @$(CMD) info-exception diff --git a/python/pymake/tests/native-command-return-fail1.mk b/python/pymake/tests/native-command-return-fail1.mk new file mode 100644 index 000000000..0cf085ae2 --- /dev/null +++ b/python/pymake/tests/native-command-return-fail1.mk @@ -0,0 +1,8 @@ +#T gmake skip +#T returncode: 2 + +CMD = %pycmd asplode_return +PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir + +all: + $(CMD) 1 diff --git a/python/pymake/tests/native-command-return-fail2.mk b/python/pymake/tests/native-command-return-fail2.mk new file mode 100644 index 000000000..c071fc879 --- /dev/null +++ b/python/pymake/tests/native-command-return-fail2.mk @@ -0,0 +1,8 @@ +#T gmake skip +#T returncode: 2 + +CMD = %pycmd asplode_return +PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir + +all: + $(CMD) not-an-integer diff --git a/python/pymake/tests/native-command-return.mk b/python/pymake/tests/native-command-return.mk new file mode 100644 index 000000000..3e4d2e0c4 --- /dev/null +++ b/python/pymake/tests/native-command-return.mk @@ -0,0 +1,11 @@ +#T gmake skip + +CMD = %pycmd asplode_return +PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir + +all: + $(CMD) 0 + -$(CMD) 1 + $(CMD) None + -$(CMD) not-an-integer + @echo TEST-PASS diff --git a/python/pymake/tests/native-command-shell-glob.mk b/python/pymake/tests/native-command-shell-glob.mk new file mode 100644 index 000000000..4bcdad8b9 --- /dev/null +++ b/python/pymake/tests/native-command-shell-glob.mk @@ -0,0 +1,11 @@ +#T gmake skip +all: + mkdir shell-glob-test + touch shell-glob-test/foo.txt + touch shell-glob-test/bar.txt + touch shell-glob-test/a.foo + touch shell-glob-test/b.foo + $(RM) shell-glob-test/*.txt + $(RM) shell-glob-test/?.foo + rmdir shell-glob-test + @echo TEST-PASS diff --git a/python/pymake/tests/native-command-sys-exit-fail1.mk b/python/pymake/tests/native-command-sys-exit-fail1.mk new file mode 100644 index 000000000..8e74800ed --- /dev/null +++ b/python/pymake/tests/native-command-sys-exit-fail1.mk @@ -0,0 +1,8 @@ +#T gmake skip +#T returncode: 2 + +CMD = %pycmd asplode +PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir + +all: + $(CMD) 1 diff --git a/python/pymake/tests/native-command-sys-exit-fail2.mk b/python/pymake/tests/native-command-sys-exit-fail2.mk new file mode 100644 index 000000000..0a04395ad --- /dev/null +++ b/python/pymake/tests/native-command-sys-exit-fail2.mk @@ -0,0 +1,8 @@ +#T gmake skip +#T returncode: 2 + +CMD = %pycmd asplode +PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir + +all: + $(CMD) not-an-integer diff --git a/python/pymake/tests/native-command-sys-exit.mk b/python/pymake/tests/native-command-sys-exit.mk new file mode 100644 index 000000000..c04913aca --- /dev/null +++ b/python/pymake/tests/native-command-sys-exit.mk @@ -0,0 +1,11 @@ +#T gmake skip + +CMD = %pycmd asplode +PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir + +all: + $(CMD) 0 + -$(CMD) 1 + $(CMD) None + -$(CMD) not-an-integer + @echo TEST-PASS diff --git a/python/pymake/tests/native-environment.mk b/python/pymake/tests/native-environment.mk new file mode 100644 index 000000000..36bd5894a --- /dev/null +++ b/python/pymake/tests/native-environment.mk @@ -0,0 +1,11 @@ +#T gmake skip +export EXPECTED := some data + +PYCOMMANDPATH = $(TESTPATH) + +all: + %pycmd writeenvtofile results EXPECTED + test "$$(cat results)" = "$(EXPECTED)" + %pycmd writesubprocessenvtofile results EXPECTED + test "$$(cat results)" = "$(EXPECTED)" + @echo TEST-PASS diff --git a/python/pymake/tests/native-pycommandpath-sep.mk b/python/pymake/tests/native-pycommandpath-sep.mk new file mode 100644 index 000000000..b1c2c2b97 --- /dev/null +++ b/python/pymake/tests/native-pycommandpath-sep.mk @@ -0,0 +1,21 @@ +#T gmake skip +EXPECTED := some data + +# verify that we can load native command modules from +# multiple directories in PYCOMMANDPATH separated by the native +# path separator +ifdef __WIN32__ +PS:=; +else +PS:=: +endif +CMD = %pycmd writetofile +CMD2 = %pymod writetofile +PYCOMMANDPATH = $(TESTPATH)$(PS)$(TESTPATH)/subdir + +all: + $(CMD) results $(EXPECTED) + test "$$(cat results)" = "$(EXPECTED)" + $(CMD2) results2 $(EXPECTED) + test "$$(cat results2)" = "$(EXPECTED)" + @echo TEST-PASS diff --git a/python/pymake/tests/native-pycommandpath.mk b/python/pymake/tests/native-pycommandpath.mk new file mode 100644 index 000000000..dd0fbc9f9 --- /dev/null +++ b/python/pymake/tests/native-pycommandpath.mk @@ -0,0 +1,15 @@ +#T gmake skip +EXPECTED := some data + +# verify that we can load native command modules from +# multiple space-separated directories in PYCOMMANDPATH +CMD = %pycmd writetofile +CMD2 = %pymod writetofile +PYCOMMANDPATH = $(TESTPATH) $(TESTPATH)/subdir + +all: + $(CMD) results $(EXPECTED) + test "$$(cat results)" = "$(EXPECTED)" + $(CMD2) results2 $(EXPECTED) + test "$$(cat results2)" = "$(EXPECTED)" + @echo TEST-PASS diff --git a/python/pymake/tests/native-simple.mk b/python/pymake/tests/native-simple.mk new file mode 100644 index 000000000..626a58670 --- /dev/null +++ b/python/pymake/tests/native-simple.mk @@ -0,0 +1,12 @@ +ifndef TOUCH +TOUCH = touch +endif + +all: testfile {testfile2} (testfile3) + test -f testfile + test -f {testfile2} + test -f "(testfile3)" + @echo TEST-PASS + +testfile {testfile2} (testfile3): + $(TOUCH) "$@" diff --git a/python/pymake/tests/native-touch.mk b/python/pymake/tests/native-touch.mk new file mode 100644 index 000000000..811161ece --- /dev/null +++ b/python/pymake/tests/native-touch.mk @@ -0,0 +1,15 @@ +TOUCH ?= touch + +foo: + $(TOUCH) bar + $(TOUCH) baz + $(MAKE) -f $(TESTPATH)/native-touch.mk baz + $(TOUCH) -t 198007040802 baz + $(MAKE) -f $(TESTPATH)/native-touch.mk baz + +bar: + $(TOUCH) $@ + +baz: bar + echo TEST-PASS + $(TOUCH) $@ diff --git a/python/pymake/tests/newlines.mk b/python/pymake/tests/newlines.mk new file mode 100644 index 000000000..5d8195c94 --- /dev/null +++ b/python/pymake/tests/newlines.mk @@ -0,0 +1,30 @@ +#T gmake skip + +# Test that we handle \\\n properly + +all: dep1 dep2 dep3 + cat testfile + test `cat testfile` = "data"; + test "$$(cat results)" = "$(EXPECTED)"; + @echo TEST-PASS + +# Test that something that still needs to go to the shell works +testfile: + printf "data" \ + >>$@ + +dep1: testfile + +# Test that something that does not need to go to the shell works +dep2: + $(echo foo) \ + $(echo bar) + +export EXPECTED := some data + +CMD = %pycmd writeenvtofile +PYCOMMANDPATH = $(TESTPATH) + +dep3: + $(CMD) \ + results EXPECTED diff --git a/python/pymake/tests/no-remake.mk b/python/pymake/tests/no-remake.mk new file mode 100644 index 000000000..c8df81bc3 --- /dev/null +++ b/python/pymake/tests/no-remake.mk @@ -0,0 +1,7 @@ +$(shell date >testfile) + +all: testfile + @echo TEST-PASS + +testfile: + @echo TEST-FAIL "We shouldn't have remade this!" diff --git a/python/pymake/tests/nosuchfile.mk b/python/pymake/tests/nosuchfile.mk new file mode 100644 index 000000000..cca9ce1e9 --- /dev/null +++ b/python/pymake/tests/nosuchfile.mk @@ -0,0 +1,4 @@ +#T returncode: 2 + +all: + reallythereisnosuchcommand diff --git a/python/pymake/tests/notargets.mk b/python/pymake/tests/notargets.mk new file mode 100644 index 000000000..8e55d944f --- /dev/null +++ b/python/pymake/tests/notargets.mk @@ -0,0 +1,5 @@ +$(NULL): foo.c + @echo TEST-FAIL + +all: + @echo TEST-PASS diff --git a/python/pymake/tests/notparallel.mk b/python/pymake/tests/notparallel.mk new file mode 100644 index 000000000..4fd8b1a8d --- /dev/null +++ b/python/pymake/tests/notparallel.mk @@ -0,0 +1,8 @@ +#T commandline: ['-j3'] + +include $(TESTPATH)/serial-rule-execution.mk + +all:: + $(MAKE) -f $(TESTPATH)/parallel-simple.mk + +.NOTPARALLEL: diff --git a/python/pymake/tests/oneline-command-continuations.mk b/python/pymake/tests/oneline-command-continuations.mk new file mode 100644 index 000000000..c11f3df52 --- /dev/null +++ b/python/pymake/tests/oneline-command-continuations.mk @@ -0,0 +1,5 @@ +all: test + @echo TEST-PASS + +test: ; test "Hello \ + world" = "Hello world" diff --git a/python/pymake/tests/override-propagate.mk b/python/pymake/tests/override-propagate.mk new file mode 100644 index 000000000..a1663ff41 --- /dev/null +++ b/python/pymake/tests/override-propagate.mk @@ -0,0 +1,37 @@ +#T commandline: ['-w', 'OVAR=oval'] + +OVAR=mval + +all: vartest run-override + $(MAKE) -f $(TESTPATH)/override-propagate.mk vartest + @echo TEST-PASS + +CLINE := OVAR=oval TESTPATH=$(TESTPATH) NATIVE_TESTPATH=$(NATIVE_TESTPATH) +ifdef __WIN32__ +CLINE += __WIN32__=1 +endif + +SORTED_CLINE := $(subst \,\\,$(sort $(CLINE))) + +vartest: + @echo MAKELEVEL: '$(MAKELEVEL)' + test '$(value MAKEFLAGS)' = 'w -- $$(MAKEOVERRIDES)' + test '$(origin MAKEFLAGS)' = 'file' + test '$(value MAKEOVERRIDES)' = '$${-*-command-variables-*-}' + test "$(sort $(MAKEOVERRIDES))" = "$(SORTED_CLINE)" + test '$(origin MAKEOVERRIDES)' = 'environment' + test '$(origin -*-command-variables-*-)' = 'automatic' + test "$(origin OVAR)" = "command line" + test "$(OVAR)" = "oval" + +run-override: MAKEOVERRIDES= +run-override: + test "$(OVAR)" = "oval" + $(MAKE) -f $(TESTPATH)/override-propagate.mk otest + +otest: + test '$(value MAKEFLAGS)' = 'w' + test '$(value MAKEOVERRIDES)' = '$${-*-command-variables-*-}' + test '$(MAKEOVERRIDES)' = '' + test '$(origin -*-command-variables-*-)' = 'undefined' + test "$(OVAR)" = "mval" diff --git a/python/pymake/tests/parallel-dep-resolution.mk b/python/pymake/tests/parallel-dep-resolution.mk new file mode 100644 index 000000000..7967eba2d --- /dev/null +++ b/python/pymake/tests/parallel-dep-resolution.mk @@ -0,0 +1,8 @@ +#T commandline: ['-j3'] +#T returncode: 2 + +all: t1 t2 + +t1: + sleep 1 + touch t1 t2 diff --git a/python/pymake/tests/parallel-dep-resolution2.mk b/python/pymake/tests/parallel-dep-resolution2.mk new file mode 100644 index 000000000..7d61e6b3e --- /dev/null +++ b/python/pymake/tests/parallel-dep-resolution2.mk @@ -0,0 +1,9 @@ +#T commandline: ['-j3'] +#T returncode: 2 + +all:: + sleep 1 + touch somefile + +all:: somefile + @echo TEST-PASS diff --git a/python/pymake/tests/parallel-native.mk b/python/pymake/tests/parallel-native.mk new file mode 100644 index 000000000..d50cfbdbb --- /dev/null +++ b/python/pymake/tests/parallel-native.mk @@ -0,0 +1,21 @@ +#T commandline: ['-j2'] + +# ensure that calling python commands doesn't block other targets +ifndef SLEEP +SLEEP := sleep +endif + +PRINTF = printf "$@:0:" >>results +EXPECTED = target2:0:target1:0: + +all:: target1 target2 + cat results + test "$$(cat results)" = "$(EXPECTED)" + @echo TEST-PASS + +target1: + $(SLEEP) 0.1 + $(PRINTF) + +target2: + $(PRINTF) diff --git a/python/pymake/tests/parallel-simple.mk b/python/pymake/tests/parallel-simple.mk new file mode 100644 index 000000000..f1aafc5f1 --- /dev/null +++ b/python/pymake/tests/parallel-simple.mk @@ -0,0 +1,27 @@ +#T commandline: ['-j2'] + +# CAUTION: this makefile is also used by serial-toparallel.mk + +define SLOWMAKE +printf "$@:0:" >>results +sleep 0.5 +printf "$@:1:" >>results +sleep 0.5 +printf "$@:2:" >>results +endef + +EXPECTED = target1:0:target2:0:target1:1:target2:1:target1:2:target2:2: + +all:: target1 target2 + cat results + test "$$(cat results)" = "$(EXPECTED)" + @echo TEST-PASS + +target1: + $(SLOWMAKE) + +target2: + sleep 0.1 + $(SLOWMAKE) + +.PHONY: all diff --git a/python/pymake/tests/parallel-submake.mk b/python/pymake/tests/parallel-submake.mk new file mode 100644 index 000000000..65cb2cf7c --- /dev/null +++ b/python/pymake/tests/parallel-submake.mk @@ -0,0 +1,17 @@ +#T commandline: ['-j2'] + +# A submake shouldn't return control to the parent until it has actually finished doing everything. + +all: + -$(MAKE) -f $(TESTPATH)/parallel-submake.mk subtarget + cat results + test "$$(cat results)" = "0123" + @echo TEST-PASS + +subtarget: succeed-slowly fail-quickly + +succeed-slowly: + printf 0 >>results; sleep 1; printf 1 >>results; sleep 1; printf 2 >>results; sleep 1; printf 3 >>results + +fail-quickly: + exit 1 diff --git a/python/pymake/tests/parallel-toserial.mk b/python/pymake/tests/parallel-toserial.mk new file mode 100644 index 000000000..9a355eb33 --- /dev/null +++ b/python/pymake/tests/parallel-toserial.mk @@ -0,0 +1,31 @@ +#T commandline: ['-j4'] + +# Test that -j1 in a submake has the proper effect. + +define SLOWCMD +printf "$@:0:" >>$(RFILE) +sleep 0.5 +printf "$@:1:" >>$(RFILE) +endef + +all: p1 p2 +subtarget: s1 s2 + +p1 p2: RFILE = presult +s1 s2: RFILE = sresult + +p1 s1: + $(SLOWCMD) + +p2 s2: + sleep 0.1 + $(SLOWCMD) + +all: + $(MAKE) -j1 -f $(TESTPATH)/parallel-toserial.mk subtarget + printf "presult: %s\n" "$$(cat presult)" + test "$$(cat presult)" = "p1:0:p2:0:p1:1:p2:1:" + printf "sresult: %s\n" "$$(cat sresult)" + test "$$(cat sresult)" = "s1:0:s1:1:s2:0:s2:1:" + @echo TEST-PASS + diff --git a/python/pymake/tests/parallel-waiting.mk b/python/pymake/tests/parallel-waiting.mk new file mode 100644 index 000000000..40a6e0d50 --- /dev/null +++ b/python/pymake/tests/parallel-waiting.mk @@ -0,0 +1,21 @@ +#T commandline: ['-j2'] + +EXPECTED = target1:before:target2:1:target2:2:target2:3:target1:after + +all:: target1 target2 + cat results + test "$$(cat results)" = "$(EXPECTED)" + @echo TEST-PASS + +target1: + printf "$@:before:" >>results + sleep 4 + printf "$@:after" >>results + +target2: + sleep 0.2 + printf "$@:1:" >>results + sleep 0.1 + printf "$@:2:" >>results + sleep 0.1 + printf "$@:3:" >>results diff --git a/python/pymake/tests/parentheses.mk b/python/pymake/tests/parentheses.mk new file mode 100644 index 000000000..f207234ff --- /dev/null +++ b/python/pymake/tests/parentheses.mk @@ -0,0 +1,2 @@ +all: + @(echo TEST-PASS) diff --git a/python/pymake/tests/parsertests.py b/python/pymake/tests/parsertests.py new file mode 100644 index 000000000..ab6406be0 --- /dev/null +++ b/python/pymake/tests/parsertests.py @@ -0,0 +1,314 @@ +import pymake.data, pymake.parser, pymake.parserdata, pymake.functions +import unittest +import logging + +from cStringIO import StringIO + +def multitest(cls): + for name in cls.testdata.iterkeys(): + def m(self, name=name): + return self.runSingle(*self.testdata[name]) + + setattr(cls, 'test_%s' % name, m) + return cls + +class TestBase(unittest.TestCase): + def assertEqual(self, a, b, msg=""): + """Actually print the values which weren't equal, if things don't work out!""" + unittest.TestCase.assertEqual(self, a, b, "%s got %r expected %r" % (msg, a, b)) + +class DataTest(TestBase): + testdata = { + 'oneline': + ("He\tllo", "f", 1, 0, + ((0, "f", 1, 0), (2, "f", 1, 2), (3, "f", 1, 4))), + 'twoline': + ("line1 \n\tl\tine2", "f", 1, 4, + ((0, "f", 1, 4), (5, "f", 1, 9), (6, "f", 1, 10), (7, "f", 2, 0), (8, "f", 2, 4), (10, "f", 2, 8), (13, "f", 2, 11))), + } + + def runSingle(self, data, filename, line, col, results): + d = pymake.parser.Data(data, 0, len(data), pymake.parserdata.Location(filename, line, col)) + for pos, file, lineno, col in results: + loc = d.getloc(pos) + self.assertEqual(loc.path, file, "data file offset %i" % pos) + self.assertEqual(loc.line, lineno, "data line offset %i" % pos) + self.assertEqual(loc.column, col, "data col offset %i" % pos) +multitest(DataTest) + +class LineEnumeratorTest(TestBase): + testdata = { + 'simple': ( + 'Hello, world', [ + ('Hello, world', 1), + ] + ), + 'multi': ( + 'Hello\nhappy \n\nworld\n', [ + ('Hello', 1), + ('happy ', 2), + ('', 3), + ('world', 4), + ('', 5), + ] + ), + 'continuation': ( + 'Hello, \\\n world\nJellybeans!', [ + ('Hello, \\\n world', 1), + ('Jellybeans!', 3), + ] + ), + 'multislash': ( + 'Hello, \\\\\n world', [ + ('Hello, \\\\', 1), + (' world', 2), + ] + ) + } + + def runSingle(self, s, lines): + gotlines = [(d.s[d.lstart:d.lend], d.loc.line) for d in pymake.parser.enumeratelines(s, 'path')] + self.assertEqual(gotlines, lines) + +multitest(LineEnumeratorTest) + +class IterTest(TestBase): + testdata = { + 'plaindata': ( + pymake.parser.iterdata, + "plaindata # test\n", + "plaindata # test\n" + ), + 'makecomment': ( + pymake.parser.itermakefilechars, + "VAR = val # comment", + "VAR = val " + ), + 'makeescapedcomment': ( + pymake.parser.itermakefilechars, + "VAR = val \# escaped hash", + "VAR = val # escaped hash" + ), + 'makeescapedslash': ( + pymake.parser.itermakefilechars, + "VAR = val\\\\", + "VAR = val\\\\", + ), + 'makecontinuation': ( + pymake.parser.itermakefilechars, + "VAR = VAL \\\n continuation # comment \\\n continuation", + "VAR = VAL continuation " + ), + 'makecontinuation2': ( + pymake.parser.itermakefilechars, + "VAR = VAL \\ \\\n continuation", + "VAR = VAL \\ continuation" + ), + 'makeawful': ( + pymake.parser.itermakefilechars, + "VAR = VAL \\\\# comment\n", + "VAR = VAL \\" + ), + 'command': ( + pymake.parser.itercommandchars, + "echo boo # comment", + "echo boo # comment", + ), + 'commandcomment': ( + pymake.parser.itercommandchars, + "echo boo \# comment", + "echo boo \# comment", + ), + 'commandcontinue': ( + pymake.parser.itercommandchars, + "echo boo # \\\n\t command 2", + "echo boo # \\\n command 2" + ), + } + + def runSingle(self, ifunc, idata, expected): + d = pymake.parser.Data.fromstring(idata, 'IterTest data') + + it = pymake.parser._alltokens.finditer(d.s, 0, d.lend) + actual = ''.join( [c for c, t, o, oo in ifunc(d, 0, ('dummy-token',), it)] ) + self.assertEqual(actual, expected) + + if ifunc == pymake.parser.itermakefilechars: + print "testing %r" % expected + self.assertEqual(pymake.parser.flattenmakesyntax(d, 0), expected) + +multitest(IterTest) + + +# 'define': ( +# pymake.parser.iterdefinechars, +# "endef", +# "" +# ), +# 'definenesting': ( +# pymake.parser.iterdefinechars, +# """define BAR # comment +#random text +#endef not what you think! +#endef # comment is ok\n""", +# """define BAR # comment +#random text +#endef not what you think!""" +# ), +# 'defineescaped': ( +# pymake.parser.iterdefinechars, +# """value \\ +#endef +#endef\n""", +# "value endef" +# ), + +class MakeSyntaxTest(TestBase): + # (string, startat, stopat, stopoffset, expansion + testdata = { + 'text': ('hello world', 0, (), None, ['hello world']), + 'singlechar': ('hello $W', 0, (), None, + ['hello ', + {'type': 'VariableRef', + '.vname': ['W']} + ]), + 'stopat': ('hello: world', 0, (':', '='), 6, ['hello']), + 'funccall': ('h $(flavor FOO)', 0, (), None, + ['h ', + {'type': 'FlavorFunction', + '[0]': ['FOO']} + ]), + 'escapedollar': ('hello$$world', 0, (), None, ['hello$world']), + 'varref': ('echo $(VAR)', 0, (), None, + ['echo ', + {'type': 'VariableRef', + '.vname': ['VAR']} + ]), + 'dynamicvarname': ('echo $($(VARNAME):.c=.o)', 0, (':',), None, + ['echo ', + {'type': 'SubstitutionRef', + '.vname': [{'type': 'VariableRef', + '.vname': ['VARNAME']} + ], + '.substfrom': ['.c'], + '.substto': ['.o']} + ]), + 'substref': (' $(VAR:VAL) := $(VAL)', 0, (':=', '+=', '=', ':'), 15, + [' ', + {'type': 'VariableRef', + '.vname': ['VAR:VAL']}, + ' ']), + 'vadsubstref': (' $(VAR:VAL) = $(VAL)', 15, (), None, + [{'type': 'VariableRef', + '.vname': ['VAL']}, + ]), + } + + def compareRecursive(self, actual, expected, path): + self.assertEqual(len(actual), len(expected), + "compareRecursive: %s %r" % (path, actual)) + for i in xrange(0, len(actual)): + ipath = path + [i] + + a, isfunc = actual[i] + e = expected[i] + if isinstance(e, str): + self.assertEqual(a, e, "compareRecursive: %s" % (ipath,)) + else: + self.assertEqual(type(a), getattr(pymake.functions, e['type']), + "compareRecursive: %s" % (ipath,)) + for k, v in e.iteritems(): + if k == 'type': + pass + elif k[0] == '[': + item = int(k[1:-1]) + proppath = ipath + [item] + self.compareRecursive(a[item], v, proppath) + elif k[0] == '.': + item = k[1:] + proppath = ipath + [item] + self.compareRecursive(getattr(a, item), v, proppath) + else: + raise Exception("Unexpected property at %s: %s" % (ipath, k)) + + def runSingle(self, s, startat, stopat, stopoffset, expansion): + d = pymake.parser.Data.fromstring(s, pymake.parserdata.Location('testdata', 1, 0)) + + a, t, offset = pymake.parser.parsemakesyntax(d, startat, stopat, pymake.parser.itermakefilechars) + self.compareRecursive(a, expansion, []) + self.assertEqual(offset, stopoffset) + +multitest(MakeSyntaxTest) + +class VariableTest(TestBase): + testdata = """ + VAR = value + VARNAME = TESTVAR + $(VARNAME) = testvalue + $(VARNAME:VAR=VAL) = moretesting + IMM := $(VARNAME) # this is a comment + MULTIVAR = val1 \\ + val2 + VARNAME = newname + """ + expected = {'VAR': 'value', + 'VARNAME': 'newname', + 'TESTVAR': 'testvalue', + 'TESTVAL': 'moretesting', + 'IMM': 'TESTVAR ', + 'MULTIVAR': 'val1 val2', + 'UNDEF': None} + + def runTest(self): + stmts = pymake.parser.parsestring(self.testdata, 'VariableTest') + + m = pymake.data.Makefile() + stmts.execute(m) + for k, v in self.expected.iteritems(): + flavor, source, val = m.variables.get(k) + if val is None: + self.assertEqual(val, v, 'variable named %s' % k) + else: + self.assertEqual(val.resolvestr(m, m.variables), v, 'variable named %s' % k) + +class SimpleRuleTest(TestBase): + testdata = """ + VAR = value +TSPEC = dummy +all: TSPEC = myrule +all:: test test2 $(VAR) + echo "Hello, $(TSPEC)" + +%.o: %.c + $(CC) -o $@ $< +""" + + def runTest(self): + stmts = pymake.parser.parsestring(self.testdata, 'SimpleRuleTest') + + m = pymake.data.Makefile() + stmts.execute(m) + self.assertEqual(m.defaulttarget, 'all', "Default target") + + self.assertTrue(m.hastarget('all'), "Has 'all' target") + target = m.gettarget('all') + rules = target.rules + self.assertEqual(len(rules), 1, "Number of rules") + prereqs = rules[0].prerequisites + self.assertEqual(prereqs, ['test', 'test2', 'value'], "Prerequisites") + commands = rules[0].commands + self.assertEqual(len(commands), 1, "Number of commands") + expanded = commands[0].resolvestr(m, target.variables) + self.assertEqual(expanded, 'echo "Hello, myrule"') + + irules = m.implicitrules + self.assertEqual(len(irules), 1, "Number of implicit rules") + + irule = irules[0] + self.assertEqual(len(irule.targetpatterns), 1, "%.o target pattern count") + self.assertEqual(len(irule.prerequisites), 1, "%.o prerequisite count") + self.assertEqual(irule.targetpatterns[0].match('foo.o'), 'foo', "%.o stem") + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + unittest.main() diff --git a/python/pymake/tests/path-length.mk b/python/pymake/tests/path-length.mk new file mode 100644 index 000000000..10c33b5ed --- /dev/null +++ b/python/pymake/tests/path-length.mk @@ -0,0 +1,9 @@ +#T gmake skip + +$(shell \ +mkdir foo; \ +touch tfile; \ +) + +all: foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../foo/../tfile + @echo TEST-PASS diff --git a/python/pymake/tests/pathdir/pathtest b/python/pymake/tests/pathdir/pathtest new file mode 100644 index 000000000..17037159f --- /dev/null +++ b/python/pymake/tests/pathdir/pathtest @@ -0,0 +1,2 @@ +#!/bin/sh +echo Called shell script: 2f7cdd0b-7277-48c1-beaf-56cb0dbacb24 diff --git a/python/pymake/tests/pathdir/pathtest.exe b/python/pymake/tests/pathdir/pathtest.exe new file mode 100644 index 000000000..3178db9a9 Binary files /dev/null and b/python/pymake/tests/pathdir/pathtest.exe differ diff --git a/python/pymake/tests/pathdir/src/Makefile b/python/pymake/tests/pathdir/src/Makefile new file mode 100644 index 000000000..6c24bd8f9 --- /dev/null +++ b/python/pymake/tests/pathdir/src/Makefile @@ -0,0 +1,2 @@ +pathtest.exe: pathtest.cpp + cl -EHsc -MT $^ diff --git a/python/pymake/tests/pathdir/src/pathtest.cpp b/python/pymake/tests/pathdir/src/pathtest.cpp new file mode 100644 index 000000000..bef8d8a11 --- /dev/null +++ b/python/pymake/tests/pathdir/src/pathtest.cpp @@ -0,0 +1,6 @@ +#include + +int main() { + std::printf("Called Windows executable: 2f7cdd0b-7277-48c1-beaf-56cb0dbacb24\n"); + return 0; +} diff --git a/python/pymake/tests/patsubst.mk b/python/pymake/tests/patsubst.mk new file mode 100644 index 000000000..0c3efdc4b --- /dev/null +++ b/python/pymake/tests/patsubst.mk @@ -0,0 +1,7 @@ +all: + test "$(patsubst foo,%.bar,foo)" = "%.bar" + test "$(patsubst \%word,replace,word %word other)" = "word replace other" + test "$(patsubst %.c,\%%.o,foo.c bar.o baz.cpp)" = "%foo.o bar.o baz.cpp" + test "$(patsubst host_%.c,host_%.o,dir/host_foo.c host_bar.c)" = "dir/host_foo.c host_bar.o" + test "$(patsubst foo,bar,dir/foo foo baz)" = "dir/foo bar baz" + @echo TEST-PASS diff --git a/python/pymake/tests/phony.mk b/python/pymake/tests/phony.mk new file mode 100644 index 000000000..36db4d121 --- /dev/null +++ b/python/pymake/tests/phony.mk @@ -0,0 +1,10 @@ +$(shell \ +touch dep; \ +sleep 2; \ +touch all; \ +) + +all:: dep + @echo TEST-PASS + +.PHONY: all diff --git a/python/pymake/tests/pycmd.py b/python/pymake/tests/pycmd.py new file mode 100644 index 000000000..83b9b966b --- /dev/null +++ b/python/pymake/tests/pycmd.py @@ -0,0 +1,38 @@ +import os, sys, subprocess + +def writetofile(args): + with open(args[0], 'w') as f: + f.write(' '.join(args[1:])) + +def writeenvtofile(args): + with open(args[0], 'w') as f: + f.write(os.environ[args[1]]) + +def writesubprocessenvtofile(args): + with open(args[0], 'w') as f: + p = subprocess.Popen([sys.executable, "-c", + "import os; print os.environ['%s']" % args[1]], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + assert p.returncode == 0 + f.write(stdout) + +def convertasplode(arg): + try: + return int(arg) + except: + return (None if arg == "None" else arg) + +def asplode(args): + arg0 = convertasplode(args[0]) + sys.exit(arg0) + +def asplode_return(args): + arg0 = convertasplode(args[0]) + return arg0 + +def asplode_raise(args): + raise Exception(args[0]) + +def delayloadfn(args): + import delayload diff --git a/python/pymake/tests/recursive-set.mk b/python/pymake/tests/recursive-set.mk new file mode 100644 index 000000000..853f90463 --- /dev/null +++ b/python/pymake/tests/recursive-set.mk @@ -0,0 +1,7 @@ +#T returncode: 2 + +FOO = $(FOO) + +all: + echo $(FOO) + @echo TEST-FAIL diff --git a/python/pymake/tests/recursive-set2.mk b/python/pymake/tests/recursive-set2.mk new file mode 100644 index 000000000..b68e34f0d --- /dev/null +++ b/python/pymake/tests/recursive-set2.mk @@ -0,0 +1,8 @@ +#T returncode: 2 + +FOO = $(BAR) +BAR = $(FOO) + +all: + echo $(FOO) + @echo TEST-FAIL diff --git a/python/pymake/tests/remake-mtime.mk b/python/pymake/tests/remake-mtime.mk new file mode 100644 index 000000000..47c775b93 --- /dev/null +++ b/python/pymake/tests/remake-mtime.mk @@ -0,0 +1,14 @@ +# mtime(dep1) < mtime(target) so the target should not be made +$(shell touch dep1; sleep 1; touch target) + +all: target + echo TEST-PASS + +target: dep1 + echo TEST-FAIL target should not have been made + +dep1: dep2 + @echo "Remaking dep1 (actually not)" + +dep2: + @echo "Making dep2 (actually not)" diff --git a/python/pymake/tests/rm-fail.mk b/python/pymake/tests/rm-fail.mk new file mode 100644 index 000000000..1a9aefb57 --- /dev/null +++ b/python/pymake/tests/rm-fail.mk @@ -0,0 +1,7 @@ +#T returncode: 2 +all: + mkdir newdir + test -d newdir + touch newdir/newfile + $(RM) newdir + @echo TEST-PASS diff --git a/python/pymake/tests/rm.mk b/python/pymake/tests/rm.mk new file mode 100644 index 000000000..6c7140e39 --- /dev/null +++ b/python/pymake/tests/rm.mk @@ -0,0 +1,21 @@ +all: +# $(RM) defaults to -f + $(RM) nosuchfile + touch newfile + test -f newfile + $(RM) newfile + test ! -f newfile + mkdir newdir + test -d newdir + touch newdir/newfile + mkdir newdir/subdir + $(RM) -r newdir/subdir + test ! -d newdir/subdir + test -d newdir + mkdir newdir/subdir1 newdir/subdir2 + $(RM) -r newdir/subdir1 newdir/subdir2 + test ! -d newdir/subdir1 -a ! -d newdir/subdir2 + test -d newdir + $(RM) -r newdir + test ! -d newdir + @echo TEST-PASS diff --git a/python/pymake/tests/runtests.py b/python/pymake/tests/runtests.py new file mode 100644 index 000000000..ab149ecfb --- /dev/null +++ b/python/pymake/tests/runtests.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python +""" +Run the test(s) listed on the command line. If a directory is listed, the script will recursively +walk the directory for files named .mk and run each. + +For each test, we run gmake -f test.mk. By default, make must exit with an exit code of 0, and must print 'TEST-PASS'. + +Each test is run in an empty directory. + +The test file may contain lines at the beginning to alter the default behavior. These are all evaluated as python: + +#T commandline: ['extra', 'params', 'here'] +#T returncode: 2 +#T returncode-on: {'win32': 2} +#T environment: {'VAR': 'VALUE} +#T grep-for: "text" +""" + +from subprocess import Popen, PIPE, STDOUT +from optparse import OptionParser +import os, re, sys, shutil, glob + +class ParentDict(dict): + def __init__(self, parent, **kwargs): + self.d = dict(kwargs) + self.parent = parent + + def __setitem__(self, k, v): + self.d[k] = v + + def __getitem__(self, k): + if k in self.d: + return self.d[k] + + return self.parent[k] + +thisdir = os.path.dirname(os.path.abspath(__file__)) + +pymake = [sys.executable, os.path.join(os.path.dirname(thisdir), 'make.py')] +manifest = os.path.join(thisdir, 'tests.manifest') + +o = OptionParser() +o.add_option('-g', '--gmake', + dest="gmake", default="gmake") +o.add_option('-d', '--tempdir', + dest="tempdir", default="_mktests") +opts, args = o.parse_args() + +if len(args) == 0: + args = [thisdir] + +makefiles = [] +for a in args: + if os.path.isfile(a): + makefiles.append(a) + elif os.path.isdir(a): + makefiles.extend(sorted(glob.glob(os.path.join(a, '*.mk')))) + +def runTest(makefile, make, logfile, options): + """ + Given a makefile path, test it with a given `make` and return + (pass, message). + """ + + if os.path.exists(opts.tempdir): shutil.rmtree(opts.tempdir) + os.mkdir(opts.tempdir, 0755) + + logfd = open(logfile, 'w') + p = Popen(make + options['commandline'], stdout=logfd, stderr=STDOUT, env=options['env']) + logfd.close() + retcode = p.wait() + + if retcode != options['returncode']: + return False, "FAIL (returncode=%i)" % retcode + + logfd = open(logfile) + stdout = logfd.read() + logfd.close() + + if stdout.find('TEST-FAIL') != -1: + print stdout + return False, "FAIL (TEST-FAIL printed)" + + if options['grepfor'] and stdout.find(options['grepfor']) == -1: + print stdout + return False, "FAIL (%s not in output)" % options['grepfor'] + + if options['returncode'] == 0 and stdout.find('TEST-PASS') == -1: + print stdout + return False, 'FAIL (No TEST-PASS printed)' + + if options['returncode'] != 0: + return True, 'PASS (retcode=%s)' % retcode + + return True, 'PASS' + +print "%-30s%-28s%-28s" % ("Test:", "gmake:", "pymake:") + +gmakefails = 0 +pymakefails = 0 + +tre = re.compile('^#T (gmake |pymake )?([a-z-]+)(?:: (.*))?$') + +for makefile in makefiles: + # For some reason, MAKEFILE_LIST uses native paths in GNU make on Windows + # (even in MSYS!) so we pass both TESTPATH and NATIVE_TESTPATH + cline = ['-C', opts.tempdir, '-f', os.path.abspath(makefile), 'TESTPATH=%s' % thisdir.replace('\\','/'), 'NATIVE_TESTPATH=%s' % thisdir] + if sys.platform == 'win32': + #XXX: hack so we can specialize the separator character on windows. + # we really shouldn't need this, but y'know + cline += ['__WIN32__=1'] + + options = { + 'returncode': 0, + 'grepfor': None, + 'env': dict(os.environ), + 'commandline': cline, + 'pass': True, + 'skip': False, + } + + gmakeoptions = ParentDict(options) + pymakeoptions = ParentDict(options) + + dmap = {None: options, 'gmake ': gmakeoptions, 'pymake ': pymakeoptions} + + mdata = open(makefile) + for line in mdata: + line = line.strip() + m = tre.search(line) + if m is None: + break + + make, key, data = m.group(1, 2, 3) + d = dmap[make] + if data is not None: + data = eval(data) + if key == 'commandline': + assert make is None + d['commandline'].extend(data) + elif key == 'returncode': + d['returncode'] = data + elif key == 'returncode-on': + if sys.platform in data: + d['returncode'] = data[sys.platform] + elif key == 'environment': + for k, v in data.iteritems(): + d['env'][k] = v + elif key == 'grep-for': + d['grepfor'] = data + elif key == 'fail': + d['pass'] = False + elif key == 'skip': + d['skip'] = True + else: + print >>sys.stderr, "%s: Unexpected #T key: %s" % (makefile, key) + sys.exit(1) + + mdata.close() + + if gmakeoptions['skip']: + gmakepass, gmakemsg = True, '' + else: + gmakepass, gmakemsg = runTest(makefile, [opts.gmake], + makefile + '.gmakelog', gmakeoptions) + + if gmakeoptions['pass']: + if not gmakepass: + gmakefails += 1 + else: + if gmakepass: + gmakefails += 1 + gmakemsg = "UNEXPECTED PASS" + else: + gmakemsg = "KNOWN FAIL" + + if pymakeoptions['skip']: + pymakepass, pymakemsg = True, '' + else: + pymakepass, pymakemsg = runTest(makefile, pymake, + makefile + '.pymakelog', pymakeoptions) + + if pymakeoptions['pass']: + if not pymakepass: + pymakefails += 1 + else: + if pymakepass: + pymakefails += 1 + pymakemsg = "UNEXPECTED PASS" + else: + pymakemsg = "OK (known fail)" + + print "%-30.30s%-28.28s%-28.28s" % (os.path.basename(makefile), + gmakemsg, pymakemsg) + +print +print "Summary:" +print "%-30s%-28s%-28s" % ("", "gmake:", "pymake:") + +if gmakefails == 0: + gmakemsg = 'PASS' +else: + gmakemsg = 'FAIL (%i failures)' % gmakefails + +if pymakefails == 0: + pymakemsg = 'PASS' +else: + pymakemsg = 'FAIL (%i failures)' % pymakefails + +print "%-30.30s%-28.28s%-28.28s" % ('', gmakemsg, pymakemsg) + +shutil.rmtree(opts.tempdir) + +if gmakefails or pymakefails: + sys.exit(1) diff --git a/python/pymake/tests/serial-dep-resolution.mk b/python/pymake/tests/serial-dep-resolution.mk new file mode 100644 index 000000000..e65f1ed03 --- /dev/null +++ b/python/pymake/tests/serial-dep-resolution.mk @@ -0,0 +1,5 @@ +all: t1 t2 + @echo TEST-PASS + +t1: + touch t1 t2 diff --git a/python/pymake/tests/serial-doublecolon-execution.mk b/python/pymake/tests/serial-doublecolon-execution.mk new file mode 100644 index 000000000..1871cb13a --- /dev/null +++ b/python/pymake/tests/serial-doublecolon-execution.mk @@ -0,0 +1,18 @@ +#T commandline: ['-j3'] + +# Commands of double-colon rules are always executed in order. + +all: dc + cat status + test "$$(cat status)" = "all1:all2:" + @echo TEST-PASS + +dc:: slowt + printf "all1:" >> status + +dc:: + sleep 0.2 + printf "all2:" >> status + +slowt: + sleep 1 diff --git a/python/pymake/tests/serial-rule-execution.mk b/python/pymake/tests/serial-rule-execution.mk new file mode 100644 index 000000000..da5b177de --- /dev/null +++ b/python/pymake/tests/serial-rule-execution.mk @@ -0,0 +1,5 @@ +all:: + touch somefile + +all:: somefile + @echo TEST-PASS diff --git a/python/pymake/tests/serial-rule-execution2.mk b/python/pymake/tests/serial-rule-execution2.mk new file mode 100644 index 000000000..252a7df83 --- /dev/null +++ b/python/pymake/tests/serial-rule-execution2.mk @@ -0,0 +1,13 @@ +#T returncode: 2 + +# The dependencies of the command rule of a single-colon target are resolved before the rules without commands. + +all: export + +export: + sleep 1 + touch somefile + +all: somefile + test -f somefile + @echo TEST-PASS diff --git a/python/pymake/tests/serial-toparallel.mk b/python/pymake/tests/serial-toparallel.mk new file mode 100644 index 000000000..a980badc7 --- /dev/null +++ b/python/pymake/tests/serial-toparallel.mk @@ -0,0 +1,5 @@ +all:: + $(MAKE) -j2 -f $(TESTPATH)/parallel-simple.mk + +all:: results + @echo TEST-PASS diff --git a/python/pymake/tests/shellfunc.mk b/python/pymake/tests/shellfunc.mk new file mode 100644 index 000000000..1e408dbac --- /dev/null +++ b/python/pymake/tests/shellfunc.mk @@ -0,0 +1,7 @@ +all: testfile + test "$(shell cat $<)" = "Hello world" + test "$(shell printf "\n")" = "" + @echo TEST-PASS + +testfile: + printf "Hello\nworld\n" > $@ diff --git a/python/pymake/tests/simple-makeflags.mk b/python/pymake/tests/simple-makeflags.mk new file mode 100644 index 000000000..c7c92ec9d --- /dev/null +++ b/python/pymake/tests/simple-makeflags.mk @@ -0,0 +1,10 @@ +# There once was a time when MAKEFLAGS=w without any following spaces would +# cause us to treat w as a target, not a flag. Silly! + +MAKEFLAGS=w + +all: + $(MAKE) -f $(TESTPATH)/simple-makeflags.mk subt + @echo TEST-PASS + +subt: diff --git a/python/pymake/tests/sort.mk b/python/pymake/tests/sort.mk new file mode 100644 index 000000000..e1313ad5c --- /dev/null +++ b/python/pymake/tests/sort.mk @@ -0,0 +1,4 @@ +# sort should remove duplicates +all: + @test "$(sort x a y b z c a z b x c y)" = "a b c x y z" + @echo "TEST-PASS" diff --git a/python/pymake/tests/specified-target.mk b/python/pymake/tests/specified-target.mk new file mode 100644 index 000000000..3b23fbf69 --- /dev/null +++ b/python/pymake/tests/specified-target.mk @@ -0,0 +1,7 @@ +#T commandline: ['VAR=all', '$(VAR)'] + +all: + @echo TEST-FAIL: unexpected target 'all' + +$$(VAR): + @echo TEST-PASS: expected target '$$(VAR)' diff --git a/python/pymake/tests/static-pattern.mk b/python/pymake/tests/static-pattern.mk new file mode 100644 index 000000000..f613b8c9a --- /dev/null +++ b/python/pymake/tests/static-pattern.mk @@ -0,0 +1,5 @@ +#T returncode: 2 + +out/host_foo.o: host_%.o: host_%.c out + cp $< $@ + @echo TEST-FAIL diff --git a/python/pymake/tests/static-pattern2.mk b/python/pymake/tests/static-pattern2.mk new file mode 100644 index 000000000..08ed834fd --- /dev/null +++ b/python/pymake/tests/static-pattern2.mk @@ -0,0 +1,10 @@ +all: foo.out + test -f $^ + @echo TEST-PASS + +foo.out: %.out: %.in + test "$*" = "foo" + cp $^ $@ + +foo.in: + touch $@ diff --git a/python/pymake/tests/subdir/delayload.py b/python/pymake/tests/subdir/delayload.py new file mode 100644 index 000000000..bdd6669db --- /dev/null +++ b/python/pymake/tests/subdir/delayload.py @@ -0,0 +1 @@ +# This module exists to test delay importing of modules at run-time. diff --git a/python/pymake/tests/subdir/pymod.py b/python/pymake/tests/subdir/pymod.py new file mode 100644 index 000000000..1a47d8af2 --- /dev/null +++ b/python/pymake/tests/subdir/pymod.py @@ -0,0 +1,5 @@ +import testmodule + +def writetofile(args): + with open(args[0], 'w') as f: + f.write(' '.join(args[1:])) diff --git a/python/pymake/tests/subdir/testmodule.py b/python/pymake/tests/subdir/testmodule.py new file mode 100644 index 000000000..05b2f821a --- /dev/null +++ b/python/pymake/tests/subdir/testmodule.py @@ -0,0 +1,3 @@ +# This is an empty module. It is imported by pymod.py to test that if a module +# is loaded from the PYCOMMANDPATH, it can import other modules from the same +# directory correctly. diff --git a/python/pymake/tests/submake-path.makefile2 b/python/pymake/tests/submake-path.makefile2 new file mode 100644 index 000000000..1266db7d1 --- /dev/null +++ b/python/pymake/tests/submake-path.makefile2 @@ -0,0 +1,11 @@ +# -*- Mode: Makefile -*- + +shellresult := $(shell pathtest) +ifneq (2f7cdd0b-7277-48c1-beaf-56cb0dbacb24,$(filter $(shellresult),2f7cdd0b-7277-48c1-beaf-56cb0dbacb24)) +$(error pathtest not found in submake shell function) +endif + +all: + @pathtest + @pathtest | grep -q 2f7cdd0b-7277-48c1-beaf-56cb0dbacb24 + @echo TEST-PASS diff --git a/python/pymake/tests/submake-path.mk b/python/pymake/tests/submake-path.mk new file mode 100644 index 000000000..b6432276d --- /dev/null +++ b/python/pymake/tests/submake-path.mk @@ -0,0 +1,16 @@ +#T gmake skip +#T grep-for: "2f7cdd0b-7277-48c1-beaf-56cb0dbacb24" + +ifdef __WIN32__ +PS:=; +else +PS:=: +endif + +export PATH := $(TESTPATH)/pathdir$(PS)$(PATH) + +# This is similar to subprocess-path.mk, except we also check $(shell) +# invocations since they're affected by exported environment variables too, +# but only in submakes! +all: + $(MAKE) -f $(TESTPATH)/submake-path.makefile2 diff --git a/python/pymake/tests/submake.makefile2 b/python/pymake/tests/submake.makefile2 new file mode 100644 index 000000000..12ce94834 --- /dev/null +++ b/python/pymake/tests/submake.makefile2 @@ -0,0 +1,24 @@ +# -*- Mode: Makefile -*- + +$(info MAKEFLAGS = '$(MAKEFLAGS)') +$(info MAKE = '$(MAKE)') +$(info value MAKE = "$(value MAKE)") + +shellresult := $(shell echo -n $$EVAR) +ifneq ($(shellresult),eval) +$(error EVAR should be eval, is instead $(shellresult)) +endif + +all: + env + test "$(MAKELEVEL)" = "1" + echo "value(MAKE)" '$(value MAKE)' + echo "value(MAKE_COMMAND)" = '$(value MAKE_COMMAND)' + test "$(origin CVAR)" = "command line" + test "$(CVAR)" = "c val=spac\ed" + test "$(origin EVAR)" = "environment" + test "$(EVAR)" = "eval" + test "$(OVAL)" = "cline" + test "$(OVAL2)" = "cline2" + test "$(ALLVAR)" = "allspecific" + @echo TEST-PASS diff --git a/python/pymake/tests/submake.mk b/python/pymake/tests/submake.mk new file mode 100644 index 000000000..41e47134b --- /dev/null +++ b/python/pymake/tests/submake.mk @@ -0,0 +1,16 @@ +#T commandline: ['CVAR=c val=spac\\ed', 'OVAL=cline', 'OVAL2=cline2'] + +export EVAR = eval +override OVAL = makefile + +# exporting an override variable doesn't mean it's an override variable +override OVAL2 = makefile2 +export OVAL2 + +export ALLVAR +ALLVAR = general +all: ALLVAR = allspecific + +all: + test "$(MAKELEVEL)" = "0" + $(MAKE) -f $(TESTPATH)/submake.makefile2 diff --git a/python/pymake/tests/subprocess-path.mk b/python/pymake/tests/subprocess-path.mk new file mode 100644 index 000000000..f63921414 --- /dev/null +++ b/python/pymake/tests/subprocess-path.mk @@ -0,0 +1,32 @@ +#T gmake skip +#T grep-for: "2f7cdd0b-7277-48c1-beaf-56cb0dbacb24" + +ifdef __WIN32__ +PS:=; +else +PS:=: +endif + +export PATH := $(TESTPATH)/pathdir$(PS)$(PATH) + +# Test two commands. The first one shouldn't go through the shell and the +# second one should. The pathdir subdirectory has a Windows executable called +# pathtest.exe and a shell script called pathtest. We don't care which one is +# run, just that one of the two is (we use a uuid + grep-for to make sure +# that happens). +# +# FAQ: +# Q. Why skip GNU Make? +# A. Because $(TESTPATH) is a Windows-style path, and MSYS make doesn't take +# too kindly to Windows paths in the PATH environment variable. +# +# Q. Why use an exe and not a batch file? +# A. The use cases here were all exe files without the extension. Batch file +# lookup has broken semantics if the .bat extension isn't passed. +# +# Q. Why are the commands silent? +# A. So that we don't pass the grep-for test by mistake. +all: + @pathtest + @pathtest | grep -q 2f7cdd0b-7277-48c1-beaf-56cb0dbacb24 + @echo TEST-PASS diff --git a/python/pymake/tests/tab-intro.mk b/python/pymake/tests/tab-intro.mk new file mode 100644 index 000000000..1c25ce747 --- /dev/null +++ b/python/pymake/tests/tab-intro.mk @@ -0,0 +1,16 @@ +# Initial tab characters should be treated well. + + THIS = a value + + ifdef THIS + VAR = conditional value + endif + +all: + test "$(THIS)" = "another value" + test "$(VAR)" = "conditional value" + @echo TEST-PASS + +THAT = makefile syntax + + THIS = another value diff --git a/python/pymake/tests/target-specific.mk b/python/pymake/tests/target-specific.mk new file mode 100644 index 000000000..217ed155e --- /dev/null +++ b/python/pymake/tests/target-specific.mk @@ -0,0 +1,30 @@ +TESTVAR = anonval + +all: target.suffix target.suffix2 dummy host_test.py my.test1 my.test2 + @echo TEST-PASS + +target.suffix: TESTVAR = testval + +%.suffix: + test "$(TESTVAR)" = "testval" + +%.suffix2: TESTVAR = testval2 + +%.suffix2: + test "$(TESTVAR)" = "testval2" + +%my: TESTVAR = dummyval + +dummy: + test "$(TESTVAR)" = "dummyval" + +%.py: TESTVAR = pyval +host_%.py: TESTVAR = hostval + +host_test.py: + test "$(TESTVAR)" = "hostval" + +%.test1 %.test2: TESTVAR = %val + +my.test1 my.test2: + test "$(TESTVAR)" = "%val" diff --git a/python/pymake/tests/unexport.mk b/python/pymake/tests/unexport.mk new file mode 100644 index 000000000..424411603 --- /dev/null +++ b/python/pymake/tests/unexport.mk @@ -0,0 +1,15 @@ +#T environment: {'ENVVAR': 'envval'} + +VAR1 = val1 +VAR2 = val2 +VAR3 = val3 + +unexport VAR3 +export VAR1 VAR2 VAR3 +unexport VAR2 ENVVAR +unexport + +all: + test "$(ENVVAR)" = "envval" # unexport.mk + $(MAKE) -f $(TESTPATH)/unexport.submk + @echo TEST-PASS diff --git a/python/pymake/tests/unexport.submk b/python/pymake/tests/unexport.submk new file mode 100644 index 000000000..8db6163de --- /dev/null +++ b/python/pymake/tests/unexport.submk @@ -0,0 +1,15 @@ +# -@- Mode: Makefile -@- + +unexport VAR1 + +all: + env + test "$(VAR1)" = "val1" + test "$(origin VAR1)" = "environment" + test "$(VAR2)" = "" # VAR2 + test "$(VAR3)" = "val3" + test "$(ENVVAR)" = "" + $(MAKE) -f $(TESTPATH)/unexport.submk subt + +subt: + test "$(VAR1)" = "" diff --git a/python/pymake/tests/unterminated-dollar.mk b/python/pymake/tests/unterminated-dollar.mk new file mode 100644 index 000000000..dee9a207b --- /dev/null +++ b/python/pymake/tests/unterminated-dollar.mk @@ -0,0 +1,6 @@ +VAR = value$ +VAR2 = other + +all: + test "$(VAR)" = "value" + @echo TEST-PASS diff --git a/python/pymake/tests/var-change-flavor.mk b/python/pymake/tests/var-change-flavor.mk new file mode 100644 index 000000000..0cccf0bd6 --- /dev/null +++ b/python/pymake/tests/var-change-flavor.mk @@ -0,0 +1,12 @@ +VAR = value1 +VAR := value2 + +VAR2 := val1 +VAR2 = val2 + +default: + test "$(flavor VAR)" = "simple" + test "$(VAR)" = "value2" + test "$(flavor VAR2)" = "recursive" + test "$(VAR2)" = "val2" + @echo "TEST-PASS" diff --git a/python/pymake/tests/var-commandline.mk b/python/pymake/tests/var-commandline.mk new file mode 100644 index 000000000..e2cdad457 --- /dev/null +++ b/python/pymake/tests/var-commandline.mk @@ -0,0 +1,8 @@ +#T commandline: ['TESTVAR=$(MAKEVAL)', 'TESTVAR2:=$(MAKEVAL)'] + +MAKEVAL=testvalue + +all: + test "$(TESTVAR)" = "testvalue" + test "$(TESTVAR2)" = "" + @echo "TEST-PASS" \ No newline at end of file diff --git a/python/pymake/tests/var-overrides.mk b/python/pymake/tests/var-overrides.mk new file mode 100644 index 000000000..bd0765d19 --- /dev/null +++ b/python/pymake/tests/var-overrides.mk @@ -0,0 +1,21 @@ +#T commandline: ['CLINEVAR=clineval', 'CLINEVAR2=clineval2'] + +# this doesn't actually test overrides yet, because they aren't implemented in pymake, +# but testing origins in general is important + +MVAR = mval +CLINEVAR = deadbeef + +override CLINEVAR2 = mval2 + +all: + test "$(origin NOVAR)" = "undefined" + test "$(CLINEVAR)" = "clineval" + test "$(origin CLINEVAR)" = "command line" + test "$(MVAR)" = "mval" + test "$(origin MVAR)" = "file" + test "$(@)" = "all" + test "$(origin @)" = "automatic" + test "$(origin CLINEVAR2)" = "override" + test "$(CLINEVAR2)" = "mval2" + @echo TEST-PASS diff --git a/python/pymake/tests/var-ref.mk b/python/pymake/tests/var-ref.mk new file mode 100644 index 000000000..3bc1886f9 --- /dev/null +++ b/python/pymake/tests/var-ref.mk @@ -0,0 +1,19 @@ +VAR = value +VAR2 == value + +VAR5 = $(NULL) $(NULL) +VARC = value # comment + +$(VAR3) + $(VAR4) +$(VAR5) + +VAR6$(VAR5) = val6 + +all: + test "$( VAR)" = "" + test "$(VAR2)" = "= value" + test "${VAR2}" = "= value" + test "$(VAR6 )" = "val6" + test "$(VARC)" = "value " + @echo TEST-PASS diff --git a/python/pymake/tests/var-set.mk b/python/pymake/tests/var-set.mk new file mode 100644 index 000000000..1603e7a35 --- /dev/null +++ b/python/pymake/tests/var-set.mk @@ -0,0 +1,55 @@ +#T commandline: ['OBASIC=oval'] + +BASIC = val + +TEST = $(TEST) + +TEST2 = $(TES +TEST2 += T) + +TES T = val + +RECVAR = foo +RECVAR += var baz + +IMMVAR := bloo +IMMVAR += $(RECVAR) + +BASIC ?= notval + +all: BASIC = valall +all: RECVAR += $(BASIC) +all: IMMVAR += $(BASIC) +all: UNSET += more +all: OBASIC += allmore + +CHECKLIT = $(NULL) check +all: CHECKLIT += appendliteral + +RECVAR = blimey + +TESTEMPTY = \ + $(NULL) + +all: other + test "$(TEST2)" = "val" + test '$(value TEST2)' = '$$(TES T)' + test "$(RECVAR)" = "blimey valall" + test "$(IMMVAR)" = "bloo foo var baz valall" + test "$(UNSET)" = "more" + test "$(OBASIC)" = "oval" + test "$(CHECKLIT)" = " check appendliteral" + test "$(TESTEMPTY)" = "" + @echo TEST-PASS + +OVAR = oval +OVAR ?= onotval + +other: OVAR ?= ooval +other: LATERVAR ?= lateroverride + +LATERVAR = olater + +other: + test "$(OVAR)" = "oval" + test "$(LATERVAR)" = "lateroverride" diff --git a/python/pymake/tests/var-substitutions.mk b/python/pymake/tests/var-substitutions.mk new file mode 100644 index 000000000..d5627d7bd --- /dev/null +++ b/python/pymake/tests/var-substitutions.mk @@ -0,0 +1,49 @@ +SIMPLEVAR = aabb.cc +SIMPLEPERCENT = test_value%extra + +SIMPLE3SUBSTNAME = SIMPLEVAR:.dd +$(SIMPLE3SUBSTNAME) = weirdval + +PERCENT = dummy + +SIMPLESUBST = $(SIMPLEVAR:.cc=.dd) +SIMPLE2SUBST = $(SIMPLEVAR:.cc) +SIMPLE3SUBST = $(SIMPLEVAR:.dd) +SIMPLE4SUBST = $(SIMPLEVAR:.cc=.dd=.ee) +SIMPLE5SUBST = $(SIMPLEVAR:.cc=%.dd) +PERCENTSUBST = $(SIMPLEVAR:%.cc=%.ee) +PERCENT2SUBST = $(SIMPLEVAR:aa%.cc=ff%.f) +PERCENT3SUBST = $(SIMPLEVAR:aa%.dd=gg%.gg) +PERCENT4SUBST = $(SIMPLEVAR:aa%.cc=gg) +PERCENT5SUBST = $(SIMPLEVAR:aa) +PERCENT6SUBST = $(SIMPLEVAR:%.cc=%.dd=%.ee) +PERCENT7SUBST = $(SIMPLEVAR:$(PERCENT).cc=%.dd) +PERCENT8SUBST = $(SIMPLEVAR:%.cc=$(PERCENT).dd) +PERCENT9SUBST = $(SIMPLEVAR:$(PERCENT).cc=$(PERCENT).dd) +PERCENT10SUBST = $(SIMPLEVAR:%%.bb.cc=zz.bb.cc) +PERCENT11SUBST = $(SIMPLEPERCENT:test%value%extra=other%value%extra) + +SPACEDVAR = $(NULL) ex1.c ex2.c $(NULL) +SPACEDSUBST = $(SPACEDVAR:.c=.o) + +all: + test "$(SIMPLESUBST)" = "aabb.dd" + test "$(SIMPLE2SUBST)" = "" + test "$(SIMPLE3SUBST)" = "weirdval" + test "$(SIMPLE4SUBST)" = "aabb.dd=.ee" + test "$(SIMPLE5SUBST)" = "aabb%.dd" + test "$(PERCENTSUBST)" = "aabb.ee" + test "$(PERCENT2SUBST)" = "ffbb.f" + test "$(PERCENT3SUBST)" = "aabb.cc" + test "$(PERCENT4SUBST)" = "gg" + test "$(PERCENT5SUBST)" = "" + test "$(PERCENT6SUBST)" = "aabb.dd=%.ee" + test "$(PERCENT7SUBST)" = "aabb.dd" + test "$(PERCENT8SUBST)" = "aabb.dd" + test "$(PERCENT9SUBST)" = "aabb.dd" + test "$(PERCENT10SUBST)" = "aabb.cc" + test "$(PERCENT11SUBST)" = "other_value%extra" + test "$(SPACEDSUBST)" = "ex1.o ex2.o" + @echo TEST-PASS + +PERCENT = % diff --git a/python/pymake/tests/vpath-directive-dynamic.mk b/python/pymake/tests/vpath-directive-dynamic.mk new file mode 100644 index 000000000..9aa1bf956 --- /dev/null +++ b/python/pymake/tests/vpath-directive-dynamic.mk @@ -0,0 +1,12 @@ +$(shell \ +mkdir subd1; \ +touch subd1/test.in; \ +) + +VVAR = %.in subd1 + +vpath $(VVAR) + +all: test.in + test "$<" = "subd1/test.in" + @echo TEST-PASS diff --git a/python/pymake/tests/vpath-directive.mk b/python/pymake/tests/vpath-directive.mk new file mode 100644 index 000000000..4c7d4bf39 --- /dev/null +++ b/python/pymake/tests/vpath-directive.mk @@ -0,0 +1,31 @@ +# On Windows, MSYS make takes Unix paths but Pymake takes Windows paths +VPSEP := $(if $(and $(__WIN32__),$(.PYMAKE)),;,:) + +$(shell \ +mkdir subd1 subd2 subd3; \ +printf "reallybaddata" >subd1/foo.in; \ +printf "gooddata" >subd2/foo.in; \ +printf "baddata" >subd3/foo.in; \ +touch subd1/foo.in2 subd2/foo.in2 subd3/foo.in2; \ +) + +vpath %.in subd + +vpath +vpath %.in subd2$(VPSEP)subd3 + +vpath %.in2 subd0 +vpath f%.in2 subd1 +vpath %.in2 $(VPSEP)subd2 + +%.out: %.in + test "$<" = "subd2/foo.in" + cp $< $@ + +%.out2: %.in2 + test "$<" = "subd1/foo.in2" + cp $< $@ + +all: foo.out foo.out2 + test "$$(cat foo.out)" = "gooddata" + @echo TEST-PASS diff --git a/python/pymake/tests/vpath.mk b/python/pymake/tests/vpath.mk new file mode 100644 index 000000000..06f52180c --- /dev/null +++ b/python/pymake/tests/vpath.mk @@ -0,0 +1,18 @@ +VPATH = foo bar + +$(shell \ +mkdir foo; touch foo/tfile1; \ +mkdir bar; touch bar/tfile2 bar/tfile3 bar/test.objtest; \ +sleep 2; \ +touch bar/test.source; \ +) + +all: tfile1 tfile2 tfile3 test.objtest test.source + test "$^" = "foo/tfile1 bar/tfile2 tfile3 test.objtest bar/test.source" + @echo TEST-PASS + +tfile3: test.objtest + +%.objtest: %.source + test "$<" = bar/test.source + test "$@" = test.objtest diff --git a/python/pymake/tests/vpath2.mk b/python/pymake/tests/vpath2.mk new file mode 100644 index 000000000..be73ffe5c --- /dev/null +++ b/python/pymake/tests/vpath2.mk @@ -0,0 +1,18 @@ +VPATH = foo bar + +$(shell \ +mkdir bar; touch bar/test.source; \ +sleep 2; \ +mkdir foo; touch foo/tfile1; \ +touch bar/tfile2 bar/tfile3 bar/test.objtest; \ +) + +all: tfile1 tfile2 tfile3 test.objtest test.source + test "$^" = "foo/tfile1 bar/tfile2 bar/tfile3 bar/test.objtest bar/test.source" + @echo TEST-PASS + +tfile3: test.objtest + +%.objtest: %.source + test "$<" = bar/test.source + test "$@" = test.objtest diff --git a/python/pymake/tests/wildcards.mk b/python/pymake/tests/wildcards.mk new file mode 100644 index 000000000..24ff3f14c --- /dev/null +++ b/python/pymake/tests/wildcards.mk @@ -0,0 +1,22 @@ +$(shell \ +mkdir foo; \ +touch a.c b.c c.out foo/d.c; \ +sleep 2; \ +touch c.in; \ +) + +VPATH = foo + +all: c.out prog + cat $< + test "$$(cat $<)" = "remadec.out" + @echo TEST-PASS + +*.out: %.out: %.in + test "$@" = c.out + test "$<" = c.in + printf "remade$@" >$@ + +prog: *.c + test "$^" = "a.c b.c" + touch $@ diff --git a/python/pymake/tests/windows-paths.mk b/python/pymake/tests/windows-paths.mk new file mode 100644 index 000000000..5f33a9050 --- /dev/null +++ b/python/pymake/tests/windows-paths.mk @@ -0,0 +1,5 @@ +all: + touch file.in + printf "%s: %s\n\ttrue" '$(CURDIR)/file.out' '$(CURDIR)/file.in' >test.mk + $(MAKE) -f test.mk $(CURDIR)/file.out + @echo TEST-PASS -- cgit v1.2.3