summaryrefslogtreecommitdiff
path: root/testing
diff options
context:
space:
mode:
authorMatt A. Tobin <email@mattatobin.com>2021-12-04 12:45:34 -0500
committerMatt A. Tobin <email@mattatobin.com>2021-12-04 12:45:34 -0500
commitb85c84d1cf9feedf7bbbd99f2d31d14df9191c1f (patch)
tree1e3725aebb75007f0013c50fa8452bdb1166d1ff /testing
parent7b24ed38db2bce57686c79648235f17005bcf184 (diff)
downloadaura-central-b85c84d1cf9feedf7bbbd99f2d31d14df9191c1f.tar.gz
Issue %3021 - Remove testing/mozharness
Diffstat (limited to 'testing')
-rw-r--r--testing/mozharness/LICENSE373
-rw-r--r--testing/mozharness/README.txt32
-rw-r--r--testing/mozharness/configs/balrog/docker-worker.py18
-rw-r--r--testing/mozharness/configs/balrog/production.py28
-rw-r--r--testing/mozharness/configs/balrog/staging.py14
-rw-r--r--testing/mozharness/configs/beetmover/en_us_build.yml.tmpl191
-rw-r--r--testing/mozharness/configs/beetmover/en_us_signing.yml.tmpl66
-rw-r--r--testing/mozharness/configs/beetmover/l10n_changesets.tmpl11
-rw-r--r--testing/mozharness/configs/beetmover/partials.yml.tmpl16
-rw-r--r--testing/mozharness/configs/beetmover/repacks.yml.tmpl65
-rw-r--r--testing/mozharness/configs/beetmover/snap.yml.tmpl11
-rw-r--r--testing/mozharness/configs/beetmover/snap_checksums.yml.tmpl14
-rw-r--r--testing/mozharness/configs/beetmover/source.yml.tmpl14
-rw-r--r--testing/mozharness/configs/beetmover/source_checksums.yml.tmpl14
-rw-r--r--testing/mozharness/configs/builds/branch_specifics.py469
-rw-r--r--testing/mozharness/configs/builds/build_pool_specifics.py44
-rw-r--r--testing/mozharness/configs/builds/releng_base_linux_32_builds.py160
-rw-r--r--testing/mozharness/configs/builds/releng_base_linux_64_builds.py139
-rw-r--r--testing/mozharness/configs/builds/releng_base_mac_64_builds.py79
-rw-r--r--testing/mozharness/configs/builds/releng_base_mac_64_cross_builds.py83
-rw-r--r--testing/mozharness/configs/builds/releng_base_windows_32_builds.py95
-rw-r--r--testing/mozharness/configs/builds/releng_base_windows_64_builds.py93
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/32_artifact.py116
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/32_debug.py45
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/32_debug_artifact.py122
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_add-on-devel.py43
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_artifact.py98
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan.py48
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_and_debug.py49
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_tc.py48
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_tc_and_debug.py49
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_code_coverage.py45
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_debug.py45
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_debug_artifact.py96
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_source.py20
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_debug.py50
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_opt.py88
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_tsan.py46
-rw-r--r--testing/mozharness/configs/builds/releng_sub_linux_configs/64_valgrind.py49
-rw-r--r--testing/mozharness/configs/builds/releng_sub_mac_configs/64_add-on-devel.py44
-rw-r--r--testing/mozharness/configs/builds/releng_sub_mac_configs/64_artifact.py65
-rw-r--r--testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_debug.py43
-rw-r--r--testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_opt.py39
-rw-r--r--testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_universal.py4
-rw-r--r--testing/mozharness/configs/builds/releng_sub_mac_configs/64_debug.py44
-rw-r--r--testing/mozharness/configs/builds/releng_sub_mac_configs/64_debug_artifact.py65
-rw-r--r--testing/mozharness/configs/builds/releng_sub_mac_configs/64_stat_and_debug.py48
-rw-r--r--testing/mozharness/configs/builds/releng_sub_windows_configs/32_add-on-devel.py38
-rw-r--r--testing/mozharness/configs/builds/releng_sub_windows_configs/32_artifact.py81
-rw-r--r--testing/mozharness/configs/builds/releng_sub_windows_configs/32_debug.py40
-rw-r--r--testing/mozharness/configs/builds/releng_sub_windows_configs/32_debug_artifact.py86
-rw-r--r--testing/mozharness/configs/builds/releng_sub_windows_configs/32_stat_and_debug.py44
-rw-r--r--testing/mozharness/configs/builds/releng_sub_windows_configs/64_add-on-devel.py37
-rw-r--r--testing/mozharness/configs/builds/releng_sub_windows_configs/64_artifact.py79
-rw-r--r--testing/mozharness/configs/builds/releng_sub_windows_configs/64_debug.py39
-rw-r--r--testing/mozharness/configs/builds/releng_sub_windows_configs/64_debug_artifact.py85
-rw-r--r--testing/mozharness/configs/builds/taskcluster_firefox_win32_debug.py91
-rw-r--r--testing/mozharness/configs/builds/taskcluster_firefox_win32_opt.py89
-rw-r--r--testing/mozharness/configs/builds/taskcluster_firefox_win64_debug.py87
-rw-r--r--testing/mozharness/configs/builds/taskcluster_firefox_win64_opt.py85
-rw-r--r--testing/mozharness/configs/developer_config.py49
-rw-r--r--testing/mozharness/configs/disable_signing.py3
-rw-r--r--testing/mozharness/configs/firefox_ui_tests/qa_jenkins.py19
-rw-r--r--testing/mozharness/configs/firefox_ui_tests/releng_release.py33
-rw-r--r--testing/mozharness/configs/firefox_ui_tests/taskcluster.py11
-rw-r--r--testing/mozharness/configs/hazards/build_browser.py4
-rw-r--r--testing/mozharness/configs/hazards/build_shell.py4
-rw-r--r--testing/mozharness/configs/hazards/common.py104
-rw-r--r--testing/mozharness/configs/marionette/prod_config.py56
-rw-r--r--testing/mozharness/configs/marionette/test_config.py29
-rw-r--r--testing/mozharness/configs/marionette/windows_config.py57
-rw-r--r--testing/mozharness/configs/marionette/windows_taskcluster_config.py56
-rw-r--r--testing/mozharness/configs/mediatests/buildbot_posix_config.py50
-rw-r--r--testing/mozharness/configs/mediatests/buildbot_windows_config.py56
-rwxr-xr-xtesting/mozharness/configs/mediatests/jenkins_config.py48
-rw-r--r--testing/mozharness/configs/mediatests/taskcluster_posix_config.py47
-rw-r--r--testing/mozharness/configs/mediatests/taskcluster_windows_config.py50
-rw-r--r--testing/mozharness/configs/partner_repacks/release_mozilla-esr52_desktop.py6
-rw-r--r--testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py6
-rw-r--r--testing/mozharness/configs/partner_repacks/staging_release_mozilla-release_desktop.py6
-rw-r--r--testing/mozharness/configs/platform_supports_post_upload_to_latest.py3
-rw-r--r--testing/mozharness/configs/releases/bouncer_firefox_beta.py148
-rw-r--r--testing/mozharness/configs/releases/bouncer_firefox_esr.py136
-rw-r--r--testing/mozharness/configs/releases/bouncer_firefox_release.py191
-rw-r--r--testing/mozharness/configs/releases/bouncer_thunderbird.py169
-rw-r--r--testing/mozharness/configs/releases/dev_bouncer_firefox_beta.py133
-rw-r--r--testing/mozharness/configs/releases/dev_postrelease_firefox_beta.py20
-rw-r--r--testing/mozharness/configs/releases/dev_postrelease_firefox_release.py22
-rw-r--r--testing/mozharness/configs/releases/dev_updates_firefox_beta.py39
-rw-r--r--testing/mozharness/configs/releases/dev_updates_firefox_release.py50
-rw-r--r--testing/mozharness/configs/releases/postrelease_firefox_beta.py18
-rw-r--r--testing/mozharness/configs/releases/postrelease_firefox_esr52.py22
-rw-r--r--testing/mozharness/configs/releases/postrelease_firefox_release.py22
-rw-r--r--testing/mozharness/configs/releases/updates_firefox_beta.py35
-rw-r--r--testing/mozharness/configs/releases/updates_firefox_esr52.py35
-rw-r--r--testing/mozharness/configs/releases/updates_firefox_release.py47
-rw-r--r--testing/mozharness/configs/releng_infra_configs/builders.py47
-rw-r--r--testing/mozharness/configs/releng_infra_configs/linux.py5
-rw-r--r--testing/mozharness/configs/releng_infra_configs/linux64.py5
-rw-r--r--testing/mozharness/configs/releng_infra_configs/macosx64.py5
-rw-r--r--testing/mozharness/configs/releng_infra_configs/testers.py67
-rw-r--r--testing/mozharness/configs/releng_infra_configs/win32.py5
-rw-r--r--testing/mozharness/configs/releng_infra_configs/win64.py5
-rw-r--r--testing/mozharness/configs/remove_executables.py8
-rw-r--r--testing/mozharness/configs/routes.json18
-rw-r--r--testing/mozharness/configs/selfserve/production.py3
-rw-r--r--testing/mozharness/configs/selfserve/staging.py3
-rw-r--r--testing/mozharness/configs/single_locale/alder.py46
-rw-r--r--testing/mozharness/configs/single_locale/ash.py46
-rw-r--r--testing/mozharness/configs/single_locale/dev-mozilla-beta.py37
-rw-r--r--testing/mozharness/configs/single_locale/dev-mozilla-release.py37
-rw-r--r--testing/mozharness/configs/single_locale/linux.py123
l---------testing/mozharness/configs/single_locale/linux32.py1
-rw-r--r--testing/mozharness/configs/single_locale/linux64.py103
-rw-r--r--testing/mozharness/configs/single_locale/macosx64.py72
-rw-r--r--testing/mozharness/configs/single_locale/mozilla-aurora.py29
-rw-r--r--testing/mozharness/configs/single_locale/mozilla-beta.py37
-rw-r--r--testing/mozharness/configs/single_locale/mozilla-central.py29
-rw-r--r--testing/mozharness/configs/single_locale/mozilla-esr52.py37
-rw-r--r--testing/mozharness/configs/single_locale/mozilla-release.py37
-rw-r--r--testing/mozharness/configs/single_locale/production.py14
-rw-r--r--testing/mozharness/configs/single_locale/staging.py17
-rw-r--r--testing/mozharness/configs/single_locale/tc_linux32.py24
-rw-r--r--testing/mozharness/configs/single_locale/tc_linux64.py24
-rw-r--r--testing/mozharness/configs/single_locale/try.py42
-rw-r--r--testing/mozharness/configs/single_locale/win32.py77
-rw-r--r--testing/mozharness/configs/single_locale/win64.py77
-rw-r--r--testing/mozharness/configs/talos/linux_config.py46
-rw-r--r--testing/mozharness/configs/talos/mac_config.py56
-rw-r--r--testing/mozharness/configs/talos/windows_config.py48
-rw-r--r--testing/mozharness/configs/taskcluster_nightly.py5
-rw-r--r--testing/mozharness/configs/test/example_config1.json5
-rw-r--r--testing/mozharness/configs/test/example_config2.py5
-rw-r--r--testing/mozharness/configs/test/test.illegal_suffix20
-rw-r--r--testing/mozharness/configs/test/test.json20
-rw-r--r--testing/mozharness/configs/test/test.py22
-rw-r--r--testing/mozharness/configs/test/test_malformed.json20
-rw-r--r--testing/mozharness/configs/test/test_malformed.py22
-rw-r--r--testing/mozharness/configs/test/test_optional.py4
-rw-r--r--testing/mozharness/configs/test/test_override.py7
-rw-r--r--testing/mozharness/configs/test/test_override2.py6
-rw-r--r--testing/mozharness/configs/unittests/linux_unittest.py307
-rw-r--r--testing/mozharness/configs/unittests/mac_unittest.py257
-rw-r--r--testing/mozharness/configs/unittests/thunderbird_extra.py17
-rw-r--r--testing/mozharness/configs/unittests/win_taskcluster_unittest.py274
-rw-r--r--testing/mozharness/configs/unittests/win_unittest.py281
-rw-r--r--testing/mozharness/configs/users/aki/gaia_json.py42
-rw-r--r--testing/mozharness/configs/users/sfink/mock.py3
-rw-r--r--testing/mozharness/configs/users/sfink/spidermonkey.py38
-rw-r--r--testing/mozharness/configs/web_platform_tests/prod_config.py47
-rw-r--r--testing/mozharness/configs/web_platform_tests/prod_config_windows.py48
-rw-r--r--testing/mozharness/configs/web_platform_tests/prod_config_windows_taskcluster.py48
-rw-r--r--testing/mozharness/configs/web_platform_tests/test_config.py32
-rw-r--r--testing/mozharness/configs/web_platform_tests/test_config_windows.py43
-rw-r--r--testing/mozharness/docs/Makefile177
-rw-r--r--testing/mozharness/docs/android_emulator_build.rst7
-rw-r--r--testing/mozharness/docs/android_emulator_unittest.rst7
-rw-r--r--testing/mozharness/docs/bouncer_submitter.rst8
-rw-r--r--testing/mozharness/docs/bump_gaia_json.rst7
-rw-r--r--testing/mozharness/docs/conf.py268
-rw-r--r--testing/mozharness/docs/configtest.rst7
-rw-r--r--testing/mozharness/docs/desktop_l10n.rst7
-rw-r--r--testing/mozharness/docs/desktop_unittest.rst7
-rw-r--r--testing/mozharness/docs/fx_desktop_build.rst7
-rw-r--r--testing/mozharness/docs/gaia_build_integration.rst7
-rw-r--r--testing/mozharness/docs/gaia_integration.rst7
-rw-r--r--testing/mozharness/docs/gaia_unit.rst7
-rw-r--r--testing/mozharness/docs/index.rst24
-rw-r--r--testing/mozharness/docs/marionette.rst7
-rw-r--r--testing/mozharness/docs/mobile_l10n.rst7
-rw-r--r--testing/mozharness/docs/mobile_partner_repack.rst7
-rw-r--r--testing/mozharness/docs/modules.rst13
-rw-r--r--testing/mozharness/docs/mozharness.base.rst101
-rw-r--r--testing/mozharness/docs/mozharness.base.vcs.rst46
-rw-r--r--testing/mozharness/docs/mozharness.mozilla.building.rst22
-rw-r--r--testing/mozharness/docs/mozharness.mozilla.l10n.rst30
-rw-r--r--testing/mozharness/docs/mozharness.mozilla.rst111
-rw-r--r--testing/mozharness/docs/mozharness.mozilla.testing.rst62
-rw-r--r--testing/mozharness/docs/mozharness.rst18
-rw-r--r--testing/mozharness/docs/multil10n.rst7
-rw-r--r--testing/mozharness/docs/scripts.rst22
-rw-r--r--testing/mozharness/docs/spidermonkey_build.rst7
-rw-r--r--testing/mozharness/docs/talos_script.rst7
-rw-r--r--testing/mozharness/docs/web_platform_tests.rst7
-rwxr-xr-xtesting/mozharness/examples/action_config_script.py130
-rwxr-xr-xtesting/mozharness/examples/silent_script.py22
-rwxr-xr-xtesting/mozharness/examples/venv.py41
-rwxr-xr-xtesting/mozharness/examples/verbose_script.py63
-rw-r--r--testing/mozharness/external_tools/__init__.py0
-rwxr-xr-xtesting/mozharness/external_tools/clobberer.py280
-rwxr-xr-xtesting/mozharness/external_tools/count_and_reboot.py62
-rw-r--r--testing/mozharness/external_tools/detect_repo.py52
-rwxr-xr-xtesting/mozharness/external_tools/download_file.py69
-rw-r--r--testing/mozharness/external_tools/extract_and_run_command.py205
-rwxr-xr-xtesting/mozharness/external_tools/git-ssh-wrapper.sh12
-rwxr-xr-xtesting/mozharness/external_tools/gittool.py94
-rw-r--r--testing/mozharness/external_tools/machine-configuration.json12
-rwxr-xr-xtesting/mozharness/external_tools/mouse_and_screen_resolution.py153
-rw-r--r--testing/mozharness/external_tools/performance-artifact-schema.json164
-rw-r--r--testing/mozharness/external_tools/robustcheckout.py451
-rw-r--r--testing/mozharness/external_tools/virtualenv/AUTHORS.txt91
-rw-r--r--testing/mozharness/external_tools/virtualenv/LICENSE.txt22
-rw-r--r--testing/mozharness/external_tools/virtualenv/MANIFEST.in12
-rw-r--r--testing/mozharness/external_tools/virtualenv/PKG-INFO87
-rw-r--r--testing/mozharness/external_tools/virtualenv/README.rst31
-rwxr-xr-xtesting/mozharness/external_tools/virtualenv/bin/rebuild-script.py73
-rw-r--r--testing/mozharness/external_tools/virtualenv/docs/Makefile130
-rw-r--r--testing/mozharness/external_tools/virtualenv/docs/changes.rst985
-rw-r--r--testing/mozharness/external_tools/virtualenv/docs/conf.py153
-rw-r--r--testing/mozharness/external_tools/virtualenv/docs/development.rst61
-rw-r--r--testing/mozharness/external_tools/virtualenv/docs/index.rst137
-rw-r--r--testing/mozharness/external_tools/virtualenv/docs/installation.rst58
-rw-r--r--testing/mozharness/external_tools/virtualenv/docs/make.bat170
-rw-r--r--testing/mozharness/external_tools/virtualenv/docs/reference.rst261
-rw-r--r--testing/mozharness/external_tools/virtualenv/docs/userguide.rst258
-rw-r--r--testing/mozharness/external_tools/virtualenv/scripts/virtualenv3
-rw-r--r--testing/mozharness/external_tools/virtualenv/setup.cfg8
-rw-r--r--testing/mozharness/external_tools/virtualenv/setup.py123
-rw-r--r--testing/mozharness/external_tools/virtualenv/site.py760
-rw-r--r--testing/mozharness/external_tools/virtualenv/tests/__init__.py0
-rwxr-xr-xtesting/mozharness/external_tools/virtualenv/tests/test_activate.sh96
-rw-r--r--testing/mozharness/external_tools/virtualenv/tests/test_activate_output.expected2
-rw-r--r--testing/mozharness/external_tools/virtualenv/tests/test_cmdline.py44
-rw-r--r--testing/mozharness/external_tools/virtualenv/tests/test_virtualenv.py139
-rwxr-xr-xtesting/mozharness/external_tools/virtualenv/virtualenv.py2329
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.bat30
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.csh36
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.fish76
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.ps1150
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.sh78
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate_this.py34
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_embedded/deactivate.bat19
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils-init.py101
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils.cfg6
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_embedded/python-config78
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_embedded/site.py758
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_support/__init__.py0
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_support/argparse-1.4.0-py2.py3-none-any.whlbin23000 -> 0 bytes
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_support/pip-8.1.2-py2.py3-none-any.whlbin1198961 -> 0 bytes
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_support/setuptools-25.2.0-py2.py3-none-any.whlbin442860 -> 0 bytes
-rw-r--r--testing/mozharness/external_tools/virtualenv/virtualenv_support/wheel-0.29.0-py2.py3-none-any.whlbin66878 -> 0 bytes
-rw-r--r--testing/mozharness/mach_commands.py196
-rw-r--r--testing/mozharness/mozfile/__init__.py5
-rw-r--r--testing/mozharness/mozfile/mozfile.py372
-rw-r--r--testing/mozharness/mozharness/__init__.py2
-rw-r--r--testing/mozharness/mozharness/base/__init__.py0
-rw-r--r--testing/mozharness/mozharness/base/config.py569
-rw-r--r--testing/mozharness/mozharness/base/diskutils.py156
-rwxr-xr-xtesting/mozharness/mozharness/base/errors.py213
-rwxr-xr-xtesting/mozharness/mozharness/base/log.py694
-rwxr-xr-xtesting/mozharness/mozharness/base/parallel.py36
-rw-r--r--testing/mozharness/mozharness/base/python.py743
-rwxr-xr-xtesting/mozharness/mozharness/base/script.py2273
-rwxr-xr-xtesting/mozharness/mozharness/base/signing.py164
-rwxr-xr-xtesting/mozharness/mozharness/base/transfer.py123
-rw-r--r--testing/mozharness/mozharness/base/vcs/__init__.py0
-rw-r--r--testing/mozharness/mozharness/base/vcs/gittool.py95
-rwxr-xr-xtesting/mozharness/mozharness/base/vcs/mercurial.py497
-rw-r--r--testing/mozharness/mozharness/base/vcs/tcvcs.py49
-rwxr-xr-xtesting/mozharness/mozharness/base/vcs/vcsbase.py149
-rw-r--r--testing/mozharness/mozharness/base/vcs/vcssync.py101
-rw-r--r--testing/mozharness/mozharness/lib/__init__.py0
-rw-r--r--testing/mozharness/mozharness/lib/python/__init__.py0
-rw-r--r--testing/mozharness/mozharness/lib/python/authentication.py53
-rw-r--r--testing/mozharness/mozharness/mozilla/__init__.py0
-rw-r--r--testing/mozharness/mozharness/mozilla/aws.py11
-rw-r--r--testing/mozharness/mozharness/mozilla/blob_upload.py109
-rw-r--r--testing/mozharness/mozharness/mozilla/bouncer/__init__.py0
-rw-r--r--testing/mozharness/mozharness/mozilla/bouncer/submitter.py114
-rwxr-xr-xtesting/mozharness/mozharness/mozilla/buildbot.py246
-rw-r--r--testing/mozharness/mozharness/mozilla/building/__init__.py0
-rwxr-xr-xtesting/mozharness/mozharness/mozilla/building/buildbase.py2153
-rw-r--r--testing/mozharness/mozharness/mozilla/building/hazards.py241
-rw-r--r--testing/mozharness/mozharness/mozilla/checksums.py21
-rw-r--r--testing/mozharness/mozharness/mozilla/l10n/__init__.py0
-rwxr-xr-xtesting/mozharness/mozharness/mozilla/l10n/locales.py280
-rwxr-xr-xtesting/mozharness/mozharness/mozilla/l10n/multi_locale_build.py254
-rw-r--r--testing/mozharness/mozharness/mozilla/mapper.py81
-rw-r--r--testing/mozharness/mozharness/mozilla/mar.py112
-rw-r--r--testing/mozharness/mozharness/mozilla/mock.py251
-rw-r--r--testing/mozharness/mozharness/mozilla/mozbase.py39
-rw-r--r--testing/mozharness/mozharness/mozilla/proxxy.py167
-rw-r--r--testing/mozharness/mozharness/mozilla/purge.py103
-rwxr-xr-xtesting/mozharness/mozharness/mozilla/release.py72
-rw-r--r--testing/mozharness/mozharness/mozilla/repo_manifest.py226
-rw-r--r--testing/mozharness/mozharness/mozilla/repo_manupulation.py164
-rw-r--r--testing/mozharness/mozharness/mozilla/secrets.py74
-rw-r--r--testing/mozharness/mozharness/mozilla/selfserve.py47
-rwxr-xr-xtesting/mozharness/mozharness/mozilla/signing.py101
-rw-r--r--testing/mozharness/mozharness/mozilla/structuredlog.py173
-rw-r--r--testing/mozharness/mozharness/mozilla/taskcluster_helper.py274
-rw-r--r--testing/mozharness/mozharness/mozilla/testing/__init__.py0
-rw-r--r--testing/mozharness/mozharness/mozilla/testing/codecoverage.py78
-rw-r--r--testing/mozharness/mozharness/mozilla/testing/device.py738
-rw-r--r--testing/mozharness/mozharness/mozilla/testing/errors.py118
-rw-r--r--testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py289
-rw-r--r--testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py300
-rw-r--r--testing/mozharness/mozharness/mozilla/testing/mozpool.py134
-rwxr-xr-xtesting/mozharness/mozharness/mozilla/testing/talos.py430
-rwxr-xr-xtesting/mozharness/mozharness/mozilla/testing/testbase.py862
-rw-r--r--testing/mozharness/mozharness/mozilla/testing/try_tools.py258
-rwxr-xr-xtesting/mozharness/mozharness/mozilla/testing/unittest.py262
-rw-r--r--testing/mozharness/mozharness/mozilla/tooltool.py129
-rw-r--r--testing/mozharness/mozharness/mozilla/updates/__init__.py0
-rw-r--r--testing/mozharness/mozharness/mozilla/updates/balrog.py149
-rw-r--r--testing/mozharness/mozharness/mozilla/vcstools.py57
-rw-r--r--testing/mozharness/mozinfo/__init__.py56
-rwxr-xr-xtesting/mozharness/mozinfo/mozinfo.py209
-rw-r--r--testing/mozharness/mozprocess/__init__.py5
-rwxr-xr-xtesting/mozharness/mozprocess/pid.py88
-rw-r--r--testing/mozharness/mozprocess/processhandler.py921
-rw-r--r--testing/mozharness/mozprocess/qijo.py140
-rw-r--r--testing/mozharness/mozprocess/winprocess.py457
-rw-r--r--testing/mozharness/mozprocess/wpk.py54
-rw-r--r--testing/mozharness/requirements.txt25
-rw-r--r--testing/mozharness/scripts/android_emulator_unittest.py733
-rwxr-xr-xtesting/mozharness/scripts/bouncer_submitter.py192
-rwxr-xr-xtesting/mozharness/scripts/configtest.py142
-rwxr-xr-xtesting/mozharness/scripts/desktop_l10n.py1152
-rwxr-xr-xtesting/mozharness/scripts/desktop_partner_repacks.py198
-rwxr-xr-xtesting/mozharness/scripts/desktop_unittest.py742
-rw-r--r--testing/mozharness/scripts/firefox_media_tests_buildbot.py122
-rwxr-xr-xtesting/mozharness/scripts/firefox_media_tests_jenkins.py48
-rw-r--r--testing/mozharness/scripts/firefox_media_tests_taskcluster.py110
-rwxr-xr-xtesting/mozharness/scripts/firefox_ui_tests/functional.py20
-rwxr-xr-xtesting/mozharness/scripts/firefox_ui_tests/update.py20
-rwxr-xr-xtesting/mozharness/scripts/firefox_ui_tests/update_release.py323
-rwxr-xr-xtesting/mozharness/scripts/fx_desktop_build.py235
-rwxr-xr-xtesting/mozharness/scripts/gaia_build_integration.py56
-rwxr-xr-xtesting/mozharness/scripts/gaia_build_unit.py56
-rw-r--r--testing/mozharness/scripts/gaia_integration.py75
-rwxr-xr-xtesting/mozharness/scripts/gaia_linter.py148
-rwxr-xr-xtesting/mozharness/scripts/gaia_unit.py109
-rwxr-xr-xtesting/mozharness/scripts/marionette.py358
-rw-r--r--testing/mozharness/scripts/marionette_harness_tests.py141
-rwxr-xr-xtesting/mozharness/scripts/merge_day/gecko_migration.py545
-rwxr-xr-xtesting/mozharness/scripts/mobile_l10n.py714
-rwxr-xr-xtesting/mozharness/scripts/mobile_partner_repack.py327
-rwxr-xr-xtesting/mozharness/scripts/multil10n.py21
-rw-r--r--testing/mozharness/scripts/openh264_build.py250
-rw-r--r--testing/mozharness/scripts/release/antivirus.py192
-rwxr-xr-xtesting/mozharness/scripts/release/beet_mover.py371
-rw-r--r--testing/mozharness/scripts/release/generate-checksums.py284
-rw-r--r--testing/mozharness/scripts/release/postrelease_bouncer_aliases.py107
-rw-r--r--testing/mozharness/scripts/release/postrelease_mark_as_shipped.py110
-rw-r--r--testing/mozharness/scripts/release/postrelease_version_bump.py184
-rw-r--r--testing/mozharness/scripts/release/publish_balrog.py119
-rw-r--r--testing/mozharness/scripts/release/push-candidate-to-releases.py199
-rw-r--r--testing/mozharness/scripts/release/updates.py299
-rw-r--r--testing/mozharness/scripts/release/uptake_monitoring.py188
-rwxr-xr-xtesting/mozharness/scripts/spidermonkey/build.b2g8
-rwxr-xr-xtesting/mozharness/scripts/spidermonkey/build.browser10
-rwxr-xr-xtesting/mozharness/scripts/spidermonkey/build.shell6
-rwxr-xr-xtesting/mozharness/scripts/spidermonkey_build.py482
-rwxr-xr-xtesting/mozharness/scripts/talos_script.py21
-rwxr-xr-xtesting/mozharness/scripts/web_platform_tests.py258
-rw-r--r--testing/mozharness/setup.cfg2
-rw-r--r--testing/mozharness/setup.py35
-rw-r--r--testing/mozharness/test/README2
-rw-r--r--testing/mozharness/test/helper_files/.noserc2
-rw-r--r--testing/mozharness/test/helper_files/archives/archive.tarbin10240 -> 0 bytes
-rw-r--r--testing/mozharness/test/helper_files/archives/archive.tar.bz2bin256 -> 0 bytes
-rw-r--r--testing/mozharness/test/helper_files/archives/archive.tar.gzbin260 -> 0 bytes
-rw-r--r--testing/mozharness/test/helper_files/archives/archive.zipbin517 -> 0 bytes
-rw-r--r--testing/mozharness/test/helper_files/archives/archive_invalid_filename.zipbin166 -> 0 bytes
-rwxr-xr-xtesting/mozharness/test/helper_files/archives/reference/bin/script.sh3
-rw-r--r--testing/mozharness/test/helper_files/archives/reference/lorem.txt1
-rwxr-xr-xtesting/mozharness/test/helper_files/create_archives.sh11
-rwxr-xr-xtesting/mozharness/test/helper_files/init_hgrepo.sh24
-rw-r--r--testing/mozharness/test/helper_files/locales.json18
-rw-r--r--testing/mozharness/test/helper_files/locales.txt4
-rw-r--r--testing/mozharness/test/hgrc9
-rw-r--r--testing/mozharness/test/pip-freeze.example.txt19
-rw-r--r--testing/mozharness/test/test_base_config.py308
-rw-r--r--testing/mozharness/test/test_base_diskutils.py84
-rw-r--r--testing/mozharness/test/test_base_log.py42
-rw-r--r--testing/mozharness/test/test_base_parallel.py26
-rw-r--r--testing/mozharness/test/test_base_python.py37
-rw-r--r--testing/mozharness/test/test_base_script.py898
-rw-r--r--testing/mozharness/test/test_base_transfer.py127
-rw-r--r--testing/mozharness/test/test_base_vcs_mercurial.py440
-rw-r--r--testing/mozharness/test/test_l10n_locales.py132
-rw-r--r--testing/mozharness/test/test_mozilla_blob_upload.py103
-rw-r--r--testing/mozharness/test/test_mozilla_buildbot.py62
-rw-r--r--testing/mozharness/test/test_mozilla_release.py42
-rw-r--r--testing/mozharness/tox.ini27
-rwxr-xr-xtesting/mozharness/unit.sh85
387 files changed, 0 insertions, 48916 deletions
diff --git a/testing/mozharness/LICENSE b/testing/mozharness/LICENSE
deleted file mode 100644
index a612ad981..000000000
--- a/testing/mozharness/LICENSE
+++ /dev/null
@@ -1,373 +0,0 @@
-Mozilla Public License Version 2.0
-==================================
-
-1. Definitions
---------------
-
-1.1. "Contributor"
- means each individual or legal entity that creates, contributes to
- the creation of, or owns Covered Software.
-
-1.2. "Contributor Version"
- means the combination of the Contributions of others (if any) used
- by a Contributor and that particular Contributor's Contribution.
-
-1.3. "Contribution"
- means Covered Software of a particular Contributor.
-
-1.4. "Covered Software"
- means Source Code Form to which the initial Contributor has attached
- the notice in Exhibit A, the Executable Form of such Source Code
- Form, and Modifications of such Source Code Form, in each case
- including portions thereof.
-
-1.5. "Incompatible With Secondary Licenses"
- means
-
- (a) that the initial Contributor has attached the notice described
- in Exhibit B to the Covered Software; or
-
- (b) that the Covered Software was made available under the terms of
- version 1.1 or earlier of the License, but not also under the
- terms of a Secondary License.
-
-1.6. "Executable Form"
- means any form of the work other than Source Code Form.
-
-1.7. "Larger Work"
- means a work that combines Covered Software with other material, in
- a separate file or files, that is not Covered Software.
-
-1.8. "License"
- means this document.
-
-1.9. "Licensable"
- means having the right to grant, to the maximum extent possible,
- whether at the time of the initial grant or subsequently, any and
- all of the rights conveyed by this License.
-
-1.10. "Modifications"
- means any of the following:
-
- (a) any file in Source Code Form that results from an addition to,
- deletion from, or modification of the contents of Covered
- Software; or
-
- (b) any new file in Source Code Form that contains any Covered
- Software.
-
-1.11. "Patent Claims" of a Contributor
- means any patent claim(s), including without limitation, method,
- process, and apparatus claims, in any patent Licensable by such
- Contributor that would be infringed, but for the grant of the
- License, by the making, using, selling, offering for sale, having
- made, import, or transfer of either its Contributions or its
- Contributor Version.
-
-1.12. "Secondary License"
- means either the GNU General Public License, Version 2.0, the GNU
- Lesser General Public License, Version 2.1, the GNU Affero General
- Public License, Version 3.0, or any later versions of those
- licenses.
-
-1.13. "Source Code Form"
- means the form of the work preferred for making modifications.
-
-1.14. "You" (or "Your")
- means an individual or a legal entity exercising rights under this
- License. For legal entities, "You" includes any entity that
- controls, is controlled by, or is under common control with You. For
- purposes of this definition, "control" means (a) the power, direct
- or indirect, to cause the direction or management of such entity,
- whether by contract or otherwise, or (b) ownership of more than
- fifty percent (50%) of the outstanding shares or beneficial
- ownership of such entity.
-
-2. License Grants and Conditions
---------------------------------
-
-2.1. Grants
-
-Each Contributor hereby grants You a world-wide, royalty-free,
-non-exclusive license:
-
-(a) under intellectual property rights (other than patent or trademark)
- Licensable by such Contributor to use, reproduce, make available,
- modify, display, perform, distribute, and otherwise exploit its
- Contributions, either on an unmodified basis, with Modifications, or
- as part of a Larger Work; and
-
-(b) under Patent Claims of such Contributor to make, use, sell, offer
- for sale, have made, import, and otherwise transfer either its
- Contributions or its Contributor Version.
-
-2.2. Effective Date
-
-The licenses granted in Section 2.1 with respect to any Contribution
-become effective for each Contribution on the date the Contributor first
-distributes such Contribution.
-
-2.3. Limitations on Grant Scope
-
-The licenses granted in this Section 2 are the only rights granted under
-this License. No additional rights or licenses will be implied from the
-distribution or licensing of Covered Software under this License.
-Notwithstanding Section 2.1(b) above, no patent license is granted by a
-Contributor:
-
-(a) for any code that a Contributor has removed from Covered Software;
- or
-
-(b) for infringements caused by: (i) Your and any other third party's
- modifications of Covered Software, or (ii) the combination of its
- Contributions with other software (except as part of its Contributor
- Version); or
-
-(c) under Patent Claims infringed by Covered Software in the absence of
- its Contributions.
-
-This License does not grant any rights in the trademarks, service marks,
-or logos of any Contributor (except as may be necessary to comply with
-the notice requirements in Section 3.4).
-
-2.4. Subsequent Licenses
-
-No Contributor makes additional grants as a result of Your choice to
-distribute the Covered Software under a subsequent version of this
-License (see Section 10.2) or under the terms of a Secondary License (if
-permitted under the terms of Section 3.3).
-
-2.5. Representation
-
-Each Contributor represents that the Contributor believes its
-Contributions are its original creation(s) or it has sufficient rights
-to grant the rights to its Contributions conveyed by this License.
-
-2.6. Fair Use
-
-This License is not intended to limit any rights You have under
-applicable copyright doctrines of fair use, fair dealing, or other
-equivalents.
-
-2.7. Conditions
-
-Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
-in Section 2.1.
-
-3. Responsibilities
--------------------
-
-3.1. Distribution of Source Form
-
-All distribution of Covered Software in Source Code Form, including any
-Modifications that You create or to which You contribute, must be under
-the terms of this License. You must inform recipients that the Source
-Code Form of the Covered Software is governed by the terms of this
-License, and how they can obtain a copy of this License. You may not
-attempt to alter or restrict the recipients' rights in the Source Code
-Form.
-
-3.2. Distribution of Executable Form
-
-If You distribute Covered Software in Executable Form then:
-
-(a) such Covered Software must also be made available in Source Code
- Form, as described in Section 3.1, and You must inform recipients of
- the Executable Form how they can obtain a copy of such Source Code
- Form by reasonable means in a timely manner, at a charge no more
- than the cost of distribution to the recipient; and
-
-(b) You may distribute such Executable Form under the terms of this
- License, or sublicense it under different terms, provided that the
- license for the Executable Form does not attempt to limit or alter
- the recipients' rights in the Source Code Form under this License.
-
-3.3. Distribution of a Larger Work
-
-You may create and distribute a Larger Work under terms of Your choice,
-provided that You also comply with the requirements of this License for
-the Covered Software. If the Larger Work is a combination of Covered
-Software with a work governed by one or more Secondary Licenses, and the
-Covered Software is not Incompatible With Secondary Licenses, this
-License permits You to additionally distribute such Covered Software
-under the terms of such Secondary License(s), so that the recipient of
-the Larger Work may, at their option, further distribute the Covered
-Software under the terms of either this License or such Secondary
-License(s).
-
-3.4. Notices
-
-You may not remove or alter the substance of any license notices
-(including copyright notices, patent notices, disclaimers of warranty,
-or limitations of liability) contained within the Source Code Form of
-the Covered Software, except that You may alter any license notices to
-the extent required to remedy known factual inaccuracies.
-
-3.5. Application of Additional Terms
-
-You may choose to offer, and to charge a fee for, warranty, support,
-indemnity or liability obligations to one or more recipients of Covered
-Software. However, You may do so only on Your own behalf, and not on
-behalf of any Contributor. You must make it absolutely clear that any
-such warranty, support, indemnity, or liability obligation is offered by
-You alone, and You hereby agree to indemnify every Contributor for any
-liability incurred by such Contributor as a result of warranty, support,
-indemnity or liability terms You offer. You may include additional
-disclaimers of warranty and limitations of liability specific to any
-jurisdiction.
-
-4. Inability to Comply Due to Statute or Regulation
----------------------------------------------------
-
-If it is impossible for You to comply with any of the terms of this
-License with respect to some or all of the Covered Software due to
-statute, judicial order, or regulation then You must: (a) comply with
-the terms of this License to the maximum extent possible; and (b)
-describe the limitations and the code they affect. Such description must
-be placed in a text file included with all distributions of the Covered
-Software under this License. Except to the extent prohibited by statute
-or regulation, such description must be sufficiently detailed for a
-recipient of ordinary skill to be able to understand it.
-
-5. Termination
---------------
-
-5.1. The rights granted under this License will terminate automatically
-if You fail to comply with any of its terms. However, if You become
-compliant, then the rights granted under this License from a particular
-Contributor are reinstated (a) provisionally, unless and until such
-Contributor explicitly and finally terminates Your grants, and (b) on an
-ongoing basis, if such Contributor fails to notify You of the
-non-compliance by some reasonable means prior to 60 days after You have
-come back into compliance. Moreover, Your grants from a particular
-Contributor are reinstated on an ongoing basis if such Contributor
-notifies You of the non-compliance by some reasonable means, this is the
-first time You have received notice of non-compliance with this License
-from such Contributor, and You become compliant prior to 30 days after
-Your receipt of the notice.
-
-5.2. If You initiate litigation against any entity by asserting a patent
-infringement claim (excluding declaratory judgment actions,
-counter-claims, and cross-claims) alleging that a Contributor Version
-directly or indirectly infringes any patent, then the rights granted to
-You by any and all Contributors for the Covered Software under Section
-2.1 of this License shall terminate.
-
-5.3. In the event of termination under Sections 5.1 or 5.2 above, all
-end user license agreements (excluding distributors and resellers) which
-have been validly granted by You or Your distributors under this License
-prior to termination shall survive termination.
-
-************************************************************************
-* *
-* 6. Disclaimer of Warranty *
-* ------------------------- *
-* *
-* Covered Software is provided under this License on an "as is" *
-* basis, without warranty of any kind, either expressed, implied, or *
-* statutory, including, without limitation, warranties that the *
-* Covered Software is free of defects, merchantable, fit for a *
-* particular purpose or non-infringing. The entire risk as to the *
-* quality and performance of the Covered Software is with You. *
-* Should any Covered Software prove defective in any respect, You *
-* (not any Contributor) assume the cost of any necessary servicing, *
-* repair, or correction. This disclaimer of warranty constitutes an *
-* essential part of this License. No use of any Covered Software is *
-* authorized under this License except under this disclaimer. *
-* *
-************************************************************************
-
-************************************************************************
-* *
-* 7. Limitation of Liability *
-* -------------------------- *
-* *
-* Under no circumstances and under no legal theory, whether tort *
-* (including negligence), contract, or otherwise, shall any *
-* Contributor, or anyone who distributes Covered Software as *
-* permitted above, be liable to You for any direct, indirect, *
-* special, incidental, or consequential damages of any character *
-* including, without limitation, damages for lost profits, loss of *
-* goodwill, work stoppage, computer failure or malfunction, or any *
-* and all other commercial damages or losses, even if such party *
-* shall have been informed of the possibility of such damages. This *
-* limitation of liability shall not apply to liability for death or *
-* personal injury resulting from such party's negligence to the *
-* extent applicable law prohibits such limitation. Some *
-* jurisdictions do not allow the exclusion or limitation of *
-* incidental or consequential damages, so this exclusion and *
-* limitation may not apply to You. *
-* *
-************************************************************************
-
-8. Litigation
--------------
-
-Any litigation relating to this License may be brought only in the
-courts of a jurisdiction where the defendant maintains its principal
-place of business and such litigation shall be governed by laws of that
-jurisdiction, without reference to its conflict-of-law provisions.
-Nothing in this Section shall prevent a party's ability to bring
-cross-claims or counter-claims.
-
-9. Miscellaneous
-----------------
-
-This License represents the complete agreement concerning the subject
-matter hereof. If any provision of this License is held to be
-unenforceable, such provision shall be reformed only to the extent
-necessary to make it enforceable. Any law or regulation which provides
-that the language of a contract shall be construed against the drafter
-shall not be used to construe this License against a Contributor.
-
-10. Versions of the License
----------------------------
-
-10.1. New Versions
-
-Mozilla Foundation is the license steward. Except as provided in Section
-10.3, no one other than the license steward has the right to modify or
-publish new versions of this License. Each version will be given a
-distinguishing version number.
-
-10.2. Effect of New Versions
-
-You may distribute the Covered Software under the terms of the version
-of the License under which You originally received the Covered Software,
-or under the terms of any subsequent version published by the license
-steward.
-
-10.3. Modified Versions
-
-If you create software not governed by this License, and you want to
-create a new license for such software, you may create and use a
-modified version of this License if you rename the license and remove
-any references to the name of the license steward (except to note that
-such modified license differs from this License).
-
-10.4. Distributing Source Code Form that is Incompatible With Secondary
-Licenses
-
-If You choose to distribute Source Code Form that is Incompatible With
-Secondary Licenses under the terms of this version of the License, the
-notice described in Exhibit B of this License must be attached.
-
-Exhibit A - Source Code Form License Notice
--------------------------------------------
-
- This Source Code Form is subject to the terms of the Mozilla Public
- License, v. 2.0. If a copy of the MPL was not distributed with this
- file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-If it is not possible or desirable to put the notice in a particular
-file, then You may include the notice in a location (such as a LICENSE
-file in a relevant directory) where a recipient would be likely to look
-for such a notice.
-
-You may add additional accurate notices of copyright ownership.
-
-Exhibit B - "Incompatible With Secondary Licenses" Notice
----------------------------------------------------------
-
- This Source Code Form is "Incompatible With Secondary Licenses", as
- defined by the Mozilla Public License, v. 2.0.
diff --git a/testing/mozharness/README.txt b/testing/mozharness/README.txt
deleted file mode 100644
index d2a2ce60a..000000000
--- a/testing/mozharness/README.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-# Mozharness
-
-## Docs
-* https://developer.mozilla.org/en-US/docs/Mozharness_FAQ
-* https://wiki.mozilla.org/ReleaseEngineering/Mozharness
-* http://moz-releng-mozharness.readthedocs.org/en/latest/mozharness.mozilla.html
-* http://moz-releng-docs.readthedocs.org/en/latest/software.html#mozharness
-
-## Submitting changes
-Like any Gecko change, please create a patch or submit to Mozreview and
-open a Bugzilla ticket under the Mozharness component:
-https://bugzilla.mozilla.org/enter_bug.cgi?product=Release%20Engineering&component=Mozharness
-
-This bug will get triaged by Release Engineering
-
-## Run unit tests
-To run the unit tests of mozharness the `tox` package needs to be installed:
-
-```
-pip install tox
-```
-
-There are various ways to run the unit tests. Just make sure you are within the `$gecko_repo/testing/mozharness` directory before running one of the commands below:
-
-```
-tox # run all unit tests
-tox -- -x # run all unit tests but stop after first failure
-tox -- test/test_base_log.py # only run the base log unit test
-```
-
-Happy contributing! =)
-
diff --git a/testing/mozharness/configs/balrog/docker-worker.py b/testing/mozharness/configs/balrog/docker-worker.py
deleted file mode 100644
index 1ff1c2ac5..000000000
--- a/testing/mozharness/configs/balrog/docker-worker.py
+++ /dev/null
@@ -1,18 +0,0 @@
-config = {
- 'balrog_servers': [
- {
- 'balrog_api_root': 'http://balrog/api',
- 'ignore_failures': False,
- 'url_replacements': [
- ('http://archive.mozilla.org/pub', 'http://download.cdn.mozilla.net/pub'),
- ],
- 'balrog_usernames': {
- 'firefox': 'ffxbld',
- 'thunderbird': 'tbirdbld',
- 'mobile': 'ffxbld',
- 'Fennec': 'ffxbld',
- }
- }
- ]
-}
-
diff --git a/testing/mozharness/configs/balrog/production.py b/testing/mozharness/configs/balrog/production.py
deleted file mode 100644
index a727f77d1..000000000
--- a/testing/mozharness/configs/balrog/production.py
+++ /dev/null
@@ -1,28 +0,0 @@
-config = {
- 'balrog_servers': [
- {
- 'balrog_api_root': 'https://aus4-admin.mozilla.org/api',
- 'ignore_failures': False,
- 'url_replacements': [
- ('http://archive.mozilla.org/pub', 'http://download.cdn.mozilla.net/pub'),
- ],
- 'balrog_usernames': {
- 'firefox': 'ffxbld',
- 'thunderbird': 'tbirdbld',
- 'mobile': 'ffxbld',
- 'Fennec': 'ffxbld',
- }
- },
- # Bug 1261346 - temporarily disable staging balrog submissions
- # {
- # 'balrog_api_root': 'https://aus4-admin-dev.allizom.org/api',
- # 'ignore_failures': True,
- # 'balrog_usernames': {
- # 'firefox': 'stage-ffxbld',
- # 'thunderbird': 'stage-tbirdbld',
- # 'mobile': 'stage-ffxbld',
- # 'Fennec': 'stage-ffxbld',
- # }
- # }
- ]
-}
diff --git a/testing/mozharness/configs/balrog/staging.py b/testing/mozharness/configs/balrog/staging.py
deleted file mode 100644
index 919974122..000000000
--- a/testing/mozharness/configs/balrog/staging.py
+++ /dev/null
@@ -1,14 +0,0 @@
-config = {
- 'balrog_servers': [
- {
- 'balrog_api_root': 'https://aus4-admin-dev.allizom.org/api',
- 'ignore_failures': False,
- 'balrog_usernames': {
- 'firefox': 'stage-ffxbld',
- 'thunderbird': 'stage-tbirdbld',
- 'mobile': 'stage-ffxbld',
- 'Fennec': 'stage-ffxbld',
- }
- }
- ]
-}
diff --git a/testing/mozharness/configs/beetmover/en_us_build.yml.tmpl b/testing/mozharness/configs/beetmover/en_us_build.yml.tmpl
deleted file mode 100644
index 33287b042..000000000
--- a/testing/mozharness/configs/beetmover/en_us_build.yml.tmpl
+++ /dev/null
@@ -1,191 +0,0 @@
----
-metadata:
- name: "Beet Mover Manifest"
- description: "Maps artifact locations to s3 key names for the en-US locale"
- owner: "release@mozilla.com"
-
-mapping:
-{% for locale in locales %}
- {{ locale }}:
-
- {% if platform == "win32" %}
- buildinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.json
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.json
- mozinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.mozinfo.json
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.mozinfo.json
- socorroinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.txt
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.txt
- jsshell:
- artifact: {{ artifact_base_url }}/jsshell-{{ platform }}.zip
- s3_key: {{ s3_prefix }}jsshell-{{ platform }}.zip
- mozharness_package:
- artifact: {{ artifact_base_url }}/mozharness.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/mozharness.zip
- xpi:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.langpack.xpi
- s3_key: {{ s3_prefix }}{{ platform }}/xpi/{{ locale }}.xpi
- symbols:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
- buildid_info:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}_info.txt
- s3_key: {{ s3_prefix }}win32_info.txt
- sdk:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.sdk.zip
- s3_key: {{ s3_prefix }}firefox-{{ version }}.{{ platform }}.sdk.zip
- mar_tools_mar:
- artifact: {{ artifact_base_url }}/mar.exe
- s3_key: {{ s3_prefix }}mar-tools/win32/mar.exe
- mar_tools_mbdiff:
- artifact: {{ artifact_base_url }}/mbsdiff.exe
- s3_key: {{ s3_prefix }}mar-tools/win32/mbsdiff.exe
- {% endif %}
-
- {% if platform == "win64" %}
- buildinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.json
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.json
- mozinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.mozinfo.json
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.mozinfo.json
- socorroinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.txt
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.txt
- jsshell:
- artifact: {{ artifact_base_url }}/jsshell-{{ platform }}.zip
- s3_key: {{ s3_prefix }}jsshell-{{ platform }}.zip
- mozharness_package:
- artifact: {{ artifact_base_url }}/mozharness.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/mozharness.zip
- xpi:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.langpack.xpi
- s3_key: {{ s3_prefix }}{{ platform }}/xpi/{{ locale }}.xpi
- symbols:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
- buildid_info:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}_info.txt
- s3_key: {{ s3_prefix }}win64_info.txt
- sdk:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.sdk.zip
- s3_key: {{ s3_prefix }}firefox-{{ version }}.{{ platform }}.sdk.zip
- mar_tools_mar:
- artifact: {{ artifact_base_url }}/mar.exe
- s3_key: {{ s3_prefix }}mar-tools/win64/mar.exe
- mar_tools_mbdiff:
- artifact: {{ artifact_base_url }}/mbsdiff.exe
- s3_key: {{ s3_prefix }}mar-tools/win64/mbsdiff.exe
- {% endif %}
-
- {% if platform == "linux-i686" %}
- buildinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.json
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.json
- mozinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.mozinfo.json
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.mozinfo.json
- socorroinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.txt
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.txt
- jsshell:
- artifact: {{ artifact_base_url }}/jsshell-{{ platform }}.zip
- s3_key: {{ s3_prefix }}jsshell-{{ platform }}.zip
- mozharness_package:
- artifact: {{ artifact_base_url }}/mozharness.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/mozharness.zip
- xpi:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.langpack.xpi
- s3_key: {{ s3_prefix }}{{ platform }}/xpi/{{ locale }}.xpi
- symbols:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
- buildid_info:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}_info.txt
- s3_key: {{ s3_prefix }}linux_info.txt
- sdk:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.sdk.tar.bz2
- s3_key: {{ s3_prefix }}firefox-{{ version }}.{{ platform }}.sdk.tar.bz2
- mar_tools_mar:
- artifact: {{ artifact_base_url }}/mar
- s3_key: {{ s3_prefix }}mar-tools/linux/mar
- mar_tools_mbdiff:
- artifact: {{ artifact_base_url }}/mbsdiff
- s3_key: {{ s3_prefix }}mar-tools/linux/mbsdiff
- {% endif %}
-
- {% if platform == "linux-x86_64" %}
- buildinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.json
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.json
- mozinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.mozinfo.json
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.mozinfo.json
- socorroinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.txt
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.txt
- jsshell:
- artifact: {{ artifact_base_url }}/jsshell-{{ platform }}.zip
- s3_key: {{ s3_prefix }}jsshell-{{ platform }}.zip
- mozharness_package:
- artifact: {{ artifact_base_url }}/mozharness.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/mozharness.zip
- xpi:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.langpack.xpi
- s3_key: {{ s3_prefix }}{{ platform }}/xpi/{{ locale }}.xpi
- symbols:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.crashreporter-symbols.zip
- buildid_info:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}_info.txt
- s3_key: {{ s3_prefix }}linux64_info.txt
- sdk:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.sdk.tar.bz2
- s3_key: {{ s3_prefix }}firefox-{{ version }}.{{ platform }}.sdk.tar.bz2
- mar_tools_mar:
- artifact: {{ artifact_base_url }}/mar
- s3_key: {{ s3_prefix }}mar-tools/linux64/mar
- mar_tools_mbdiff:
- artifact: {{ artifact_base_url }}/mbsdiff
- s3_key: {{ s3_prefix }}mar-tools/linux64/mbsdiff
- {% endif %}
-
- {% if platform == "mac" %}
- buildinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.json
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.json
- mozinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.mozinfo.json
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.mozinfo.json
- socorroinfo:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.txt
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.txt
- jsshell:
- artifact: {{ artifact_base_url }}/jsshell-{{ platform }}.zip
- s3_key: {{ s3_prefix }}jsshell-{{ platform }}.zip
- mozharness_package:
- artifact: {{ artifact_base_url }}/mozharness.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/mozharness.zip
- xpi:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.langpack.xpi
- s3_key: {{ s3_prefix }}{{ platform }}/xpi/{{ locale }}.xpi
- symbols:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.crashreporter-symbols.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/Firefox {{ version }}.crashreporter-symbols.zip
- buildid_info:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}_info.txt
- s3_key: {{ s3_prefix }}macosx64_info.txt
- sdk:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}-x86_64.sdk.tar.bz2
- s3_key: {{ s3_prefix }}firefox-{{ version }}.{{ platform }}-x86_64.sdk.tar.bz2
- mar_tools_mar:
- artifact: {{ artifact_base_url }}/mar
- s3_key: {{ s3_prefix }}mar-tools/macosx64/mar
- mar_tools_mbdiff:
- artifact: {{ artifact_base_url }}/mbsdiff
- s3_key: {{ s3_prefix }}mar-tools/macosx64/mbsdiff
- {% endif %}
-
-{% endfor %}
diff --git a/testing/mozharness/configs/beetmover/en_us_signing.yml.tmpl b/testing/mozharness/configs/beetmover/en_us_signing.yml.tmpl
deleted file mode 100644
index 54fc2c792..000000000
--- a/testing/mozharness/configs/beetmover/en_us_signing.yml.tmpl
+++ /dev/null
@@ -1,66 +0,0 @@
----
-metadata:
- name: "Beet Mover Manifest"
- description: "Maps artifact locations to s3 key names for the en-US locale"
- owner: "release@mozilla.com"
-
-mapping:
-{% for locale in locales %}
- {{ locale }}:
- {% if platform == "win32" %}
- complete_mar:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.complete.mar
- s3_key: {{ s3_prefix }}update/{{ platform }}/{{ locale }}/firefox-{{ version }}.complete.mar
- full_installer:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.installer.exe
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/Firefox Setup {{ version }}.exe
- {% if "esr" not in version %}
- stub_installer:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.installer-stub.exe
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/Firefox Setup Stub {{ version }}.exe
- {% endif %}
- package:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.zip
- {% endif %}
-
- {% if platform == "win64" %}
- complete_mar:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.complete.mar
- s3_key: {{ s3_prefix }}update/{{ platform }}/{{ locale }}/firefox-{{ version }}.complete.mar
- full_installer:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.installer.exe
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/Firefox Setup {{ version }}.exe
- package:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.zip
- {% endif %}
-
- {% if platform == "linux-i686" %}
- complete_mar:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.complete.mar
- s3_key: {{ s3_prefix }}update/{{ platform }}/{{ locale }}/firefox-{{ version }}.complete.mar
- package:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.tar.bz2
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.tar.bz2
- {% endif %}
-
- {% if platform == "linux-x86_64" %}
- complete_mar:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.complete.mar
- s3_key: {{ s3_prefix }}update/{{ platform }}/{{ locale }}/firefox-{{ version }}.complete.mar
- package:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.tar.bz2
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.tar.bz2
- {% endif %}
-
- {% if platform == "mac" %}
- complete_mar:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.complete.mar
- s3_key: {{ s3_prefix }}update/{{ platform }}/{{ locale }}/firefox-{{ version }}.complete.mar
- package:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.dmg
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/Firefox {{ version }}.dmg
- {% endif %}
-
-{% endfor %}
diff --git a/testing/mozharness/configs/beetmover/l10n_changesets.tmpl b/testing/mozharness/configs/beetmover/l10n_changesets.tmpl
deleted file mode 100644
index bde4bc8a7..000000000
--- a/testing/mozharness/configs/beetmover/l10n_changesets.tmpl
+++ /dev/null
@@ -1,11 +0,0 @@
----
-metadata:
- name: "Beet Mover L10N Changesets"
- description: "Maps artifact locations to s3 key names for L10N changesets"
- owner: "release@mozilla.com"
-
-mapping:
- all:
- l10n_changesets:
- artifact: {{ artifact_base_url }}/l10n_changesets.txt
- s3_key: {{ s3_prefix }}l10n_changesets.txt
diff --git a/testing/mozharness/configs/beetmover/partials.yml.tmpl b/testing/mozharness/configs/beetmover/partials.yml.tmpl
deleted file mode 100644
index a97ac42c0..000000000
--- a/testing/mozharness/configs/beetmover/partials.yml.tmpl
+++ /dev/null
@@ -1,16 +0,0 @@
----
-metadata:
- name: "Beet Mover Manifest"
- description: "Maps artifact locations to s3 key names for partials"
- owner: "release@mozilla.com"
-
-mapping:
-{% for locale in locales %}
- {{ locale }}:
- partial_mar:
- artifact: {{ artifact_base_url }}/firefox-{{ partial_version }}-{{ version }}.{{ locale }}.{{ platform }}.partial.mar
- s3_key: {{ s3_prefix }}update/{{ platform }}/{{ locale }}/firefox-{{ partial_version }}-{{ version }}.partial.mar
- partial_mar_sig:
- artifact: {{ artifact_base_url }}/firefox-{{ partial_version }}-{{ version }}.{{ locale }}.{{ platform }}.partial.mar.asc
- s3_key: {{ s3_prefix }}update/{{ platform }}/{{ locale }}/firefox-{{ partial_version }}-{{ version }}.partial.mar.asc
-{% endfor %}
diff --git a/testing/mozharness/configs/beetmover/repacks.yml.tmpl b/testing/mozharness/configs/beetmover/repacks.yml.tmpl
deleted file mode 100644
index c275ff3e8..000000000
--- a/testing/mozharness/configs/beetmover/repacks.yml.tmpl
+++ /dev/null
@@ -1,65 +0,0 @@
----
-metadata:
- name: "Beet Mover Manifest"
- description: "Maps artifact locations to s3 key names for the non en-US locales"
- owner: "release@mozilla.com"
-
-mapping:
-{% for locale in locales %}
- # common deliverables
- {{ locale }}:
- complete_mar:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.complete.mar
- s3_key: {{ s3_prefix }}update/{{ platform }}/{{ locale }}/firefox-{{ version }}.complete.mar
- checksum:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.checksums
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.checksums
- checksum_sig:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.checksums.asc
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.checksums.asc
- xpi:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.langpack.xpi
- s3_key: {{ s3_prefix }}{{ platform }}/xpi/{{ locale }}.xpi
-
- {% if platform == "win32" %}
- full_installer:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.installer.exe
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/Firefox Setup {{ version }}.exe
- {% if "esr" not in version %}
- stub_installer:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.installer-stub.exe
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/Firefox Setup Stub {{ version }}.exe
- {% endif %}
- package:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.zip
- {% endif %}
-
- {% if platform == "win64" %}
- full_installer:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.installer.exe
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/Firefox Setup {{ version }}.exe
- package:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.zip
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.zip
- {% endif %}
-
- {% if platform == "linux-i686" %}
- package:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.tar.bz2
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.tar.bz2
- {% endif %}
-
- {% if platform == "linux-x86_64" %}
- package:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.tar.bz2
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/firefox-{{ version }}.tar.bz2
- {% endif %}
-
- {% if platform == "mac" %}
- package:
- artifact: {{ artifact_base_url }}/firefox-{{ app_version }}.{{ locale }}.{{ platform }}.dmg
- s3_key: {{ s3_prefix }}{{ platform }}/{{ locale }}/Firefox {{ version }}.dmg
- {% endif %}
-
-{% endfor %}
diff --git a/testing/mozharness/configs/beetmover/snap.yml.tmpl b/testing/mozharness/configs/beetmover/snap.yml.tmpl
deleted file mode 100644
index afc8f35ce..000000000
--- a/testing/mozharness/configs/beetmover/snap.yml.tmpl
+++ /dev/null
@@ -1,11 +0,0 @@
----
-metadata:
- name: "Beet Mover Manifest"
- description: "Maps artifact locations to s3 key names for snap iamge"
- owner: "release@mozilla.com"
-
-mapping:
- all:
- snap:
- artifact: {{ artifact_base_url }}/firefox-{{ version }}.snap
- s3_key: {{ s3_prefix }}snap/firefox-{{ version }}.snap
diff --git a/testing/mozharness/configs/beetmover/snap_checksums.yml.tmpl b/testing/mozharness/configs/beetmover/snap_checksums.yml.tmpl
deleted file mode 100644
index aa905d38d..000000000
--- a/testing/mozharness/configs/beetmover/snap_checksums.yml.tmpl
+++ /dev/null
@@ -1,14 +0,0 @@
----
-metadata:
- name: "Beet Mover Manifest"
- description: "Maps artifact locations to s3 key names for snap checksums"
- owner: "release@mozilla.com"
-
-mapping:
- all:
- snap_checksum:
- artifact: {{ artifact_base_url }}/firefox-{{ version }}.snap.checksums
- s3_key: {{ s3_prefix }}snap/firefox-{{ version }}.snap.checksums
- snap_checksum_asc:
- artifact: {{ artifact_base_url }}/firefox-{{ version }}.snap.checksums.asc
- s3_key: {{ s3_prefix }}snap/firefox-{{ version }}.snap.checksums.asc
diff --git a/testing/mozharness/configs/beetmover/source.yml.tmpl b/testing/mozharness/configs/beetmover/source.yml.tmpl
deleted file mode 100644
index f991f257c..000000000
--- a/testing/mozharness/configs/beetmover/source.yml.tmpl
+++ /dev/null
@@ -1,14 +0,0 @@
----
-metadata:
- name: "Beet Mover Manifest"
- description: "Maps artifact locations to s3 key names for source bundles"
- owner: "release@mozilla.com"
-
-mapping:
- all:
- source_bundle:
- artifact: {{ artifact_base_url }}/firefox-{{ version }}.bundle
- s3_key: {{ s3_prefix }}source/firefox-{{ version }}.bundle
- source_tar:
- artifact: {{ artifact_base_url }}/firefox-{{ version }}.source.tar.xz
- s3_key: {{ s3_prefix }}source/firefox-{{ version }}.source.tar.xz
diff --git a/testing/mozharness/configs/beetmover/source_checksums.yml.tmpl b/testing/mozharness/configs/beetmover/source_checksums.yml.tmpl
deleted file mode 100644
index 0dd228c24..000000000
--- a/testing/mozharness/configs/beetmover/source_checksums.yml.tmpl
+++ /dev/null
@@ -1,14 +0,0 @@
----
-metadata:
- name: "Beet Mover Manifest"
- description: "Maps artifact locations to s3 key names for source bundle checksums"
- owner: "release@mozilla.com"
-
-mapping:
- all:
- source_checksum:
- artifact: {{ artifact_base_url }}/firefox-{{ version }}.source.checksums
- s3_key: {{ s3_prefix }}source/firefox-{{ version }}.source.checksums
- source_checksum_asc:
- artifact: {{ artifact_base_url }}/firefox-{{ version }}.source.checksums.asc
- s3_key: {{ s3_prefix }}source/firefox-{{ version }}.source.checksums.asc
diff --git a/testing/mozharness/configs/builds/branch_specifics.py b/testing/mozharness/configs/builds/branch_specifics.py
deleted file mode 100644
index 43f14c5ad..000000000
--- a/testing/mozharness/configs/builds/branch_specifics.py
+++ /dev/null
@@ -1,469 +0,0 @@
-# this is a dict of branch specific keys/values. As this fills up and more
-# fx build factories are ported, we might deal with this differently
-
-# we should be able to port this in-tree and have the respective repos and
-# revisions handle what goes on in here. Tracking: bug 978510
-
-# example config and explanation of how it works:
-# config = {
-# # if a branch matches a key below, override items in self.config with
-# # items in the key's value.
-# # this override can be done for every platform or at a platform level
-# '<branch-name>': {
-# # global config items (applies to all platforms and build types)
-# 'repo_path': "projects/<branch-name>",
-# 'graph_server_branch_name': "Firefox",
-#
-# # platform config items (applies to specific platforms)
-# 'platform_overrides': {
-# # if a platform matches a key below, override items in
-# # self.config with items in the key's value
-# 'linux64-debug': {
-# 'upload_symbols': False,
-# },
-# 'win64': {
-# 'enable_checktests': False,
-# },
-# }
-# },
-# }
-
-config = {
- ### release branches
- "mozilla-central": {
- "repo_path": 'mozilla-central',
- "update_channel": "nightly",
- "graph_server_branch_name": "Firefox",
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- 'mozilla-release': {
- 'enable_release_promotion': True,
- 'repo_path': 'releases/mozilla-release',
- 'update_channel': 'release',
- 'branch_uses_per_checkin_strategy': True,
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- 'platform_overrides': {
- 'linux': {
- 'src_mozconfig': 'browser/config/mozconfigs/linux32/release',
- 'force_clobber': True,
- },
- 'linux64': {
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/release',
- 'force_clobber': True,
- },
- 'macosx64': {
- 'src_mozconfig': 'browser/config/mozconfigs/macosx-universal/release',
- 'force_clobber': True,
- },
- 'win32': {
- 'src_mozconfig': 'browser/config/mozconfigs/win32/release',
- 'force_clobber': True,
- },
- 'win64': {
- 'src_mozconfig': 'browser/config/mozconfigs/win64/release',
- 'force_clobber': True,
- },
- 'linux-debug': {
- 'update_channel': 'default',
- },
- 'linux64-debug': {
- 'update_channel': 'default',
- },
- 'linux64-asan-debug': {
- 'update_channel': 'default',
- },
- 'linux64-asan': {
- 'update_channel': 'default',
- },
- 'linux64-cc': {
- 'update_channel': 'default',
- },
- 'linux64-st-an-debug': {
- 'update_channel': 'default',
- },
- 'linux64-st-an': {
- 'update_channel': 'default',
- },
- 'linux64-tsan': {
- 'update_channel': 'default',
- },
- 'linux64-add-on-devel': {
- 'update_channel': 'default',
- },
- 'macosx64-debug': {
- 'update_channel': 'default',
- },
- 'macosx64-st-an': {
- 'update_channel': 'default',
- },
- 'macosx64-st-an-debug': {
- 'update_channel': 'default',
- },
- 'macosx64-add-on-devel': {
- 'update_channel': 'default',
- },
- 'win32-debug': {
- 'update_channel': 'default',
- },
- 'win32-add-on-devel': {
- 'update_channel': 'default',
- },
- 'win64-debug': {
- 'update_channel': 'default',
- },
- 'win64-add-on-devel': {
- 'update_channel': 'default',
- },
- },
- },
- 'mozilla-beta': {
- 'enable_release_promotion': 1,
- 'repo_path': 'releases/mozilla-beta',
- 'update_channel': 'beta',
- 'branch_uses_per_checkin_strategy': True,
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- 'platform_overrides': {
- 'linux': {
- 'src_mozconfig': 'browser/config/mozconfigs/linux32/beta',
- 'force_clobber': True,
- },
- 'linux64': {
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/beta',
- 'force_clobber': True,
- },
- 'macosx64': {
- 'src_mozconfig': 'browser/config/mozconfigs/macosx-universal/beta',
- 'force_clobber': True,
- },
- 'win32': {
- 'src_mozconfig': 'browser/config/mozconfigs/win32/beta',
- 'force_clobber': True,
- },
- 'win64': {
- 'src_mozconfig': 'browser/config/mozconfigs/win64/beta',
- 'force_clobber': True,
- },
- 'linux-debug': {
- 'update_channel': 'default',
- },
- 'linux64-debug': {
- 'update_channel': 'default',
- },
- 'linux64-asan-debug': {
- 'update_channel': 'default',
- },
- 'linux64-asan': {
- 'update_channel': 'default',
- },
- 'linux64-cc': {
- 'update_channel': 'default',
- },
- 'linux64-st-an-debug': {
- 'update_channel': 'default',
- },
- 'linux64-st-an': {
- 'update_channel': 'default',
- },
- 'linux64-tsan': {
- 'update_channel': 'default',
- },
- 'linux64-add-on-devel': {
- 'update_channel': 'default',
- },
- 'macosx64-debug': {
- 'update_channel': 'default',
- },
- 'macosx64-st-an': {
- 'update_channel': 'default',
- },
- 'macosx64-st-an-debug': {
- 'update_channel': 'default',
- },
- 'macosx64-add-on-devel': {
- 'update_channel': 'default',
- },
- 'win32-debug': {
- 'update_channel': 'default',
- },
- 'win32-add-on-devel': {
- 'update_channel': 'default',
- },
- 'win64-debug': {
- 'update_channel': 'default',
- },
- 'win64-add-on-devel': {
- 'update_channel': 'default',
- },
- },
- },
- 'mozilla-esr52': {
- 'enable_release_promotion': True,
- 'repo_path': 'releases/mozilla-esr52',
- 'update_channel': 'esr',
- 'branch_uses_per_checkin_strategy': True,
- 'use_branch_in_symbols_extra_buildid': False,
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- 'platform_overrides': {
- 'linux': {
- 'src_mozconfig': 'browser/config/mozconfigs/linux32/release',
- 'force_clobber': True,
- },
- 'linux64': {
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/release',
- 'force_clobber': True,
- },
- 'macosx64': {
- 'src_mozconfig': 'browser/config/mozconfigs/macosx-universal/release',
- 'force_clobber': True,
- },
- 'win32': {
- 'src_mozconfig': 'browser/config/mozconfigs/win32/release',
- 'force_clobber': True,
- },
- 'win64': {
- 'src_mozconfig': 'browser/config/mozconfigs/win64/release',
- 'force_clobber': True,
- },
- 'linux-debug': {
- 'update_channel': 'default',
- },
- 'linux64-debug': {
- 'update_channel': 'default',
- },
- 'linux64-asan-debug': {
- 'update_channel': 'default',
- },
- 'linux64-asan': {
- 'update_channel': 'default',
- },
- 'linux64-cc': {
- 'update_channel': 'default',
- },
- 'linux64-st-an-debug': {
- 'update_channel': 'default',
- },
- 'linux64-st-an': {
- 'update_channel': 'default',
- },
- 'linux64-tsan': {
- 'update_channel': 'default',
- },
- 'macosx64-debug': {
- 'update_channel': 'default',
- },
- 'macosx64-st-an': {
- 'update_channel': 'default',
- },
- 'macosx64-st-an-debug': {
- 'update_channel': 'default',
- },
- 'win32-debug': {
- 'update_channel': 'default',
- },
- 'win64-debug': {
- 'update_channel': 'default',
- },
- },
- },
- 'mozilla-aurora': {
- 'repo_path': 'releases/mozilla-aurora',
- 'update_channel': 'aurora',
- 'branch_uses_per_checkin_strategy': True,
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- 'try': {
- 'repo_path': 'try',
- 'clone_by_revision': True,
- 'clone_with_purge': True,
- 'tinderbox_build_dir': '%(who)s-%(got_revision)s',
- 'to_tinderbox_dated': False,
- 'include_post_upload_builddir': True,
- 'release_to_try_builds': True,
- 'stage_server': 'upload.trybld.productdelivery.prod.mozaws.net',
- 'stage_username': 'trybld',
- 'stage_ssh_key': 'trybld_dsa',
- 'branch_supports_uploadsymbols': False,
- 'use_clobberer': False,
- },
-
- ### project branches
- #'fx-team': {}, #Bug 1296396
- 'gum': {
- 'branch_uses_per_checkin_strategy': True,
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- 'mozilla-inbound': {
- 'repo_path': 'integration/mozilla-inbound',
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- 'autoland': {
- 'repo_path': 'integration/autoland',
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- 'ux': {
- "graph_server_branch_name": "UX",
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- # When build promotion goes live the mozconfig changes are probably better
- # expressed once in files like configs/builds/releng_base_windows_32_builds.py
- 'date': {
- 'update_channel': 'beta-dev',
- 'enable_release_promotion': 1,
- 'platform_overrides': {
- 'linux': {
- 'src_mozconfig': 'browser/config/mozconfigs/linux32/beta',
- },
- 'linux-debug': {
- 'update_channel': 'default',
- },
- 'linux64': {
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/beta',
- },
- 'linux64-debug': {
- 'update_channel': 'default',
- },
- 'linux64-asan-debug': {
- 'update_channel': 'default',
- },
- 'linux64-asan': {
- 'update_channel': 'default',
- },
- 'linux64-cc': {
- 'update_channel': 'default',
- },
- 'linux64-st-an-debug': {
- 'update_channel': 'default',
- },
- 'linux64-st-an': {
- 'update_channel': 'default',
- },
- 'linux64-tsan': {
- 'update_channel': 'default',
- },
- 'macosx64': {
- 'src_mozconfig': 'browser/config/mozconfigs/macosx-universal/beta',
- },
- 'macosx64-debug': {
- 'update_channel': 'default',
- },
- 'macosx64-st-an': {
- 'update_channel': 'default',
- },
- 'macosx64-st-an-debug': {
- 'update_channel': 'default',
- },
- 'win32': {
- 'src_mozconfig': 'browser/config/mozconfigs/win32/beta',
- },
- 'win32-debug': {
- 'update_channel': 'default',
- },
- 'win64': {
- 'src_mozconfig': 'browser/config/mozconfigs/win64/beta',
- },
- 'win64-debug': {
- 'update_channel': 'default',
- },
- },
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- 'cypress': {
- # bug 1164935
- 'branch_uses_per_checkin_strategy': True,
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
-
- ### other branches that do not require anything special:
- 'alder': {
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- 'ash': {
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- 'birch': {
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- # 'build-system': {}
- 'cedar': {
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- 'elm': {
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- 'fig': {},
- 'graphics': {
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- # 'holly': {},
- 'jamun': {
- 'update_channel': 'release-dev',
- 'enable_release_promotion': 1,
- 'platform_overrides': {
- 'linux': {
- 'src_mozconfig': 'browser/config/mozconfigs/linux32/release',
- },
- 'linux-debug': {
- 'update_channel': 'default',
- },
- 'linux64': {
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/release',
- },
- 'linux64-debug': {
- 'update_channel': 'default',
- },
- 'linux64-asan-debug': {
- 'update_channel': 'default',
- },
- 'linux64-asan': {
- 'update_channel': 'default',
- },
- 'linux64-cc': {
- 'update_channel': 'default',
- },
- 'linux64-st-an-debug': {
- 'update_channel': 'default',
- },
- 'linux64-st-an': {
- 'update_channel': 'default',
- },
- 'linux64-tsan': {
- 'update_channel': 'default',
- },
- 'macosx64': {
- 'src_mozconfig': 'browser/config/mozconfigs/macosx-universal/release',
- },
- 'macosx64-debug': {
- 'update_channel': 'default',
- },
- 'macosx64-st-an': {
- 'update_channel': 'default',
- },
- 'macosx64-st-an-debug': {
- 'update_channel': 'default',
- },
- 'win32': {
- 'src_mozconfig': 'browser/config/mozconfigs/win32/release',
- },
- 'win32-debug': {
- 'update_channel': 'default',
- },
- 'win64': {
- 'src_mozconfig': 'browser/config/mozconfigs/win64/release',
- },
- 'win64-debug': {
- 'update_channel': 'default',
- },
- },
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- 'larch': {
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- # 'maple': {},
- 'oak': {
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
- 'pine': {
- 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- },
-}
diff --git a/testing/mozharness/configs/builds/build_pool_specifics.py b/testing/mozharness/configs/builds/build_pool_specifics.py
deleted file mode 100644
index 8559b48b7..000000000
--- a/testing/mozharness/configs/builds/build_pool_specifics.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# this is a dict of pool specific keys/values. As this fills up and more
-# fx build factories are ported, we might deal with this differently
-
-config = {
- "staging": {
- # if not clobberer_url, only clobber 'abs_work_dir'
- # if true: possibly clobber, clobberer
- # see PurgeMixin for clobber() conditions
- 'clobberer_url': 'https://api-pub-build.allizom.org/clobberer/lastclobber',
- # staging we should use MozillaTest
- # but in production we let the self.branch decide via
- # self._query_graph_server_branch_name()
- "graph_server_branch_name": "MozillaTest",
- 'graph_server': 'graphs.allizom.org',
- 'stage_server': 'upload.ffxbld.productdelivery.stage.mozaws.net',
- "sendchange_masters": ["dev-master1.srv.releng.scl3.mozilla.com:9038"],
- 'taskcluster_index': 'index.garbage.staging',
- 'post_upload_extra': ['--bucket-prefix', 'net-mozaws-stage-delivery',
- '--url-prefix', 'http://ftp.stage.mozaws.net/',
- ],
- },
- "production": {
- # if not clobberer_url, only clobber 'abs_work_dir'
- # if true: possibly clobber, clobberer
- # see PurgeMixin for clobber() conditions
- 'clobberer_url': 'https://api.pub.build.mozilla.org/clobberer/lastclobber',
- 'graph_server': 'graphs.mozilla.org',
- # bug 1216907, set this at branch level
- # 'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
- "sendchange_masters": ["buildbot-master81.build.mozilla.org:9301"],
- 'taskcluster_index': 'index',
- },
- "taskcluster": {
- 'graph_server': 'graphs.mozilla.org',
- 'stage_server': 'ignored',
- # use the relengapi proxy to talk to tooltool
- "tooltool_servers": ['http://relengapi/tooltool/'],
- "tooltool_url": 'http://relengapi/tooltool/',
- 'upload_env': {
- 'UPLOAD_HOST': 'localhost',
- 'UPLOAD_PATH': '/home/worker/artifacts',
- },
- },
-}
diff --git a/testing/mozharness/configs/builds/releng_base_linux_32_builds.py b/testing/mozharness/configs/builds/releng_base_linux_32_builds.py
deleted file mode 100644
index 393cf8983..000000000
--- a/testing/mozharness/configs/builds/releng_base_linux_32_builds.py
+++ /dev/null
@@ -1,160 +0,0 @@
-import os
-
-config = {
- #########################################################################
- ######## LINUX GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 32 bit keys/values please add them
- # below under the '32 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced with
- # releng_base_linux_64_builds.py
-
- # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- "buildbot": "/tools/buildbot/bin/buildbot",
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': True,
- # mock shtuff
- 'mock_mozilla_dir': '/builds/mock_mozilla',
- 'mock_target': 'mozilla-centos6-x86_64',
- 'mock_files': [
- ('/home/cltbld/.ssh', '/home/mock_mozilla/.ssh'),
- ('/home/cltbld/.hgrc', '/builds/.hgrc'),
- ('/home/cltbld/.boto', '/builds/.boto'),
- ('/builds/gapi.data', '/builds/gapi.data'),
- ('/builds/relengapi.tok', '/builds/relengapi.tok'),
- ('/tools/tooltool.py', '/builds/tooltool.py'),
- ('/builds/mozilla-desktop-geoloc-api.key', '/builds/mozilla-desktop-geoloc-api.key'),
- ('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
- ('/builds/adjust-sdk.token', '/builds/adjust-sdk.token'),
- ('/builds/adjust-sdk-beta.token', '/builds/adjust-sdk-beta.token'),
- ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
- ],
- 'secret_files': [
- {'filename': '/builds/gapi.data',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data',
- 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
- {'filename': '/builds/mozilla-desktop-geoloc-api.key',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key',
- 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
- {'filename': '/builds/adjust-sdk.token',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token',
- 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
- {'filename': '/builds/adjust-sdk-beta.token',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token',
- 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
- ],
- 'enable_ccache': True,
- 'vcs_share_base': '/builds/hg-shared',
- 'objdir': 'obj-firefox',
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': True,
- 'enable_talos_sendchange': True,
- 'enable_unittest_sendchange': True,
- #########################################################################
-
-
- #########################################################################
- ###### 32 bit specific ######
- 'base_name': 'Linux_%(branch)s',
- 'platform': 'linux',
- 'stage_platform': 'linux',
- 'publish_nightly_en_US_routes': True,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- # 32 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib/ccache:\
-/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:\
-/tools/python27/bin:/tools/python27-mercurial/bin:/home/cltbld/bin',
- 'LD_LIBRARY_PATH': "/tools/gcc-4.3.3/installed/lib",
- },
- 'upload_env': {
- # stage_server is dictated from build_pool_specifics.py
- 'UPLOAD_HOST': '%(stage_server)s',
- 'UPLOAD_USER': '%(stage_username)s',
- 'UPLOAD_SSH_KEY': '/home/mock_mozilla/.ssh/%(stage_ssh_key)s',
- 'UPLOAD_TO_TEMP': '1',
- },
- "check_test_env": {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/linux/minidump_stackwalk',
- 'MINIDUMP_SAVE_PATH': '%(base_work_dir)s/minidumps',
- },
- 'mock_packages': [
- 'autoconf213', 'python', 'mozilla-python27', 'zip', 'mozilla-python27-mercurial',
- 'git', 'ccache', 'perl-Test-Simple', 'perl-Config-General',
- 'yasm', 'wget',
- 'mpfr', # required for system compiler
- 'xorg-x11-font*', # fonts required for PGO
- 'imake', # required for makedepend!?!
- ### <-- from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'valgrind',
- ######## 32 bit specific ###########
- 'glibc-static.i686', 'libstdc++-static.i686',
- 'gtk2-devel.i686', 'libnotify-devel.i686',
- 'alsa-lib-devel.i686', 'libcurl-devel.i686',
- 'wireless-tools-devel.i686', 'libX11-devel.i686',
- 'libXt-devel.i686', 'mesa-libGL-devel.i686',
- 'gnome-vfs2-devel.i686', 'GConf2-devel.i686',
- 'pulseaudio-libs-devel.i686',
- 'gstreamer-devel.i686', 'gstreamer-plugins-base-devel.i686',
- # Packages already installed in the mock environment, as x86_64
- # packages.
- 'glibc-devel.i686', 'libgcc.i686', 'libstdc++-devel.i686',
- # yum likes to install .x86_64 -devel packages that satisfy .i686
- # -devel packages dependencies. So manually install the dependencies
- # of the above packages.
- 'ORBit2-devel.i686', 'atk-devel.i686', 'cairo-devel.i686',
- 'check-devel.i686', 'dbus-devel.i686', 'dbus-glib-devel.i686',
- 'fontconfig-devel.i686', 'glib2-devel.i686',
- 'hal-devel.i686', 'libICE-devel.i686', 'libIDL-devel.i686',
- 'libSM-devel.i686', 'libXau-devel.i686', 'libXcomposite-devel.i686',
- 'libXcursor-devel.i686', 'libXdamage-devel.i686',
- 'libXdmcp-devel.i686', 'libXext-devel.i686', 'libXfixes-devel.i686',
- 'libXft-devel.i686', 'libXi-devel.i686', 'libXinerama-devel.i686',
- 'libXrandr-devel.i686', 'libXrender-devel.i686',
- 'libXxf86vm-devel.i686', 'libdrm-devel.i686', 'libidn-devel.i686',
- 'libpng-devel.i686', 'libxcb-devel.i686', 'libxml2-devel.i686',
- 'pango-devel.i686', 'perl-devel.i686', 'pixman-devel.i686',
- 'zlib-devel.i686',
- # Freetype packages need to be installed be version, because a newer
- # version is available, but we don't want it for Firefox builds.
- 'freetype-2.3.11-6.el6_1.8.i686',
- 'freetype-devel-2.3.11-6.el6_1.8.i686',
- 'freetype-2.3.11-6.el6_1.8.x86_64',
- ######## 32 bit specific ###########
- ],
- 'src_mozconfig': 'browser/config/mozconfigs/linux32/nightly',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux32/\
-releng.manifest",
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_base_linux_64_builds.py b/testing/mozharness/configs/builds/releng_base_linux_64_builds.py
deleted file mode 100644
index fe04b73b5..000000000
--- a/testing/mozharness/configs/builds/releng_base_linux_64_builds.py
+++ /dev/null
@@ -1,139 +0,0 @@
-import os
-
-config = {
- #########################################################################
- ######## LINUX GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 64 bit keys/values please add them
- # below under the '64 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced with
- # releng_base_linux_64_builds.py
-
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- "buildbot": "/tools/buildbot/bin/buildbot",
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': True,
- # mock shtuff
- 'mock_mozilla_dir': '/builds/mock_mozilla',
- 'mock_target': 'mozilla-centos6-x86_64',
- 'mock_files': [
- ('/home/cltbld/.ssh', '/home/mock_mozilla/.ssh'),
- ('/home/cltbld/.hgrc', '/builds/.hgrc'),
- ('/home/cltbld/.boto', '/builds/.boto'),
- ('/builds/gapi.data', '/builds/gapi.data'),
- ('/builds/relengapi.tok', '/builds/relengapi.tok'),
- ('/tools/tooltool.py', '/builds/tooltool.py'),
- ('/builds/mozilla-desktop-geoloc-api.key', '/builds/mozilla-desktop-geoloc-api.key'),
- ('/builds/crash-stats-api.token', '/builds/crash-stats-api.token'),
- ('/builds/adjust-sdk.token', '/builds/adjust-sdk.token'),
- ('/builds/adjust-sdk-beta.token', '/builds/adjust-sdk-beta.token'),
- ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
- ],
- 'secret_files': [
- {'filename': '/builds/gapi.data',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data',
- 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
- {'filename': '/builds/mozilla-desktop-geoloc-api.key',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key',
- 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
- {'filename': '/builds/adjust-sdk.token',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk.token',
- 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
- {'filename': '/builds/adjust-sdk-beta.token',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/adjust-sdk-beta.token',
- 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
- ],
- 'enable_ccache': True,
- 'vcs_share_base': '/builds/hg-shared',
- 'objdir': 'obj-firefox',
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': True,
- 'enable_talos_sendchange': True,
- 'enable_unittest_sendchange': True,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'Linux_x86-64_%(branch)s',
- 'platform': 'linux64',
- 'stage_platform': 'linux64',
- 'publish_nightly_en_US_routes': True,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- 'LD_LIBRARY_PATH': "/tools/gcc-4.3.3/installed/lib64",
- ##
- },
- 'upload_env': {
- # stage_server is dictated from build_pool_specifics.py
- 'UPLOAD_HOST': '%(stage_server)s',
- 'UPLOAD_USER': '%(stage_username)s',
- 'UPLOAD_SSH_KEY': '/home/mock_mozilla/.ssh/%(stage_ssh_key)s',
- 'UPLOAD_TO_TEMP': '1',
- },
- "check_test_env": {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/linux64/minidump_stackwalk',
- 'MINIDUMP_SAVE_PATH': '%(base_work_dir)s/minidumps',
- },
- 'mock_packages': [
- 'autoconf213', 'python', 'mozilla-python27', 'zip', 'mozilla-python27-mercurial',
- 'git', 'ccache', 'perl-Test-Simple', 'perl-Config-General',
- 'yasm', 'wget',
- 'mpfr', # required for system compiler
- 'xorg-x11-font*', # fonts required for PGO
- 'imake', # required for makedepend!?!
- ### <-- from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'valgrind', 'dbus-x11',
- ######## 64 bit specific ###########
- 'glibc-static', 'libstdc++-static',
- 'gtk2-devel', 'libnotify-devel',
- 'alsa-lib-devel', 'libcurl-devel', 'wireless-tools-devel',
- 'libX11-devel', 'libXt-devel', 'mesa-libGL-devel', 'gnome-vfs2-devel',
- 'GConf2-devel',
- ### from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'pulseaudio-libs-devel', 'gstreamer-devel',
- 'gstreamer-plugins-base-devel', 'freetype-2.3.11-6.el6_1.8.x86_64',
- 'freetype-devel-2.3.11-6.el6_1.8.x86_64'
- ],
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/nightly',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
-releng.manifest",
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_base_mac_64_builds.py b/testing/mozharness/configs/builds/releng_base_mac_64_builds.py
deleted file mode 100644
index e6e338ada..000000000
--- a/testing/mozharness/configs/builds/releng_base_mac_64_builds.py
+++ /dev/null
@@ -1,79 +0,0 @@
-import os
-import sys
-
-config = {
- #########################################################################
- ######## MACOSX GENERIC CONFIG KEYS/VAlUES
-
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- # 'setup-mock',
- 'checkout-sources',
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- 'python2.7': sys.executable,
- "buildbot": "/tools/buildbot/bin/buildbot",
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': True,
- 'enable_ccache': True,
- 'vcs_share_base': '/builds/hg-shared',
- 'objdir': 'obj-firefox/x86_64',
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': False,
- 'enable_talos_sendchange': True,
- 'enable_unittest_sendchange': True,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'OS X 10.7 %(branch)s',
- 'platform': 'macosx64',
- 'stage_platform': 'macosx64',
- 'publish_nightly_en_US_routes': True,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'CHOWN_ROOT': '~/bin/chown_root',
- 'CHOWN_REVERT': '~/bin/chown_revert',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:'
- '/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin',
- ##
- },
- 'upload_env': {
- # stage_server is dictated from build_pool_specifics.py
- 'UPLOAD_HOST': '%(stage_server)s',
- 'UPLOAD_USER': '%(stage_username)s',
- 'UPLOAD_SSH_KEY': '/Users/cltbld/.ssh/%(stage_ssh_key)s',
- 'UPLOAD_TO_TEMP': '1',
- },
- "check_test_env": {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/osx64/minidump_stackwalk',
- 'MINIDUMP_SAVE_PATH': '%(base_work_dir)s/minidumps',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/macosx-universal/nightly',
- 'tooltool_manifest_src': 'browser/config/tooltool-manifests/macosx64/releng.manifest',
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_base_mac_64_cross_builds.py b/testing/mozharness/configs/builds/releng_base_mac_64_cross_builds.py
deleted file mode 100644
index 47738e1ce..000000000
--- a/testing/mozharness/configs/builds/releng_base_mac_64_cross_builds.py
+++ /dev/null
@@ -1,83 +0,0 @@
-import os
-import sys
-
-config = {
- #########################################################################
- ######## MACOSX CROSS GENERIC CONFIG KEYS/VAlUES
-
- # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'build',
- 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- 'python2.7': sys.executable,
- "buildbot": "/tools/buildbot/bin/buildbot",
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': True,
- 'secret_files': [
- {'filename': '/builds/gapi.data',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data',
- 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
- {'filename': '/builds/mozilla-desktop-geoloc-api.key',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key',
- 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
- ],
- 'enable_ccache': True,
- 'enable_check_test': False,
- 'vcs_share_base': '/builds/hg-shared',
- 'objdir': 'obj-firefox/',
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': False,
- 'enable_talos_sendchange': True,
- 'enable_unittest_sendchange': True,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'OS X 10.7 %(branch)s',
- 'platform': 'macosx64',
- 'stage_platform': 'macosx64',
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- ##
- },
- 'upload_env': {
- # stage_server is dictated from build_pool_specifics.py
- 'UPLOAD_HOST': '%(stage_server)s',
- 'UPLOAD_USER': '%(stage_username)s',
- 'UPLOAD_SSH_KEY': '/Users/cltbld/.ssh/%(stage_ssh_key)s',
- 'UPLOAD_TO_TEMP': '1',
- },
- "check_test_env": {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/linux64/minidump_stackwalk',
- 'MINIDUMP_SAVE_PATH': '%(base_work_dir)s/minidumps',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/macosx64/nightly',
- 'tooltool_manifest_src': 'browser/config/tooltool-manifests/macosx64/cross-releng.manifest',
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_base_windows_32_builds.py b/testing/mozharness/configs/builds/releng_base_windows_32_builds.py
deleted file mode 100644
index 0a6708a1f..000000000
--- a/testing/mozharness/configs/builds/releng_base_windows_32_builds.py
+++ /dev/null
@@ -1,95 +0,0 @@
-import os
-import sys
-
-config = {
- #########################################################################
- ######## WINDOWS GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 32 bit keys/values please add them
- # below under the '32 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced with
- # releng_base_windows_32_builds.py
-
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock', windows do not use mock
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- 'python2.7': sys.executable,
- "buildbot": [
- sys.executable,
- 'c:\\mozilla-build\\buildbotve\\scripts\\buildbot'
- ],
- "make": [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'build', 'pymake', 'make.py'
- )
- ],
- 'virtualenv': [
- sys.executable,
- 'c:/mozilla-build/buildbotve/virtualenv.py'
- ],
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': True,
- 'enable_ccache': False,
- 'vcs_share_base': 'C:/builds/hg-shared',
- 'objdir': 'obj-firefox',
- 'tooltool_script': [sys.executable,
- 'C:/mozilla-build/tooltool.py'],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': False,
- 'enable_talos_sendchange': True,
- 'enable_unittest_sendchange': True,
- 'max_build_output_timeout': 60 * 80,
- #########################################################################
-
-
- #########################################################################
- ###### 32 bit specific ######
- 'base_name': 'WINNT_5.2_%(branch)s',
- 'platform': 'win32',
- 'stage_platform': 'win32',
- 'publish_nightly_en_US_routes': True,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'BINSCOPE': 'C:/Program Files (x86)/Microsoft/SDL BinScope/BinScope.exe',
- 'HG_SHARE_BASE_DIR': 'C:/builds/hg-shared',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PATH': 'C:/mozilla-build/nsis-3.01;C:/mozilla-build/python27;'
- 'C:/mozilla-build/buildbotve/scripts;'
- '%s' % (os.environ.get('path')),
- 'PDBSTR_PATH': '/c/Program Files (x86)/Windows Kits/8.0/Debuggers/x64/srcsrv/pdbstr.exe',
- 'PROPERTIES_FILE': os.path.join(os.getcwd(), 'buildprops.json'),
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- },
- 'upload_env': {
- # stage_server is dictated from build_pool_specifics.py
- 'UPLOAD_HOST': '%(stage_server)s',
- 'UPLOAD_USER': '%(stage_username)s',
- 'UPLOAD_SSH_KEY': '/c/Users/cltbld/.ssh/%(stage_ssh_key)s',
- 'UPLOAD_TO_TEMP': '1',
- },
- "check_test_env": {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/win32/minidump_stackwalk.exe',
- 'MINIDUMP_SAVE_PATH': '%(base_work_dir)s/minidumps',
- },
- 'enable_pymake': True,
- 'src_mozconfig': 'browser/config/mozconfigs/win32/nightly',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/win32/releng.manifest",
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_base_windows_64_builds.py b/testing/mozharness/configs/builds/releng_base_windows_64_builds.py
deleted file mode 100644
index ab12fc982..000000000
--- a/testing/mozharness/configs/builds/releng_base_windows_64_builds.py
+++ /dev/null
@@ -1,93 +0,0 @@
-import os
-import sys
-
-config = {
- #########################################################################
- ######## WINDOWS GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 32 bit keys/values please add them
- # below under the '32 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced with
- # releng_base_windows_64_builds.py
-
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock', windows do not use mock
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- 'python2.7': sys.executable,
- "buildbot": [
- sys.executable,
- 'c:\\mozilla-build\\buildbotve\\scripts\\buildbot'
- ],
- "make": [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'build', 'pymake', 'make.py'
- )
- ],
- 'virtualenv': [
- sys.executable,
- 'c:/mozilla-build/buildbotve/virtualenv.py'
- ],
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': True,
- 'enable_ccache': False,
- 'vcs_share_base': 'C:/builds/hg-shared',
- 'objdir': 'obj-firefox',
- 'tooltool_script': [sys.executable,
- 'C:/mozilla-build/tooltool.py'],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': False,
- 'enable_talos_sendchange': True,
- 'enable_unittest_sendchange': True,
- 'max_build_output_timeout': 60 * 80,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'WINNT_6.1_x86-64_%(branch)s',
- 'platform': 'win64',
- 'stage_platform': 'win64',
- 'publish_nightly_en_US_routes': True,
- 'env': {
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': 'C:/builds/hg-shared',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PATH': 'C:/mozilla-build/nsis-3.01;C:/mozilla-build/python27;'
- 'C:/mozilla-build/buildbotve/scripts;'
- '%s' % (os.environ.get('path')),
- 'PDBSTR_PATH': '/c/Program Files (x86)/Windows Kits/8.0/Debuggers/x64/srcsrv/pdbstr.exe',
- 'PROPERTIES_FILE': os.path.join(os.getcwd(), 'buildprops.json'),
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- },
- 'upload_env': {
- # stage_server is dictated from build_pool_specifics.py
- 'UPLOAD_HOST': '%(stage_server)s',
- 'UPLOAD_USER': '%(stage_username)s',
- 'UPLOAD_SSH_KEY': '/c/Users/cltbld/.ssh/%(stage_ssh_key)s',
- 'UPLOAD_TO_TEMP': '1',
- },
- "check_test_env": {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/win64/minidump_stackwalk.exe',
- 'MINIDUMP_SAVE_PATH': '%(base_work_dir)s/minidumps',
- },
- 'enable_pymake': True,
- 'src_mozconfig': 'browser/config/mozconfigs/win64/nightly',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/win64/releng.manifest",
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/32_artifact.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/32_artifact.py
deleted file mode 100644
index f016d5606..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/32_artifact.py
+++ /dev/null
@@ -1,116 +0,0 @@
-import os
-
-config = {
- #########################################################################
- ######## LINUX GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 32 bit keys/values please add them
- # below under the '32 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced with
- # releng_base_linux_64_builds.py
-
- # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'sendchange',
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- "buildbot": "/tools/buildbot/bin/buildbot",
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': False,
- 'enable_ccache': True,
- 'vcs_share_base': '/builds/hg-shared',
- 'objdir': 'obj-firefox',
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': True,
- 'enable_talos_sendchange': False,
- # allows triggering of test jobs when --artifact try syntax is detected on buildbot
- 'enable_unittest_sendchange': True,
- #########################################################################
-
-
- #########################################################################
- ###### 32 bit specific ######
- 'base_name': 'Linux_%(branch)s_Artifact_build',
- 'platform': 'linux',
- 'stage_platform': 'linux',
- 'publish_nightly_en_US_routes': False,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- # 32 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib/ccache:\
-/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:\
-/tools/python27/bin:/tools/python27-mercurial/bin:/home/cltbld/bin',
- 'LD_LIBRARY_PATH': "/tools/gcc-4.3.3/installed/lib",
- },
- 'mock_packages': [
- 'autoconf213', 'python', 'mozilla-python27', 'zip', 'mozilla-python27-mercurial',
- 'git', 'ccache', 'perl-Test-Simple', 'perl-Config-General',
- 'yasm', 'wget',
- 'mpfr', # required for system compiler
- 'xorg-x11-font*', # fonts required for PGO
- 'imake', # required for makedepend!?!
- ### <-- from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'valgrind',
- ######## 32 bit specific ###########
- 'glibc-static.i686', 'libstdc++-static.i686',
- 'gtk2-devel.i686', 'libnotify-devel.i686',
- 'alsa-lib-devel.i686', 'libcurl-devel.i686',
- 'wireless-tools-devel.i686', 'libX11-devel.i686',
- 'libXt-devel.i686', 'mesa-libGL-devel.i686',
- 'gnome-vfs2-devel.i686', 'GConf2-devel.i686',
- 'pulseaudio-libs-devel.i686',
- 'gstreamer-devel.i686', 'gstreamer-plugins-base-devel.i686',
- # Packages already installed in the mock environment, as x86_64
- # packages.
- 'glibc-devel.i686', 'libgcc.i686', 'libstdc++-devel.i686',
- # yum likes to install .x86_64 -devel packages that satisfy .i686
- # -devel packages dependencies. So manually install the dependencies
- # of the above packages.
- 'ORBit2-devel.i686', 'atk-devel.i686', 'cairo-devel.i686',
- 'check-devel.i686', 'dbus-devel.i686', 'dbus-glib-devel.i686',
- 'fontconfig-devel.i686', 'glib2-devel.i686',
- 'hal-devel.i686', 'libICE-devel.i686', 'libIDL-devel.i686',
- 'libSM-devel.i686', 'libXau-devel.i686', 'libXcomposite-devel.i686',
- 'libXcursor-devel.i686', 'libXdamage-devel.i686',
- 'libXdmcp-devel.i686', 'libXext-devel.i686', 'libXfixes-devel.i686',
- 'libXft-devel.i686', 'libXi-devel.i686', 'libXinerama-devel.i686',
- 'libXrandr-devel.i686', 'libXrender-devel.i686',
- 'libXxf86vm-devel.i686', 'libdrm-devel.i686', 'libidn-devel.i686',
- 'libpng-devel.i686', 'libxcb-devel.i686', 'libxml2-devel.i686',
- 'pango-devel.i686', 'perl-devel.i686', 'pixman-devel.i686',
- 'zlib-devel.i686',
- # Freetype packages need to be installed be version, because a newer
- # version is available, but we don't want it for Firefox builds.
- 'freetype-2.3.11-6.el6_1.8.i686',
- 'freetype-devel-2.3.11-6.el6_1.8.i686',
- 'freetype-2.3.11-6.el6_1.8.x86_64',
- ######## 32 bit specific ###########
- ],
- 'src_mozconfig': 'browser/config/mozconfigs/linux32/artifact',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux32/\
-releng.manifest",
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/32_debug.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/32_debug.py
deleted file mode 100644
index 914bfdfe3..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/32_debug.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- # 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- 'debug_build': True,
- 'stage_platform': 'linux-debug',
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- #### 32 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': MOZ_OBJDIR,
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- # 32 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- 'LD_LIBRARY_PATH': '/tools/gcc-4.3.3/installed/lib:\
-%s/dist/bin' % (MOZ_OBJDIR,),
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- 'TINDERBOX_OUTPUT': '1',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/linux32/debug',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/32_debug_artifact.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/32_debug_artifact.py
deleted file mode 100644
index 88ff8450a..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/32_debug_artifact.py
+++ /dev/null
@@ -1,122 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- #########################################################################
- ######## LINUX GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 32 bit keys/values please add them
- # below under the '32 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced with
- # releng_base_linux_64_builds.py
-
- # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'sendchange',
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- "buildbot": "/tools/buildbot/bin/buildbot",
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': False,
- 'enable_ccache': True,
- 'vcs_share_base': '/builds/hg-shared',
- 'objdir': MOZ_OBJDIR,
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': True,
- # debug specific
- 'debug_build': True,
- 'enable_talos_sendchange': False,
- # allows triggering of test jobs when --artifact try syntax is detected on buildbot
- 'enable_unittest_sendchange': True,
- #########################################################################
-
-
- #########################################################################
- ###### 32 bit specific ######
- 'base_name': 'Linux_%(branch)s_Artifact_build',
- 'platform': 'linux',
- 'stage_platform': 'linux-debug',
- 'publish_nightly_en_US_routes': False,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': MOZ_OBJDIR,
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- # debug-specific
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- # 32 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib/ccache:\
-/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:\
-/tools/python27/bin:/tools/python27-mercurial/bin:/home/cltbld/bin',
- 'LD_LIBRARY_PATH': "/tools/gcc-4.3.3/installed/lib",
- },
- 'mock_packages': [
- 'autoconf213', 'python', 'mozilla-python27', 'zip', 'mozilla-python27-mercurial',
- 'git', 'ccache', 'perl-Test-Simple', 'perl-Config-General',
- 'yasm', 'wget',
- 'mpfr', # required for system compiler
- 'xorg-x11-font*', # fonts required for PGO
- 'imake', # required for makedepend!?!
- ### <-- from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'valgrind',
- ######## 32 bit specific ###########
- 'glibc-static.i686', 'libstdc++-static.i686',
- 'gtk2-devel.i686', 'libnotify-devel.i686',
- 'alsa-lib-devel.i686', 'libcurl-devel.i686',
- 'wireless-tools-devel.i686', 'libX11-devel.i686',
- 'libXt-devel.i686', 'mesa-libGL-devel.i686',
- 'gnome-vfs2-devel.i686', 'GConf2-devel.i686',
- 'pulseaudio-libs-devel.i686',
- 'gstreamer-devel.i686', 'gstreamer-plugins-base-devel.i686',
- # Packages already installed in the mock environment, as x86_64
- # packages.
- 'glibc-devel.i686', 'libgcc.i686', 'libstdc++-devel.i686',
- # yum likes to install .x86_64 -devel packages that satisfy .i686
- # -devel packages dependencies. So manually install the dependencies
- # of the above packages.
- 'ORBit2-devel.i686', 'atk-devel.i686', 'cairo-devel.i686',
- 'check-devel.i686', 'dbus-devel.i686', 'dbus-glib-devel.i686',
- 'fontconfig-devel.i686', 'glib2-devel.i686',
- 'hal-devel.i686', 'libICE-devel.i686', 'libIDL-devel.i686',
- 'libSM-devel.i686', 'libXau-devel.i686', 'libXcomposite-devel.i686',
- 'libXcursor-devel.i686', 'libXdamage-devel.i686',
- 'libXdmcp-devel.i686', 'libXext-devel.i686', 'libXfixes-devel.i686',
- 'libXft-devel.i686', 'libXi-devel.i686', 'libXinerama-devel.i686',
- 'libXrandr-devel.i686', 'libXrender-devel.i686',
- 'libXxf86vm-devel.i686', 'libdrm-devel.i686', 'libidn-devel.i686',
- 'libpng-devel.i686', 'libxcb-devel.i686', 'libxml2-devel.i686',
- 'pango-devel.i686', 'perl-devel.i686', 'pixman-devel.i686',
- 'zlib-devel.i686',
- # Freetype packages need to be installed be version, because a newer
- # version is available, but we don't want it for Firefox builds.
- 'freetype-2.3.11-6.el6_1.8.i686',
- 'freetype-devel-2.3.11-6.el6_1.8.i686',
- 'freetype-2.3.11-6.el6_1.8.x86_64',
- ######## 32 bit specific ###########
- ],
- 'src_mozconfig': 'browser/config/mozconfigs/linux32/debug-artifact',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux32/\
-releng.manifest",
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_add-on-devel.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_add-on-devel.py
deleted file mode 100644
index 98462a62f..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_add-on-devel.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import os
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files',
-# 'sendchange',
- 'check-test',
- # 'generate-build-stats',
- # 'update',
- ],
- 'stage_platform': 'linux64-add-on-devel',
- 'publish_nightly_en_US_routes': False,
- 'build_type': 'add-on-devel',
- 'platform_supports_post_upload_to_latest': False,
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/home/worker/workspace/build/src/gcc/bin:/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/add-on-devel',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_artifact.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_artifact.py
deleted file mode 100644
index 5cbc70ade..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_artifact.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import os
-
-config = {
- # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'sendchange',
- # 'generate-build-stats',
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- "buildbot": "/tools/buildbot/bin/buildbot",
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': False,
- 'secret_files': [
- {'filename': '/builds/gapi.data',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/gapi.data',
- 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
- {'filename': '/builds/mozilla-desktop-geoloc-api.key',
- 'secret_name': 'project/releng/gecko/build/level-%(scm-level)s/mozilla-desktop-geoloc-api.key',
- 'min_scm_level': 2, 'default': 'try-build-has-no-secrets'},
- ],
- 'enable_ccache': True,
- 'vcs_share_base': '/builds/hg-shared',
- 'objdir': 'obj-firefox',
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': True,
- 'enable_talos_sendchange': False,
- # allows triggering of test jobs when --artifact try syntax is detected on buildbot
- 'enable_unittest_sendchange': True,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'Linux_x86-64_%(branch)s_Artifact_build',
- 'platform': 'linux64',
- 'stage_platform': 'linux64',
- 'publish_nightly_en_US_routes': False,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- 'LD_LIBRARY_PATH': "/tools/gcc-4.3.3/installed/lib64",
- ##
- },
- 'mock_packages': [
- 'autoconf213', 'python', 'mozilla-python27', 'zip', 'mozilla-python27-mercurial',
- 'git', 'ccache', 'perl-Test-Simple', 'perl-Config-General',
- 'yasm', 'wget',
- 'mpfr', # required for system compiler
- 'xorg-x11-font*', # fonts required for PGO
- 'imake', # required for makedepend!?!
- ### <-- from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'valgrind', 'dbus-x11',
- ######## 64 bit specific ###########
- 'glibc-static', 'libstdc++-static',
- 'gtk2-devel', 'libnotify-devel',
- 'alsa-lib-devel', 'libcurl-devel', 'wireless-tools-devel',
- 'libX11-devel', 'libXt-devel', 'mesa-libGL-devel', 'gnome-vfs2-devel',
- 'GConf2-devel',
- ### from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'pulseaudio-libs-devel', 'gstreamer-devel',
- 'gstreamer-plugins-base-devel', 'freetype-2.3.11-6.el6_1.8.x86_64',
- 'freetype-devel-2.3.11-6.el6_1.8.x86_64'
- ],
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/artifact',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
-releng.manifest",
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan.py
deleted file mode 100644
index 0f57520b5..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan.py
+++ /dev/null
@@ -1,48 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- # 'generate-build-stats',
- # 'update',
- ],
- 'stage_platform': 'linux64-asan',
- 'publish_nightly_en_US_routes': False,
- 'build_type': 'asan',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
-asan.manifest",
- 'platform_supports_post_upload_to_latest': False,
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/nightly-asan',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_and_debug.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_and_debug.py
deleted file mode 100644
index 4ff6a9d2c..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_and_debug.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- # 'generate-build-stats',
- # 'update',
- ],
- 'stage_platform': 'linux64-asan-debug',
- 'publish_nightly_en_US_routes': False,
- 'build_type': 'asan-debug',
- 'debug_build': True,
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
-asan.manifest",
- 'platform_supports_post_upload_to_latest': False,
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/debug-asan',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_tc.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_tc.py
deleted file mode 100644
index 0f57520b5..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_tc.py
+++ /dev/null
@@ -1,48 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- # 'generate-build-stats',
- # 'update',
- ],
- 'stage_platform': 'linux64-asan',
- 'publish_nightly_en_US_routes': False,
- 'build_type': 'asan',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
-asan.manifest",
- 'platform_supports_post_upload_to_latest': False,
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/nightly-asan',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_tc_and_debug.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_tc_and_debug.py
deleted file mode 100644
index 4ff6a9d2c..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_asan_tc_and_debug.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- # 'generate-build-stats',
- # 'update',
- ],
- 'stage_platform': 'linux64-asan-debug',
- 'publish_nightly_en_US_routes': False,
- 'build_type': 'asan-debug',
- 'debug_build': True,
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
-asan.manifest",
- 'platform_supports_post_upload_to_latest': False,
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/debug-asan',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_code_coverage.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_code_coverage.py
deleted file mode 100644
index 3ab4f25a3..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_code_coverage.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- # 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- 'stage_platform': 'linux64-ccov',
- 'platform_supports_post_upload_to_latest': False,
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- 'enable_count_ctors': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/code-coverage',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_debug.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_debug.py
deleted file mode 100644
index e97c82fcd..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_debug.py
+++ /dev/null
@@ -1,45 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- # 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- 'stage_platform': 'linux64-debug',
- 'debug_build': True,
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': MOZ_OBJDIR,
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- # 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- 'LD_LIBRARY_PATH': '/tools/gcc-4.3.3/installed/lib64:\
-%s/dist/bin' % (MOZ_OBJDIR,),
- 'TINDERBOX_OUTPUT': '1',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/debug',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_debug_artifact.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_debug_artifact.py
deleted file mode 100644
index d3a82e476..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_debug_artifact.py
+++ /dev/null
@@ -1,96 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'sendchange',
- # 'generate-build-stats',
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- "buildbot": "/tools/buildbot/bin/buildbot",
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- 'enable_ccache': True,
- 'vcs_share_base': '/builds/hg-shared',
- 'objdir': MOZ_OBJDIR,
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': True,
- # debug specific
- 'debug_build': True,
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- # allows triggering of test jobs when --artifact try syntax is detected on buildbot
- 'enable_unittest_sendchange': True,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'Linux_x86-64_%(branch)s_Artifact_build',
- 'platform': 'linux64',
- 'stage_platform': 'linux64-debug',
- 'publish_nightly_en_US_routes': False,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': MOZ_OBJDIR,
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- # debug-specific
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- ## 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- 'LD_LIBRARY_PATH': "/tools/gcc-4.3.3/installed/lib64",
- ##
- },
- 'mock_packages': [
- 'autoconf213', 'python', 'mozilla-python27', 'zip', 'mozilla-python27-mercurial',
- 'git', 'ccache', 'perl-Test-Simple', 'perl-Config-General',
- 'yasm', 'wget',
- 'mpfr', # required for system compiler
- 'xorg-x11-font*', # fonts required for PGO
- 'imake', # required for makedepend!?!
- ### <-- from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'valgrind', 'dbus-x11',
- ######## 64 bit specific ###########
- 'glibc-static', 'libstdc++-static',
- 'gtk2-devel', 'libnotify-devel',
- 'alsa-lib-devel', 'libcurl-devel', 'wireless-tools-devel',
- 'libX11-devel', 'libXt-devel', 'mesa-libGL-devel', 'gnome-vfs2-devel',
- 'GConf2-devel',
- ### from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'pulseaudio-libs-devel', 'gstreamer-devel',
- 'gstreamer-plugins-base-devel', 'freetype-2.3.11-6.el6_1.8.x86_64',
- 'freetype-devel-2.3.11-6.el6_1.8.x86_64'
- ],
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/debug-artifact',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
-releng.manifest",
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_source.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_source.py
deleted file mode 100644
index dfc87cdf1..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_source.py
+++ /dev/null
@@ -1,20 +0,0 @@
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'package-source',
- 'generate-source-signing-manifest',
- ],
- 'stage_platform': 'source', # Not used, but required by the script
- 'buildbot_json_path': 'buildprops.json',
- 'app_ini_path': 'FAKE', # Not used, but required by the script
- 'objdir': 'obj-firefox',
- 'env': {
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'LC_ALL': 'C',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/source',
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_debug.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_debug.py
deleted file mode 100644
index d4de036de..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_debug.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- # 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- 'debug_build': True,
- 'stage_platform': 'linux64-st-an-debug',
- 'build_type': 'st-an-debug',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
-clang.manifest",
- 'platform_supports_post_upload_to_latest': False,
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- 'enable_unittest_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': MOZ_OBJDIR,
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- # 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/\
-debug-static-analysis-clang',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_opt.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_opt.py
deleted file mode 100644
index 496d89f96..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_stat_and_opt.py
+++ /dev/null
@@ -1,88 +0,0 @@
-import os
-
-config = {
- # note: overridden by MOZHARNESS_ACTIONS in TaskCluster tasks
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- # 'generate-build-stats',
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- "buildbot": "/tools/buildbot/bin/buildbot",
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': False,
- 'enable_ccache': True,
- 'vcs_share_base': '/builds/hg-shared',
- 'objdir': 'obj-firefox',
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': True,
- 'enable_talos_sendchange': False,
- 'enable_unittest_sendchange': False,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'Linux_x86-64_%(branch)s_Static_Analysis',
- 'platform': 'linux64',
- 'stage_platform': 'linux64-st-an',
- 'publish_nightly_en_US_routes': False,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- 'LD_LIBRARY_PATH': "/tools/gcc-4.3.3/installed/lib64",
- ##
- },
- 'mock_packages': [
- 'autoconf213', 'python', 'mozilla-python27', 'zip', 'mozilla-python27-mercurial',
- 'git', 'ccache', 'perl-Test-Simple', 'perl-Config-General',
- 'yasm', 'wget',
- 'mpfr', # required for system compiler
- 'xorg-x11-font*', # fonts required for PGO
- 'imake', # required for makedepend!?!
- ### <-- from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'valgrind', 'dbus-x11',
- ######## 64 bit specific ###########
- 'glibc-static', 'libstdc++-static',
- 'gtk2-devel', 'libnotify-devel',
- 'alsa-lib-devel', 'libcurl-devel', 'wireless-tools-devel',
- 'libX11-devel', 'libXt-devel', 'mesa-libGL-devel', 'gnome-vfs2-devel',
- 'GConf2-devel',
- ### from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'pulseaudio-libs-devel', 'gstreamer-devel',
- 'gstreamer-plugins-base-devel', 'freetype-2.3.11-6.el6_1.8.x86_64',
- 'freetype-devel-2.3.11-6.el6_1.8.x86_64'
- ],
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/opt-static-analysis-clang',
- 'tooltool_manifest_src': 'browser/config/tooltool-manifests/linux64/\
-clang.manifest.centos6',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_tsan.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_tsan.py
deleted file mode 100644
index ae8ed6278..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_tsan.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- # 'check-test',
- # 'generate-build-stats',
- # 'update',
- ],
- 'stage_platform': 'linux64-tsan',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
-tsan.manifest",
- 'platform_supports_post_upload_to_latest': False,
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/opt-tsan',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_valgrind.py b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_valgrind.py
deleted file mode 100644
index 97ffd84f8..000000000
--- a/testing/mozharness/configs/builds/releng_sub_linux_configs/64_valgrind.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- #'setup-mock',
- 'build',
- #'upload-files',
- #'sendchange',
- 'check-test',
- 'valgrind-test',
- #'generate-build-stats',
- #'update',
- ],
- 'stage_platform': 'linux64-valgrind',
- 'publish_nightly_en_US_routes': False,
- 'build_type': 'valgrind',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/linux64/\
-releng.manifest",
- 'platform_supports_post_upload_to_latest': False,
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'DISPLAY': ':2',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/buildbot/bin:/usr/local/bin:/usr/lib64/ccache:/bin:\
-/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/tools/git/bin:/tools/python27/bin:\
-/tools/python27-mercurial/bin:/home/cltbld/bin',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/linux64/valgrind',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_add-on-devel.py b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_add-on-devel.py
deleted file mode 100644
index d54c4d3a6..000000000
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_add-on-devel.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import os
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
-# 'setup-mock',
- 'build',
- 'upload-files',
-# 'sendchange',
- 'check-test',
-# 'generate-build-stats',
-# 'update',
- ],
- 'stage_platform': 'macosx64-add-on-devel',
- 'publish_nightly_en_US_routes': False,
- 'build_type': 'add-on-devel',
- 'platform_supports_post_upload_to_latest': False,
- 'objdir': 'obj-firefox',
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:'
- '/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin',
- ##
- },
- 'src_mozconfig': 'browser/config/mozconfigs/macosx64/add-on-devel',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_artifact.py b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_artifact.py
deleted file mode 100644
index c4d74c145..000000000
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_artifact.py
+++ /dev/null
@@ -1,65 +0,0 @@
-import os
-import sys
-
-config = {
- #########################################################################
- ######## MACOSX GENERIC CONFIG KEYS/VAlUES
-
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- # 'setup-mock',
- 'checkout-sources',
- 'build',
- 'sendchange',
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- 'python2.7': sys.executable,
- "buildbot": "/tools/buildbot/bin/buildbot",
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': False,
- 'enable_ccache': True,
- 'vcs_share_base': '/builds/hg-shared',
- 'objdir': 'obj-firefox',
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': False,
- 'enable_talos_sendchange': False,
- # allows triggering of test jobs when --artifact try syntax is detected on buildbot
- 'enable_unittest_sendchange': True,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'OS X 10.7 %(branch)s_Artifact_build',
- 'platform': 'macosx64',
- 'stage_platform': 'macosx64',
- 'publish_nightly_en_US_routes': False,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'CHOWN_ROOT': '~/bin/chown_root',
- 'CHOWN_REVERT': '~/bin/chown_revert',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- ## 64 bit specific
- 'PATH': '/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:'
- '/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin',
- ##
- },
- 'src_mozconfig': 'browser/config/mozconfigs/macosx64/artifact',
- 'tooltool_manifest_src': 'browser/config/tooltool-manifests/macosx64/releng.manifest',
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_debug.py b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_debug.py
deleted file mode 100644
index 91cbdb62d..000000000
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_debug.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- # 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- 'stage_platform': 'macosx64-debug',
- 'debug_build': True,
- 'objdir': 'obj-firefox',
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- ## 64 bit specific
- 'PATH': '/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:'
- '/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin',
- ##
- },
- 'src_mozconfig': 'browser/config/mozconfigs/macosx64/debug',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_opt.py b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_opt.py
deleted file mode 100644
index f29800f14..000000000
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_opt.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock',
- 'build',
- ],
- 'stage_platform': 'macosx64-st-an',
- 'debug_build': False,
- 'objdir': 'obj-firefox',
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- ## 64 bit specific
- 'PATH': '/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:'
- '/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin',
- ##
- },
- 'src_mozconfig': 'browser/config/mozconfigs/macosx64/opt-static-analysis',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_universal.py b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_universal.py
deleted file mode 100644
index c399b4f4d..000000000
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_cross_universal.py
+++ /dev/null
@@ -1,4 +0,0 @@
-config = {
- 'objdir': 'obj-firefox/x86_64',
- 'src_mozconfig': 'browser/config/mozconfigs/macosx-universal/nightly',
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_debug.py b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_debug.py
deleted file mode 100644
index 374dc12d1..000000000
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_debug.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- 'stage_platform': 'macosx64-debug',
- 'debug_build': True,
- 'objdir': 'obj-firefox',
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- ## 64 bit specific
- 'PATH': '/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:'
- '/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin',
- ##
- },
- 'src_mozconfig': 'browser/config/mozconfigs/macosx64/debug',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_debug_artifact.py b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_debug_artifact.py
deleted file mode 100644
index 937ca1291..000000000
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_debug_artifact.py
+++ /dev/null
@@ -1,65 +0,0 @@
-import os
-import sys
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- #########################################################################
- ######## MACOSX GENERIC CONFIG KEYS/VAlUES
-
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- # 'setup-mock',
- 'checkout-sources',
- 'build',
- 'sendchange',
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- 'python2.7': sys.executable,
- "buildbot": "/tools/buildbot/bin/buildbot",
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': False,
- 'enable_ccache': True,
- 'vcs_share_base': '/builds/hg-shared',
- 'objdir': MOZ_OBJDIR,
- # debug specific
- 'debug_build': True,
- 'enable_talos_sendchange': False,
- # allows triggering of test jobs when --artifact try syntax is detected on buildbot
- 'enable_unittest_sendchange': True,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'OS X 10.7 %(branch)s_Artifact_build',
- 'platform': 'macosx64',
- 'stage_platform': 'macosx64-debug',
- 'publish_nightly_en_US_routes': False,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': MOZ_OBJDIR,
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- # debug-specific
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- ## 64 bit specific
- 'PATH': '/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:'
- '/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin',
- ##
- },
- 'src_mozconfig': 'browser/config/mozconfigs/macosx64/debug-artifact',
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_stat_and_debug.py b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_stat_and_debug.py
deleted file mode 100644
index 6dccae7ab..000000000
--- a/testing/mozharness/configs/builds/releng_sub_mac_configs/64_stat_and_debug.py
+++ /dev/null
@@ -1,48 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock',
- 'build',
- 'upload-files',
- 'sendchange',
- # 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- 'debug_build': True,
- 'stage_platform': 'macosx64-st-an-debug',
- 'build_type': 'st-an-debug',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/macosx64/\
-clang.manifest",
- 'platform_supports_post_upload_to_latest': False,
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- 'enable_unittest_sendchange': False,
- 'objdir': MOZ_OBJDIR,
- #### 64 bit build specific #####
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': '/builds/hg-shared',
- 'MOZ_OBJDIR': MOZ_OBJDIR,
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'CCACHE_DIR': '/builds/ccache',
- 'CCACHE_COMPRESS': '1',
- 'CCACHE_UMASK': '002',
- 'LC_ALL': 'C',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- # 64 bit specific
- 'PATH': '/tools/python/bin:/tools/buildbot/bin:/opt/local/bin:/usr/bin:'
- '/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/macosx64/debug-static-analysis',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_add-on-devel.py b/testing/mozharness/configs/builds/releng_sub_windows_configs/32_add-on-devel.py
deleted file mode 100644
index ba108ab1f..000000000
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_add-on-devel.py
+++ /dev/null
@@ -1,38 +0,0 @@
-import os
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock', windows do not use mock
- 'build',
- 'upload-files',
-# 'sendchange',
- 'check-test',
-# 'generate-build-stats',
-# 'update',
- ],
- 'stage_platform': 'win32-add-on-devel',
- 'build_type': 'add-on-devel',
- 'enable_talos_sendchange': False,
- #### 32 bit build specific #####
- 'env': {
- 'BINSCOPE': 'C:/Program Files (x86)/Microsoft/SDL BinScope/BinScope.exe',
- 'HG_SHARE_BASE_DIR': 'C:/builds/hg-shared',
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PATH': 'C:/mozilla-build/nsis-3.01;C:/mozilla-build/python27;'
- 'C:/mozilla-build/buildbotve/scripts;'
- '%s' % (os.environ.get('path')),
- 'PROPERTIES_FILE': os.path.join(os.getcwd(), 'buildprops.json'),
- 'TINDERBOX_OUTPUT': '1',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/win32/add-on-devel',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_artifact.py b/testing/mozharness/configs/builds/releng_sub_windows_configs/32_artifact.py
deleted file mode 100644
index 8bf35fba3..000000000
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_artifact.py
+++ /dev/null
@@ -1,81 +0,0 @@
-import os
-import sys
-
-config = {
- #########################################################################
- ######## WINDOWS GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 32 bit keys/values please add them
- # below under the '32 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced with
- # releng_base_windows_32_builds.py
-
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock', windows do not use mock
- 'build',
- 'sendchange',
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- 'python2.7': sys.executable,
- "buildbot": [
- sys.executable,
- 'c:\\mozilla-build\\buildbotve\\scripts\\buildbot'
- ],
- "make": [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'build', 'pymake', 'make.py'
- )
- ],
- 'virtualenv': [
- sys.executable,
- 'c:/mozilla-build/buildbotve/virtualenv.py'
- ],
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': False,
- 'enable_ccache': False,
- 'vcs_share_base': 'C:/builds/hg-shared',
- 'objdir': 'obj-firefox',
- 'tooltool_script': [sys.executable,
- 'C:/mozilla-build/tooltool.py'],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': False,
- 'enable_talos_sendchange': False,
- # allows triggering of test jobs when --artifact try syntax is detected on buildbot
- 'enable_unittest_sendchange': True,
- 'max_build_output_timeout': 60 * 80,
- #########################################################################
-
-
- #########################################################################
- ###### 32 bit specific ######
- 'base_name': 'WINNT_5.2_%(branch)s_Artifact_build',
- 'platform': 'win32',
- 'stage_platform': 'win32',
- 'publish_nightly_en_US_routes': False,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'BINSCOPE': 'C:/Program Files (x86)/Microsoft/SDL BinScope/BinScope.exe',
- 'HG_SHARE_BASE_DIR': 'C:/builds/hg-shared',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PATH': 'C:/mozilla-build/nsis-3.0b1;C:/mozilla-build/python27;'
- 'C:/mozilla-build/buildbotve/scripts;'
- '%s' % (os.environ.get('path')),
- 'PDBSTR_PATH': '/c/Program Files (x86)/Windows Kits/8.0/Debuggers/x64/srcsrv/pdbstr.exe',
- 'PROPERTIES_FILE': os.path.join(os.getcwd(), 'buildprops.json'),
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- },
- 'enable_pymake': True,
- 'src_mozconfig': 'browser/config/mozconfigs/win32/artifact',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/win32/releng.manifest",
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_debug.py b/testing/mozharness/configs/builds/releng_sub_windows_configs/32_debug.py
deleted file mode 100644
index d9b769505..000000000
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_debug.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock', windows do not use mock
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- 'stage_platform': 'win32-debug',
- 'debug_build': True,
- 'enable_talos_sendchange': False,
- #### 32 bit build specific #####
- 'env': {
- 'BINSCOPE': 'C:/Program Files (x86)/Microsoft/SDL BinScope/BinScope.exe',
- 'HG_SHARE_BASE_DIR': 'C:/builds/hg-shared',
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PATH': 'C:/mozilla-build/nsis-3.01;C:/mozilla-build/python27;'
- 'C:/mozilla-build/buildbotve/scripts;'
- '%s' % (os.environ.get('path')),
- 'PROPERTIES_FILE': os.path.join(os.getcwd(), 'buildprops.json'),
- 'TINDERBOX_OUTPUT': '1',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/win32/debug',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_debug_artifact.py b/testing/mozharness/configs/builds/releng_sub_windows_configs/32_debug_artifact.py
deleted file mode 100644
index ad9b2eeaf..000000000
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_debug_artifact.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import os
-import sys
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- #########################################################################
- ######## WINDOWS GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 32 bit keys/values please add them
- # below under the '32 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced with
- # releng_base_windows_32_builds.py
-
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock', windows do not use mock
- 'build',
- 'sendchange',
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- 'python2.7': sys.executable,
- "buildbot": [
- sys.executable,
- 'c:\\mozilla-build\\buildbotve\\scripts\\buildbot'
- ],
- "make": [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'build', 'pymake', 'make.py'
- )
- ],
- 'virtualenv': [
- sys.executable,
- 'c:/mozilla-build/buildbotve/virtualenv.py'
- ],
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': False,
- 'enable_ccache': False,
- 'vcs_share_base': 'C:/builds/hg-shared',
- 'objdir': MOZ_OBJDIR,
- 'tooltool_script': [sys.executable,
- 'C:/mozilla-build/tooltool.py'],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': False,
- # debug specific
- 'debug_build': True,
- 'enable_talos_sendchange': False,
- # allows triggering of test jobs when --artifact try syntax is detected on buildbot
- 'enable_unittest_sendchange': True,
- 'max_build_output_timeout': 60 * 80,
- #########################################################################
-
-
- #########################################################################
- ###### 32 bit specific ######
- 'base_name': 'WINNT_5.2_%(branch)s_Artifact_build',
- 'platform': 'win32',
- 'stage_platform': 'win32-debug',
- 'publish_nightly_en_US_routes': False,
- 'env': {
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'BINSCOPE': 'C:/Program Files (x86)/Microsoft/SDL BinScope/BinScope.exe',
- 'HG_SHARE_BASE_DIR': 'C:/builds/hg-shared',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': MOZ_OBJDIR,
- 'PATH': 'C:/mozilla-build/nsis-3.0b1;C:/mozilla-build/python27;'
- 'C:/mozilla-build/buildbotve/scripts;'
- '%s' % (os.environ.get('path')),
- 'PROPERTIES_FILE': os.path.join(os.getcwd(), 'buildprops.json'),
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- # debug-specific
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- },
- 'enable_pymake': True,
- 'src_mozconfig': 'browser/config/mozconfigs/win32/debug-artifact',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/win32/releng.manifest",
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_stat_and_debug.py b/testing/mozharness/configs/builds/releng_sub_windows_configs/32_stat_and_debug.py
deleted file mode 100644
index e02703462..000000000
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/32_stat_and_debug.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock', windows do not use mock
- 'build',
- # 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- 'stage_platform': 'win32-st-an-debug',
- 'debug_build': True,
- 'enable_signing': False,
- 'enable_talos_sendchange': False,
- 'enable_unittest_sendchange': False,
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/win32/\
-clang.manifest",
- 'platform_supports_post_upload_to_latest': False,
- 'objdir': MOZ_OBJDIR,
- #### 32 bit build specific #####
- 'env': {
- 'BINSCOPE': 'C:/Program Files (x86)/Microsoft/SDL BinScope/BinScope.exe',
- 'HG_SHARE_BASE_DIR': 'C:/builds/hg-shared',
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PATH': 'C:/mozilla-build/nsis-3.01;C:/mozilla-build/python27;'
- 'C:/mozilla-build/buildbotve/scripts;'
- '%s' % (os.environ.get('path')),
- 'PROPERTIES_FILE': os.path.join(os.getcwd(), 'buildprops.json'),
- 'TINDERBOX_OUTPUT': '1',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/win32/debug-static-analysis',
- 'purge_minsize': 9,
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_windows_configs/64_add-on-devel.py b/testing/mozharness/configs/builds/releng_sub_windows_configs/64_add-on-devel.py
deleted file mode 100644
index 8567c7e72..000000000
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/64_add-on-devel.py
+++ /dev/null
@@ -1,37 +0,0 @@
-import os
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock', windows do not use mock
- 'build',
- 'upload-files',
-# 'sendchange',
- 'check-test',
-# 'generate-build-stats',
-# 'update',
- ],
- 'stage_platform': 'win64-add-on-devel',
- 'build_type': 'add-on-devel',
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'BINSCOPE': 'C:/Program Files (x86)/Microsoft/SDL BinScope/BinScope.exe',
- 'HG_SHARE_BASE_DIR': 'C:/builds/hg-shared',
- 'MOZ_AUTOMATION': '1',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PATH': 'C:/mozilla-build/nsis-3.01;C:/mozilla-build/python27;'
- 'C:/mozilla-build/buildbotve/scripts;'
- '%s' % (os.environ.get('path')),
- 'PROPERTIES_FILE': os.path.join(os.getcwd(), 'buildprops.json'),
- 'TINDERBOX_OUTPUT': '1',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/win64/add-on-devel',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_windows_configs/64_artifact.py b/testing/mozharness/configs/builds/releng_sub_windows_configs/64_artifact.py
deleted file mode 100644
index b99ebb6b3..000000000
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/64_artifact.py
+++ /dev/null
@@ -1,79 +0,0 @@
-import os
-import sys
-
-config = {
- #########################################################################
- ######## WINDOWS GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 32 bit keys/values please add them
- # below under the '32 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced with
- # releng_base_windows_64_builds.py
-
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock', windows do not use mock
- 'build',
- 'sendchange',
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- 'python2.7': sys.executable,
- "buildbot": [
- sys.executable,
- 'c:\\mozilla-build\\buildbotve\\scripts\\buildbot'
- ],
- "make": [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'build', 'pymake', 'make.py'
- )
- ],
- 'virtualenv': [
- sys.executable,
- 'c:/mozilla-build/buildbotve/virtualenv.py'
- ],
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': False,
- 'enable_ccache': False,
- 'vcs_share_base': 'C:/builds/hg-shared',
- 'objdir': 'obj-firefox',
- 'tooltool_script': [sys.executable,
- 'C:/mozilla-build/tooltool.py'],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': False,
- 'enable_talos_sendchange': False,
- # allows triggering of test jobs when --artifact try syntax is detected on buildbot
- 'enable_unittest_sendchange': True,
- 'max_build_output_timeout': 60 * 80,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'WINNT_6.1_x86-64_%(branch)s_Artifact_build',
- 'platform': 'win64',
- 'stage_platform': 'win64',
- 'publish_nightly_en_US_routes': False,
- 'env': {
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': 'C:/builds/hg-shared',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PATH': 'C:/mozilla-build/nsis-3.0b1;C:/mozilla-build/python27;'
- 'C:/mozilla-build/buildbotve/scripts;'
- '%s' % (os.environ.get('path')),
- 'PDBSTR_PATH': '/c/Program Files (x86)/Windows Kits/8.0/Debuggers/x64/srcsrv/pdbstr.exe',
- 'PROPERTIES_FILE': os.path.join(os.getcwd(), 'buildprops.json'),
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- },
- 'enable_pymake': True,
- 'src_mozconfig': 'browser/config/mozconfigs/win64/artifact',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/win64/releng.manifest",
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_windows_configs/64_debug.py b/testing/mozharness/configs/builds/releng_sub_windows_configs/64_debug.py
deleted file mode 100644
index e8145dea9..000000000
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/64_debug.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import os
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock', windows do not use mock
- 'build',
- 'upload-files',
- 'sendchange',
- 'check-test',
- 'generate-build-stats',
- 'update', # decided by query_is_nightly()
- ],
- 'stage_platform': 'win64-debug',
- 'debug_build': True,
- 'enable_talos_sendchange': False,
- #### 64 bit build specific #####
- 'env': {
- 'BINSCOPE': 'C:/Program Files (x86)/Microsoft/SDL BinScope/BinScope.exe',
- 'HG_SHARE_BASE_DIR': 'C:/builds/hg-shared',
- 'MOZ_AUTOMATION': '1',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PATH': 'C:/mozilla-build/nsis-3.01;C:/mozilla-build/python27;'
- 'C:/mozilla-build/buildbotve/scripts;'
- '%s' % (os.environ.get('path')),
- 'PROPERTIES_FILE': os.path.join(os.getcwd(), 'buildprops.json'),
- 'TINDERBOX_OUTPUT': '1',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- },
- 'src_mozconfig': 'browser/config/mozconfigs/win64/debug',
- #######################
-}
diff --git a/testing/mozharness/configs/builds/releng_sub_windows_configs/64_debug_artifact.py b/testing/mozharness/configs/builds/releng_sub_windows_configs/64_debug_artifact.py
deleted file mode 100644
index 892a6622d..000000000
--- a/testing/mozharness/configs/builds/releng_sub_windows_configs/64_debug_artifact.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import os
-import sys
-
-MOZ_OBJDIR = 'obj-firefox'
-
-config = {
- #########################################################################
- ######## WINDOWS GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 32 bit keys/values please add them
- # below under the '32 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced with
- # releng_base_windows_64_builds.py
-
- 'default_actions': [
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- # 'setup-mock', windows do not use mock
- 'build',
- 'sendchange',
- ],
- "buildbot_json_path": "buildprops.json",
- 'exes': {
- 'python2.7': sys.executable,
- "buildbot": [
- sys.executable,
- 'c:\\mozilla-build\\buildbotve\\scripts\\buildbot'
- ],
- "make": [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'build', 'pymake', 'make.py'
- )
- ],
- 'virtualenv': [
- sys.executable,
- 'c:/mozilla-build/buildbotve/virtualenv.py'
- ],
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': False,
- 'enable_ccache': False,
- 'vcs_share_base': 'C:/builds/hg-shared',
- 'objdir': MOZ_OBJDIR,
- 'tooltool_script': [sys.executable,
- 'C:/mozilla-build/tooltool.py'],
- 'tooltool_bootstrap': "setup.sh",
- 'enable_count_ctors': False,
- # debug specific
- 'debug_build': True,
- 'enable_talos_sendchange': False,
- # allows triggering of test jobs when --artifact try syntax is detected on buildbot
- 'enable_unittest_sendchange': True,
- 'max_build_output_timeout': 60 * 80,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'WINNT_6.1_x86-64_%(branch)s_Artifact_build',
- 'platform': 'win64',
- 'stage_platform': 'win64-debug',
- 'publish_nightly_en_US_routes': False,
- 'env': {
- 'BINSCOPE': 'C:/Program Files (x86)/Microsoft/SDL BinScope/BinScope.exe',
- 'MOZ_AUTOMATION': '1',
- 'HG_SHARE_BASE_DIR': 'C:/builds/hg-shared',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': MOZ_OBJDIR,
- 'PATH': 'C:/mozilla-build/nsis-3.0b1;C:/mozilla-build/python27;'
- 'C:/mozilla-build/buildbotve/scripts;'
- '%s' % (os.environ.get('path')),
- 'PROPERTIES_FILE': os.path.join(os.getcwd(), 'buildprops.json'),
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- # debug-specific
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- },
- 'enable_pymake': True,
- 'src_mozconfig': 'browser/config/mozconfigs/win64/debug-artifact',
- 'tooltool_manifest_src': "browser/config/tooltool-manifests/win64/releng.manifest",
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/taskcluster_firefox_win32_debug.py b/testing/mozharness/configs/builds/taskcluster_firefox_win32_debug.py
deleted file mode 100644
index ed53474ad..000000000
--- a/testing/mozharness/configs/builds/taskcluster_firefox_win32_debug.py
+++ /dev/null
@@ -1,91 +0,0 @@
-import os
-import sys
-
-config = {
- #########################################################################
- ######## WINDOWS GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 32 bit keys/values please add them
- # below under the '32 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced between:
- # - taskcluster_firefox_win32_debug
- # - taskcluster_firefox_win32_opt
- # - taskcluster_firefox_win64_debug
- # - taskcluster_firefox_win64_opt
-
- 'default_actions': [
- 'clone-tools',
- 'build',
- 'check-test',
- ],
- 'exes': {
- 'python2.7': sys.executable,
- 'make': [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'build', 'pymake', 'make.py'
- )
- ],
- 'virtualenv': [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'python', 'virtualenv', 'virtualenv.py'
- )
- ],
- 'mach-build': [
- os.path.join(os.environ['MOZILLABUILD'], 'msys', 'bin', 'bash.exe'),
- os.path.join(os.getcwd(), 'build', 'src', 'mach'),
- '--log-no-times', 'build', '-v'
- ],
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': True,
- 'enable_ccache': False,
- 'vcs_share_base': os.path.join('y:', os.sep, 'hg-shared'),
- 'objdir': 'obj-firefox',
- 'tooltool_script': [
- sys.executable,
- os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')
- ],
- 'tooltool_bootstrap': 'setup.sh',
- 'enable_count_ctors': False,
- 'max_build_output_timeout': 60 * 80,
- #########################################################################
-
-
- #########################################################################
- ###### 32 bit specific ######
- 'base_name': 'WINNT_5.2_%(branch)s',
- 'platform': 'win32',
- 'stage_platform': 'win32-debug',
- 'debug_build': True,
- 'publish_nightly_en_US_routes': True,
- 'env': {
- 'BINSCOPE': os.path.join(
- os.environ['ProgramFiles(x86)'], 'Microsoft', 'SDL BinScope', 'BinScope.exe'
- ),
- 'HG_SHARE_BASE_DIR': os.path.join('y:', os.sep, 'hg-shared'),
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PDBSTR_PATH': '/c/Program Files (x86)/Windows Kits/10/Debuggers/x86/srcsrv/pdbstr.exe',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- 'MSYSTEM': 'MINGW32',
- },
- 'upload_env': {
- 'UPLOAD_HOST': 'localhost',
- 'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
- },
- "check_test_env": {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s\\breakpad\\win32\\minidump_stackwalk.exe',
- 'MINIDUMP_SAVE_PATH': '%(base_work_dir)s\\minidumps',
- },
- 'enable_pymake': True,
- 'src_mozconfig': 'browser\\config\\mozconfigs\\win32\\debug',
- 'tooltool_manifest_src': 'browser\\config\\tooltool-manifests\\win32\\releng.manifest',
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/taskcluster_firefox_win32_opt.py b/testing/mozharness/configs/builds/taskcluster_firefox_win32_opt.py
deleted file mode 100644
index 4a6502dce..000000000
--- a/testing/mozharness/configs/builds/taskcluster_firefox_win32_opt.py
+++ /dev/null
@@ -1,89 +0,0 @@
-import os
-import sys
-
-config = {
- #########################################################################
- ######## WINDOWS GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 32 bit keys/values please add them
- # below under the '32 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced between:
- # - taskcluster_firefox_win32_debug
- # - taskcluster_firefox_win32_opt
- # - taskcluster_firefox_win64_debug
- # - taskcluster_firefox_win64_opt
-
- 'default_actions': [
- 'clone-tools',
- 'build',
- 'check-test',
- ],
- 'exes': {
- 'python2.7': sys.executable,
- 'make': [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'build', 'pymake', 'make.py'
- )
- ],
- 'virtualenv': [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'python', 'virtualenv', 'virtualenv.py'
- )
- ],
- 'mach-build': [
- os.path.join(os.environ['MOZILLABUILD'], 'msys', 'bin', 'bash.exe'),
- os.path.join(os.getcwd(), 'build', 'src', 'mach'),
- '--log-no-times', 'build', '-v'
- ],
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': True,
- 'enable_ccache': False,
- 'vcs_share_base': os.path.join('y:', os.sep, 'hg-shared'),
- 'objdir': 'obj-firefox',
- 'tooltool_script': [
- sys.executable,
- os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')
- ],
- 'tooltool_bootstrap': 'setup.sh',
- 'enable_count_ctors': False,
- 'max_build_output_timeout': 60 * 80,
- #########################################################################
-
-
- #########################################################################
- ###### 32 bit specific ######
- 'base_name': 'WINNT_5.2_%(branch)s',
- 'platform': 'win32',
- 'stage_platform': 'win32',
- 'publish_nightly_en_US_routes': True,
- 'env': {
- 'BINSCOPE': os.path.join(
- os.environ['ProgramFiles(x86)'], 'Microsoft', 'SDL BinScope', 'BinScope.exe'
- ),
- 'HG_SHARE_BASE_DIR': os.path.join('y:', os.sep, 'hg-shared'),
- 'MOZBUILD_STATE_PATH': os.path.join(os.getcwd(), '.mozbuild'),
- 'MOZ_AUTOMATION': '1',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PDBSTR_PATH': '/c/Program Files (x86)/Windows Kits/10/Debuggers/x86/srcsrv/pdbstr.exe',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- 'MSYSTEM': 'MINGW32',
- },
- 'upload_env': {
- 'UPLOAD_HOST': 'localhost',
- 'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
- },
- "check_test_env": {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s\\breakpad\\win32\\minidump_stackwalk.exe',
- 'MINIDUMP_SAVE_PATH': '%(base_work_dir)s\\minidumps',
- },
- 'enable_pymake': True,
- 'src_mozconfig': 'browser\\config\\mozconfigs\\win32\\nightly',
- 'tooltool_manifest_src': 'browser\\config\\tooltool-manifests\\win32\\releng.manifest',
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/taskcluster_firefox_win64_debug.py b/testing/mozharness/configs/builds/taskcluster_firefox_win64_debug.py
deleted file mode 100644
index 687cf13c6..000000000
--- a/testing/mozharness/configs/builds/taskcluster_firefox_win64_debug.py
+++ /dev/null
@@ -1,87 +0,0 @@
-import os
-import sys
-
-config = {
- #########################################################################
- ######## WINDOWS GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 64 bit keys/values please add them
- # below under the '64 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced between:
- # - taskcluster_firefox_win32_debug
- # - taskcluster_firefox_win32_opt
- # - taskcluster_firefox_win64_debug
- # - taskcluster_firefox_win64_opt
-
- 'default_actions': [
- 'clone-tools',
- 'build',
- 'check-test',
- ],
- 'exes': {
- 'python2.7': sys.executable,
- 'make': [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'build', 'pymake', 'make.py'
- )
- ],
- 'virtualenv': [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'python', 'virtualenv', 'virtualenv.py'
- )
- ],
- 'mach-build': [
- os.path.join(os.environ['MOZILLABUILD'], 'msys', 'bin', 'bash.exe'),
- os.path.join(os.getcwd(), 'build', 'src', 'mach'),
- '--log-no-times', 'build', '-v'
- ],
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': True,
- 'enable_ccache': False,
- 'vcs_share_base': os.path.join('y:', os.sep, 'hg-shared'),
- 'objdir': 'obj-firefox',
- 'tooltool_script': [
- sys.executable,
- os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')
- ],
- 'tooltool_bootstrap': 'setup.sh',
- 'enable_count_ctors': False,
- 'max_build_output_timeout': 60 * 80,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'WINNT_6.1_x86-64_%(branch)s',
- 'platform': 'win64',
- 'stage_platform': 'win64-debug',
- 'debug_build': True,
- 'publish_nightly_en_US_routes': True,
- 'env': {
- 'HG_SHARE_BASE_DIR': os.path.join('y:', os.sep, 'hg-shared'),
- 'MOZ_AUTOMATION': '1',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PDBSTR_PATH': '/c/Program Files (x86)/Windows Kits/10/Debuggers/x64/srcsrv/pdbstr.exe',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- 'XPCOM_DEBUG_BREAK': 'stack-and-abort',
- 'MSYSTEM': 'MINGW32',
- },
- 'upload_env': {
- 'UPLOAD_HOST': 'localhost',
- 'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
- },
- "check_test_env": {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s\\breakpad\\win64\\minidump_stackwalk.exe',
- 'MINIDUMP_SAVE_PATH': '%(base_work_dir)s\\minidumps',
- },
- 'enable_pymake': True,
- 'src_mozconfig': 'browser\\config\\mozconfigs\\win64\\debug',
- 'tooltool_manifest_src': 'browser\\config\\tooltool-manifests\\win64\\releng.manifest',
- #########################################################################
-}
diff --git a/testing/mozharness/configs/builds/taskcluster_firefox_win64_opt.py b/testing/mozharness/configs/builds/taskcluster_firefox_win64_opt.py
deleted file mode 100644
index ba9cc9350..000000000
--- a/testing/mozharness/configs/builds/taskcluster_firefox_win64_opt.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import os
-import sys
-
-config = {
- #########################################################################
- ######## WINDOWS GENERIC CONFIG KEYS/VAlUES
- # if you are updating this with custom 64 bit keys/values please add them
- # below under the '64 bit specific' code block otherwise, update in this
- # code block and also make sure this is synced between:
- # - taskcluster_firefox_win32_debug
- # - taskcluster_firefox_win32_opt
- # - taskcluster_firefox_win64_debug
- # - taskcluster_firefox_win64_opt
-
- 'default_actions': [
- 'clone-tools',
- 'build',
- 'check-test',
- ],
- 'exes': {
- 'python2.7': sys.executable,
- 'make': [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'build', 'pymake', 'make.py'
- )
- ],
- 'virtualenv': [
- sys.executable,
- os.path.join(
- os.getcwd(), 'build', 'src', 'python', 'virtualenv', 'virtualenv.py'
- )
- ],
- 'mach-build': [
- os.path.join(os.environ['MOZILLABUILD'], 'msys', 'bin', 'bash.exe'),
- os.path.join(os.getcwd(), 'build', 'src', 'mach'),
- '--log-no-times', 'build', '-v'
- ],
- },
- 'app_ini_path': '%(obj_dir)s/dist/bin/application.ini',
- # decides whether we want to use moz_sign_cmd in env
- 'enable_signing': True,
- 'enable_ccache': False,
- 'vcs_share_base': os.path.join('y:', os.sep, 'hg-shared'),
- 'objdir': 'obj-firefox',
- 'tooltool_script': [
- sys.executable,
- os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')
- ],
- 'tooltool_bootstrap': 'setup.sh',
- 'enable_count_ctors': False,
- 'max_build_output_timeout': 60 * 80,
- #########################################################################
-
-
- #########################################################################
- ###### 64 bit specific ######
- 'base_name': 'WINNT_6.1_x86-64_%(branch)s',
- 'platform': 'win64',
- 'stage_platform': 'win64',
- 'publish_nightly_en_US_routes': True,
- 'env': {
- 'HG_SHARE_BASE_DIR': os.path.join('y:', os.sep, 'hg-shared'),
- 'MOZ_AUTOMATION': '1',
- 'MOZ_CRASHREPORTER_NO_REPORT': '1',
- 'MOZ_OBJDIR': 'obj-firefox',
- 'PDBSTR_PATH': '/c/Program Files (x86)/Windows Kits/10/Debuggers/x64/srcsrv/pdbstr.exe',
- 'TINDERBOX_OUTPUT': '1',
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- 'MSYSTEM': 'MINGW32',
- },
- 'upload_env': {
- 'UPLOAD_HOST': 'localhost',
- 'UPLOAD_PATH': os.path.join(os.getcwd(), 'public', 'build'),
- },
- "check_test_env": {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s\\breakpad\\win64\\minidump_stackwalk.exe',
- 'MINIDUMP_SAVE_PATH': '%(base_work_dir)s\\minidumps',
- },
- 'enable_pymake': True,
- 'src_mozconfig': 'browser\\config\\mozconfigs\\win64\\nightly',
- 'tooltool_manifest_src': 'browser\\config\\tooltool-manifests\\win64\\releng.manifest',
- #########################################################################
-}
diff --git a/testing/mozharness/configs/developer_config.py b/testing/mozharness/configs/developer_config.py
deleted file mode 100644
index 49ddb6eb7..000000000
--- a/testing/mozharness/configs/developer_config.py
+++ /dev/null
@@ -1,49 +0,0 @@
-"""
-This config file can be appended to any other mozharness job
-running under treeherder. The purpose of this config is to
-override values that are specific to Release Engineering machines
-that can reach specific hosts within their network.
-In other words, this config allows you to run any job
-outside of the Release Engineering network
-
-Using this config file should be accompanied with using
---test-url and --installer-url where appropiate
-"""
-
-import os
-LOCAL_WORKDIR = os.path.expanduser("~/.mozilla/releng")
-
-config = {
- # Developer mode values
- "developer_mode": True,
- "local_workdir": LOCAL_WORKDIR,
- "replace_urls": [
- ("http://pvtbuilds.pvt.build", "https://pvtbuilds"),
- ],
-
- # General local variable overwrite
- "exes": {
- "gittool.py": os.path.join(LOCAL_WORKDIR, "gittool.py"),
- },
-
- # Pip
- "find_links": ["http://pypi.pub.build.mozilla.org/pub"],
- "pip_index": False,
-
- # Talos related
- "python_webserver": True,
- "virtualenv_path": '%s/build/venv' % os.getcwd(),
- "preflight_run_cmd_suites": [],
- "postflight_run_cmd_suites": [],
-
- # Tooltool related
- "download_tooltool": True,
- "tooltool_cache": os.path.join(LOCAL_WORKDIR, "builds/tooltool_cache"),
- "tooltool_cache_path": os.path.join(LOCAL_WORKDIR, "builds/tooltool_cache"),
-
- # VCS tools
- "gittool.py": 'http://hg.mozilla.org/build/puppet/raw-file/faaf5abd792e/modules/packages/files/gittool.py',
-
- # Android related
- "host_utils_url": "https://api.pub.build.mozilla.org/tooltool/sha512/372c89f9dccaf5ee3b9d35fd1cfeb089e1e5db3ff1c04e35aa3adc8800bc61a2ae10e321f37ae7bab20b56e60941f91bb003bcb22035902a73d70872e7bd3282",
-}
diff --git a/testing/mozharness/configs/disable_signing.py b/testing/mozharness/configs/disable_signing.py
deleted file mode 100644
index 77fc85f2d..000000000
--- a/testing/mozharness/configs/disable_signing.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
- 'enable_signing': False,
-}
diff --git a/testing/mozharness/configs/firefox_ui_tests/qa_jenkins.py b/testing/mozharness/configs/firefox_ui_tests/qa_jenkins.py
deleted file mode 100644
index 5f6911b81..000000000
--- a/testing/mozharness/configs/firefox_ui_tests/qa_jenkins.py
+++ /dev/null
@@ -1,19 +0,0 @@
-# Default configuration as used by Mozmill CI (Jenkins)
-
-
-config = {
- # Tests run in mozmill-ci do not use RelEng infra
- 'developer_mode': True,
-
- # PIP
- 'find_links': ['http://pypi.pub.build.mozilla.org/pub'],
- 'pip_index': False,
-
- # mozcrash support
- 'download_minidump_stackwalk': True,
- 'download_symbols': 'ondemand',
- 'download_tooltool': True,
-
- # Disable proxxy because it isn't present in the QA environment.
- 'proxxy': {},
-}
diff --git a/testing/mozharness/configs/firefox_ui_tests/releng_release.py b/testing/mozharness/configs/firefox_ui_tests/releng_release.py
deleted file mode 100644
index 28baf6aef..000000000
--- a/testing/mozharness/configs/firefox_ui_tests/releng_release.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# Default configuration as used by Release Engineering for testing release/beta builds
-
-import os
-import sys
-
-import mozharness
-
-
-external_tools_path = os.path.join(
- os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
- 'external_tools',
-)
-
-
-config = {
- # General local variable overwrite
- 'exes': {
- 'gittool.py': [
- # Bug 1227079 - Python executable eeded to get it executed on Windows
- sys.executable,
- os.path.join(external_tools_path, 'gittool.py')
- ],
- },
-
- # PIP
- 'find_links': ['http://pypi.pub.build.mozilla.org/pub'],
- 'pip_index': False,
-
- # mozcrash support
- 'download_minidump_stackwalk': True,
- 'download_symbols': 'ondemand',
- 'download_tooltool': True,
-}
diff --git a/testing/mozharness/configs/firefox_ui_tests/taskcluster.py b/testing/mozharness/configs/firefox_ui_tests/taskcluster.py
deleted file mode 100644
index 66fc72935..000000000
--- a/testing/mozharness/configs/firefox_ui_tests/taskcluster.py
+++ /dev/null
@@ -1,11 +0,0 @@
-# Config file for firefox ui tests run via TaskCluster.
-
-config = {
- "find_links": [
- "http://pypi.pub.build.mozilla.org/pub",
- ],
-
- "pip_index": False,
-
- "tooltool_cache": "/builds/tooltool_cache",
-}
diff --git a/testing/mozharness/configs/hazards/build_browser.py b/testing/mozharness/configs/hazards/build_browser.py
deleted file mode 100644
index a08efe925..000000000
--- a/testing/mozharness/configs/hazards/build_browser.py
+++ /dev/null
@@ -1,4 +0,0 @@
-config = {
- 'build_command': "build.browser",
- 'expect_file': "expect.browser.json",
-}
diff --git a/testing/mozharness/configs/hazards/build_shell.py b/testing/mozharness/configs/hazards/build_shell.py
deleted file mode 100644
index 16135705a..000000000
--- a/testing/mozharness/configs/hazards/build_shell.py
+++ /dev/null
@@ -1,4 +0,0 @@
-config = {
- 'build_command': "build.shell",
- 'expect_file': "expect.shell.json",
-}
diff --git a/testing/mozharness/configs/hazards/common.py b/testing/mozharness/configs/hazards/common.py
deleted file mode 100644
index f8d751044..000000000
--- a/testing/mozharness/configs/hazards/common.py
+++ /dev/null
@@ -1,104 +0,0 @@
-import os
-
-HG_SHARE_BASE_DIR = "/builds/hg-shared"
-
-PYTHON_DIR = "/tools/python27"
-SRCDIR = "source"
-
-config = {
- "platform": "linux64",
- "build_type": "br-haz",
- "log_name": "hazards",
- "shell-objdir": "obj-opt-js",
- "analysis-dir": "analysis",
- "analysis-objdir": "obj-analyzed",
- "srcdir": SRCDIR,
- "analysis-scriptdir": "js/src/devtools/rootAnalysis",
-
- # These paths are relative to the tooltool checkout location
- "sixgill": "sixgill/usr/libexec/sixgill",
- "sixgill_bin": "sixgill/usr/bin",
-
- "python": "python",
-
- "exes": {
- 'gittool.py': '%(abs_tools_dir)s/buildfarm/utils/gittool.py',
- 'tooltool.py': '/tools/tooltool.py',
- "virtualenv": [PYTHON_DIR + "/bin/python", "/tools/misc-python/virtualenv.py"],
- },
-
- "force_clobber": True,
- 'vcs_share_base': HG_SHARE_BASE_DIR,
-
- "repos": [{
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools"
- }],
-
- "upload_remote_baseuri": 'https://ftp-ssl.mozilla.org/',
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
-
- "virtualenv_path": '%s/venv' % os.getcwd(),
- 'tools_dir': "/tools",
- 'compiler_manifest': "build/gcc.manifest",
- 'b2g_compiler_manifest': "build/gcc-b2g.manifest",
- 'sixgill_manifest': "build/sixgill.manifest",
-
- # Mock.
- "mock_packages": [
- "autoconf213", "mozilla-python27-mercurial", "ccache",
- "zip", "zlib-devel", "glibc-static",
- "openssh-clients", "mpfr", "wget", "rsync",
-
- # For building the JS shell
- "gmp-devel", "nspr", "nspr-devel",
-
- # For building the browser
- "dbus-devel", "dbus-glib-devel", "hal-devel",
- "libICE-devel", "libIDL-devel",
-
- # For mach resource-usage
- "python-psutil",
-
- 'zip', 'git',
- 'libstdc++-static', 'perl-Test-Simple', 'perl-Config-General',
- 'gtk2-devel', 'libnotify-devel', 'yasm',
- 'alsa-lib-devel', 'libcurl-devel',
- 'wireless-tools-devel', 'libX11-devel',
- 'libXt-devel', 'mesa-libGL-devel',
- 'gnome-vfs2-devel', 'GConf2-devel', 'wget',
- 'mpfr', # required for system compiler
- 'xorg-x11-font*', # fonts required for PGO
- 'imake', # required for makedepend!?!
- 'pulseaudio-libs-devel',
- 'freetype-2.3.11-6.el6_1.8.x86_64',
- 'freetype-devel-2.3.11-6.el6_1.8.x86_64',
- 'gstreamer-devel', 'gstreamer-plugins-base-devel',
- ],
- "mock_files": [
- ("/home/cltbld/.ssh", "/home/mock_mozilla/.ssh"),
- ('/home/cltbld/.hgrc', '/builds/.hgrc'),
- ('/builds/relengapi.tok', '/builds/relengapi.tok'),
- ("/tools/tooltool.py", "/tools/tooltool.py"),
- ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
- ],
- "env_replacements": {
- "pythondir": PYTHON_DIR,
- "gccdir": "%(abs_work_dir)s/gcc",
- "sixgilldir": "%(abs_work_dir)s/sixgill",
- },
- "partial_env": {
- "PATH": "%(pythondir)s/bin:%(gccdir)s/bin:%(PATH)s",
- "LD_LIBRARY_PATH": "%(sixgilldir)s/usr/lib64",
-
- # Suppress the mercurial-setup check. When running in automation, this
- # is redundant with MOZ_AUTOMATION, but a local developer-mode build
- # will have the mach state directory set to a nonstandard location and
- # therefore will always claim that mercurial-setup has not been run.
- "I_PREFER_A_SUBOPTIMAL_MERCURIAL_EXPERIENCE": "1",
- },
-}
diff --git a/testing/mozharness/configs/marionette/prod_config.py b/testing/mozharness/configs/marionette/prod_config.py
deleted file mode 100644
index 0d71c1cc3..000000000
--- a/testing/mozharness/configs/marionette/prod_config.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# This is a template config file for marionette production.
-import os
-
-HG_SHARE_BASE_DIR = "/builds/hg-shared"
-
-config = {
- # marionette options
- "marionette_address": "localhost:2828",
- "test_manifest": "unit-tests.ini",
-
- "vcs_share_base": HG_SHARE_BASE_DIR,
- "exes": {
- 'python': '/tools/buildbot/bin/python',
- 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
- 'tooltool.py': "/tools/tooltool.py",
- },
-
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
-
- "buildbot_json_path": "buildprops.json",
-
- "default_actions": [
- 'clobber',
- 'read-buildbot-config',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-tests',
- ],
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
- "download_symbols": "ondemand",
- "download_minidump_stackwalk": True,
- "tooltool_cache": "/builds/tooltool_cache",
- "suite_definitions": {
- "marionette_desktop": {
- "options": [
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--log-html=%(html_report_file)s",
- "--binary=%(binary)s",
- "--address=%(address)s",
- "--symbols-path=%(symbols_path)s"
- ],
- "run_filename": "",
- "testsdir": ""
- }
- },
- "structured_output": True,
-}
diff --git a/testing/mozharness/configs/marionette/test_config.py b/testing/mozharness/configs/marionette/test_config.py
deleted file mode 100644
index 6a0f3eee3..000000000
--- a/testing/mozharness/configs/marionette/test_config.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# This is a template config file for marionette test.
-
-config = {
- # marionette options
- "marionette_address": "localhost:2828",
- "test_manifest": "unit-tests.ini",
-
- "default_actions": [
- 'clobber',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-tests',
- ],
- "suite_definitions": {
- "marionette_desktop": {
- "options": [
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--log-html=%(html_report_file)s",
- "--binary=%(binary)s",
- "--address=%(address)s",
- "--symbols-path=%(symbols_path)s"
- ],
- "run_filename": "",
- "testsdir": ""
- },
- },
-}
diff --git a/testing/mozharness/configs/marionette/windows_config.py b/testing/mozharness/configs/marionette/windows_config.py
deleted file mode 100644
index 039a459b2..000000000
--- a/testing/mozharness/configs/marionette/windows_config.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# This is a template config file for marionette production on Windows.
-import os
-import sys
-
-config = {
- # marionette options
- "marionette_address": "localhost:2828",
- "test_manifest": "unit-tests.ini",
-
- "virtualenv_python_dll": 'c:/mozilla-build/python27/python27.dll',
- "virtualenv_path": 'venv',
- "exes": {
- 'python': 'c:/mozilla-build/python27/python',
- 'virtualenv': ['c:/mozilla-build/python27/python', 'c:/mozilla-build/buildbotve/virtualenv.py'],
- 'hg': 'c:/mozilla-build/hg/hg',
- 'mozinstall': ['%s/build/venv/scripts/python' % os.getcwd(),
- '%s/build/venv/scripts/mozinstall-script.py' % os.getcwd()],
- 'tooltool.py': [sys.executable, 'C:/mozilla-build/tooltool.py'],
- },
-
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
-
- "buildbot_json_path": "buildprops.json",
-
- "default_actions": [
- 'clobber',
- 'read-buildbot-config',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-tests',
- ],
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
- "download_minidump_stackwalk": True,
- "download_symbols": "ondemand",
- "suite_definitions": {
- "marionette_desktop": {
- "options": [
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--log-html=%(html_report_file)s",
- "--binary=%(binary)s",
- "--address=%(address)s",
- "--symbols-path=%(symbols_path)s"
- ],
- "run_filename": "",
- "testsdir": ""
- },
- },
-}
diff --git a/testing/mozharness/configs/marionette/windows_taskcluster_config.py b/testing/mozharness/configs/marionette/windows_taskcluster_config.py
deleted file mode 100644
index fe3ed0c62..000000000
--- a/testing/mozharness/configs/marionette/windows_taskcluster_config.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# This is a template config file for marionette production on Windows.
-import os
-import sys
-
-config = {
- # marionette options
- "marionette_address": "localhost:2828",
- "test_manifest": "unit-tests.ini",
-
- "virtualenv_python_dll": os.path.join(os.path.dirname(sys.executable), 'python27.dll'),
- "virtualenv_path": 'venv',
- "exes": {
- 'python': sys.executable,
- 'virtualenv': [
- sys.executable,
- os.path.join(os.path.dirname(sys.executable), 'Lib', 'site-packages', 'virtualenv.py')
- ],
- 'mozinstall': ['build/venv/scripts/python', 'build/venv/scripts/mozinstall-script.py'],
- 'tooltool.py': [sys.executable, os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')],
- 'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg')
- },
-
- "proxxy": {},
- "find_links": [
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
-
- "default_actions": [
- 'clobber',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-tests',
- ],
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "blob_uploader_auth_file" : 'C:/builds/oauth.txt',
- "download_minidump_stackwalk": True,
- "download_symbols": "ondemand",
- "suite_definitions": {
- "marionette_desktop": {
- "options": [
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--log-html=%(html_report_file)s",
- "--binary=%(binary)s",
- "--address=%(address)s",
- "--symbols-path=%(symbols_path)s"
- ],
- "run_filename": "",
- "testsdir": ""
- },
- },
-}
diff --git a/testing/mozharness/configs/mediatests/buildbot_posix_config.py b/testing/mozharness/configs/mediatests/buildbot_posix_config.py
deleted file mode 100644
index 8c30a9f28..000000000
--- a/testing/mozharness/configs/mediatests/buildbot_posix_config.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import os
-import mozharness
-
-external_tools_path = os.path.join(
- os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
- 'external_tools',
-)
-
-config = {
- "virtualenv_path": 'venv',
- "exes": {
- 'python': '/tools/buildbot/bin/python',
- 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
- 'tooltool.py': "/tools/tooltool.py",
- },
-
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
-
- "buildbot_json_path": "buildprops.json",
-
- "default_actions": [
- 'clobber',
- 'read-buildbot-config',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-media-tests',
- ],
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
- "download_minidump_stackwalk": True,
- "download_symbols": "ondemand",
-
- "suite_definitions": {
- "media-tests": {
- "options": [],
- },
- "media-youtube-tests": {
- "options": [
- "%(test_manifest)s"
- ],
- },
- },
-}
diff --git a/testing/mozharness/configs/mediatests/buildbot_windows_config.py b/testing/mozharness/configs/mediatests/buildbot_windows_config.py
deleted file mode 100644
index 270938378..000000000
--- a/testing/mozharness/configs/mediatests/buildbot_windows_config.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import os
-import sys
-import mozharness
-
-external_tools_path = os.path.join(
- os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
- 'external_tools',
-)
-
-config = {
- "virtualenv_python_dll": 'c:/mozilla-build/python27/python27.dll',
- "virtualenv_path": 'venv',
- "exes": {
- 'python': 'c:/mozilla-build/python27/python',
- 'virtualenv': ['c:/mozilla-build/python27/python', 'c:/mozilla-build/buildbotve/virtualenv.py'],
- 'hg': 'c:/mozilla-build/hg/hg',
- 'mozinstall': ['%s/build/venv/scripts/python' % os.getcwd(),
- '%s/build/venv/scripts/mozinstall-script.py' % os.getcwd()],
- 'tooltool.py': [sys.executable, 'C:/mozilla-build/tooltool.py'],
- },
-
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
-
- "buildbot_json_path": "buildprops.json",
-
- "default_actions": [
- 'clobber',
- 'read-buildbot-config',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-media-tests',
- ],
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
- "in_tree_config": "config/mozharness/marionette.py",
- "download_minidump_stackwalk": True,
- "download_symbols": "ondemand",
-
- "suite_definitions": {
- "media-tests": {
- "options": [],
- },
- "media-youtube-tests": {
- "options": [
- "%(test_manifest)s"
- ],
- },
- },
-}
diff --git a/testing/mozharness/configs/mediatests/jenkins_config.py b/testing/mozharness/configs/mediatests/jenkins_config.py
deleted file mode 100755
index 52de7221d..000000000
--- a/testing/mozharness/configs/mediatests/jenkins_config.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Default configuration as used by Mozmill CI (Jenkins)
-
-import os
-import platform
-import sys
-
-import mozharness
-
-
-external_tools_path = os.path.join(
- os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
- 'external_tools',
-)
-
-config = {
- # PIP
- 'find_links': ['http://pypi.pub.build.mozilla.org/pub'],
- 'pip_index': False,
-
- # mozcrash support
- 'download_minidump_stackwalk': True,
- 'download_symbols': 'ondemand',
- 'download_tooltool': True,
-
- # Default test suite
- 'test_suite': 'media-tests',
-
- 'suite_definitions': {
- 'media-tests': {
- 'options': [],
- },
- 'media-youtube-tests': {
- 'options': [
- '%(test_manifest)s'
- ],
- },
- },
-
- 'default_actions': [
- 'clobber',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-media-tests',
- ],
-
-}
-
diff --git a/testing/mozharness/configs/mediatests/taskcluster_posix_config.py b/testing/mozharness/configs/mediatests/taskcluster_posix_config.py
deleted file mode 100644
index d02effa3d..000000000
--- a/testing/mozharness/configs/mediatests/taskcluster_posix_config.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import os
-import mozharness
-
-external_tools_path = os.path.join(
- os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
- 'external_tools',
-)
-
-config = {
- # Python env
- "virtualenv_path": 'venv',
- "exes": {
- 'python': '/tools/buildbot/bin/python',
- 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
- 'tooltool.py': "/tools/tooltool.py",
- },
-
- # PIP
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
-
- #mozcrash support
- "download_minidump_stackwalk": True,
- "download_symbols": "ondemand",
-
- "default_actions": [
- 'clobber',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-media-tests',
- ],
-
- "suite_definitions": {
- "media-tests": {
- "options": [],
- },
- "media-youtube-tests": {
- "options": [
- "%(test_manifest)s"
- ],
- },
- },
-}
diff --git a/testing/mozharness/configs/mediatests/taskcluster_windows_config.py b/testing/mozharness/configs/mediatests/taskcluster_windows_config.py
deleted file mode 100644
index 85bf8b525..000000000
--- a/testing/mozharness/configs/mediatests/taskcluster_windows_config.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import os
-import sys
-import mozharness
-
-external_tools_path = os.path.join(
- os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
- 'external_tools',
-)
-
-config = {
- "virtualenv_python_dll": os.path.join(os.path.dirname(sys.executable), 'python27.dll'),
- "virtualenv_path": 'venv',
- "exes": {
- 'python': sys.executable,
- 'virtualenv': [
- sys.executable,
- os.path.join(os.path.dirname(sys.executable), 'Lib', 'site-packages', 'virtualenv.py')
- ],
- 'mozinstall': ['build/venv/scripts/python', 'build/venv/scripts/mozinstall-script.py'],
- 'tooltool.py': [sys.executable, os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')],
- 'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg')
- },
- "proxxy": {},
- "find_links": [
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
-
- "download_minidump_stackwalk": True,
- "download_symbols": "ondemand",
-
- "default_actions": [
- 'clobber',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-media-tests',
- ],
-
- "suite_definitions": {
- "media-tests": {
- "options": [],
- },
- "media-youtube-tests": {
- "options": [
- "%(test_manifest)s"
- ],
- },
- },
-}
diff --git a/testing/mozharness/configs/partner_repacks/release_mozilla-esr52_desktop.py b/testing/mozharness/configs/partner_repacks/release_mozilla-esr52_desktop.py
deleted file mode 100644
index 604407e6a..000000000
--- a/testing/mozharness/configs/partner_repacks/release_mozilla-esr52_desktop.py
+++ /dev/null
@@ -1,6 +0,0 @@
-config = {
- "appName": "Firefox",
- "log_name": "partner_repack",
- "repack_manifests_url": "https://github.com/mozilla-partners/mozilla-sha1-manifest",
- "repo_file": "https://raw.githubusercontent.com/mozilla/git-repo/master/repo",
-}
diff --git a/testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py b/testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py
deleted file mode 100644
index 229c2bb44..000000000
--- a/testing/mozharness/configs/partner_repacks/release_mozilla-release_desktop.py
+++ /dev/null
@@ -1,6 +0,0 @@
-config = {
- "appName": "Firefox",
- "log_name": "partner_repack",
- "repack_manifests_url": "git@github.com:mozilla-partners/repack-manifests.git",
- "repo_file": "https://raw.githubusercontent.com/mozilla/git-repo/master/repo",
-}
diff --git a/testing/mozharness/configs/partner_repacks/staging_release_mozilla-release_desktop.py b/testing/mozharness/configs/partner_repacks/staging_release_mozilla-release_desktop.py
deleted file mode 100644
index 229c2bb44..000000000
--- a/testing/mozharness/configs/partner_repacks/staging_release_mozilla-release_desktop.py
+++ /dev/null
@@ -1,6 +0,0 @@
-config = {
- "appName": "Firefox",
- "log_name": "partner_repack",
- "repack_manifests_url": "git@github.com:mozilla-partners/repack-manifests.git",
- "repo_file": "https://raw.githubusercontent.com/mozilla/git-repo/master/repo",
-}
diff --git a/testing/mozharness/configs/platform_supports_post_upload_to_latest.py b/testing/mozharness/configs/platform_supports_post_upload_to_latest.py
deleted file mode 100644
index 6ed654ed1..000000000
--- a/testing/mozharness/configs/platform_supports_post_upload_to_latest.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
- 'platform_supports_post_upload_to_latest': False,
-}
diff --git a/testing/mozharness/configs/releases/bouncer_firefox_beta.py b/testing/mozharness/configs/releases/bouncer_firefox_beta.py
deleted file mode 100644
index 6c563124c..000000000
--- a/testing/mozharness/configs/releases/bouncer_firefox_beta.py
+++ /dev/null
@@ -1,148 +0,0 @@
-# lint_ignore=E501
-config = {
- "shipped-locales-url": "https://hg.mozilla.org/%(repo)s/raw-file/%(revision)s/browser/locales/shipped-locales",
- "products": {
- "installer": {
- "product-name": "Firefox-%(version)s",
- "check_uptake": True,
- "alias": "firefox-beta-latest",
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/linux-i686/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/linux-x86_64/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/mac/:lang/Firefox%%20%(version)s.dmg",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/win64/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win64",
- },
- },
- },
- "installer-ssl": {
- "product-name": "Firefox-%(version)s-SSL",
- "check_uptake": True,
- "alias": "firefox-beta-latest-ssl",
- "ssl-only": True,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/linux-i686/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/linux-x86_64/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/mac/:lang/Firefox%%20%(version)s.dmg",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/win64/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win64",
- },
- },
- },
- "stub-installer": {
- "product-name": "Firefox-%(version)s-stub",
- "check_uptake": True,
- "alias": "firefox-beta-stub",
- "ssl-only": True,
- "add-locales": True,
- "paths": {
- "win32": {
- "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20Stub%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- },
- },
- "sha1-installer": {
- "product-name": "Firefox-%(version)s-sha1",
- "check_uptake": True,
- "alias": "firefox-beta-sha1",
- "ssl-only": True,
- "add-locales": True,
- "paths": {
- "win32": {
- "path": "/firefox/releases/%(version)s/win32-sha1/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- },
- },
- "complete-mar": {
- "product-name": "Firefox-%(version)s-Complete",
- "check_uptake": True,
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/update/linux-i686/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/update/linux-x86_64/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/update/mac/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/update/win32/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/update/win64/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "win64",
- },
- },
- },
- },
- "partials": {
- "releases-dir": {
- "product-name": "Firefox-%(version)s-Partial-%(prev_version)s",
- "check_uptake": True,
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/update/linux-i686/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/update/linux-x86_64/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/update/mac/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/update/win32/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/update/win64/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "win64",
- },
- },
- },
- },
-}
diff --git a/testing/mozharness/configs/releases/bouncer_firefox_esr.py b/testing/mozharness/configs/releases/bouncer_firefox_esr.py
deleted file mode 100644
index 747ff5664..000000000
--- a/testing/mozharness/configs/releases/bouncer_firefox_esr.py
+++ /dev/null
@@ -1,136 +0,0 @@
-# lint_ignore=E501
-config = {
- "shipped-locales-url": "https://hg.mozilla.org/%(repo)s/raw-file/%(revision)s/browser/locales/shipped-locales",
- "products": {
- "installer": {
- "product-name": "Firefox-%(version)s",
- "check_uptake": True,
- "alias": "firefox-esr-latest",
- "ssl-only": True,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/linux-i686/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/linux-x86_64/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/mac/:lang/Firefox%%20%(version)s.dmg",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/win64/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win64",
- },
- },
- },
- "installer-ssl": {
- "product-name": "Firefox-%(version)s-SSL",
- "check_uptake": True,
- "alias": "firefox-esr-latest-ssl",
- "ssl-only": True,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/linux-i686/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/linux-x86_64/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/mac/:lang/Firefox%%20%(version)s.dmg",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/win64/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win64",
- },
- },
- },
- "sha1-installer": {
- "product-name": "Firefox-%(version)s-sha1",
- "check_uptake": True,
- # XP/Vista Release users are redicted to ESR52
- "alias": "firefox-sha1",
- "ssl-only": True,
- "add-locales": True,
- "paths": {
- "win32": {
- "path": "/firefox/releases/%(version)s/win32-sha1/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- },
- },
- "complete-mar": {
- "product-name": "Firefox-%(version)s-Complete",
- "check_uptake": True,
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/update/linux-i686/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/update/linux-x86_64/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/update/mac/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/update/win32/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/update/win64/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "win64",
- },
- },
- },
- },
- "partials": {
- "releases-dir": {
- "product-name": "Firefox-%(version)s-Partial-%(prev_version)s",
- "check_uptake": True,
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/update/linux-i686/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/update/linux-x86_64/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/update/mac/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/update/win32/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/update/win64/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "win64",
- },
- },
- },
- },
-}
diff --git a/testing/mozharness/configs/releases/bouncer_firefox_release.py b/testing/mozharness/configs/releases/bouncer_firefox_release.py
deleted file mode 100644
index 59ecd20a2..000000000
--- a/testing/mozharness/configs/releases/bouncer_firefox_release.py
+++ /dev/null
@@ -1,191 +0,0 @@
-# lint_ignore=E501
-config = {
- "shipped-locales-url": "https://hg.mozilla.org/%(repo)s/raw-file/%(revision)s/browser/locales/shipped-locales",
- "products": {
- "installer": {
- "product-name": "Firefox-%(version)s",
- "check_uptake": True,
- "alias": "firefox-latest",
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/linux-i686/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/linux-x86_64/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/mac/:lang/Firefox%%20%(version)s.dmg",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/win64/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win64",
- },
- },
- },
- "installer-ssl": {
- "product-name": "Firefox-%(version)s-SSL",
- "check_uptake": True,
- "alias": "firefox-latest-ssl",
- "ssl-only": True,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/linux-i686/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/linux-x86_64/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/mac/:lang/Firefox%%20%(version)s.dmg",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/win64/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win64",
- },
- },
- },
- "stub-installer": {
- "product-name": "Firefox-%(version)s-stub",
- "check_uptake": True,
- "alias": "firefox-stub",
- "ssl-only": True,
- "add-locales": True,
- "paths": {
- "win32": {
- "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20Stub%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- },
- },
- "complete-mar": {
- "product-name": "Firefox-%(version)s-Complete",
- "check_uptake": True,
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/update/linux-i686/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/update/linux-x86_64/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/update/mac/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/update/win32/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/update/win64/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "win64",
- },
- },
- },
- "complete-mar-candidates": {
- "product-name": "Firefox-%(version)sbuild%(build_number)s-Complete",
- "check_uptake": False,
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/linux-i686/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/linux-x86_64/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/mac/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/win32/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/win64/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "win64",
- },
- },
- },
- },
- "partials": {
- "releases-dir": {
- "product-name": "Firefox-%(version)s-Partial-%(prev_version)s",
- "check_uptake": True,
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/update/linux-i686/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/update/linux-x86_64/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/update/mac/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/update/win32/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/update/win64/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "win64",
- },
- },
- },
- "candidates-dir": {
- "product-name": "Firefox-%(version)sbuild%(build_number)s-Partial-%(prev_version)sbuild%(prev_build_number)s",
- "check_uptake": False,
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/linux-i686/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/linux-x86_64/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/mac/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/win32/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/candidates/%(version)s-candidates/build%(build_number)s/update/win64/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "win64",
- },
- },
- },
- },
-}
diff --git a/testing/mozharness/configs/releases/bouncer_thunderbird.py b/testing/mozharness/configs/releases/bouncer_thunderbird.py
deleted file mode 100644
index 5d0548a59..000000000
--- a/testing/mozharness/configs/releases/bouncer_thunderbird.py
+++ /dev/null
@@ -1,169 +0,0 @@
-# lint_ignore=E501
-config = {
- "shipped-locales-url": "https://hg.mozilla.org/%(repo)s/raw-file/%(revision)s/mail/locales/shipped-locales",
- "products": {
- "installer": {
- "product-name": "Thunderbird-%(version)s",
- "check_uptake": True,
- "alias": "thunderbird-latest",
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/thunderbird/releases/%(version)s/linux-i686/:lang/thunderbird-%(version)s.tar.bz2",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/thunderbird/releases/%(version)s/linux-x86_64/:lang/thunderbird-%(version)s.tar.bz2",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/thunderbird/releases/%(version)s/mac/:lang/Thunderbird%%20%(version)s.dmg",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/thunderbird/releases/%(version)s/win32/:lang/Thunderbird%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- "opensolaris-i386": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(version)s.en-US.opensolaris-i386.tar.bz2",
- "bouncer-platform": "opensolaris-i386",
- },
- "opensolaris-sparc": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(version)s.en-US.opensolaris-sparc.tar.bz2",
- "bouncer-platform": "opensolaris-sparc",
- },
- "solaris-i386": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(version)s.en-US.solaris-i386.tar.bz2",
- "bouncer-platform": "solaris-i386",
- },
- "solaris-sparc": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(version)s.en-US.solaris-sparc.tar.bz2",
- "bouncer-platform": "solaris-sparc",
- },
- },
- },
- "installer-ssl": {
- "product-name": "Thunderbird-%(version)s-SSL",
- "check_uptake": True,
- "ssl-only": True,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/thunderbird/releases/%(version)s/linux-i686/:lang/thunderbird-%(version)s.tar.bz2",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/thunderbird/releases/%(version)s/linux-x86_64/:lang/thunderbird-%(version)s.tar.bz2",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/thunderbird/releases/%(version)s/mac/:lang/Thunderbird%%20%(version)s.dmg",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/thunderbird/releases/%(version)s/win32/:lang/Thunderbird%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- "opensolaris-i386": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(version)s.en-US.opensolaris-i386.tar.bz2",
- "bouncer-platform": "opensolaris-i386",
- },
- "opensolaris-sparc": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(version)s.en-US.opensolaris-sparc.tar.bz2",
- "bouncer-platform": "opensolaris-sparc",
- },
- "solaris-i386": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(version)s.en-US.solaris-i386.tar.bz2",
- "bouncer-platform": "solaris-i386",
- },
- "solaris-sparc": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(version)s.en-US.solaris-sparc.tar.bz2",
- "bouncer-platform": "solaris-sparc",
- },
- },
- },
- "complete-mar": {
- "product-name": "Thunderbird-%(version)s-Complete",
- "check_uptake": True,
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/thunderbird/releases/%(version)s/update/linux-i686/:lang/thunderbird-%(version)s.complete.mar",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/thunderbird/releases/%(version)s/update/linux-x86_64/:lang/thunderbird-%(version)s.complete.mar",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/thunderbird/releases/%(version)s/update/mac/:lang/thunderbird-%(version)s.complete.mar",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/thunderbird/releases/%(version)s/update/win32/:lang/thunderbird-%(version)s.complete.mar",
- "bouncer-platform": "win",
- },
- "opensolaris-i386": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(version)s.en-US.opensolaris-i386.complete.mar",
- "bouncer-platform": "opensolaris-i386",
- },
- "opensolaris-sparc": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(version)s.en-US.opensolaris-sparc.complete.mar",
- "bouncer-platform": "opensolaris-sparc",
- },
- "solaris-i386": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(version)s.en-US.solaris-i386.complete.mar",
- "bouncer-platform": "solaris-i386",
- },
- "solaris-sparc": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(version)s.en-US.solaris-sparc.complete.mar",
- "bouncer-platform": "solaris-sparc",
- },
- },
- },
- },
- "partials": {
- "releases-dir": {
- "product-name": "Thunderbird-%(version)s-Partial-%(prev_version)s",
- "check_uptake": True,
- "ssl-only": False,
- "add-locales": True,
- "paths": {
- "linux": {
- "path": "/thunderbird/releases/%(version)s/update/linux-i686/:lang/thunderbird-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/thunderbird/releases/%(version)s/update/linux-x86_64/:lang/thunderbird-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/thunderbird/releases/%(version)s/update/mac/:lang/thunderbird-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/thunderbird/releases/%(version)s/update/win32/:lang/thunderbird-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "win",
- },
- "opensolaris-i386": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(prev_version)s-%(version)s.en-US.opensolaris-i386.partial.mar",
- "bouncer-platform": "opensolaris-i386",
- },
- "opensolaris-sparc": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(prev_version)s-%(version)s.en-US.opensolaris-sparc.partial.mar",
- "bouncer-platform": "opensolaris-sparc",
- },
- "solaris-i386": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(prev_version)s-%(version)s.en-US.solaris-i386.partial.mar",
- "bouncer-platform": "solaris-i386",
- },
- "solaris-sparc": {
- "path": "/thunderbird/releases/%(version)s/contrib/solaris_tarball/thunderbird-%(prev_version)s-%(version)s.en-US.solaris-sparc.partial.mar",
- "bouncer-platform": "solaris-sparc",
- },
- },
- },
- },
-}
diff --git a/testing/mozharness/configs/releases/dev_bouncer_firefox_beta.py b/testing/mozharness/configs/releases/dev_bouncer_firefox_beta.py
deleted file mode 100644
index 29c6e6cfb..000000000
--- a/testing/mozharness/configs/releases/dev_bouncer_firefox_beta.py
+++ /dev/null
@@ -1,133 +0,0 @@
-# lint_ignore=E501
-config = {
- "products": {
- "installer": {
- "product-name": "Firefox-%(version)s",
- "check_uptake": True,
- "alias": "firefox-beta-latest",
- "ssl-only": False,
- "add-locales": False,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/linux-i686/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/linux-x86_64/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/mac/:lang/Firefox%%20%(version)s.dmg",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/win64/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win64",
- },
- },
- },
- "installer-ssl": {
- "product-name": "Firefox-%(version)s-SSL",
- "check_uptake": True,
- "ssl-only": True,
- "add-locales": False,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/linux-i686/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/linux-x86_64/:lang/firefox-%(version)s.tar.bz2",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/mac/:lang/Firefox%%20%(version)s.dmg",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/win64/:lang/Firefox%%20Setup%%20%(version)s.exe",
- "bouncer-platform": "win64",
- },
- },
- },
- "stub-installer": {
- "product-name": "Firefox-%(version)s-stub",
- "check_uptake": True,
- "alias": "firefox-beta-stub",
- "ssl-only": True,
- "add-locales": False,
- "paths": {
- "win32": {
- "path": "/firefox/releases/%(version)s/win32/:lang/Firefox%%20Setup%%20Stub%%20%(version)s.exe",
- "bouncer-platform": "win",
- },
- },
- },
- "complete-mar": {
- "product-name": "Firefox-%(version)s-Complete",
- "check_uptake": True,
- "ssl-only": False,
- "add-locales": False,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/update/linux-i686/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/update/linux-x86_64/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/update/mac/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/update/win32/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/update/win64/:lang/firefox-%(version)s.complete.mar",
- "bouncer-platform": "win64",
- },
- },
- },
- },
- "partials": {
- "releases-dir": {
- "product-name": "Firefox-%(version)s-Partial-%(prev_version)s",
- "check_uptake": True,
- "ssl-only": False,
- "add-locales": False,
- "paths": {
- "linux": {
- "path": "/firefox/releases/%(version)s/update/linux-i686/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "linux",
- },
- "linux64": {
- "path": "/firefox/releases/%(version)s/update/linux-x86_64/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "linux64",
- },
- "macosx64": {
- "path": "/firefox/releases/%(version)s/update/mac/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "osx",
- },
- "win32": {
- "path": "/firefox/releases/%(version)s/update/win32/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "win",
- },
- "win64": {
- "path": "/firefox/releases/%(version)s/update/win64/:lang/firefox-%(prev_version)s-%(version)s.partial.mar",
- "bouncer-platform": "win64",
- },
- },
- },
- },
-}
diff --git a/testing/mozharness/configs/releases/dev_postrelease_firefox_beta.py b/testing/mozharness/configs/releases/dev_postrelease_firefox_beta.py
deleted file mode 100644
index 4ecd32349..000000000
--- a/testing/mozharness/configs/releases/dev_postrelease_firefox_beta.py
+++ /dev/null
@@ -1,20 +0,0 @@
-config = {
- # date is used for staging mozilla-beta
- "log_name": "bump_date",
- "version_files": [{"file": "browser/config/version_display.txt"}],
- "repo": {
- # date is used for staging mozilla-beta
- "repo": "https://hg.mozilla.org/projects/date",
- "branch": "default",
- "dest": "date",
- "vcs": "hg",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- },
- # date is used for staging mozilla-beta
- "push_dest": "ssh://hg.mozilla.org/projects/date",
- "ignore_no_changes": True,
- "ssh_user": "ffxbld",
- "ssh_key": "~/.ssh/ffxbld_rsa",
- "ship_it_root": "https://ship-it-dev.allizom.org",
- "ship_it_username": "ship_it-stage-ffxbld",
-}
diff --git a/testing/mozharness/configs/releases/dev_postrelease_firefox_release.py b/testing/mozharness/configs/releases/dev_postrelease_firefox_release.py
deleted file mode 100644
index 0a1497595..000000000
--- a/testing/mozharness/configs/releases/dev_postrelease_firefox_release.py
+++ /dev/null
@@ -1,22 +0,0 @@
-config = {
- "log_name": "bump_release_dev",
- "version_files": [
- {"file": "browser/config/version.txt"},
- {"file": "browser/config/version_display.txt"},
- {"file": "config/milestone.txt"},
- ],
- "repo": {
- # jamun is used for staging mozilla-release
- "repo": "https://hg.mozilla.org/projects/jamun",
- "branch": "default",
- "dest": "jamun",
- "vcs": "hg",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- },
- "push_dest": "ssh://hg.mozilla.org/projects/jamun",
- "ignore_no_changes": True,
- "ssh_user": "ffxbld",
- "ssh_key": "~/.ssh/ffxbld_rsa",
- "ship_it_root": "https://ship-it-dev.allizom.org",
- "ship_it_username": "ship_it-stage-ffxbld",
-}
diff --git a/testing/mozharness/configs/releases/dev_updates_firefox_beta.py b/testing/mozharness/configs/releases/dev_updates_firefox_beta.py
deleted file mode 100644
index 40b87c57b..000000000
--- a/testing/mozharness/configs/releases/dev_updates_firefox_beta.py
+++ /dev/null
@@ -1,39 +0,0 @@
-
-config = {
- "log_name": "bump_beta_dev",
- # TODO: use real repo
- "repo": {
- "repo": "https://hg.mozilla.org/users/raliiev_mozilla.com/tools",
- "branch": "default",
- "dest": "tools",
- "vcs": "hg",
- },
- "vcs_share_base": "/builds/hg-shared",
- # TODO: use real repo
- "push_dest": "ssh://hg.mozilla.org/users/raliiev_mozilla.com/tools",
- # date repo used for staging beta
- "shipped-locales-url": "https://hg.mozilla.org/projects/date/raw-file/{revision}/browser/locales/shipped-locales",
- "ignore_no_changes": True,
- "ssh_user": "ffxbld",
- "ssh_key": "~/.ssh/ffxbld_rsa",
- "archive_domain": "ftp.stage.mozaws.net",
- "archive_prefix": "https://ftp.stage.mozaws.net/pub",
- "previous_archive_prefix": "https://archive.mozilla.org/pub",
- "download_domain": "download.mozilla.org",
- "balrog_url": "http://ec2-54-241-39-23.us-west-1.compute.amazonaws.com",
- "balrog_username": "balrog-stage-ffxbld",
- "update_channels": {
- "beta-dev": {
- "version_regex": r"^(\d+\.\d+(b\d+)?)$",
- "requires_mirrors": True,
- # TODO - when we use a real repo, rename this file # s/MozDate/MozBeta-dev/
- "patcher_config": "mozDate-branch-patcher2.cfg",
- "update_verify_channel": "beta-dev-localtest",
- "mar_channel_ids": [],
- "channel_names": ["beta-dev", "beta-dev-localtest", "beta-dev-cdntest"],
- "rules_to_update": ["firefox-beta-dev-cdntest", "firefox-beta-dev-localtest"],
- "publish_rules": ["firefox-beta"],
- }
- },
- "balrog_use_dummy_suffix": False,
-}
diff --git a/testing/mozharness/configs/releases/dev_updates_firefox_release.py b/testing/mozharness/configs/releases/dev_updates_firefox_release.py
deleted file mode 100644
index 8c2696b5b..000000000
--- a/testing/mozharness/configs/releases/dev_updates_firefox_release.py
+++ /dev/null
@@ -1,50 +0,0 @@
-
-config = {
- "log_name": "updates_release_dev",
- # TODO: use real repo
- "repo": {
- "repo": "https://hg.mozilla.org/users/raliiev_mozilla.com/tools",
- "branch": "default",
- "dest": "tools",
- "vcs": "hg",
- },
- "vcs_share_base": "/builds/hg-shared",
- # TODO: use real repo
- "push_dest": "ssh://hg.mozilla.org/users/raliiev_mozilla.com/tools",
- # jamun repo used for staging release
- "shipped-locales-url": "https://hg.mozilla.org/projects/jamun/raw-file/{revision}/browser/locales/shipped-locales",
- "ignore_no_changes": True,
- "ssh_user": "ffxbld",
- "ssh_key": "~/.ssh/ffxbld_rsa",
- "archive_domain": "ftp.stage.mozaws.net",
- "archive_prefix": "https://ftp.stage.mozaws.net/pub",
- "previous_archive_prefix": "https://archive.mozilla.org/pub",
- "download_domain": "download.mozilla.org",
- "balrog_url": "http://ec2-54-241-39-23.us-west-1.compute.amazonaws.com",
- "balrog_username": "balrog-stage-ffxbld",
- "update_channels": {
- "beta-dev": {
- "version_regex": r"^(\d+\.\d+(b\d+)?)$",
- "requires_mirrors": False,
- "patcher_config": "mozDate-branch-patcher2.cfg",
- "update_verify_channel": "beta-dev-localtest",
- "mar_channel_ids": [
- "firefox-mozilla-beta-dev", "firefox-mozilla-release-dev",
- ],
- "channel_names": ["beta-dev", "beta-dev-localtest", "beta-dev-cdntest"],
- "rules_to_update": ["firefox-beta-dev-cdntest", "firefox-beta-dev-localtest"],
- "publish_rules": ["firefox-beta"],
- },
- "release-dev": {
- "version_regex": r"^\d+\.\d+(\.\d+)?$",
- "requires_mirrors": True,
- "patcher_config": "mozJamun-branch-patcher2.cfg",
- "update_verify_channel": "release-dev-localtest",
- "mar_channel_ids": [],
- "channel_names": ["release-dev", "release-dev-localtest", "release-dev-cdntest"],
- "rules_to_update": ["firefox-release-dev-cdntest", "firefox-release-dev-localtest"],
- "publish_rules": ["firefox-release"],
- },
- },
- "balrog_use_dummy_suffix": False,
-}
diff --git a/testing/mozharness/configs/releases/postrelease_firefox_beta.py b/testing/mozharness/configs/releases/postrelease_firefox_beta.py
deleted file mode 100644
index b72302d91..000000000
--- a/testing/mozharness/configs/releases/postrelease_firefox_beta.py
+++ /dev/null
@@ -1,18 +0,0 @@
-config = {
- "log_name": "bump_beta",
- "version_files": [{"file": "browser/config/version_display.txt"}],
- "repo": {
- "repo": "https://hg.mozilla.org/releases/mozilla-beta",
- "branch": "default",
- "dest": "mozilla-beta",
- "vcs": "hg",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- },
- "vcs_share_base": "/builds/hg-shared",
- "push_dest": "ssh://hg.mozilla.org/releases/mozilla-beta",
- "ignore_no_changes": True,
- "ssh_user": "ffxbld",
- "ssh_key": "~/.ssh/ffxbld_rsa",
- "ship_it_root": "https://ship-it.mozilla.org",
- "ship_it_username": "ship_it-ffxbld",
-}
diff --git a/testing/mozharness/configs/releases/postrelease_firefox_esr52.py b/testing/mozharness/configs/releases/postrelease_firefox_esr52.py
deleted file mode 100644
index ab461c0c8..000000000
--- a/testing/mozharness/configs/releases/postrelease_firefox_esr52.py
+++ /dev/null
@@ -1,22 +0,0 @@
-config = {
- "log_name": "bump_esr52",
- "version_files": [
- {"file": "browser/config/version.txt"},
- {"file": "browser/config/version_display.txt"},
- {"file": "config/milestone.txt"},
- ],
- "repo": {
- "repo": "https://hg.mozilla.org/releases/mozilla-esr52",
- "branch": "default",
- "dest": "mozilla-esr52",
- "vcs": "hg",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- },
- "vcs_share_base": "/builds/hg-shared",
- "push_dest": "ssh://hg.mozilla.org/releases/mozilla-esr52",
- "ignore_no_changes": True,
- "ssh_user": "ffxbld",
- "ssh_key": "~/.ssh/ffxbld_rsa",
- "ship_it_root": "https://ship-it.mozilla.org",
- "ship_it_username": "ship_it-ffxbld",
-}
diff --git a/testing/mozharness/configs/releases/postrelease_firefox_release.py b/testing/mozharness/configs/releases/postrelease_firefox_release.py
deleted file mode 100644
index 31a1b2774..000000000
--- a/testing/mozharness/configs/releases/postrelease_firefox_release.py
+++ /dev/null
@@ -1,22 +0,0 @@
-config = {
- "log_name": "bump_release",
- "version_files": [
- {"file": "browser/config/version.txt"},
- {"file": "browser/config/version_display.txt"},
- {"file": "config/milestone.txt"},
- ],
- "repo": {
- "repo": "https://hg.mozilla.org/releases/mozilla-release",
- "branch": "default",
- "dest": "mozilla-release",
- "vcs": "hg",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- },
- "vcs_share_base": "/builds/hg-shared",
- "push_dest": "ssh://hg.mozilla.org/releases/mozilla-release",
- "ignore_no_changes": True,
- "ssh_user": "ffxbld",
- "ssh_key": "~/.ssh/ffxbld_rsa",
- "ship_it_root": "https://ship-it.mozilla.org",
- "ship_it_username": "ship_it-ffxbld",
-}
diff --git a/testing/mozharness/configs/releases/updates_firefox_beta.py b/testing/mozharness/configs/releases/updates_firefox_beta.py
deleted file mode 100644
index fa81e085f..000000000
--- a/testing/mozharness/configs/releases/updates_firefox_beta.py
+++ /dev/null
@@ -1,35 +0,0 @@
-
-config = {
- "log_name": "updates_beta",
- "repo": {
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- "vcs": "hg",
- },
- "vcs_share_base": "/builds/hg-shared",
- "push_dest": "ssh://hg.mozilla.org/build/tools",
- "shipped-locales-url": "https://hg.mozilla.org/releases/mozilla-beta/raw-file/{revision}/browser/locales/shipped-locales",
- "ignore_no_changes": True,
- "ssh_user": "ffxbld",
- "ssh_key": "~/.ssh/ffxbld_rsa",
- "archive_domain": "archive.mozilla.org",
- "archive_prefix": "https://archive.mozilla.org/pub",
- "previous_archive_prefix": "https://archive.mozilla.org/pub",
- "download_domain": "download.mozilla.org",
- "balrog_url": "https://aus5.mozilla.org",
- "balrog_username": "balrog-ffxbld",
- "update_channels": {
- "beta": {
- "version_regex": r"^(\d+\.\d+(b\d+)?)$",
- "requires_mirrors": True,
- "patcher_config": "mozBeta-branch-patcher2.cfg",
- "update_verify_channel": "beta-localtest",
- "mar_channel_ids": [],
- "channel_names": ["beta", "beta-localtest", "beta-cdntest"],
- "rules_to_update": ["firefox-beta-cdntest", "firefox-beta-localtest"],
- "publish_rules": ["firefox-beta"],
- },
- },
- "balrog_use_dummy_suffix": False,
-}
diff --git a/testing/mozharness/configs/releases/updates_firefox_esr52.py b/testing/mozharness/configs/releases/updates_firefox_esr52.py
deleted file mode 100644
index 6c5a05cf9..000000000
--- a/testing/mozharness/configs/releases/updates_firefox_esr52.py
+++ /dev/null
@@ -1,35 +0,0 @@
-
-config = {
- "log_name": "updates_esr52",
- "repo": {
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- "vcs": "hg",
- },
- "vcs_share_base": "/builds/hg-shared",
- "push_dest": "ssh://hg.mozilla.org/build/tools",
- "shipped-locales-url": "https://hg.mozilla.org/releases/mozilla-esr52/raw-file/{revision}/browser/locales/shipped-locales",
- "ignore_no_changes": True,
- "ssh_user": "ffxbld",
- "ssh_key": "~/.ssh/ffxbld_rsa",
- "archive_domain": "archive.mozilla.org",
- "archive_prefix": "https://archive.mozilla.org/pub",
- "previous_archive_prefix": "https://archive.mozilla.org/pub",
- "download_domain": "download.mozilla.org",
- "balrog_url": "https://aus5.mozilla.org",
- "balrog_username": "balrog-ffxbld",
- "update_channels": {
- "esr": {
- "version_regex": r".*",
- "requires_mirrors": True,
- "patcher_config": "mozEsr52-branch-patcher2.cfg",
- "update_verify_channel": "esr-localtest",
- "mar_channel_ids": [],
- "channel_names": ["esr", "esr-localtest", "esr-cdntest"],
- "rules_to_update": ["esr52-cdntest", "esr52-localtest"],
- "publish_rules": [521],
- },
- },
- "balrog_use_dummy_suffix": False,
-}
diff --git a/testing/mozharness/configs/releases/updates_firefox_release.py b/testing/mozharness/configs/releases/updates_firefox_release.py
deleted file mode 100644
index 58210d371..000000000
--- a/testing/mozharness/configs/releases/updates_firefox_release.py
+++ /dev/null
@@ -1,47 +0,0 @@
-
-config = {
- "log_name": "updates_release",
- "repo": {
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- "vcs": "hg",
- },
- "vcs_share_base": "/builds/hg-shared",
- "push_dest": "ssh://hg.mozilla.org/build/tools",
- "shipped-locales-url": "https://hg.mozilla.org/releases/mozilla-release/raw-file/{revision}/browser/locales/shipped-locales",
- "ignore_no_changes": True,
- "ssh_user": "ffxbld",
- "ssh_key": "~/.ssh/ffxbld_rsa",
- "archive_domain": "archive.mozilla.org",
- "archive_prefix": "https://archive.mozilla.org/pub",
- "previous_archive_prefix": "https://archive.mozilla.org/pub",
- "download_domain": "download.mozilla.org",
- "balrog_url": "https://aus5.mozilla.org",
- "balrog_username": "balrog-ffxbld",
- "update_channels": {
- "beta": {
- "version_regex": r"^(\d+\.\d+(b\d+)?)$",
- "requires_mirrors": False,
- "patcher_config": "mozBeta-branch-patcher2.cfg",
- "update_verify_channel": "beta-localtest",
- "mar_channel_ids": [
- "firefox-mozilla-beta", "firefox-mozilla-release",
- ],
- "channel_names": ["beta", "beta-localtest", "beta-cdntest"],
- "rules_to_update": ["firefox-beta-cdntest", "firefox-beta-localtest"],
- "publish_rules": ["firefox-beta"],
- },
- "release": {
- "version_regex": r"^\d+\.\d+(\.\d+)?$",
- "requires_mirrors": True,
- "patcher_config": "mozRelease-branch-patcher2.cfg",
- "update_verify_channel": "release-localtest",
- "mar_channel_ids": [],
- "channel_names": ["release", "release-localtest", "release-cdntest"],
- "rules_to_update": ["firefox-release-cdntest", "firefox-release-localtest"],
- "publish_rules": ["firefox-release"],
- },
- },
- "balrog_use_dummy_suffix": False,
-}
diff --git a/testing/mozharness/configs/releng_infra_configs/builders.py b/testing/mozharness/configs/releng_infra_configs/builders.py
deleted file mode 100644
index 3a6a8b595..000000000
--- a/testing/mozharness/configs/releng_infra_configs/builders.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# This config file has generic values needed for any job and any platform running
-# on Release Engineering machines inside the VPN
-from mozharness.base.script import platform_name
-
-# These are values specific to each platform on Release Engineering machines
-PYTHON_WIN32 = 'c:/mozilla-build/python27/python.exe'
-# These are values specific to running machines on Release Engineering machines
-# to run it locally on your machines append --cfg developer_config.py
-PLATFORM_CONFIG = {
- 'linux64': {
- 'exes': {
- 'gittool.py': '/usr/local/bin/gittool.py',
- 'python': '/tools/buildbot/bin/python',
- 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
- },
- 'env': {
- 'DISPLAY': ':2',
- }
- },
- 'macosx': {
- 'exes': {
- 'gittool.py': '/usr/local/bin/gittool.py',
- 'python': '/tools/buildbot/bin/python',
- 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
- },
- },
- 'win32': {
- "exes": {
- 'gittool.py': [PYTHON_WIN32, 'c:/builds/hg-shared/build/tools/buildfarm/utils/gittool.py'],
- # Otherwise, depending on the PATH we can pick python 2.6 up
- 'python': PYTHON_WIN32,
- 'virtualenv': [PYTHON_WIN32, 'c:/mozilla-build/buildbotve/virtualenv.py'],
- }
- }
-}
-
-config = PLATFORM_CONFIG[platform_name()]
-# Generic values
-config.update({
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- 'pip_index': False,
- 'virtualenv_path': 'venv',
-})
-
diff --git a/testing/mozharness/configs/releng_infra_configs/linux.py b/testing/mozharness/configs/releng_infra_configs/linux.py
deleted file mode 100644
index dbac47935..000000000
--- a/testing/mozharness/configs/releng_infra_configs/linux.py
+++ /dev/null
@@ -1,5 +0,0 @@
-config = {
- 'env': {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/linux/minidump_stackwalk',
- }
-}
diff --git a/testing/mozharness/configs/releng_infra_configs/linux64.py b/testing/mozharness/configs/releng_infra_configs/linux64.py
deleted file mode 100644
index d7e97d6e8..000000000
--- a/testing/mozharness/configs/releng_infra_configs/linux64.py
+++ /dev/null
@@ -1,5 +0,0 @@
-config = {
- 'env': {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/linux64/minidump_stackwalk',
- }
-}
diff --git a/testing/mozharness/configs/releng_infra_configs/macosx64.py b/testing/mozharness/configs/releng_infra_configs/macosx64.py
deleted file mode 100644
index c0b5948cc..000000000
--- a/testing/mozharness/configs/releng_infra_configs/macosx64.py
+++ /dev/null
@@ -1,5 +0,0 @@
-config = {
- 'env': {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/osx64/minidump_stackwalk',
- }
-}
diff --git a/testing/mozharness/configs/releng_infra_configs/testers.py b/testing/mozharness/configs/releng_infra_configs/testers.py
deleted file mode 100644
index 7f0ce2a7f..000000000
--- a/testing/mozharness/configs/releng_infra_configs/testers.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# This config file has generic values needed for any job and any platform running
-# on Release Engineering machines inside the VPN
-import os
-
-import mozharness
-
-from mozharness.base.script import platform_name
-
-external_tools_path = os.path.join(
- os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
- 'external_tools',
-)
-
-# These are values specific to each platform on Release Engineering machines
-PYTHON_WIN32 = 'c:/mozilla-build/python27/python.exe'
-# These are values specific to running machines on Release Engineering machines
-# to run it locally on your machines append --cfg developer_config.py
-PLATFORM_CONFIG = {
- 'linux': {
- 'exes': {
- 'gittool.py': os.path.join(external_tools_path, 'gittool.py'),
- 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
- },
- 'env': {
- 'DISPLAY': ':0',
- 'PATH': '%(PATH)s:' + external_tools_path,
- }
- },
- 'linux64': {
- 'exes': {
- 'gittool.py': os.path.join(external_tools_path, 'gittool.py'),
- 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
- },
- 'env': {
- 'DISPLAY': ':0',
- 'PATH': '%(PATH)s:' + external_tools_path,
- }
- },
- 'macosx': {
- 'exes': {
- 'gittool.py': os.path.join(external_tools_path, 'gittool.py'),
- 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
- },
- 'env': {
- 'PATH': '%(PATH)s:' + external_tools_path,
- }
- },
- 'win32': {
- "exes": {
- 'gittool.py': [PYTHON_WIN32, os.path.join(external_tools_path, 'gittool.py')],
- # Otherwise, depending on the PATH we can pick python 2.6 up
- 'python': PYTHON_WIN32,
- 'virtualenv': [PYTHON_WIN32, 'c:/mozilla-build/buildbotve/virtualenv.py'],
- }
- }
-}
-
-config = PLATFORM_CONFIG[platform_name()]
-# Generic values
-config.update({
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- 'pip_index': False,
- 'virtualenv_path': 'venv',
-})
diff --git a/testing/mozharness/configs/releng_infra_configs/win32.py b/testing/mozharness/configs/releng_infra_configs/win32.py
deleted file mode 100644
index 778fa00d9..000000000
--- a/testing/mozharness/configs/releng_infra_configs/win32.py
+++ /dev/null
@@ -1,5 +0,0 @@
-config = {
- 'env': {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/win32/minidump_stackwalk',
- }
-}
diff --git a/testing/mozharness/configs/releng_infra_configs/win64.py b/testing/mozharness/configs/releng_infra_configs/win64.py
deleted file mode 100644
index 97968793e..000000000
--- a/testing/mozharness/configs/releng_infra_configs/win64.py
+++ /dev/null
@@ -1,5 +0,0 @@
-config = {
- 'env': {
- 'MINIDUMP_STACKWALK': '%(abs_tools_dir)s/breakpad/win64/minidump_stackwalk',
- }
-}
diff --git a/testing/mozharness/configs/remove_executables.py b/testing/mozharness/configs/remove_executables.py
deleted file mode 100644
index dec7a2965..000000000
--- a/testing/mozharness/configs/remove_executables.py
+++ /dev/null
@@ -1,8 +0,0 @@
-config = {
- # We bake this directly into the tester image now...
- "download_minidump_stackwalk": False,
- "minidump_stackwalk_path": "/usr/local/bin/linux64-minidump_stackwalk",
- "download_nodejs": False,
- "nodejs_path": "/usr/local/bin/node",
- "exes": {}
-}
diff --git a/testing/mozharness/configs/routes.json b/testing/mozharness/configs/routes.json
deleted file mode 100644
index 9596f4c97..000000000
--- a/testing/mozharness/configs/routes.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "routes": [
- "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
- "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}.{build_name}-{build_type}",
- "{index}.gecko.v2.{project}.latest.{build_product}.{build_name}-{build_type}"
- ],
- "nightly": [
- "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
- "{index}.gecko.v2.{project}.nightly.{year}.{month}.{day}.latest.{build_product}.{build_name}-{build_type}",
- "{index}.gecko.v2.{project}.nightly.revision.{head_rev}.{build_product}.{build_name}-{build_type}",
- "{index}.gecko.v2.{project}.nightly.latest.{build_product}.{build_name}-{build_type}"
- ],
- "l10n": [
- "{index}.gecko.v2.{project}.revision.{head_rev}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
- "{index}.gecko.v2.{project}.pushdate.{year}.{month}.{day}.{pushdate}.{build_product}-l10n.{build_name}-{build_type}.{locale}",
- "{index}.gecko.v2.{project}.latest.{build_product}-l10n.{build_name}-{build_type}.{locale}"
- ]
-}
diff --git a/testing/mozharness/configs/selfserve/production.py b/testing/mozharness/configs/selfserve/production.py
deleted file mode 100644
index f28c6c1ff..000000000
--- a/testing/mozharness/configs/selfserve/production.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
- "selfserve_url": "https://secure.pub.build.mozilla.org/buildapi/self-serve",
-}
diff --git a/testing/mozharness/configs/selfserve/staging.py b/testing/mozharness/configs/selfserve/staging.py
deleted file mode 100644
index e0ab70090..000000000
--- a/testing/mozharness/configs/selfserve/staging.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
- "selfserve_url": "https://secure-pub-build.allizom.org/buildapi/self-serve",
-}
diff --git a/testing/mozharness/configs/single_locale/alder.py b/testing/mozharness/configs/single_locale/alder.py
deleted file mode 100644
index e2fc0e6a3..000000000
--- a/testing/mozharness/configs/single_locale/alder.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# This configuration uses mozilla-central binaries (en-US, localized complete
-# mars) and urls but it generates 'alder' artifacts. With this setup, binaries
-# generated on alder are NOT overwriting mozilla-central files.
-# Using this configuration, on a successful build, artifacts will be uploaded
-# here:
-#
-# * http://dev-stage01.srv.releng.scl3.mozilla.com/pub/mozilla.org/firefox/nightly/latest-alder-l10n/
-# (in staging environment)
-# * https://ftp.mozilla.org/pub/firefox/nightly/latest-alder-l10n/
-# (in production environment)
-#
-# If you really want to have localized alder builds, use the use the following
-# values:
-# * "en_us_binary_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/alder-%(platform)s/latest/",
-# * "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/alder-%(platform)s/latest/",
-# * "repo": "https://hg.mozilla.org/projects/alder",
-#
-
-config = {
- "nightly_build": True,
- "branch": "alder",
- "en_us_binary_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/",
- "update_channel": "nightly",
-
- # l10n
- "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
-
- # mar
- "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/mar-tools/%(platform)s",
-
- # repositories
- "mozilla_dir": "alder",
- "repos": [{
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- }, {
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/mozilla-central",
- "branch": "default",
- "dest": "alder",
- }],
- # purge options
- 'is_automation': True,
-}
diff --git a/testing/mozharness/configs/single_locale/ash.py b/testing/mozharness/configs/single_locale/ash.py
deleted file mode 100644
index 3036d4fba..000000000
--- a/testing/mozharness/configs/single_locale/ash.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# This configuration uses mozilla-central binaries (en-US, localized complete
-# mars) and urls but it generates 'ash' artifacts. With this setup, binaries
-# generated on ash are NOT overwriting mozilla-central files.
-# Using this configuration, on a successful build, artifacts will be uploaded
-# here:
-#
-# * http://dev-stage01.srv.releng.scl3.mozilla.com/pub/mozilla.org/firefox/nightly/latest-ash-l10n/
-# (in staging environment)
-# * https://ftp.mozilla.org/pub/firefox/nightly/latest-ash-l10n/
-# (in production environment)
-#
-# If you really want to have localized ash builds, use the use the following
-# values:
-# * "en_us_binary_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/ash-%(platform)s/latest/",
-# * "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/tinderbox-builds/ash-%(platform)s/latest/",
-# * "repo": "https://hg.mozilla.org/projects/ash",
-#
-
-config = {
- "nightly_build": True,
- "branch": "ash",
- "en_us_binary_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/",
- "update_channel": "nightly",
-
- # l10n
- "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
-
- # mar
- "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/mar-tools/%(platform)s",
-
- # repositories
- "mozilla_dir": "ash",
- "repos": [{
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- }, {
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/mozilla-central",
- "branch": "default",
- "dest": "ash",
- }],
- # purge options
- 'is_automation': True,
-}
diff --git a/testing/mozharness/configs/single_locale/dev-mozilla-beta.py b/testing/mozharness/configs/single_locale/dev-mozilla-beta.py
deleted file mode 100644
index ef96b9b7c..000000000
--- a/testing/mozharness/configs/single_locale/dev-mozilla-beta.py
+++ /dev/null
@@ -1,37 +0,0 @@
-config = {
- "branch": "date",
- "nightly_build": True,
- "update_channel": "beta-dev",
-
- # l10n
- "hg_l10n_base": "https://hg.mozilla.org/releases/l10n/mozilla-beta",
-
- # repositories
- # staging beta dev releases use date repo for now
- "mozilla_dir": "date",
- "repos": [{
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- }, {
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/projects/date",
- "branch": "%(revision)s",
- "dest": "date",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- }],
- # purge options
- 'is_automation': True,
- 'purge_minsize': 12,
- 'default_actions': [
- "clobber",
- "pull",
- "clone-locales",
- "list-locales",
- "setup",
- "repack",
- "taskcluster-upload",
- "summary",
- ],
-}
diff --git a/testing/mozharness/configs/single_locale/dev-mozilla-release.py b/testing/mozharness/configs/single_locale/dev-mozilla-release.py
deleted file mode 100644
index 09048310b..000000000
--- a/testing/mozharness/configs/single_locale/dev-mozilla-release.py
+++ /dev/null
@@ -1,37 +0,0 @@
-config = {
- "branch": "jamun",
- "nightly_build": True,
- "update_channel": "release-dev",
-
- # l10n
- "hg_l10n_base": "https://hg.mozilla.org/releases/l10n/mozilla-release",
-
- # repositories
- # staging release uses jamun
- "mozilla_dir": "jamun",
- "repos": [{
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- }, {
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/projects/jamun",
- "branch": "%(revision)s",
- "dest": "jamun",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- }],
- # purge options
- 'purge_minsize': 12,
- 'is_automation': True,
- 'default_actions': [
- "clobber",
- "pull",
- "clone-locales",
- "list-locales",
- "setup",
- "repack",
- "taskcluster-upload",
- "summary",
- ],
-}
diff --git a/testing/mozharness/configs/single_locale/linux.py b/testing/mozharness/configs/single_locale/linux.py
deleted file mode 100644
index 3aa2c0349..000000000
--- a/testing/mozharness/configs/single_locale/linux.py
+++ /dev/null
@@ -1,123 +0,0 @@
-import os
-
-config = {
- "platform": "linux",
- "stage_product": "firefox",
- "update_platform": "Linux_x86-gcc3",
- "mozconfig": "%(branch)s/browser/config/mozconfigs/linux32/l10n-mozconfig",
- "bootstrap_env": {
- "MOZ_OBJDIR": "obj-l10n",
- "EN_US_BINARY_URL": "%(en_us_binary_url)s",
- "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
- "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
- "DIST": "%(abs_objdir)s",
- "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
- "L10NBASEDIR": "../../l10n",
- "MOZ_MAKE_COMPLETE_MAR": "1",
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- },
- "ssh_key_dir": "/home/mock_mozilla/.ssh",
- "log_name": "single_locale",
- "objdir": "obj-l10n",
- "js_src_dir": "js/src",
- "vcs_share_base": "/builds/hg-shared",
-
- # tooltool
- 'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'tooltool_manifest_src': 'browser/config/tooltool-manifests/linux32/releng.manifest',
- # balrog credential file:
- 'balrog_credentials_file': 'oauth.txt',
-
- # l10n
- "ignore_locales": ["en-US", "ja-JP-mac"],
- "l10n_dir": "l10n",
- "locales_file": "%(branch)s/browser/locales/all-locales",
- "locales_dir": "browser/locales",
- "hg_l10n_tag": "default",
- "merge_locales": True,
-
- # MAR
- "previous_mar_dir": "dist/previous",
- "current_mar_dir": "dist/current",
- "update_mar_dir": "dist/update", # sure?
- "previous_mar_filename": "previous.mar",
- "current_work_mar_dir": "current.work",
- "package_base_dir": "dist/l10n-stage",
- "application_ini": "application.ini",
- "buildid_section": 'App',
- "buildid_option": "BuildID",
- "unpack_script": "tools/update-packaging/unwrap_full_update.pl",
- "incremental_update_script": "tools/update-packaging/make_incremental_update.sh",
- "balrog_release_pusher_script": "scripts/updates/balrog-release-pusher.py",
- "update_packaging_dir": "tools/update-packaging",
- "local_mar_tool_dir": "dist/host/bin",
- "mar": "mar",
- "mbsdiff": "mbsdiff",
- "current_mar_filename": "firefox-%(version)s.%(locale)s.linux-i686.complete.mar",
- "complete_mar": "firefox-%(version)s.en-US.linux-i686.complete.mar",
- "localized_mar": "firefox-%(version)s.%(locale)s.linux-i686.complete.mar",
- "partial_mar": "firefox-%(version)s.%(locale)s.linux-i686.partial.%(from_buildid)s-%(to_buildid)s.mar",
- 'installer_file': "firefox-%(version)s.en-US.linux-i686.tar.bz2",
-
- # Mock
- 'mock_target': 'mozilla-centos6-x86_64',
- 'mock_packages': [
- 'autoconf213', 'python', 'mozilla-python27', 'zip', 'mozilla-python27-mercurial',
- 'git', 'ccache', 'perl-Test-Simple', 'perl-Config-General',
- 'yasm', 'wget',
- 'mpfr', # required for system compiler
- 'xorg-x11-font*', # fonts required for PGO
- 'imake', # required for makedepend!?!
- ### <-- from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'valgrind',
- ######## 32 bit specific ###########
- 'glibc-static.i686', 'libstdc++-static.i686',
- 'gtk2-devel.i686', 'libnotify-devel.i686',
- 'alsa-lib-devel.i686', 'libcurl-devel.i686',
- 'wireless-tools-devel.i686', 'libX11-devel.i686',
- 'libXt-devel.i686', 'mesa-libGL-devel.i686',
- 'gnome-vfs2-devel.i686', 'GConf2-devel.i686',
- 'pulseaudio-libs-devel.i686',
- 'gstreamer-devel.i686', 'gstreamer-plugins-base-devel.i686',
- # Packages already installed in the mock environment, as x86_64
- # packages.
- 'glibc-devel.i686', 'libgcc.i686', 'libstdc++-devel.i686',
- # yum likes to install .x86_64 -devel packages that satisfy .i686
- # -devel packages dependencies. So manually install the dependencies
- # of the above packages.
- 'ORBit2-devel.i686', 'atk-devel.i686', 'cairo-devel.i686',
- 'check-devel.i686', 'dbus-devel.i686', 'dbus-glib-devel.i686',
- 'fontconfig-devel.i686', 'glib2-devel.i686',
- 'hal-devel.i686', 'libICE-devel.i686', 'libIDL-devel.i686',
- 'libSM-devel.i686', 'libXau-devel.i686', 'libXcomposite-devel.i686',
- 'libXcursor-devel.i686', 'libXdamage-devel.i686',
- 'libXdmcp-devel.i686', 'libXext-devel.i686', 'libXfixes-devel.i686',
- 'libXft-devel.i686', 'libXi-devel.i686', 'libXinerama-devel.i686',
- 'libXrandr-devel.i686', 'libXrender-devel.i686',
- 'libXxf86vm-devel.i686', 'libdrm-devel.i686', 'libidn-devel.i686',
- 'libpng-devel.i686', 'libxcb-devel.i686', 'libxml2-devel.i686',
- 'pango-devel.i686', 'perl-devel.i686', 'pixman-devel.i686',
- 'zlib-devel.i686',
- # Freetype packages need to be installed be version, because a newer
- # version is available, but we don't want it for Firefox builds.
- 'freetype-2.3.11-6.el6_1.8.i686',
- 'freetype-devel-2.3.11-6.el6_1.8.i686',
- 'freetype-2.3.11-6.el6_1.8.x86_64',
- ######## 32 bit specific ###########
- ],
- 'mock_files': [
- ('/home/cltbld/.ssh', '/home/mock_mozilla/.ssh'),
- ('/home/cltbld/.hgrc', '/builds/.hgrc'),
- ('/home/cltbld/.boto', '/builds/.boto'),
- ('/builds/gapi.data', '/builds/gapi.data'),
- ('/builds/relengapi.tok', '/builds/relengapi.tok'),
- ('/tools/tooltool.py', '/builds/tooltool.py'),
- ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
- ],
-}
diff --git a/testing/mozharness/configs/single_locale/linux32.py b/testing/mozharness/configs/single_locale/linux32.py
deleted file mode 120000
index e9866bbbf..000000000
--- a/testing/mozharness/configs/single_locale/linux32.py
+++ /dev/null
@@ -1 +0,0 @@
-linux.py \ No newline at end of file
diff --git a/testing/mozharness/configs/single_locale/linux64.py b/testing/mozharness/configs/single_locale/linux64.py
deleted file mode 100644
index 8a511e56d..000000000
--- a/testing/mozharness/configs/single_locale/linux64.py
+++ /dev/null
@@ -1,103 +0,0 @@
-import os
-
-config = {
- "platform": "linux64",
- "stage_product": "firefox",
- "update_platform": "Linux_x86_64-gcc3",
- "mozconfig": "%(branch)s/browser/config/mozconfigs/linux64/l10n-mozconfig",
- "bootstrap_env": {
- "MOZ_OBJDIR": "obj-l10n",
- "EN_US_BINARY_URL": "%(en_us_binary_url)s",
- "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
- "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
- "DIST": "%(abs_objdir)s",
- "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
- "L10NBASEDIR": "../../l10n",
- "MOZ_MAKE_COMPLETE_MAR": "1",
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- },
- "ssh_key_dir": "/home/mock_mozilla/.ssh",
- "log_name": "single_locale",
- "objdir": "obj-l10n",
- "js_src_dir": "js/src",
- "vcs_share_base": "/builds/hg-shared",
-
- # tooltool
- 'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'tooltool_manifest_src': 'browser/config/tooltool-manifests/linux64/releng.manifest',
- # balrog credential file:
- 'balrog_credentials_file': 'oauth.txt',
-
- # l10n
- "ignore_locales": ["en-US", "ja-JP-mac"],
- "l10n_dir": "l10n",
- "locales_file": "%(branch)s/browser/locales/all-locales",
- "locales_dir": "browser/locales",
- "hg_l10n_tag": "default",
- "merge_locales": True,
-
- # MAR
- "previous_mar_dir": "dist/previous",
- "current_mar_dir": "dist/current",
- "update_mar_dir": "dist/update", # sure?
- "previous_mar_filename": "previous.mar",
- "current_work_mar_dir": "current.work",
- "package_base_dir": "dist/l10n-stage",
- "application_ini": "application.ini",
- "buildid_section": 'App',
- "buildid_option": "BuildID",
- "unpack_script": "tools/update-packaging/unwrap_full_update.pl",
- "incremental_update_script": "tools/update-packaging/make_incremental_update.sh",
- "balrog_release_pusher_script": "scripts/updates/balrog-release-pusher.py",
- "update_packaging_dir": "tools/update-packaging",
- "local_mar_tool_dir": "dist/host/bin",
- "mar": "mar",
- "mbsdiff": "mbsdiff",
- "current_mar_filename": "firefox-%(version)s.%(locale)s.linux-x86_64.complete.mar",
- "complete_mar": "firefox-%(version)s.en-US.linux-x86_64.complete.mar",
- "localized_mar": "firefox-%(version)s.%(locale)s.linux-x86_64.complete.mar",
- "partial_mar": "firefox-%(version)s.%(locale)s.linux-x86_64.partial.%(from_buildid)s-%(to_buildid)s.mar",
- "installer_file": "firefox-%(version)s.en-US.linux-x86_64.tar.bz2",
-
- # Mock
- 'mock_target': 'mozilla-centos6-x86_64',
-
- 'mock_packages': [
- 'autoconf213', 'python', 'mozilla-python27', 'zip', 'mozilla-python27-mercurial',
- 'git', 'ccache', 'perl-Test-Simple', 'perl-Config-General',
- 'yasm', 'wget',
- 'mpfr', # required for system compiler
- 'xorg-x11-font*', # fonts required for PGO
- 'imake', # required for makedepend!?!
- ### <-- from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'valgrind', 'dbus-x11',
- ######## 64 bit specific ###########
- 'glibc-static', 'libstdc++-static',
- 'gtk2-devel', 'libnotify-devel',
- 'alsa-lib-devel', 'libcurl-devel', 'wireless-tools-devel',
- 'libX11-devel', 'libXt-devel', 'mesa-libGL-devel', 'gnome-vfs2-devel',
- 'GConf2-devel',
- ### from releng repo
- 'gcc45_0moz3', 'gcc454_0moz1', 'gcc472_0moz1', 'gcc473_0moz1',
- 'yasm', 'ccache',
- ###
- 'pulseaudio-libs-devel', 'gstreamer-devel',
- 'gstreamer-plugins-base-devel', 'freetype-2.3.11-6.el6_1.8.x86_64',
- 'freetype-devel-2.3.11-6.el6_1.8.x86_64'
- ],
- 'mock_files': [
- ('/home/cltbld/.ssh', '/home/mock_mozilla/.ssh'),
- ('/home/cltbld/.hgrc', '/builds/.hgrc'),
- ('/home/cltbld/.boto', '/builds/.boto'),
- ('/builds/gapi.data', '/builds/gapi.data'),
- ('/builds/relengapi.tok', '/builds/relengapi.tok'),
- ('/tools/tooltool.py', '/builds/tooltool.py'),
- ('/usr/local/lib/hgext', '/usr/local/lib/hgext'),
- ],
-}
diff --git a/testing/mozharness/configs/single_locale/macosx64.py b/testing/mozharness/configs/single_locale/macosx64.py
deleted file mode 100644
index c2ee47674..000000000
--- a/testing/mozharness/configs/single_locale/macosx64.py
+++ /dev/null
@@ -1,72 +0,0 @@
-import os
-
-config = {
- # mozconfig file to use, it depends on branch and platform names
- "platform": "macosx64",
- "stage_product": "firefox",
- "update_platform": "Darwin_x86_64-gcc3",
- "mozconfig": "%(branch)s/browser/config/mozconfigs/macosx-universal/l10n-mozconfig",
- "bootstrap_env": {
- "SHELL": '/bin/bash',
- "MOZ_OBJDIR": "obj-l10n",
- "EN_US_BINARY_URL": "%(en_us_binary_url)s",
- "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
- "MOZ_PKG_PLATFORM": "mac",
- # "IS_NIGHTLY": "yes",
- "DIST": "%(abs_objdir)s",
- "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
- "L10NBASEDIR": "../../l10n",
- "MOZ_MAKE_COMPLETE_MAR": "1",
- "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
- 'TOOLTOOL_CACHE': '/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/builds',
- },
- "ssh_key_dir": "~/.ssh",
- "log_name": "single_locale",
- "objdir": "obj-l10n",
- "js_src_dir": "js/src",
- "vcs_share_base": "/builds/hg-shared",
-
- "upload_env_extra": {
- "MOZ_PKG_PLATFORM": "mac",
- },
-
- # tooltool
- 'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
- 'tooltool_script': ["/builds/tooltool.py"],
- 'tooltool_bootstrap': "setup.sh",
- 'tooltool_manifest_src': 'browser/config/tooltool-manifests/macosx64/releng.manifest',
- # balrog credential file:
- 'balrog_credentials_file': 'oauth.txt',
-
- # l10n
- "ignore_locales": ["en-US", "ja"],
- "l10n_dir": "l10n",
- "locales_file": "%(branch)s/browser/locales/all-locales",
- "locales_dir": "browser/locales",
- "hg_l10n_tag": "default",
- "merge_locales": True,
-
- # MAR
- "previous_mar_dir": "dist/previous",
- "current_mar_dir": "dist/current",
- "update_mar_dir": "dist/update", # sure?
- "previous_mar_filename": "previous.mar",
- "current_work_mar_dir": "current.work",
- "package_base_dir": "dist/l10n-stage",
- "application_ini": "Contents/Resources/application.ini",
- "buildid_section": 'App',
- "buildid_option": "BuildID",
- "unpack_script": "tools/update-packaging/unwrap_full_update.pl",
- "incremental_update_script": "tools/update-packaging/make_incremental_update.sh",
- "balrog_release_pusher_script": "scripts/updates/balrog-release-pusher.py",
- "update_packaging_dir": "tools/update-packaging",
- "local_mar_tool_dir": "dist/host/bin",
- "mar": "mar",
- "mbsdiff": "mbsdiff",
- "current_mar_filename": "firefox-%(version)s.%(locale)s.mac.complete.mar",
- "complete_mar": "firefox-%(version)s.en-US.mac.complete.mar",
- "localized_mar": "firefox-%(version)s.%(locale)s.mac.complete.mar",
- "partial_mar": "firefox-%(version)s.%(locale)s.mac.partial.%(from_buildid)s-%(to_buildid)s.mar",
- 'installer_file': "firefox-%(version)s.en-US.mac.dmg",
-}
diff --git a/testing/mozharness/configs/single_locale/mozilla-aurora.py b/testing/mozharness/configs/single_locale/mozilla-aurora.py
deleted file mode 100644
index 1ce85f726..000000000
--- a/testing/mozharness/configs/single_locale/mozilla-aurora.py
+++ /dev/null
@@ -1,29 +0,0 @@
-config = {
- "nightly_build": True,
- "branch": "mozilla-aurora",
- "en_us_binary_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-aurora/",
- "update_channel": "aurora",
-
- # l10n
- "hg_l10n_base": "https://hg.mozilla.org/releases/l10n/mozilla-aurora",
-
- # mar
- "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-aurora/mar-tools/%(platform)s",
-
- # repositories
- "mozilla_dir": "mozilla-aurora",
- "repos": [{
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- }, {
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/releases/mozilla-aurora",
- "branch": "default",
- "dest": "mozilla-aurora",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- }],
- # purge options
- 'is_automation': True,
-}
diff --git a/testing/mozharness/configs/single_locale/mozilla-beta.py b/testing/mozharness/configs/single_locale/mozilla-beta.py
deleted file mode 100644
index 90ff23027..000000000
--- a/testing/mozharness/configs/single_locale/mozilla-beta.py
+++ /dev/null
@@ -1,37 +0,0 @@
-config = {
- "nightly_build": True,
- "branch": "mozilla-beta",
- "en_us_binary_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-beta/",
- "update_channel": "beta",
-
- # l10n
- "hg_l10n_base": "https://hg.mozilla.org/releases/l10n/mozilla-beta",
-
- # repositories
- "mozilla_dir": "mozilla-beta",
- "repos": [{
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- }, {
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/releases/mozilla-beta",
- "revision": "%(revision)s",
- "dest": "mozilla-beta",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- }],
- # purge options
- 'purge_minsize': 12,
- 'is_automation': True,
- 'default_actions': [
- "clobber",
- "pull",
- "clone-locales",
- "list-locales",
- "setup",
- "repack",
- "taskcluster-upload",
- "summary",
- ],
-}
diff --git a/testing/mozharness/configs/single_locale/mozilla-central.py b/testing/mozharness/configs/single_locale/mozilla-central.py
deleted file mode 100644
index c2bf974d6..000000000
--- a/testing/mozharness/configs/single_locale/mozilla-central.py
+++ /dev/null
@@ -1,29 +0,0 @@
-config = {
- "nightly_build": True,
- "branch": "mozilla-central",
- "en_us_binary_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/",
- "update_channel": "nightly",
-
- # l10n
- "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
-
- # mar
- "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/mar-tools/%(platform)s",
-
- # repositories
- "mozilla_dir": "mozilla-central",
- "repos": [{
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- }, {
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/mozilla-central",
- "revision": "%(revision)s",
- "dest": "mozilla-central",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- }],
- # purge options
- 'is_automation': True,
-}
diff --git a/testing/mozharness/configs/single_locale/mozilla-esr52.py b/testing/mozharness/configs/single_locale/mozilla-esr52.py
deleted file mode 100644
index 0d01f1340..000000000
--- a/testing/mozharness/configs/single_locale/mozilla-esr52.py
+++ /dev/null
@@ -1,37 +0,0 @@
-config = {
- "nightly_build": True,
- "branch": "mozilla-esr52",
- "en_us_binary_url": "https://archive.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-esr52/",
- "update_channel": "esr",
-
- # l10n
- "hg_l10n_base": "https://hg.mozilla.org/releases/l10n/mozilla-release",
-
- # repositories
- "mozilla_dir": "mozilla-esr52",
- "repos": [{
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- }, {
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/releases/mozilla-esr52",
- "revision": "%(revision)s",
- "dest": "mozilla-esr52",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- }],
- # purge options
- 'purge_minsize': 12,
- 'is_automation': True,
- 'default_actions': [
- "clobber",
- "pull",
- "clone-locales",
- "list-locales",
- "setup",
- "repack",
- "taskcluster-upload",
- "summary",
- ],
-}
diff --git a/testing/mozharness/configs/single_locale/mozilla-release.py b/testing/mozharness/configs/single_locale/mozilla-release.py
deleted file mode 100644
index f02ea2ca9..000000000
--- a/testing/mozharness/configs/single_locale/mozilla-release.py
+++ /dev/null
@@ -1,37 +0,0 @@
-config = {
- "nightly_build": True,
- "branch": "mozilla-release",
- "en_us_binary_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-release/",
- "update_channel": "release",
-
- # l10n
- "hg_l10n_base": "https://hg.mozilla.org/releases/l10n/mozilla-release",
-
- # repositories
- "mozilla_dir": "mozilla-release",
- "repos": [{
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- }, {
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/releases/mozilla-release",
- "revision": "%(revision)s",
- "dest": "mozilla-release",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- }],
- # purge options
- 'purge_minsize': 12,
- 'is_automation': True,
- 'default_actions': [
- "clobber",
- "pull",
- "clone-locales",
- "list-locales",
- "setup",
- "repack",
- "taskcluster-upload",
- "summary",
- ],
-}
diff --git a/testing/mozharness/configs/single_locale/production.py b/testing/mozharness/configs/single_locale/production.py
deleted file mode 100644
index fe97fe361..000000000
--- a/testing/mozharness/configs/single_locale/production.py
+++ /dev/null
@@ -1,14 +0,0 @@
-config = {
- "upload_environment": "prod",
- "upload_env": {
- "UPLOAD_USER": "ffxbld",
- # ssh_key_dir is defined per platform: it is "~/.ssh" for every platform
- # except when mock is in use, in this case, ssh_key_dir is
- # /home/mock_mozilla/.ssh
- "UPLOAD_SSH_KEY": "%(ssh_key_dir)s/ffxbld_rsa",
- "UPLOAD_HOST": "upload.ffxbld.productdelivery.prod.mozaws.net",
- "POST_UPLOAD_CMD": "post_upload.py -b %(branch)s-l10n -p %(stage_product)s -i %(buildid)s --release-to-latest --release-to-dated",
- "UPLOAD_TO_TEMP": "1"
- },
- 'taskcluster_index': 'index',
-}
diff --git a/testing/mozharness/configs/single_locale/staging.py b/testing/mozharness/configs/single_locale/staging.py
deleted file mode 100644
index 82caa8dda..000000000
--- a/testing/mozharness/configs/single_locale/staging.py
+++ /dev/null
@@ -1,17 +0,0 @@
-config = {
- "upload_environment": "stage",
- "upload_env": {
- "UPLOAD_USER": "ffxbld",
- # ssh_key_dir is defined per platform: it is "~/.ssh" for every platform
- # except when mock is in use, in this case, ssh_key_dir is
- # /home/mock_mozilla/.ssh
- "UPLOAD_SSH_KEY": "%(ssh_key_dir)s/ffxbld_rsa",
- "UPLOAD_HOST": "upload.ffxbld.productdelivery.stage.mozaws.net",
- "POST_UPLOAD_CMD": "post_upload.py -b %(branch)s-l10n -p %(stage_product)s -i %(buildid)s --release-to-latest --release-to-dated %(post_upload_extra)s",
- "UPLOAD_TO_TEMP": "1"
- },
- 'taskcluster_index': 'index.garbage.staging',
- 'post_upload_extra': ['--bucket-prefix', 'net-mozaws-stage-delivery',
- '--url-prefix', 'http://ftp.stage.mozaws.net/',
- ],
-}
diff --git a/testing/mozharness/configs/single_locale/tc_linux32.py b/testing/mozharness/configs/single_locale/tc_linux32.py
deleted file mode 100644
index 3045138f8..000000000
--- a/testing/mozharness/configs/single_locale/tc_linux32.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import os
-
-config = {
- "locales_file": "src/browser/locales/all-locales",
- "tools_repo": "https://hg.mozilla.org/build/tools",
- "mozconfig": "src/browser/config/mozconfigs/linux32/l10n-mozconfig",
- "bootstrap_env": {
- "NO_MERCURIAL_SETUP_CHECK": "1",
- "MOZ_OBJDIR": "obj-l10n",
- "EN_US_BINARY_URL": "%(en_us_binary_url)s",
- "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
- "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
- "DIST": "%(abs_objdir)s",
- "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
- "L10NBASEDIR": "../../l10n",
- "MOZ_MAKE_COMPLETE_MAR": "1",
- 'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
- },
- "upload_env": {
- 'UPLOAD_HOST': 'localhost',
- 'UPLOAD_PATH': '/home/worker/artifacts/',
- },
- "mozilla_dir": "src/",
-}
diff --git a/testing/mozharness/configs/single_locale/tc_linux64.py b/testing/mozharness/configs/single_locale/tc_linux64.py
deleted file mode 100644
index 28a4c6f56..000000000
--- a/testing/mozharness/configs/single_locale/tc_linux64.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import os
-
-config = {
- "locales_file": "src/browser/locales/all-locales",
- "tools_repo": "https://hg.mozilla.org/build/tools",
- "mozconfig": "src/browser/config/mozconfigs/linux64/l10n-mozconfig",
- "bootstrap_env": {
- "NO_MERCURIAL_SETUP_CHECK": "1",
- "MOZ_OBJDIR": "obj-l10n",
- "EN_US_BINARY_URL": "%(en_us_binary_url)s",
- "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
- "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
- "DIST": "%(abs_objdir)s",
- "LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
- "L10NBASEDIR": "../../l10n",
- "MOZ_MAKE_COMPLETE_MAR": "1",
- 'TOOLTOOL_CACHE': os.environ.get('TOOLTOOL_CACHE'),
- },
- "upload_env": {
- 'UPLOAD_HOST': 'localhost',
- 'UPLOAD_PATH': '/home/worker/artifacts/',
- },
- "mozilla_dir": "src/",
-}
diff --git a/testing/mozharness/configs/single_locale/try.py b/testing/mozharness/configs/single_locale/try.py
deleted file mode 100644
index 369159111..000000000
--- a/testing/mozharness/configs/single_locale/try.py
+++ /dev/null
@@ -1,42 +0,0 @@
-config = {
- "nightly_build": False,
- "branch": "try",
- "en_us_binary_url": "http://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central",
- "update_channel": "nightly",
- "update_gecko_source_to_enUS": False,
-
- # l10n
- "hg_l10n_base": "https://hg.mozilla.org/l10n-central",
-
- # mar
- "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/mar-tools/%(platform)s",
-
- # repositories
- "mozilla_dir": "try",
- "repos": [{
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools",
- }, {
- "vcs": "hg",
- "repo": "https://hg.mozilla.org/try",
- "revision": "%(revision)s",
- "dest": "try",
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- "clone_by_revision": True,
- "clone_with_purge": True,
- }],
- # purge options
- 'is_automation': True,
- "upload_env": {
- "UPLOAD_USER": "trybld",
- # ssh_key_dir is defined per platform: it is "~/.ssh" for every platform
- # except when mock is in use, in this case, ssh_key_dir is
- # /home/mock_mozilla/.ssh
- "UPLOAD_SSH_KEY": "%(ssh_key_dir)s/trybld_dsa",
- "UPLOAD_HOST": "upload.trybld.productdelivery.%(upload_environment)s.mozaws.net",
- "POST_UPLOAD_CMD": "post_upload.py --who %(who)s --builddir %(branch)s-%(platform)s --tinderbox-builds-dir %(who)s-%(revision)s -p %(stage_product)s -i %(buildid)s --revision %(revision)s --release-to-try-builds %(post_upload_extra)s",
- "UPLOAD_TO_TEMP": "1"
- },
-}
diff --git a/testing/mozharness/configs/single_locale/win32.py b/testing/mozharness/configs/single_locale/win32.py
deleted file mode 100644
index ea07fff86..000000000
--- a/testing/mozharness/configs/single_locale/win32.py
+++ /dev/null
@@ -1,77 +0,0 @@
-import os
-import sys
-
-config = {
- "platform": "win32",
- "stage_product": "firefox",
- "update_platform": "WINNT_x86-msvc",
- "mozconfig": "%(branch)s/browser/config/mozconfigs/win32/l10n-mozconfig",
- "bootstrap_env": {
- "MOZ_OBJDIR": "obj-l10n",
- "EN_US_BINARY_URL": "%(en_us_binary_url)s",
- "LOCALE_MERGEDIR": "%(abs_merge_dir)s",
- "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
- "DIST": "%(abs_objdir)s",
- "L10NBASEDIR": "../../l10n",
- "MOZ_MAKE_COMPLETE_MAR": "1",
- "PATH": 'C:\\mozilla-build\\nsis-3.01;'
- '%s' % (os.environ.get('path')),
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- },
- "ssh_key_dir": "~/.ssh",
- "log_name": "single_locale",
- "objdir": "obj-l10n",
- "js_src_dir": "js/src",
- "vcs_share_base": "c:/builds/hg-shared",
-
- # tooltool
- 'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
- 'tooltool_script': [sys.executable,
- 'C:/mozilla-build/tooltool.py'],
- 'tooltool_bootstrap': "setup.sh",
- 'tooltool_manifest_src': 'browser/config/tooltool-manifests/win32/releng.manifest',
- # balrog credential file:
- 'balrog_credentials_file': 'oauth.txt',
-
- # l10n
- "ignore_locales": ["en-US", "ja-JP-mac"],
- "l10n_dir": "l10n",
- "locales_file": "%(branch)s/browser/locales/all-locales",
- "locales_dir": "browser/locales",
- "hg_l10n_tag": "default",
- "merge_locales": True,
-
- # MAR
- "previous_mar_dir": "dist\\previous",
- "current_mar_dir": "dist\\current",
- "update_mar_dir": "dist\\update", # sure?
- "previous_mar_filename": "previous.mar",
- "current_work_mar_dir": "current.work",
- "package_base_dir": "dist\\l10n-stage",
- "application_ini": "application.ini",
- "buildid_section": 'App',
- "buildid_option": "BuildID",
- "unpack_script": "tools\\update-packaging\\unwrap_full_update.pl",
- "incremental_update_script": "tools\\update-packaging\\make_incremental_update.sh",
- "balrog_release_pusher_script": "scripts\\updates\\balrog-release-pusher.py",
- "update_packaging_dir": "tools\\update-packaging",
- "local_mar_tool_dir": "dist\\host\\bin",
- "mar": "mar.exe",
- "mbsdiff": "mbsdiff.exe",
- "current_mar_filename": "firefox-%(version)s.%(locale)s.win32.complete.mar",
- "complete_mar": "firefox-%(version)s.en-US.win32.complete.mar",
- "localized_mar": "firefox-%(version)s.%(locale)s.win32.complete.mar",
- "partial_mar": "firefox-%(version)s.%(locale)s.win32.partial.%(from_buildid)s-%(to_buildid)s.mar",
- 'installer_file': "firefox-%(version)s.en-US.win32.installer.exe",
-
- # use mozmake?
- "enable_mozmake": True,
- 'exes': {
- 'python2.7': sys.executable,
- 'virtualenv': [
- sys.executable,
- 'c:/mozilla-build/buildbotve/virtualenv.py'
- ],
- }
-}
diff --git a/testing/mozharness/configs/single_locale/win64.py b/testing/mozharness/configs/single_locale/win64.py
deleted file mode 100644
index df553018f..000000000
--- a/testing/mozharness/configs/single_locale/win64.py
+++ /dev/null
@@ -1,77 +0,0 @@
-import os
-import sys
-
-config = {
- "platform": "win64",
- "stage_product": "firefox",
- "update_platform": "WINNT_x86_64-msvc",
- "mozconfig": "%(branch)s/browser/config/mozconfigs/win64/l10n-mozconfig",
- "bootstrap_env": {
- "MOZ_OBJDIR": "obj-l10n",
- "EN_US_BINARY_URL": "%(en_us_binary_url)s",
- "MOZ_UPDATE_CHANNEL": "%(update_channel)s",
- "DIST": "%(abs_objdir)s",
- "LOCALE_MERGEDIR": "%(abs_merge_dir)s",
- "L10NBASEDIR": "../../l10n",
- "MOZ_MAKE_COMPLETE_MAR": "1",
- "PATH": 'C:\\mozilla-build\\nsis-3.01;'
- '%s' % (os.environ.get('path')),
- 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
- 'TOOLTOOL_HOME': '/c/builds',
- },
- "ssh_key_dir": "~/.ssh",
- "log_name": "single_locale",
- "objdir": "obj-l10n",
- "js_src_dir": "js/src",
- "vcs_share_base": "c:/builds/hg-shared",
-
- # tooltool
- 'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
- 'tooltool_script': [sys.executable,
- 'C:/mozilla-build/tooltool.py'],
- 'tooltool_bootstrap': "setup.sh",
- 'tooltool_manifest_src': 'browser/config/tooltool-manifests/win64/releng.manifest',
- # balrog credential file:
- 'balrog_credentials_file': 'oauth.txt',
-
- # l10n
- "ignore_locales": ["en-US", "ja-JP-mac"],
- "l10n_dir": "l10n",
- "locales_file": "%(branch)s/browser/locales/all-locales",
- "locales_dir": "browser/locales",
- "hg_l10n_tag": "default",
- "merge_locales": True,
-
- # MAR
- "previous_mar_dir": "dist\\previous",
- "current_mar_dir": "dist\\current",
- "update_mar_dir": "dist\\update", # sure?
- "previous_mar_filename": "previous.mar",
- "current_work_mar_dir": "current.work",
- "package_base_dir": "dist\\l10n-stage",
- "application_ini": "application.ini",
- "buildid_section": 'App',
- "buildid_option": "BuildID",
- "unpack_script": "tools\\update-packaging\\unwrap_full_update.pl",
- "incremental_update_script": "tools\\update-packaging\\make_incremental_update.sh",
- "balrog_release_pusher_script": "scripts\\updates\\balrog-release-pusher.py",
- "update_packaging_dir": "tools\\update-packaging",
- "local_mar_tool_dir": "dist\\host\\bin",
- "mar": "mar.exe",
- "mbsdiff": "mbsdiff.exe",
- "current_mar_filename": "firefox-%(version)s.%(locale)s.win64.complete.mar",
- "complete_mar": "firefox-%(version)s.en-US.win64.complete.mar",
- "localized_mar": "firefox-%(version)s.%(locale)s.win64.complete.mar",
- "partial_mar": "firefox-%(version)s.%(locale)s.win64.partial.%(from_buildid)s-%(to_buildid)s.mar",
- 'installer_file': "firefox-%(version)s.en-US.win64.installer.exe",
-
- # use mozmake?
- "enable_mozmake": True,
- 'exes': {
- 'python2.7': sys.executable,
- 'virtualenv': [
- sys.executable,
- 'c:/mozilla-build/buildbotve/virtualenv.py'
- ],
- }
-}
diff --git a/testing/mozharness/configs/talos/linux_config.py b/testing/mozharness/configs/talos/linux_config.py
deleted file mode 100644
index 192de17c6..000000000
--- a/testing/mozharness/configs/talos/linux_config.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import os
-import platform
-
-PYTHON = '/tools/buildbot/bin/python'
-VENV_PATH = '%s/build/venv' % os.getcwd()
-if platform.architecture()[0] == '64bit':
- TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
- MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
-else:
- TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/releng.manifest"
- MINIDUMP_STACKWALK_PATH = "linux32-minidump_stackwalk"
-
-config = {
- "log_name": "talos",
- "buildbot_json_path": "buildprops.json",
- "installer_path": "installer.exe",
- "virtualenv_path": VENV_PATH,
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
- "exes": {
- 'python': PYTHON,
- 'virtualenv': [PYTHON, '/tools/misc-python/virtualenv.py'],
- 'tooltool.py': "/tools/tooltool.py",
- },
- "title": os.uname()[1].lower().split('.')[0],
- "default_actions": [
- "clobber",
- "read-buildbot-config",
- "download-and-extract",
- "populate-webroot",
- "create-virtualenv",
- "install",
- "run-tests",
- ],
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
- "download_minidump_stackwalk": True,
- "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
- "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
- "tooltool_cache": "/builds/tooltool_cache",
-}
diff --git a/testing/mozharness/configs/talos/mac_config.py b/testing/mozharness/configs/talos/mac_config.py
deleted file mode 100644
index 56876dbdd..000000000
--- a/testing/mozharness/configs/talos/mac_config.py
+++ /dev/null
@@ -1,56 +0,0 @@
-ENABLE_SCREEN_RESOLUTION_CHECK = True
-
-SCREEN_RESOLUTION_CHECK = {
- "name": "check_screen_resolution",
- "cmd": ["bash", "-c", "screenresolution get && screenresolution list && system_profiler SPDisplaysDataType"],
- "architectures": ["32bit", "64bit"],
- "halt_on_failure": False,
- "enabled": ENABLE_SCREEN_RESOLUTION_CHECK
-}
-
-import os
-
-PYTHON = '/tools/buildbot/bin/python'
-VENV_PATH = '%s/build/venv' % os.getcwd()
-
-config = {
- "log_name": "talos",
- "buildbot_json_path": "buildprops.json",
- "installer_path": "installer.exe",
- "virtualenv_path": VENV_PATH,
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
- "exes": {
- 'python': PYTHON,
- 'virtualenv': [PYTHON, '/tools/misc-python/virtualenv.py'],
- 'tooltool.py': "/tools/tooltool.py",
- },
- "title": os.uname()[1].lower().split('.')[0],
- "default_actions": [
- "clobber",
- "read-buildbot-config",
- "download-and-extract",
- "populate-webroot",
- "create-virtualenv",
- "install",
- "run-tests",
- ],
- "run_cmd_checks_enabled": True,
- "preflight_run_cmd_suites": [
- SCREEN_RESOLUTION_CHECK,
- ],
- "postflight_run_cmd_suites": [
- SCREEN_RESOLUTION_CHECK,
- ],
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
- "download_minidump_stackwalk": True,
- "minidump_stackwalk_path": "macosx64-minidump_stackwalk",
- "minidump_tooltool_manifest_path": "config/tooltool-manifests/macosx64/releng.manifest",
- "tooltool_cache": "/builds/tooltool_cache",
-}
diff --git a/testing/mozharness/configs/talos/windows_config.py b/testing/mozharness/configs/talos/windows_config.py
deleted file mode 100644
index 50c924c44..000000000
--- a/testing/mozharness/configs/talos/windows_config.py
+++ /dev/null
@@ -1,48 +0,0 @@
-import os
-import socket
-
-PYTHON = 'c:/mozilla-build/python27/python.exe'
-PYTHON_DLL = 'c:/mozilla-build/python27/python27.dll'
-VENV_PATH = os.path.join(os.getcwd(), 'build/venv')
-
-config = {
- "log_name": "talos",
- "buildbot_json_path": "buildprops.json",
- "installer_path": "installer.exe",
- "virtualenv_path": VENV_PATH,
- "virtualenv_python_dll": PYTHON_DLL,
- "pip_index": False,
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "virtualenv_modules": ['pywin32', 'talos', 'mozinstall'],
- "exes": {
- 'python': PYTHON,
- 'virtualenv': [PYTHON, 'c:/mozilla-build/buildbotve/virtualenv.py'],
- 'easy_install': ['%s/scripts/python' % VENV_PATH,
- '%s/scripts/easy_install-2.7-script.py' % VENV_PATH],
- 'mozinstall': ['%s/scripts/python' % VENV_PATH,
- '%s/scripts/mozinstall-script.py' % VENV_PATH],
- 'hg': 'c:/mozilla-build/hg/hg',
- 'tooltool.py': [PYTHON, 'C:/mozilla-build/tooltool.py'],
- },
- "title": socket.gethostname().split('.')[0],
- "default_actions": [
- "clobber",
- "read-buildbot-config",
- "download-and-extract",
- "populate-webroot",
- "create-virtualenv",
- "install",
- "run-tests",
- ],
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
- "metro_harness_path_frmt": "%(metro_base_path)s/metro/metrotestharness.exe",
- "download_minidump_stackwalk": True,
- "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
- "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
-}
diff --git a/testing/mozharness/configs/taskcluster_nightly.py b/testing/mozharness/configs/taskcluster_nightly.py
deleted file mode 100644
index 6c4e4a754..000000000
--- a/testing/mozharness/configs/taskcluster_nightly.py
+++ /dev/null
@@ -1,5 +0,0 @@
-config = {
- 'nightly_build': True,
- 'taskcluster_nightly': True,
-}
-
diff --git a/testing/mozharness/configs/test/example_config1.json b/testing/mozharness/configs/test/example_config1.json
deleted file mode 100644
index ca73466ba..000000000
--- a/testing/mozharness/configs/test/example_config1.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
- "beverage": "fizzy drink",
- "long_sleep_time": 1800,
- "random_config_key1": "spectacular"
-}
diff --git a/testing/mozharness/configs/test/example_config2.py b/testing/mozharness/configs/test/example_config2.py
deleted file mode 100644
index 958543b60..000000000
--- a/testing/mozharness/configs/test/example_config2.py
+++ /dev/null
@@ -1,5 +0,0 @@
-config = {
- "beverage": "cider",
- "long_sleep_time": 300,
- "random_config_key2": "wunderbar",
-}
diff --git a/testing/mozharness/configs/test/test.illegal_suffix b/testing/mozharness/configs/test/test.illegal_suffix
deleted file mode 100644
index 7d9a4d96d..000000000
--- a/testing/mozharness/configs/test/test.illegal_suffix
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "log_name": "test",
- "log_dir": "test_logs",
- "log_to_console": false,
- "key1": "value1",
- "key2": "value2",
- "section1": {
-
- "subsection1": {
- "key1": "value1",
- "key2": "value2"
- },
-
- "subsection2": {
- "key1": "value1",
- "key2": "value2"
- }
-
- }
-}
diff --git a/testing/mozharness/configs/test/test.json b/testing/mozharness/configs/test/test.json
deleted file mode 100644
index 7d9a4d96d..000000000
--- a/testing/mozharness/configs/test/test.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "log_name": "test",
- "log_dir": "test_logs",
- "log_to_console": false,
- "key1": "value1",
- "key2": "value2",
- "section1": {
-
- "subsection1": {
- "key1": "value1",
- "key2": "value2"
- },
-
- "subsection2": {
- "key1": "value1",
- "key2": "value2"
- }
-
- }
-}
diff --git a/testing/mozharness/configs/test/test.py b/testing/mozharness/configs/test/test.py
deleted file mode 100644
index 84fc357b2..000000000
--- a/testing/mozharness/configs/test/test.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-config = {
- "log_name": "test",
- "log_dir": "test_logs",
- "log_to_console": False,
- "key1": "value1",
- "key2": "value2",
- "section1": {
-
- "subsection1": {
- "key1": "value1",
- "key2": "value2"
- },
-
- "subsection2": {
- "key1": "value1",
- "key2": "value2"
- },
-
- },
- "opt_override": "some stuff",
-}
diff --git a/testing/mozharness/configs/test/test_malformed.json b/testing/mozharness/configs/test/test_malformed.json
deleted file mode 100644
index 260be45b8..000000000
--- a/testing/mozharness/configs/test/test_malformed.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "log_name": "test",
- "log_dir": "test_logs",
- "log_to_console": false,
- "key1": "value1",
- "key2": "value2",
- "section1": {
-
- "subsection1": {
- "key1": "value1",
- "key2": "value2"
- },
-
- "subsection2": {
- "key1": "value1",
- "key2": "value2"
- },
-
- }
-}
diff --git a/testing/mozharness/configs/test/test_malformed.py b/testing/mozharness/configs/test/test_malformed.py
deleted file mode 100644
index e7ccefd15..000000000
--- a/testing/mozharness/configs/test/test_malformed.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-config = {
- "log_name": "test",
- "log_dir": "test_logs",
- "log_to_console": False,
- "key1": "value1",
- "key2": "value2",
- "section1": {
-
- "subsection1": {
- "key1": "value1",
- "key2": "value2"
- },
-
-a;sldkfjas;dfkljasdf;kjasdf;ljkadsflkjsdfkweoi
- "subsection2": {
- "key1": "value1",
- "key2": "value2"
- },
-
- },
-}
diff --git a/testing/mozharness/configs/test/test_optional.py b/testing/mozharness/configs/test/test_optional.py
deleted file mode 100644
index 4eb13b3df..000000000
--- a/testing/mozharness/configs/test/test_optional.py
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/usr/bin/env python
-config = {
- "opt_override": "new stuff",
-}
diff --git a/testing/mozharness/configs/test/test_override.py b/testing/mozharness/configs/test/test_override.py
deleted file mode 100644
index 00db5220a..000000000
--- a/testing/mozharness/configs/test/test_override.py
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env python
-config = {
- "override_string": "TODO",
- "override_list": ['to', 'do'],
- "override_dict": {'to': 'do'},
- "keep_string": "don't change me",
-}
diff --git a/testing/mozharness/configs/test/test_override2.py b/testing/mozharness/configs/test/test_override2.py
deleted file mode 100644
index 27091d453..000000000
--- a/testing/mozharness/configs/test/test_override2.py
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/env python
-config = {
- "override_string": "yay",
- "override_list": ["yay", 'worked'],
- "override_dict": {"yay": 'worked'},
-}
diff --git a/testing/mozharness/configs/unittests/linux_unittest.py b/testing/mozharness/configs/unittests/linux_unittest.py
deleted file mode 100644
index b591fa559..000000000
--- a/testing/mozharness/configs/unittests/linux_unittest.py
+++ /dev/null
@@ -1,307 +0,0 @@
-import os
-import platform
-
-# OS Specifics
-ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
-BINARY_PATH = os.path.join(ABS_WORK_DIR, "application", "firefox", "firefox-bin")
-INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.tar.bz2")
-XPCSHELL_NAME = "xpcshell"
-EXE_SUFFIX = ""
-DISABLE_SCREEN_SAVER = True
-ADJUST_MOUSE_AND_SCREEN = False
-
-# Note: keep these Valgrind .sup file names consistent with those
-# in testing/mochitest/mochitest_options.py.
-VALGRIND_SUPP_DIR = os.path.join(os.getcwd(), "build/tests/mochitest")
-VALGRIND_SUPP_CROSS_ARCH = os.path.join(VALGRIND_SUPP_DIR,
- "cross-architecture.sup")
-VALGRIND_SUPP_ARCH = None
-
-if platform.architecture()[0] == "64bit":
- TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
- MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
- VALGRIND_SUPP_ARCH = os.path.join(VALGRIND_SUPP_DIR,
- "x86_64-redhat-linux-gnu.sup")
- NODEJS_PATH = "node-linux-x64/bin/node"
- NODEJS_TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/nodejs.manifest"
-else:
- TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/releng.manifest"
- MINIDUMP_STACKWALK_PATH = "linux32-minidump_stackwalk"
- VALGRIND_SUPP_ARCH = os.path.join(VALGRIND_SUPP_DIR,
- "i386-redhat-linux-gnu.sup")
- NODEJS_PATH = "node-linux-x86/bin/node"
- NODEJS_TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/nodejs.manifest"
-
-#####
-config = {
- "buildbot_json_path": "buildprops.json",
- "exes": {
- "python": "/tools/buildbot/bin/python",
- "virtualenv": ["/tools/buildbot/bin/python", "/tools/misc-python/virtualenv.py"],
- "tooltool.py": "/tools/tooltool.py",
- },
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
- ###
- "installer_path": INSTALLER_PATH,
- "binary_path": BINARY_PATH,
- "xpcshell_name": XPCSHELL_NAME,
- "exe_suffix": EXE_SUFFIX,
- "run_file_names": {
- "mochitest": "runtests.py",
- "reftest": "runreftest.py",
- "xpcshell": "runxpcshelltests.py",
- "cppunittest": "runcppunittests.py",
- "gtest": "rungtests.py",
- "jittest": "jit_test.py",
- "mozbase": "test.py",
- "mozmill": "runtestlist.py",
- },
- "minimum_tests_zip_dirs": [
- "bin/*",
- "certs/*",
- "config/*",
- "mach",
- "marionette/*",
- "modules/*",
- "mozbase/*",
- "tools/*",
- ],
- "specific_tests_zip_dirs": {
- "mochitest": ["mochitest/*"],
- "reftest": ["reftest/*", "jsreftest/*"],
- "xpcshell": ["xpcshell/*"],
- "cppunittest": ["cppunittest/*"],
- "gtest": ["gtest/*"],
- "jittest": ["jit-test/*"],
- "mozbase": ["mozbase/*"],
- "mozmill": ["mozmill/*"],
- },
- "suite_definitions": {
- "cppunittest": {
- "options": [
- "--symbols-path=%(symbols_path)s",
- "--xre-path=%(abs_app_dir)s"
- ],
- "run_filename": "runcppunittests.py",
- "testsdir": "cppunittest"
- },
- "jittest": {
- "options": [
- "tests/bin/js",
- "--no-slow",
- "--no-progress",
- "--format=automation",
- "--jitflags=all",
- "--timeout=970" # Keep in sync with run_timeout below.
- ],
- "run_filename": "jit_test.py",
- "testsdir": "jit-test/jit-test",
- "run_timeout": 1000 # Keep in sync with --timeout above.
- },
- "mochitest": {
- "options": [
- "--appname=%(binary_path)s",
- "--utility-path=tests/bin",
- "--extra-profile-file=tests/bin/plugins",
- "--symbols-path=%(symbols_path)s",
- "--certificate-path=tests/certs",
- "--setpref=webgl.force-enabled=true",
- "--quiet",
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--use-test-media-devices",
- "--screenshot-on-fail",
- "--cleanup-crashes",
- "--marionette-startup-timeout=180",
- ],
- "run_filename": "runtests.py",
- "testsdir": "mochitest"
- },
- "mozbase": {
- "options": [
- "-b",
- "%(binary_path)s"
- ],
- "run_filename": "test.py",
- "testsdir": "mozbase"
- },
- "mozmill": {
- "options": [
- "--binary=%(binary_path)s",
- "--testing-modules-dir=test/modules",
- "--plugins-path=%(test_plugin_path)s",
- "--symbols-path=%(symbols_path)s"
- ],
- "run_filename": "runtestlist.py",
- "testsdir": "mozmill"
- },
- "reftest": {
- "options": [
- "--appname=%(binary_path)s",
- "--utility-path=tests/bin",
- "--extra-profile-file=tests/bin/plugins",
- "--symbols-path=%(symbols_path)s",
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--cleanup-crashes",
- ],
- "run_filename": "runreftest.py",
- "testsdir": "reftest"
- },
- "xpcshell": {
- "options": [
- "--symbols-path=%(symbols_path)s",
- "--test-plugin-path=%(test_plugin_path)s",
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--utility-path=tests/bin",
- ],
- "run_filename": "runxpcshelltests.py",
- "testsdir": "xpcshell"
- },
- "gtest": {
- "options": [
- "--xre-path=%(abs_res_dir)s",
- "--cwd=%(gtest_dir)s",
- "--symbols-path=%(symbols_path)s",
- "--utility-path=tests/bin",
- "%(binary_path)s",
- ],
- "run_filename": "rungtests.py",
- },
- },
- # local mochi suites
- "all_mochitest_suites": {
- "valgrind-plain": ["--valgrind=/usr/bin/valgrind",
- "--valgrind-supp-files=" + VALGRIND_SUPP_ARCH +
- "," + VALGRIND_SUPP_CROSS_ARCH,
- "--timeout=900", "--max-timeouts=50"],
- "plain": [],
- "plain-gpu": ["--subsuite=gpu"],
- "plain-clipboard": ["--subsuite=clipboard"],
- "plain-chunked": ["--chunk-by-dir=4"],
- "mochitest-media": ["--subsuite=media"],
- "chrome": ["--flavor=chrome"],
- "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
- "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
- "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
- "browser-chrome": ["--flavor=browser"],
- "browser-chrome-gpu": ["--flavor=browser", "--subsuite=gpu"],
- "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
- "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
- "browser-chrome-addons": ["--flavor=browser", "--chunk-by-runtime", "--tag=addons"],
- "browser-chrome-coverage": ["--flavor=browser", "--chunk-by-runtime", "--timeout=1200"],
- "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
- "mochitest-gl": ["--subsuite=webgl"],
- "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
- "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
- "mochitest-devtools-chrome-coverage": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime", "--timeout=1200"],
- "jetpack-package": ["--flavor=jetpack-package"],
- "jetpack-package-clipboard": ["--flavor=jetpack-package", "--subsuite=clipboard"],
- "jetpack-addon": ["--flavor=jetpack-addon"],
- "a11y": ["--flavor=a11y"],
- },
- # local reftest suites
- "all_reftest_suites": {
- "crashtest": {
- "options": ["--suite=crashtest"],
- "tests": ["tests/reftest/tests/testing/crashtest/crashtests.list"]
- },
- "jsreftest": {
- "options": ["--extra-profile-file=tests/jsreftest/tests/user.js",
- "--suite=jstestbrowser"],
- "tests": ["tests/jsreftest/tests/jstests.list"]
- },
- "reftest": {
- "options": ["--suite=reftest"],
- "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
- },
- "reftest-no-accel": {
- "options": ["--suite=reftest",
- "--setpref=layers.acceleration.enabled=disabled",
- "--setpref=layers.acceleration.force=disabled"],
- "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]},
- },
- "all_xpcshell_suites": {
- "xpcshell": {
- "options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
- "--manifest=tests/xpcshell/tests/xpcshell.ini"],
- "tests": []
- },
- "xpcshell-addons": {
- "options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
- "--tag=addons",
- "--manifest=tests/xpcshell/tests/xpcshell.ini"],
- "tests": []
- },
- "xpcshell-coverage": {
- "options": ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
- "--manifest=tests/xpcshell/tests/xpcshell.ini"],
- "tests": []
- },
- },
- "all_cppunittest_suites": {
- "cppunittest": {"tests": ["tests/cppunittest"]}
- },
- "all_gtest_suites": {
- "gtest": []
- },
- "all_jittest_suites": {
- "jittest": [],
- "jittest1": ["--total-chunks=2", "--this-chunk=1"],
- "jittest2": ["--total-chunks=2", "--this-chunk=2"],
- "jittest-chunked": [],
- },
- "all_mozbase_suites": {
- "mozbase": []
- },
- "run_cmd_checks_enabled": True,
- "preflight_run_cmd_suites": [
- # NOTE 'enabled' is only here while we have unconsolidated configs
- {
- "name": "disable_screen_saver",
- "cmd": ["xset", "s", "off", "s", "reset"],
- "halt_on_failure": False,
- "architectures": ["32bit", "64bit"],
- "enabled": DISABLE_SCREEN_SAVER
- },
- {
- "name": "run mouse & screen adjustment script",
- "cmd": [
- # when configs are consolidated this python path will only show
- # for windows.
- "python", "../scripts/external_tools/mouse_and_screen_resolution.py",
- "--configuration-file",
- "../scripts/external_tools/machine-configuration.json"],
- "architectures": ["32bit"],
- "halt_on_failure": True,
- "enabled": ADJUST_MOUSE_AND_SCREEN
- },
- ],
- "vcs_output_timeout": 1000,
- "minidump_save_path": "%(abs_work_dir)s/../minidumps",
- "buildbot_max_log_size": 52428800,
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "unstructured_flavors": {"mochitest": ['jetpack'],
- "xpcshell": [],
- "gtest": [],
- "mozmill": [],
- "cppunittest": [],
- "jittest": [],
- "mozbase": [],
- },
- "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
- "download_minidump_stackwalk": True,
- "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
- "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
- "tooltool_cache": "/builds/tooltool_cache",
- "download_nodejs": True,
- "nodejs_path": NODEJS_PATH,
- "nodejs_tooltool_manifest_path": NODEJS_TOOLTOOL_MANIFEST_PATH,
-}
diff --git a/testing/mozharness/configs/unittests/mac_unittest.py b/testing/mozharness/configs/unittests/mac_unittest.py
deleted file mode 100644
index 20bbcf9f5..000000000
--- a/testing/mozharness/configs/unittests/mac_unittest.py
+++ /dev/null
@@ -1,257 +0,0 @@
-import os
-
-# OS Specifics
-INSTALLER_PATH = os.path.join(os.getcwd(), "installer.dmg")
-XPCSHELL_NAME = 'xpcshell'
-EXE_SUFFIX = ''
-DISABLE_SCREEN_SAVER = False
-ADJUST_MOUSE_AND_SCREEN = False
-#####
-config = {
- "buildbot_json_path": "buildprops.json",
- "exes": {
- 'python': '/tools/buildbot/bin/python',
- 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
- 'tooltool.py': "/tools/tooltool.py",
- },
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
- ###
- "installer_path": INSTALLER_PATH,
- "xpcshell_name": XPCSHELL_NAME,
- "exe_suffix": EXE_SUFFIX,
- "run_file_names": {
- "mochitest": "runtests.py",
- "reftest": "runreftest.py",
- "xpcshell": "runxpcshelltests.py",
- "cppunittest": "runcppunittests.py",
- "gtest": "rungtests.py",
- "jittest": "jit_test.py",
- "mozbase": "test.py",
- "mozmill": "runtestlist.py",
- },
- "minimum_tests_zip_dirs": [
- "bin/*",
- "certs/*",
- "config/*",
- "mach",
- "marionette/*",
- "modules/*",
- "mozbase/*",
- "tools/*",
- ],
- "specific_tests_zip_dirs": {
- "mochitest": ["mochitest/*"],
- "reftest": ["reftest/*", "jsreftest/*"],
- "xpcshell": ["xpcshell/*"],
- "cppunittest": ["cppunittest/*"],
- "gtest": ["gtest/*"],
- "jittest": ["jit-test/*"],
- "mozbase": ["mozbase/*"],
- "mozmill": ["mozmill/*"],
- },
- "suite_definitions": {
- "cppunittest": {
- "options": [
- "--symbols-path=%(symbols_path)s",
- "--xre-path=%(abs_res_dir)s"
- ],
- "run_filename": "runcppunittests.py",
- "testsdir": "cppunittest"
- },
- "jittest": {
- "options": [
- "tests/bin/js",
- "--no-slow",
- "--no-progress",
- "--format=automation",
- "--jitflags=all",
- "--timeout=970" # Keep in sync with run_timeout below.
- ],
- "run_filename": "jit_test.py",
- "testsdir": "jit-test/jit-test",
- "run_timeout": 1000 # Keep in sync with --timeout above.
- },
- "mochitest": {
- "options": [
- "--appname=%(binary_path)s",
- "--utility-path=tests/bin",
- "--extra-profile-file=tests/bin/plugins",
- "--symbols-path=%(symbols_path)s",
- "--certificate-path=tests/certs",
- "--quiet",
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--screenshot-on-fail",
- "--cleanup-crashes",
- ],
- "run_filename": "runtests.py",
- "testsdir": "mochitest"
- },
- "mozbase": {
- "options": [
- "-b",
- "%(binary_path)s"
- ],
- "run_filename": "test.py",
- "testsdir": "mozbase"
- },
- "mozmill": {
- "options": [
- "--binary=%(binary_path)s",
- "--testing-modules-dir=test/modules",
- "--plugins-path=%(test_plugin_path)s",
- "--symbols-path=%(symbols_path)s"
- ],
- "run_filename": "runtestlist.py",
- "testsdir": "mozmill"
- },
- "reftest": {
- "options": [
- "--appname=%(binary_path)s",
- "--utility-path=tests/bin",
- "--extra-profile-file=tests/bin/plugins",
- "--symbols-path=%(symbols_path)s",
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--cleanup-crashes",
- ],
- "run_filename": "runreftest.py",
- "testsdir": "reftest"
- },
- "xpcshell": {
- "options": [
- "--symbols-path=%(symbols_path)s",
- "--test-plugin-path=%(test_plugin_path)s",
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--utility-path=tests/bin",
- ],
- "run_filename": "runxpcshelltests.py",
- "testsdir": "xpcshell"
- },
- "gtest": {
- "options": [
- "--xre-path=%(abs_res_dir)s",
- "--cwd=%(gtest_dir)s",
- "--symbols-path=%(symbols_path)s",
- "--utility-path=tests/bin",
- "%(binary_path)s",
- ],
- "run_filename": "rungtests.py",
- },
- },
- # local mochi suites
- "all_mochitest_suites": {
- "plain": [],
- "plain-gpu": ["--subsuite=gpu"],
- "plain-clipboard": ["--subsuite=clipboard"],
- "plain-chunked": ["--chunk-by-dir=4"],
- "mochitest-media": ["--subsuite=media"],
- "chrome": ["--flavor=chrome"],
- "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
- "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
- "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
- "browser-chrome": ["--flavor=browser"],
- "browser-chrome-gpu": ["--flavor=browser", "--subsuite=gpu"],
- "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
- "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
- "browser-chrome-addons": ["--flavor=browser", "--chunk-by-runtime", "--tag=addons"],
- "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
- "mochitest-gl": ["--subsuite=webgl"],
- "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
- "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
- "jetpack-package": ["--flavor=jetpack-package"],
- "jetpack-package-clipboard": ["--flavor=jetpack-package", "--subsuite=clipboard"],
- "jetpack-addon": ["--flavor=jetpack-addon"],
- "a11y": ["--flavor=a11y"],
- },
- # local reftest suites
- "all_reftest_suites": {
- "crashtest": {
- 'options': ["--suite=crashtest"],
- 'tests': ["tests/reftest/tests/testing/crashtest/crashtests.list"]
- },
- "jsreftest": {
- 'options':["--extra-profile-file=tests/jsreftest/tests/user.js"],
- 'tests': ["tests/jsreftest/tests/jstests.list"]
- },
- "reftest": {
- 'options': ["--suite=reftest"],
- 'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
- },
- },
- "all_xpcshell_suites": {
- "xpcshell": {
- 'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
- "--manifest=tests/xpcshell/tests/xpcshell.ini"],
- 'tests': []
- },
- "xpcshell-addons": {
- 'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
- "--tag=addons",
- "--manifest=tests/xpcshell/tests/xpcshell.ini"],
- 'tests': []
- },
- },
- "all_cppunittest_suites": {
- "cppunittest": ['tests/cppunittest']
- },
- "all_gtest_suites": {
- "gtest": []
- },
- "all_jittest_suites": {
- "jittest": []
- },
- "all_mozbase_suites": {
- "mozbase": []
- },
- "run_cmd_checks_enabled": True,
- "preflight_run_cmd_suites": [
- # NOTE 'enabled' is only here while we have unconsolidated configs
- {
- "name": "disable_screen_saver",
- "cmd": ["xset", "s", "off", "s", "reset"],
- "architectures": ["32bit", "64bit"],
- "halt_on_failure": False,
- "enabled": DISABLE_SCREEN_SAVER
- },
- {
- "name": "run mouse & screen adjustment script",
- "cmd": [
- # when configs are consolidated this python path will only show
- # for windows.
- "python", "../scripts/external_tools/mouse_and_screen_resolution.py",
- "--configuration-file",
- "../scripts/external_tools/machine-configuration.json"],
- "architectures": ["32bit"],
- "halt_on_failure": True,
- "enabled": ADJUST_MOUSE_AND_SCREEN
- },
- ],
- "vcs_output_timeout": 1000,
- "minidump_save_path": "%(abs_work_dir)s/../minidumps",
- "buildbot_max_log_size": 52428800,
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "unstructured_flavors": {"mochitest": ['jetpack'],
- "xpcshell": [],
- "gtest": [],
- "mozmill": [],
- "cppunittest": [],
- "jittest": [],
- "mozbase": [],
- },
- "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
- "download_minidump_stackwalk": True,
- "minidump_stackwalk_path": "macosx64-minidump_stackwalk",
- "minidump_tooltool_manifest_path": "config/tooltool-manifests/macosx64/releng.manifest",
- "tooltool_cache": "/builds/tooltool_cache",
- "download_nodejs": True,
- "nodejs_path": "node-osx/bin/node",
- "nodejs_tooltool_manifest_path": "config/tooltool-manifests/macosx64/nodejs.manifest",
-}
diff --git a/testing/mozharness/configs/unittests/thunderbird_extra.py b/testing/mozharness/configs/unittests/thunderbird_extra.py
deleted file mode 100644
index 2021b9d55..000000000
--- a/testing/mozharness/configs/unittests/thunderbird_extra.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#####
-config = {
- "application": "thunderbird",
- "minimum_tests_zip_dirs": [
- "bin/*",
- "certs/*",
- "config/*",
- "extensions/*",
- "marionette/*",
- "modules/*",
- "mozbase/*",
- "tools/*",
- ],
- "all_mozmill_suites": {
- "mozmill": ["--list=tests/mozmill/mozmilltests.list"],
- },
-}
diff --git a/testing/mozharness/configs/unittests/win_taskcluster_unittest.py b/testing/mozharness/configs/unittests/win_taskcluster_unittest.py
deleted file mode 100644
index 2eeae22e2..000000000
--- a/testing/mozharness/configs/unittests/win_taskcluster_unittest.py
+++ /dev/null
@@ -1,274 +0,0 @@
-import os
-import sys
-
-# OS Specifics
-ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
-BINARY_PATH = os.path.join(ABS_WORK_DIR, "firefox", "firefox.exe")
-INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.zip")
-XPCSHELL_NAME = 'xpcshell.exe'
-EXE_SUFFIX = '.exe'
-DISABLE_SCREEN_SAVER = False
-ADJUST_MOUSE_AND_SCREEN = True
-#####
-config = {
- "exes": {
- 'python': sys.executable,
- 'virtualenv': [
- sys.executable,
- os.path.join(os.path.dirname(sys.executable), 'Lib', 'site-packages', 'virtualenv.py')
- ],
- 'mozinstall': ['build/venv/scripts/python', 'build/venv/scripts/mozinstall-script.py'],
- 'tooltool.py': [sys.executable, os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')],
- 'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg')
- },
- ###
- "installer_path": INSTALLER_PATH,
- "binary_path": BINARY_PATH,
- "xpcshell_name": XPCSHELL_NAME,
- "virtualenv_path": 'venv',
- "virtualenv_python_dll": os.path.join(os.path.dirname(sys.executable), "python27.dll"),
-
- "proxxy": {},
- "find_links": [
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
- "exe_suffix": EXE_SUFFIX,
- "run_file_names": {
- "mochitest": "runtests.py",
- "reftest": "runreftest.py",
- "xpcshell": "runxpcshelltests.py",
- "cppunittest": "runcppunittests.py",
- "gtest": "rungtests.py",
- "jittest": "jit_test.py",
- "mozbase": "test.py",
- "mozmill": "runtestlist.py",
- },
- "minimum_tests_zip_dirs": [
- "bin/*",
- "certs/*",
- "config/*",
- "mach",
- "marionette/*",
- "modules/*",
- "mozbase/*",
- "tools/*",
- ],
- "specific_tests_zip_dirs": {
- "mochitest": ["mochitest/*"],
- "reftest": ["reftest/*", "jsreftest/*"],
- "xpcshell": ["xpcshell/*"],
- "cppunittest": ["cppunittest/*"],
- "gtest": ["gtest/*"],
- "jittest": ["jit-test/*"],
- "mozbase": ["mozbase/*"],
- "mozmill": ["mozmill/*"],
- },
- "suite_definitions": {
- "cppunittest": {
- "options": [
- "--symbols-path=%(symbols_path)s",
- "--xre-path=%(abs_app_dir)s"
- ],
- "run_filename": "runcppunittests.py",
- "testsdir": "cppunittest"
- },
- "jittest": {
- "options": [
- "tests/bin/js",
- "--no-slow",
- "--no-progress",
- "--format=automation",
- "--jitflags=all",
- "--timeout=970" # Keep in sync with run_timeout below.
- ],
- "run_filename": "jit_test.py",
- "testsdir": "jit-test/jit-test",
- "run_timeout": 1000 # Keep in sync with --timeout above.
- },
- "mochitest": {
- "options": [
- "--appname=%(binary_path)s",
- "--utility-path=tests/bin",
- "--extra-profile-file=tests/bin/plugins",
- "--symbols-path=%(symbols_path)s",
- "--certificate-path=tests/certs",
- "--quiet",
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--screenshot-on-fail",
- "--cleanup-crashes",
- ],
- "run_filename": "runtests.py",
- "testsdir": "mochitest"
- },
- "mozbase": {
- "options": [
- "-b",
- "%(binary_path)s"
- ],
- "run_filename": "test.py",
- "testsdir": "mozbase"
- },
- "mozmill": {
- "options": [
- "--binary=%(binary_path)s",
- "--testing-modules-dir=test/modules",
- "--plugins-path=%(test_plugin_path)s",
- "--symbols-path=%(symbols_path)s"
- ],
- "run_filename": "runtestlist.py",
- "testsdir": "mozmill"
- },
- "reftest": {
- "options": [
- "--appname=%(binary_path)s",
- "--utility-path=tests/bin",
- "--extra-profile-file=tests/bin/plugins",
- "--symbols-path=%(symbols_path)s",
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--cleanup-crashes",
- ],
- "run_filename": "runreftest.py",
- "testsdir": "reftest"
- },
- "xpcshell": {
- "options": [
- "--symbols-path=%(symbols_path)s",
- "--test-plugin-path=%(test_plugin_path)s",
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--utility-path=tests/bin",
- ],
- "run_filename": "runxpcshelltests.py",
- "testsdir": "xpcshell"
- },
- "gtest": {
- "options": [
- "--xre-path=%(abs_res_dir)s",
- "--cwd=%(gtest_dir)s",
- "--symbols-path=%(symbols_path)s",
- "--utility-path=tests/bin",
- "%(binary_path)s",
- ],
- "run_filename": "rungtests.py",
- },
- },
- # local mochi suites
- "all_mochitest_suites":
- {
- "plain": [],
- "plain-gpu": ["--subsuite=gpu"],
- "plain-clipboard": ["--subsuite=clipboard"],
- "plain-chunked": ["--chunk-by-dir=4"],
- "mochitest-media": ["--subsuite=media"],
- "chrome": ["--flavor=chrome"],
- "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
- "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
- "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
- "browser-chrome": ["--flavor=browser"],
- "browser-chrome-gpu": ["--flavor=browser", "--subsuite=gpu"],
- "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
- "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
- "browser-chrome-addons": ["--flavor=browser", "--chunk-by-runtime", "--tag=addons"],
- "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
- "mochitest-gl": ["--subsuite=webgl"],
- "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
- "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
- "mochitest-metro-chrome": ["--flavor=browser", "--metro-immersive"],
- "jetpack-package": ["--flavor=jetpack-package"],
- "jetpack-package-clipboard": ["--flavor=jetpack-package", "--subsuite=clipboard"],
- "jetpack-addon": ["--flavor=jetpack-addon"],
- "a11y": ["--flavor=a11y"],
- },
- # local reftest suites
- "all_reftest_suites": {
- "crashtest": {
- 'options': ["--suite=crashtest"],
- 'tests': ["tests/reftest/tests/testing/crashtest/crashtests.list"]
- },
- "jsreftest": {
- 'options':["--extra-profile-file=tests/jsreftest/tests/user.js"],
- 'tests': ["tests/jsreftest/tests/jstests.list"]
- },
- "reftest": {
- 'options': ["--suite=reftest"],
- 'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
- },
- "reftest-gpu": {
- 'options': ["--suite=reftest",
- "--setpref=layers.gpu-process.force-enabled=true"],
- 'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
- },
- "reftest-no-accel": {
- "options": ["--suite=reftest",
- "--setpref=gfx.direct2d.disabled=true",
- "--setpref=layers.acceleration.enabled=false"],
- "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
- },
- },
- "all_xpcshell_suites": {
- "xpcshell": {
- 'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
- "--manifest=tests/xpcshell/tests/xpcshell.ini"],
- 'tests': []
- },
- "xpcshell-addons": {
- 'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
- "--tag=addons",
- "--manifest=tests/xpcshell/tests/xpcshell.ini"],
- 'tests': []
- },
- },
- "all_cppunittest_suites": {
- "cppunittest": ['tests/cppunittest']
- },
- "all_gtest_suites": {
- "gtest": []
- },
- "all_jittest_suites": {
- "jittest": []
- },
- "all_mozbase_suites": {
- "mozbase": []
- },
- "run_cmd_checks_enabled": True,
- "preflight_run_cmd_suites": [
- {
- 'name': 'disable_screen_saver',
- 'cmd': ['xset', 's', 'off', 's', 'reset'],
- 'architectures': ['32bit', '64bit'],
- 'halt_on_failure': False,
- 'enabled': DISABLE_SCREEN_SAVER
- },
- {
- 'name': 'run mouse & screen adjustment script',
- 'cmd': [
- sys.executable,
- os.path.join(os.getcwd(),
- 'mozharness', 'external_tools', 'mouse_and_screen_resolution.py'),
- '--configuration-file',
- os.path.join(os.getcwd(),
- 'mozharness', 'external_tools', 'machine-configuration.json')
- ],
- 'architectures': ['32bit'],
- 'halt_on_failure': True,
- 'enabled': ADJUST_MOUSE_AND_SCREEN
- }
- ],
- "vcs_output_timeout": 1000,
- "minidump_save_path": "%(abs_work_dir)s/../minidumps",
- "buildbot_max_log_size": 52428800,
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "structured_suites": ["reftest"],
- 'blob_uploader_auth_file': 'C:/builds/oauth.txt',
- "download_minidump_stackwalk": True,
- "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
- "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
- "download_nodejs": True,
- "nodejs_path": "node-win32.exe",
- "nodejs_tooltool_manifest_path": "config/tooltool-manifests/win32/nodejs.manifest",
-}
diff --git a/testing/mozharness/configs/unittests/win_unittest.py b/testing/mozharness/configs/unittests/win_unittest.py
deleted file mode 100644
index 1f682cc8f..000000000
--- a/testing/mozharness/configs/unittests/win_unittest.py
+++ /dev/null
@@ -1,281 +0,0 @@
-import os
-import sys
-
-# OS Specifics
-ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
-BINARY_PATH = os.path.join(ABS_WORK_DIR, "application", "firefox", "firefox.exe")
-INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.zip")
-XPCSHELL_NAME = 'xpcshell.exe'
-EXE_SUFFIX = '.exe'
-DISABLE_SCREEN_SAVER = False
-ADJUST_MOUSE_AND_SCREEN = True
-#####
-config = {
- "buildbot_json_path": "buildprops.json",
- "exes": {
- 'python': sys.executable,
- 'virtualenv': [sys.executable, 'c:/mozilla-build/buildbotve/virtualenv.py'],
- 'hg': 'c:/mozilla-build/hg/hg',
- 'mozinstall': ['%s/build/venv/scripts/python' % os.getcwd(),
- '%s/build/venv/scripts/mozinstall-script.py' % os.getcwd()],
- 'tooltool.py': [sys.executable, 'C:/mozilla-build/tooltool.py'],
- },
- ###
- "installer_path": INSTALLER_PATH,
- "binary_path": BINARY_PATH,
- "xpcshell_name": XPCSHELL_NAME,
- "virtualenv_path": 'venv',
- "virtualenv_python_dll": os.path.join(os.path.dirname(sys.executable), "python27.dll"),
- "virtualenv_modules": ['pypiwin32'],
-
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
- "exe_suffix": EXE_SUFFIX,
- "run_file_names": {
- "mochitest": "runtests.py",
- "reftest": "runreftest.py",
- "xpcshell": "runxpcshelltests.py",
- "cppunittest": "runcppunittests.py",
- "gtest": "rungtests.py",
- "jittest": "jit_test.py",
- "mozbase": "test.py",
- "mozmill": "runtestlist.py",
- },
- "minimum_tests_zip_dirs": [
- "bin/*",
- "certs/*",
- "config/*",
- "mach",
- "marionette/*",
- "modules/*",
- "mozbase/*",
- "tools/*",
- ],
- "specific_tests_zip_dirs": {
- "mochitest": ["mochitest/*"],
- "reftest": ["reftest/*", "jsreftest/*"],
- "xpcshell": ["xpcshell/*"],
- "cppunittest": ["cppunittest/*"],
- "gtest": ["gtest/*"],
- "jittest": ["jit-test/*"],
- "mozbase": ["mozbase/*"],
- "mozmill": ["mozmill/*"],
- },
- "suite_definitions": {
- "cppunittest": {
- "options": [
- "--symbols-path=%(symbols_path)s",
- "--xre-path=%(abs_app_dir)s"
- ],
- "run_filename": "runcppunittests.py",
- "testsdir": "cppunittest"
- },
- "jittest": {
- "options": [
- "tests/bin/js",
- "--no-slow",
- "--no-progress",
- "--format=automation",
- "--jitflags=all",
- "--timeout=970" # Keep in sync with run_timeout below.
- ],
- "run_filename": "jit_test.py",
- "testsdir": "jit-test/jit-test",
- "run_timeout": 1000 # Keep in sync with --timeout above.
- },
- "mochitest": {
- "options": [
- "--appname=%(binary_path)s",
- "--utility-path=tests/bin",
- "--extra-profile-file=tests/bin/plugins",
- "--symbols-path=%(symbols_path)s",
- "--certificate-path=tests/certs",
- "--quiet",
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--screenshot-on-fail",
- "--cleanup-crashes",
- ],
- "run_filename": "runtests.py",
- "testsdir": "mochitest"
- },
- "mozbase": {
- "options": [
- "-b",
- "%(binary_path)s"
- ],
- "run_filename": "test.py",
- "testsdir": "mozbase"
- },
- "mozmill": {
- "options": [
- "--binary=%(binary_path)s",
- "--testing-modules-dir=test/modules",
- "--plugins-path=%(test_plugin_path)s",
- "--symbols-path=%(symbols_path)s"
- ],
- "run_filename": "runtestlist.py",
- "testsdir": "mozmill"
- },
- "reftest": {
- "options": [
- "--appname=%(binary_path)s",
- "--utility-path=tests/bin",
- "--extra-profile-file=tests/bin/plugins",
- "--symbols-path=%(symbols_path)s",
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--cleanup-crashes",
- ],
- "run_filename": "runreftest.py",
- "testsdir": "reftest"
- },
- "xpcshell": {
- "options": [
- "--symbols-path=%(symbols_path)s",
- "--test-plugin-path=%(test_plugin_path)s",
- "--log-raw=%(raw_log_file)s",
- "--log-errorsummary=%(error_summary_file)s",
- "--utility-path=tests/bin",
- ],
- "run_filename": "runxpcshelltests.py",
- "testsdir": "xpcshell"
- },
- "gtest": {
- "options": [
- "--xre-path=%(abs_res_dir)s",
- "--cwd=%(gtest_dir)s",
- "--symbols-path=%(symbols_path)s",
- "--utility-path=tests/bin",
- "%(binary_path)s",
- ],
- "run_filename": "rungtests.py",
- },
- },
- # local mochi suites
- "all_mochitest_suites":
- {
- "plain": [],
- "plain-gpu": ["--subsuite=gpu"],
- "plain-clipboard": ["--subsuite=clipboard"],
- "plain-chunked": ["--chunk-by-dir=4"],
- "mochitest-media": ["--subsuite=media"],
- "chrome": ["--flavor=chrome"],
- "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
- "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
- "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
- "browser-chrome": ["--flavor=browser"],
- "browser-chrome-gpu": ["--flavor=browser", "--subsuite=gpu"],
- "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
- "browser-chrome-chunked": ["--flavor=browser", "--chunk-by-runtime"],
- "browser-chrome-addons": ["--flavor=browser", "--chunk-by-runtime", "--tag=addons"],
- "browser-chrome-screenshots": ["--flavor=browser", "--subsuite=screenshots"],
- "mochitest-gl": ["--subsuite=webgl"],
- "mochitest-devtools-chrome": ["--flavor=browser", "--subsuite=devtools"],
- "mochitest-devtools-chrome-chunked": ["--flavor=browser", "--subsuite=devtools", "--chunk-by-runtime"],
- "mochitest-metro-chrome": ["--flavor=browser", "--metro-immersive"],
- "jetpack-package": ["--flavor=jetpack-package"],
- "jetpack-package-clipboard": ["--flavor=jetpack-package", "--subsuite=clipboard"],
- "jetpack-addon": ["--flavor=jetpack-addon"],
- "a11y": ["--flavor=a11y"],
- },
- # local reftest suites
- "all_reftest_suites": {
- "crashtest": {
- 'options': ["--suite=crashtest"],
- 'tests': ["tests/reftest/tests/testing/crashtest/crashtests.list"]
- },
- "jsreftest": {
- 'options':["--extra-profile-file=tests/jsreftest/tests/user.js"],
- 'tests': ["tests/jsreftest/tests/jstests.list"]
- },
- "reftest": {
- 'options': ["--suite=reftest"],
- 'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
- },
- "reftest-gpu": {
- 'options': ["--suite=reftest",
- "--setpref=layers.gpu-process.force-enabled=true"],
- 'tests': ["tests/reftest/tests/layout/reftests/reftest.list"]
- },
- "reftest-no-accel": {
- "options": ["--suite=reftest",
- "--setpref=gfx.direct2d.disabled=true",
- "--setpref=layers.acceleration.enabled=false"],
- "tests": ["tests/reftest/tests/layout/reftests/reftest.list"]
- },
- },
- "all_xpcshell_suites": {
- "xpcshell": {
- 'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
- "--manifest=tests/xpcshell/tests/xpcshell.ini"],
- 'tests': []
- },
- "xpcshell-addons": {
- 'options': ["--xpcshell=%(abs_app_dir)s/" + XPCSHELL_NAME,
- "--tag=addons",
- "--manifest=tests/xpcshell/tests/xpcshell.ini"],
- 'tests': []
- },
- },
- "all_cppunittest_suites": {
- "cppunittest": ['tests/cppunittest']
- },
- "all_gtest_suites": {
- "gtest": []
- },
- "all_jittest_suites": {
- "jittest": []
- },
- "all_mozbase_suites": {
- "mozbase": []
- },
- "run_cmd_checks_enabled": True,
- "preflight_run_cmd_suites": [
- # NOTE 'enabled' is only here while we have unconsolidated configs
- {
- "name": "disable_screen_saver",
- "cmd": ["xset", "s", "off", "s", "reset"],
- "architectures": ["32bit", "64bit"],
- "halt_on_failure": False,
- "enabled": DISABLE_SCREEN_SAVER
- },
- {
- "name": "run mouse & screen adjustment script",
- "cmd": [
- # when configs are consolidated this python path will only show
- # for windows.
- sys.executable,
- "../scripts/external_tools/mouse_and_screen_resolution.py",
- "--configuration-file",
- "../scripts/external_tools/machine-configuration.json"],
- "architectures": ["32bit"],
- "halt_on_failure": True,
- "enabled": ADJUST_MOUSE_AND_SCREEN
- },
- ],
- "vcs_output_timeout": 1000,
- "minidump_save_path": "%(abs_work_dir)s/../minidumps",
- "buildbot_max_log_size": 52428800,
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
- "unstructured_flavors": {"mochitest": ['jetpack'],
- "xpcshell": [],
- "gtest": [],
- "mozmill": [],
- "cppunittest": [],
- "jittest": [],
- "mozbase": [],
- },
- "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
- "download_minidump_stackwalk": True,
- "minidump_stackwalk_path": "win32-minidump_stackwalk.exe",
- "minidump_tooltool_manifest_path": "config/tooltool-manifests/win32/releng.manifest",
- "download_nodejs": True,
- "nodejs_path": "node-win32.exe",
- "nodejs_tooltool_manifest_path": "config/tooltool-manifests/win32/nodejs.manifest",
-}
diff --git a/testing/mozharness/configs/users/aki/gaia_json.py b/testing/mozharness/configs/users/aki/gaia_json.py
deleted file mode 100644
index 4263cc908..000000000
--- a/testing/mozharness/configs/users/aki/gaia_json.py
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env python
-
-config = {
- "log_name": "gaia_bump",
- "log_max_rotate": 99,
- "ssh_key": "~/.ssh/id_rsa",
- "ssh_user": "asasaki@mozilla.com",
- "hg_user": "Test Pusher <aki@escapewindow.com>",
- "revision_file": "b2g/config/gaia.json",
- "exes": {
- # Get around the https warnings
- "hg": ['hg', "--config", "web.cacerts=/etc/pki/tls/certs/ca-bundle.crt"],
- },
- "repo_list": [{
- "polling_url": "https://hg.mozilla.org/integration/gaia-central/json-pushes?full=1",
- "branch": "default",
- "repo_url": "https://hg.mozilla.org/integration/gaia-central",
- "repo_name": "gaia-central",
- "target_push_url": "ssh://hg.mozilla.org/users/asasaki_mozilla.com/birch",
- "target_pull_url": "https://hg.mozilla.org/users/asasaki_mozilla.com/birch",
- "target_tag": "default",
- "target_repo_name": "birch",
- }, {
- "polling_url": "https://hg.mozilla.org/integration/gaia-1_2/json-pushes?full=1",
- "branch": "default",
- "repo_url": "https://hg.mozilla.org/integration/gaia-1_2",
- "repo_name": "gaia-1_2",
- "target_push_url": "ssh://hg.mozilla.org/users/asasaki_mozilla.com/mozilla-aurora",
- "target_pull_url": "https://hg.mozilla.org/users/asasaki_mozilla.com/mozilla-aurora",
- "target_tag": "default",
- "target_repo_name": "mozilla-aurora",
- }, {
- "polling_url": "https://hg.mozilla.org/integration/gaia-1_2/json-pushes?full=1",
- "branch": "default",
- "repo_url": "https://hg.mozilla.org/integration/gaia-1_2",
- "repo_name": "gaia-1_2",
- "target_push_url": "ssh://hg.mozilla.org/users/asasaki_mozilla.com/mozilla-aurora",
- "target_pull_url": "https://hg.mozilla.org/users/asasaki_mozilla.com/mozilla-aurora",
- "target_tag": "default",
- "target_repo_name": "mozilla-aurora",
- }],
-}
diff --git a/testing/mozharness/configs/users/sfink/mock.py b/testing/mozharness/configs/users/sfink/mock.py
deleted file mode 100644
index 07b5c5c43..000000000
--- a/testing/mozharness/configs/users/sfink/mock.py
+++ /dev/null
@@ -1,3 +0,0 @@
-config = {
- "mock_target": "mozilla-centos6-x86_64",
-}
diff --git a/testing/mozharness/configs/users/sfink/spidermonkey.py b/testing/mozharness/configs/users/sfink/spidermonkey.py
deleted file mode 100644
index efbc9a805..000000000
--- a/testing/mozharness/configs/users/sfink/spidermonkey.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# This config file is for locally testing spidermonkey_build.py. It provides
-# the values that would otherwise be provided by buildbot.
-
-BRANCH = "local-src"
-HOME = "/home/sfink"
-REPO = HOME + "/src/MI-GC"
-
-config = {
- "hgurl": "https://hg.mozilla.org/",
- "python": "python",
- "sixgill": HOME + "/src/sixgill",
- "sixgill_bin": HOME + "/src/sixgill/bin",
-
- "repo": REPO,
- "repos": [{
- "repo": REPO,
- "branch": "default",
- "dest": BRANCH,
- }, {
- "repo": "https://hg.mozilla.org/build/tools",
- "branch": "default",
- "dest": "tools"
- }],
-
- "tools_dir": "/tools",
-
- "mock_target": "mozilla-centos6-x86_64",
-
- "upload_remote_basepath": "/tmp/upload-base",
- "upload_ssh_server": "localhost",
- "upload_ssh_key": "/home/sfink/.ssh/id_rsa",
- "upload_ssh_user": "sfink",
- "upload_label": "linux64-br-haz",
-
- # For testing tryserver uploads (directory structure is different)
- #"branch": "try",
- #"revision": "deadbeef1234",
-}
diff --git a/testing/mozharness/configs/web_platform_tests/prod_config.py b/testing/mozharness/configs/web_platform_tests/prod_config.py
deleted file mode 100644
index f0fb0b074..000000000
--- a/testing/mozharness/configs/web_platform_tests/prod_config.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-import os
-
-config = {
- "options": [
- "--prefs-root=%(test_path)s/prefs",
- "--processes=1",
- "--config=%(test_path)s/wptrunner.ini",
- "--ca-cert-path=%(test_path)s/certs/cacert.pem",
- "--host-key-path=%(test_path)s/certs/web-platform.test.key",
- "--host-cert-path=%(test_path)s/certs/web-platform.test.pem",
- "--certutil-binary=%(test_install_path)s/bin/certutil",
- ],
-
- "exes": {
- 'python': '/tools/buildbot/bin/python',
- 'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
- 'tooltool.py': "/tools/tooltool.py",
- },
-
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
-
- "pip_index": False,
-
- "buildbot_json_path": "buildprops.json",
-
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
-
- "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
-
- "download_minidump_stackwalk": True,
-
- "download_tooltool": True,
-
- "tooltool_cache": "/builds/tooltool_cache",
-
-}
-
diff --git a/testing/mozharness/configs/web_platform_tests/prod_config_windows.py b/testing/mozharness/configs/web_platform_tests/prod_config_windows.py
deleted file mode 100644
index 7c0f525fe..000000000
--- a/testing/mozharness/configs/web_platform_tests/prod_config_windows.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-# This is a template config file for web-platform-tests test.
-
-import os
-import sys
-
-config = {
- "options": [
- "--prefs-root=%(test_path)s/prefs",
- "--processes=1",
- "--config=%(test_path)s/wptrunner.ini",
- "--ca-cert-path=%(test_path)s/certs/cacert.pem",
- "--host-key-path=%(test_path)s/certs/web-platform.test.key",
- "--host-cert-path=%(test_path)s/certs/web-platform.test.pem",
- "--certutil-binary=%(test_install_path)s/bin/certutil",
- ],
-
- "exes": {
- 'python': sys.executable,
- 'virtualenv': [sys.executable, 'c:/mozilla-build/buildbotve/virtualenv.py'],
- 'hg': 'c:/mozilla-build/hg/hg',
- 'mozinstall': ['%s/build/venv/scripts/python' % os.getcwd(),
- '%s/build/venv/scripts/mozinstall-script.py' % os.getcwd()],
- 'tooltool.py': [sys.executable, 'C:/mozilla-build/tooltool.py'],
- },
-
- "find_links": [
- "http://pypi.pvt.build.mozilla.org/pub",
- "http://pypi.pub.build.mozilla.org/pub",
- ],
-
- "pip_index": False,
-
- "buildbot_json_path": "buildprops.json",
-
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
-
- "blob_uploader_auth_file" : os.path.join(os.getcwd(), "oauth.txt"),
-
- "download_minidump_stackwalk": True,
-}
diff --git a/testing/mozharness/configs/web_platform_tests/prod_config_windows_taskcluster.py b/testing/mozharness/configs/web_platform_tests/prod_config_windows_taskcluster.py
deleted file mode 100644
index 845c66f76..000000000
--- a/testing/mozharness/configs/web_platform_tests/prod_config_windows_taskcluster.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-# This is a template config file for web-platform-tests test.
-
-import os
-import sys
-
-config = {
- "options": [
- "--prefs-root=%(test_path)s/prefs",
- "--processes=1",
- "--config=%(test_path)s/wptrunner.ini",
- "--ca-cert-path=%(test_path)s/certs/cacert.pem",
- "--host-key-path=%(test_path)s/certs/web-platform.test.key",
- "--host-cert-path=%(test_path)s/certs/web-platform.test.pem",
- "--certutil-binary=%(test_install_path)s/bin/certutil",
- ],
-
- "exes": {
- 'python': sys.executable,
- 'virtualenv': [
- sys.executable,
- os.path.join(os.path.dirname(sys.executable), 'Lib', 'site-packages', 'virtualenv.py')
- ],
- 'mozinstall': ['build/venv/scripts/python', 'build/venv/scripts/mozinstall-script.py'],
- 'tooltool.py': [sys.executable, os.path.join(os.environ['MOZILLABUILD'], 'tooltool.py')],
- 'hg': os.path.join(os.environ['PROGRAMFILES'], 'Mercurial', 'hg')
- },
-
- "proxxy": {},
- "find_links": [
- "http://pypi.pub.build.mozilla.org/pub",
- ],
-
- "pip_index": False,
-
- "default_blob_upload_servers": [
- "https://blobupload.elasticbeanstalk.com",
- ],
-
- "blob_uploader_auth_file" : 'C:/builds/oauth.txt',
-
- "download_minidump_stackwalk": True,
-}
diff --git a/testing/mozharness/configs/web_platform_tests/test_config.py b/testing/mozharness/configs/web_platform_tests/test_config.py
deleted file mode 100644
index 29dd8014b..000000000
--- a/testing/mozharness/configs/web_platform_tests/test_config.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-config = {
- "options": [
- "--prefs-root=%(test_path)s/prefs",
- "--processes=1",
- "--config=%(test_path)s/wptrunner.ini",
- "--ca-cert-path=%(test_path)s/certs/cacert.pem",
- "--host-key-path=%(test_path)s/certs/web-platform.test.key",
- "--host-cert-path=%(test_path)s/certs/web-platform.test.pem",
- "--certutil-binary=%(test_install_path)s/bin/certutil",
- ],
-
- "default_actions": [
- 'clobber',
- 'download-and-extract',
- 'create-virtualenv',
- 'pull',
- 'install',
- 'run-tests',
- ],
-
- "find_links": [
- "http://pypi.pub.build.mozilla.org/pub",
- ],
-
- "pip_index": False,
-}
diff --git a/testing/mozharness/configs/web_platform_tests/test_config_windows.py b/testing/mozharness/configs/web_platform_tests/test_config_windows.py
deleted file mode 100644
index d83c136ea..000000000
--- a/testing/mozharness/configs/web_platform_tests/test_config_windows.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-import os
-import sys
-
-config = {
- "options": [
- "--prefs-root=%(test_path)s/prefs",
- "--processes=1",
- "--config=%(test_path)s/wptrunner.ini",
- "--ca-cert-path=%(test_path)s/certs/cacert.pem",
- "--host-key-path=%(test_path)s/certs/web-platform.test.key",
- "--host-cert-path=%(test_path)s/certs/web-platform.test.pem",
- "--certutil-binary=%(test_install_path)s/bin/certutil",
- ],
-
- "exes": {
- 'python': sys.executable,
- 'virtualenv': [sys.executable, 'c:/mozilla-source/cedar/python/virtualenv/virtualenv.py'], #'c:/mozilla-build/buildbotve/virtualenv.py'],
- 'hg': 'c:/mozilla-build/hg/hg',
- 'mozinstall': ['%s/build/venv/scripts/python' % os.getcwd(),
- '%s/build/venv/scripts/mozinstall-script.py' % os.getcwd()],
- },
-
- "default_actions": [
- 'clobber',
- 'download-and-extract',
- 'create-virtualenv',
- 'pull',
- 'install',
- 'run-tests',
- ],
-
- "find_links": [
- "http://pypi.pub.build.mozilla.org/pub",
- ],
-
- "pip_index": False,
-}
diff --git a/testing/mozharness/docs/Makefile b/testing/mozharness/docs/Makefile
deleted file mode 100644
index 980ffbd3b..000000000
--- a/testing/mozharness/docs/Makefile
+++ /dev/null
@@ -1,177 +0,0 @@
-# Makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line.
-SPHINXOPTS =
-SPHINXBUILD = sphinx-build
-PAPER =
-BUILDDIR = _build
-
-# User-friendly check for sphinx-build
-ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
-$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
-endif
-
-# Internal variables.
-PAPEROPT_a4 = -D latex_paper_size=a4
-PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-# the i18n builder cannot share the environment and doctrees with the others
-I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
-
-help:
- @echo "Please use \`make <target>' where <target> is one of"
- @echo " html to make standalone HTML files"
- @echo " dirhtml to make HTML files named index.html in directories"
- @echo " singlehtml to make a single large HTML file"
- @echo " pickle to make pickle files"
- @echo " json to make JSON files"
- @echo " htmlhelp to make HTML files and a HTML help project"
- @echo " qthelp to make HTML files and a qthelp project"
- @echo " devhelp to make HTML files and a Devhelp project"
- @echo " epub to make an epub"
- @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
- @echo " latexpdf to make LaTeX files and run them through pdflatex"
- @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
- @echo " text to make text files"
- @echo " man to make manual pages"
- @echo " texinfo to make Texinfo files"
- @echo " info to make Texinfo files and run them through makeinfo"
- @echo " gettext to make PO message catalogs"
- @echo " changes to make an overview of all changed/added/deprecated items"
- @echo " xml to make Docutils-native XML files"
- @echo " pseudoxml to make pseudoxml-XML files for display purposes"
- @echo " linkcheck to check all external links for integrity"
- @echo " doctest to run all doctests embedded in the documentation (if enabled)"
-
-clean:
- rm -rf $(BUILDDIR)/*
-
-html:
- $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
- @echo
- @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
-
-dirhtml:
- $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
- @echo
- @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
-
-singlehtml:
- $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
- @echo
- @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
-
-pickle:
- $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
- @echo
- @echo "Build finished; now you can process the pickle files."
-
-json:
- $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
- @echo
- @echo "Build finished; now you can process the JSON files."
-
-htmlhelp:
- $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
- @echo
- @echo "Build finished; now you can run HTML Help Workshop with the" \
- ".hhp project file in $(BUILDDIR)/htmlhelp."
-
-qthelp:
- $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
- @echo
- @echo "Build finished; now you can run "qcollectiongenerator" with the" \
- ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
- @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/MozHarness.qhcp"
- @echo "To view the help file:"
- @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/MozHarness.qhc"
-
-devhelp:
- $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
- @echo
- @echo "Build finished."
- @echo "To view the help file:"
- @echo "# mkdir -p $$HOME/.local/share/devhelp/MozHarness"
- @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/MozHarness"
- @echo "# devhelp"
-
-epub:
- $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
- @echo
- @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
-
-latex:
- $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
- @echo
- @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
- @echo "Run \`make' in that directory to run these through (pdf)latex" \
- "(use \`make latexpdf' here to do that automatically)."
-
-latexpdf:
- $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
- @echo "Running LaTeX files through pdflatex..."
- $(MAKE) -C $(BUILDDIR)/latex all-pdf
- @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
-
-latexpdfja:
- $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
- @echo "Running LaTeX files through platex and dvipdfmx..."
- $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
- @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
-
-text:
- $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
- @echo
- @echo "Build finished. The text files are in $(BUILDDIR)/text."
-
-man:
- $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
- @echo
- @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
-
-texinfo:
- $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
- @echo
- @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
- @echo "Run \`make' in that directory to run these through makeinfo" \
- "(use \`make info' here to do that automatically)."
-
-info:
- $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
- @echo "Running Texinfo files through makeinfo..."
- make -C $(BUILDDIR)/texinfo info
- @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
-
-gettext:
- $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
- @echo
- @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
-
-changes:
- $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
- @echo
- @echo "The overview file is in $(BUILDDIR)/changes."
-
-linkcheck:
- $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
- @echo
- @echo "Link check complete; look for any errors in the above output " \
- "or in $(BUILDDIR)/linkcheck/output.txt."
-
-doctest:
- $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
- @echo "Testing of doctests in the sources finished, look at the " \
- "results in $(BUILDDIR)/doctest/output.txt."
-
-xml:
- $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
- @echo
- @echo "Build finished. The XML files are in $(BUILDDIR)/xml."
-
-pseudoxml:
- $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
- @echo
- @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
diff --git a/testing/mozharness/docs/android_emulator_build.rst b/testing/mozharness/docs/android_emulator_build.rst
deleted file mode 100644
index 4087c64d4..000000000
--- a/testing/mozharness/docs/android_emulator_build.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-android_emulator_build module
-=============================
-
-.. automodule:: android_emulator_build
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/android_emulator_unittest.rst b/testing/mozharness/docs/android_emulator_unittest.rst
deleted file mode 100644
index 7a8c42c50..000000000
--- a/testing/mozharness/docs/android_emulator_unittest.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-android_emulator_unittest module
-================================
-
-.. automodule:: android_emulator_unittest
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/bouncer_submitter.rst b/testing/mozharness/docs/bouncer_submitter.rst
deleted file mode 100644
index 5b71caca7..000000000
--- a/testing/mozharness/docs/bouncer_submitter.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-bouncer_submitter module
-========================
-
-.. automodule:: bouncer_submitter
- :members:
- :undoc-members:
- :private-members:
- :special-members:
diff --git a/testing/mozharness/docs/bump_gaia_json.rst b/testing/mozharness/docs/bump_gaia_json.rst
deleted file mode 100644
index 81b84d3a9..000000000
--- a/testing/mozharness/docs/bump_gaia_json.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-bump_gaia_json module
-=====================
-
-.. automodule:: bump_gaia_json
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/conf.py b/testing/mozharness/docs/conf.py
deleted file mode 100644
index e18c868a0..000000000
--- a/testing/mozharness/docs/conf.py
+++ /dev/null
@@ -1,268 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Moz Harness documentation build configuration file, created by
-# sphinx-quickstart on Mon Apr 14 17:35:24 2014.
-#
-# This file is execfile()d with the current directory set to its
-# containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-import sys
-import os
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
-sys.path.insert(0, os.path.abspath('../scripts'))
-sys.path.insert(0, os.path.abspath('../mozharness'))
-
-# -- General configuration ------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
- 'sphinx.ext.autodoc',
- 'sphinx.ext.intersphinx',
- 'sphinx.ext.viewcode',
-]
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# The suffix of source filenames.
-source_suffix = '.rst'
-
-# The encoding of source files.
-#source_encoding = 'utf-8-sig'
-
-# The master toctree document.
-master_doc = 'index'
-
-# General information about the project.
-project = u'Moz Harness'
-copyright = u'2014, aki and a cast of tens!'
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-version = '0.1'
-# The full version, including alpha/beta/rc tags.
-release = '0.1'
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#language = None
-
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
-#today = ''
-# Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-exclude_patterns = ['_build']
-
-# The reST default role (used for this markup: `text`) to use for all
-# documents.
-#default_role = None
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
-
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-#add_module_names = True
-
-# If true, sectionauthor and moduleauthor directives will be shown in the
-# output. They are ignored by default.
-#show_authors = False
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-# A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
-
-# If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
-
-
-# -- Options for HTML output ----------------------------------------------
-
-# The theme to use for HTML and HTML Help pages. See the documentation for
-# a list of builtin themes.
-html_theme = 'default'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further. For a list of options available for each theme, see the
-# documentation.
-#html_theme_options = {}
-
-# Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
-
-# The name for this set of Sphinx documents. If None, it defaults to
-# "<project> v<release> documentation".
-#html_title = None
-
-# A shorter title for the navigation bar. Default is the same as html_title.
-#html_short_title = None
-
-# The name of an image file (relative to this directory) to place at the top
-# of the sidebar.
-#html_logo = None
-
-# The name of an image file (within the static path) to use as favicon of the
-# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
-# pixels large.
-#html_favicon = None
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
-# Add any extra paths that contain custom files (such as robots.txt or
-# .htaccess) here, relative to this directory. These files are copied
-# directly to the root of the documentation.
-#html_extra_path = []
-
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
-
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-#html_use_smartypants = True
-
-# Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
-
-# Additional templates that should be rendered to pages, maps page names to
-# template names.
-#html_additional_pages = {}
-
-# If false, no module index is generated.
-#html_domain_indices = True
-
-# If false, no index is generated.
-#html_use_index = True
-
-# If true, the index is split into individual pages for each letter.
-#html_split_index = False
-
-# If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
-
-# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
-
-# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
-
-# If true, an OpenSearch description file will be output, and all pages will
-# contain a <link> tag referring to it. The value of this option must be the
-# base URL from which the finished HTML is served.
-#html_use_opensearch = ''
-
-# This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'MozHarnessdoc'
-
-
-# -- Options for LaTeX output ---------------------------------------------
-
-latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-# author, documentclass [howto, manual, or own class]).
-latex_documents = [
- ('index', 'MozHarness.tex', u'Moz Harness Documentation',
- u'aki and a cast of tens!', 'manual'),
-]
-
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-#latex_logo = None
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-#latex_use_parts = False
-
-# If true, show page references after internal links.
-#latex_show_pagerefs = False
-
-# If true, show URL addresses after external links.
-#latex_show_urls = False
-
-# Documents to append as an appendix to all manuals.
-#latex_appendices = []
-
-# If false, no module index is generated.
-#latex_domain_indices = True
-
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
- ('index', 'mozharness', u'Moz Harness Documentation',
- [u'aki and a cast of tens!'], 1)
-]
-
-# If true, show URL addresses after external links.
-#man_show_urls = False
-
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-# dir menu entry, description, category)
-texinfo_documents = [
- ('index', 'MozHarness', u'Moz Harness Documentation',
- u'aki and a cast of tens!', 'MozHarness', 'One line description of project.',
- 'Miscellaneous'),
-]
-
-# Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
-
-# If false, no module index is generated.
-#texinfo_domain_indices = True
-
-# How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
-
-# If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False
-
-
-# Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
diff --git a/testing/mozharness/docs/configtest.rst b/testing/mozharness/docs/configtest.rst
deleted file mode 100644
index 10e4a56c9..000000000
--- a/testing/mozharness/docs/configtest.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-configtest module
-=================
-
-.. automodule:: configtest
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/desktop_l10n.rst b/testing/mozharness/docs/desktop_l10n.rst
deleted file mode 100644
index b94dadedc..000000000
--- a/testing/mozharness/docs/desktop_l10n.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-desktop_l10n module
-===================
-
-.. automodule:: desktop_l10n
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/desktop_unittest.rst b/testing/mozharness/docs/desktop_unittest.rst
deleted file mode 100644
index f70e8d8d9..000000000
--- a/testing/mozharness/docs/desktop_unittest.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-desktop_unittest module
-=======================
-
-.. automodule:: desktop_unittest
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/fx_desktop_build.rst b/testing/mozharness/docs/fx_desktop_build.rst
deleted file mode 100644
index b5d6ac21c..000000000
--- a/testing/mozharness/docs/fx_desktop_build.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-fx_desktop_build module
-=======================
-
-.. automodule:: fx_desktop_build
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/gaia_build_integration.rst b/testing/mozharness/docs/gaia_build_integration.rst
deleted file mode 100644
index a2c15204c..000000000
--- a/testing/mozharness/docs/gaia_build_integration.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-gaia_build_integration module
-=============================
-
-.. automodule:: gaia_build_integration
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/gaia_integration.rst b/testing/mozharness/docs/gaia_integration.rst
deleted file mode 100644
index da143919a..000000000
--- a/testing/mozharness/docs/gaia_integration.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-gaia_integration module
-=======================
-
-.. automodule:: gaia_integration
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/gaia_unit.rst b/testing/mozharness/docs/gaia_unit.rst
deleted file mode 100644
index 9212b288c..000000000
--- a/testing/mozharness/docs/gaia_unit.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-gaia_unit module
-================
-
-.. automodule:: gaia_unit
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/index.rst b/testing/mozharness/docs/index.rst
deleted file mode 100644
index e2c05d34a..000000000
--- a/testing/mozharness/docs/index.rst
+++ /dev/null
@@ -1,24 +0,0 @@
-.. Moz Harness documentation master file, created by
- sphinx-quickstart on Mon Apr 14 17:35:24 2014.
- You can adapt this file completely to your liking, but it should at least
- contain the root `toctree` directive.
-
-Welcome to Moz Harness's documentation!
-=======================================
-
-Contents:
-
-.. toctree::
- :maxdepth: 2
-
- modules.rst
- scripts.rst
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
-
diff --git a/testing/mozharness/docs/marionette.rst b/testing/mozharness/docs/marionette.rst
deleted file mode 100644
index 28763406b..000000000
--- a/testing/mozharness/docs/marionette.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-marionette module
-=================
-
-.. automodule:: marionette
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/mobile_l10n.rst b/testing/mozharness/docs/mobile_l10n.rst
deleted file mode 100644
index ed53d09d3..000000000
--- a/testing/mozharness/docs/mobile_l10n.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-mobile_l10n module
-==================
-
-.. automodule:: mobile_l10n
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/mobile_partner_repack.rst b/testing/mozharness/docs/mobile_partner_repack.rst
deleted file mode 100644
index f8be0bef8..000000000
--- a/testing/mozharness/docs/mobile_partner_repack.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-mobile_partner_repack module
-============================
-
-.. automodule:: mobile_partner_repack
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/modules.rst b/testing/mozharness/docs/modules.rst
deleted file mode 100644
index 73652563b..000000000
--- a/testing/mozharness/docs/modules.rst
+++ /dev/null
@@ -1,13 +0,0 @@
-mozharness
-==========
-
-.. toctree::
- :maxdepth: 4
-
- mozharness
- mozharness.base.rst
- mozharness.base.vcs.rst
- mozharness.mozilla.building.rst
- mozharness.mozilla.l10n.rst
- mozharness.mozilla.rst
- mozharness.mozilla.testing.rst
diff --git a/testing/mozharness/docs/mozharness.base.rst b/testing/mozharness/docs/mozharness.base.rst
deleted file mode 100644
index 923e5658d..000000000
--- a/testing/mozharness/docs/mozharness.base.rst
+++ /dev/null
@@ -1,101 +0,0 @@
-mozharness.base package
-=======================
-
-Subpackages
------------
-
-.. toctree::
-
- mozharness.base.vcs
-
-Submodules
-----------
-
-mozharness.base.config module
------------------------------
-
-.. automodule:: mozharness.base.config
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.base.errors module
------------------------------
-
-.. automodule:: mozharness.base.errors
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.base.gaia_test module
---------------------------------
-
-.. automodule:: mozharness.base.gaia_test
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.base.log module
---------------------------
-
-.. automodule:: mozharness.base.log
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.base.mar module
---------------------------
-
-.. automodule:: mozharness.base.mar
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.base.parallel module
--------------------------------
-
-.. automodule:: mozharness.base.parallel
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.base.python module
------------------------------
-
-.. automodule:: mozharness.base.python
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.base.script module
------------------------------
-
-.. automodule:: mozharness.base.script
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.base.signing module
-------------------------------
-
-.. automodule:: mozharness.base.signing
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.base.transfer module
--------------------------------
-
-.. automodule:: mozharness.base.transfer
- :members:
- :undoc-members:
- :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: mozharness.base
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/mozharness.base.vcs.rst b/testing/mozharness/docs/mozharness.base.vcs.rst
deleted file mode 100644
index f262b3f7a..000000000
--- a/testing/mozharness/docs/mozharness.base.vcs.rst
+++ /dev/null
@@ -1,46 +0,0 @@
-mozharness.base.vcs package
-===========================
-
-Submodules
-----------
-
-mozharness.base.vcs.gittool module
-----------------------------------
-
-.. automodule:: mozharness.base.vcs.gittool
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.base.vcs.mercurial module
-------------------------------------
-
-.. automodule:: mozharness.base.vcs.mercurial
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.base.vcs.vcsbase module
-----------------------------------
-
-.. automodule:: mozharness.base.vcs.vcsbase
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.base.vcs.vcssync module
-----------------------------------
-
-.. automodule:: mozharness.base.vcs.vcssync
- :members:
- :undoc-members:
- :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: mozharness.base.vcs
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/mozharness.mozilla.building.rst b/testing/mozharness/docs/mozharness.mozilla.building.rst
deleted file mode 100644
index b8b6106c2..000000000
--- a/testing/mozharness/docs/mozharness.mozilla.building.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-mozharness.mozilla.building package
-===================================
-
-Submodules
-----------
-
-mozharness.mozilla.building.buildbase module
---------------------------------------------
-
-.. automodule:: mozharness.mozilla.building.buildbase
- :members:
- :undoc-members:
- :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: mozharness.mozilla.building
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/mozharness.mozilla.l10n.rst b/testing/mozharness/docs/mozharness.mozilla.l10n.rst
deleted file mode 100644
index 6951ec1a7..000000000
--- a/testing/mozharness/docs/mozharness.mozilla.l10n.rst
+++ /dev/null
@@ -1,30 +0,0 @@
-mozharness.mozilla.l10n package
-===============================
-
-Submodules
-----------
-
-mozharness.mozilla.l10n.locales module
---------------------------------------
-
-.. automodule:: mozharness.mozilla.l10n.locales
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.l10n.multi_locale_build module
--------------------------------------------------
-
-.. automodule:: mozharness.mozilla.l10n.multi_locale_build
- :members:
- :undoc-members:
- :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: mozharness.mozilla.l10n
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/mozharness.mozilla.rst b/testing/mozharness/docs/mozharness.mozilla.rst
deleted file mode 100644
index 2a869db7b..000000000
--- a/testing/mozharness/docs/mozharness.mozilla.rst
+++ /dev/null
@@ -1,111 +0,0 @@
-mozharness.mozilla package
-==========================
-
-Subpackages
------------
-
-.. toctree::
-
- mozharness.mozilla.building
- mozharness.mozilla.l10n
- mozharness.mozilla.testing
-
-Submodules
-----------
-
-mozharness.mozilla.blob_upload module
--------------------------------------
-
-.. automodule:: mozharness.mozilla.blob_upload
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.buildbot module
-----------------------------------
-
-.. automodule:: mozharness.mozilla.buildbot
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.gaia module
-------------------------------
-
-.. automodule:: mozharness.mozilla.gaia
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.mapper module
---------------------------------
-
-.. automodule:: mozharness.mozilla.mapper
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.mock module
-------------------------------
-
-.. automodule:: mozharness.mozilla.mock
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.mozbase module
----------------------------------
-
-.. automodule:: mozharness.mozilla.mozbase
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.purge module
--------------------------------
-
-.. automodule:: mozharness.mozilla.purge
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.release module
----------------------------------
-
-.. automodule:: mozharness.mozilla.release
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.repo_manifest module
----------------------------------------
-
-.. automodule:: mozharness.mozilla.repo_manifest
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.signing module
----------------------------------
-
-.. automodule:: mozharness.mozilla.signing
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.tooltool module
-----------------------------------
-
-.. automodule:: mozharness.mozilla.tooltool
- :members:
- :undoc-members:
- :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: mozharness.mozilla
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/mozharness.mozilla.testing.rst b/testing/mozharness/docs/mozharness.mozilla.testing.rst
deleted file mode 100644
index ccb57a3dd..000000000
--- a/testing/mozharness/docs/mozharness.mozilla.testing.rst
+++ /dev/null
@@ -1,62 +0,0 @@
-mozharness.mozilla.testing package
-==================================
-
-Submodules
-----------
-
-mozharness.mozilla.testing.device module
-----------------------------------------
-
-.. automodule:: mozharness.mozilla.testing.device
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.testing.errors module
-----------------------------------------
-
-.. automodule:: mozharness.mozilla.testing.errors
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.testing.mozpool module
------------------------------------------
-
-.. automodule:: mozharness.mozilla.testing.mozpool
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.testing.talos module
----------------------------------------
-
-.. automodule:: mozharness.mozilla.testing.talos
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.testing.testbase module
-------------------------------------------
-
-.. automodule:: mozharness.mozilla.testing.testbase
- :members:
- :undoc-members:
- :show-inheritance:
-
-mozharness.mozilla.testing.unittest module
-------------------------------------------
-
-.. automodule:: mozharness.mozilla.testing.unittest
- :members:
- :undoc-members:
- :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: mozharness.mozilla.testing
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/mozharness.rst b/testing/mozharness/docs/mozharness.rst
deleted file mode 100644
index f14e6b91e..000000000
--- a/testing/mozharness/docs/mozharness.rst
+++ /dev/null
@@ -1,18 +0,0 @@
-mozharness package
-==================
-
-Subpackages
------------
-
-.. toctree::
-
- mozharness.base
- mozharness.mozilla
-
-Module contents
----------------
-
-.. automodule:: mozharness
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/multil10n.rst b/testing/mozharness/docs/multil10n.rst
deleted file mode 100644
index b14e62b78..000000000
--- a/testing/mozharness/docs/multil10n.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-multil10n module
-================
-
-.. automodule:: multil10n
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/scripts.rst b/testing/mozharness/docs/scripts.rst
deleted file mode 100644
index b5258457e..000000000
--- a/testing/mozharness/docs/scripts.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-scripts
-=======
-
-.. toctree::
- android_emulator_build.rst
- android_emulator_unittest.rst
- bouncer_submitter.rst
- bump_gaia_json.rst
- configtest.rst
- desktop_l10n.rst
- desktop_unittest.rst
- fx_desktop_build.rst
- gaia_build_integration.rst
- gaia_integration.rst
- gaia_unit.rst
- marionette.rst
- mobile_l10n.rst
- mobile_partner_repack.rst
- multil10n.rst
- spidermonkey_build.rst
- talos_script.rst
- web_platform_tests.rst
diff --git a/testing/mozharness/docs/spidermonkey_build.rst b/testing/mozharness/docs/spidermonkey_build.rst
deleted file mode 100644
index 7e73c672e..000000000
--- a/testing/mozharness/docs/spidermonkey_build.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-spidermonkey_build module
-=========================
-
-.. automodule:: spidermonkey_build
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/talos_script.rst b/testing/mozharness/docs/talos_script.rst
deleted file mode 100644
index 509aac400..000000000
--- a/testing/mozharness/docs/talos_script.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-talos_script module
-===================
-
-.. automodule:: talos_script
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/docs/web_platform_tests.rst b/testing/mozharness/docs/web_platform_tests.rst
deleted file mode 100644
index 6a2887aa8..000000000
--- a/testing/mozharness/docs/web_platform_tests.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-web_platform_tests module
-=========================
-
-.. automodule:: web_platform_tests
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/testing/mozharness/examples/action_config_script.py b/testing/mozharness/examples/action_config_script.py
deleted file mode 100755
index e1135771e..000000000
--- a/testing/mozharness/examples/action_config_script.py
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/usr/bin/env python -u
-"""action_config_script.py
-
-Demonstrate actions and config.
-"""
-
-import os
-import sys
-import time
-
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.script import BaseScript
-
-
-# ActionsConfigExample {{{1
-class ActionsConfigExample(BaseScript):
- config_options = [[
- ['--beverage', ],
- {"action": "store",
- "dest": "beverage",
- "type": "string",
- "help": "Specify your beverage of choice",
- }
- ], [
- ['--ship-style', ],
- {"action": "store",
- "dest": "ship_style",
- "type": "choice",
- "choices": ["1", "2", "3"],
- "help": "Specify the type of ship",
- }
- ], [
- ['--long-sleep-time', ],
- {"action": "store",
- "dest": "long_sleep_time",
- "type": "int",
- "help": "Specify how long to sleep",
- }
- ]]
-
- def __init__(self, require_config_file=False):
- super(ActionsConfigExample, self).__init__(
- config_options=self.config_options,
- all_actions=[
- 'clobber',
- 'nap',
- 'ship-it',
- ],
- default_actions=[
- 'clobber',
- 'nap',
- 'ship-it',
- ],
- require_config_file=require_config_file,
- config={
- 'beverage': "kool-aid",
- 'long_sleep_time': 3600,
- 'ship_style': "1",
- }
- )
-
- def _sleep(self, sleep_length, interval=5):
- self.info("Sleeping %d seconds..." % sleep_length)
- counter = 0
- while counter + interval <= sleep_length:
- sys.stdout.write(".")
- try:
- time.sleep(interval)
- except:
- print
- self.error("Impatient, are we?")
- sys.exit(1)
- counter += interval
- print
- self.info("Ok, done.")
-
- def _ship1(self):
- self.info("""
- _~
- _~ )_)_~
- )_))_))_)
- _!__!__!_
- \______t/
-~~~~~~~~~~~~~
-""")
-
- def _ship2(self):
- self.info("""
- _4 _4
- _)_))_)
- _)_)_)_)
- _)_))_))_)_
- \_=__=__=_/
-~~~~~~~~~~~~~
-""")
-
- def _ship3(self):
- self.info("""
- ,;;:;,
- ;;;;;
- ,:;;:; ,'=.
- ;:;:;' .=" ,'_\\
- ':;:;,/ ,__:=@
- ';;:; =./)_
- `"=\\_ )_"`
- ``'"`
-""")
-
- def nap(self):
- for var_name in self.config.keys():
- if var_name.startswith("random_config_key"):
- self.info("This is going to be %s!" % self.config[var_name])
- sleep_time = self.config['long_sleep_time']
- if sleep_time > 60:
- self.info("Ok, grab a %s. This is going to take a while." % self.config['beverage'])
- else:
- self.info("This will be quick, but grab a %s anyway." % self.config['beverage'])
- self._sleep(self.config['long_sleep_time'])
-
- def ship_it(self):
- name = "_ship%s" % self.config['ship_style']
- if hasattr(self, name):
- getattr(self, name)()
-
-
-# __main__ {{{1
-if __name__ == '__main__':
- actions_config_example = ActionsConfigExample()
- actions_config_example.run_and_exit()
diff --git a/testing/mozharness/examples/silent_script.py b/testing/mozharness/examples/silent_script.py
deleted file mode 100755
index c73298ed7..000000000
--- a/testing/mozharness/examples/silent_script.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-""" This script is an example of why I care so much about Mozharness' 2nd core
-concept, logging. http://escapewindow.dreamwidth.org/230853.html
-"""
-
-import os
-import shutil
-
-#print "downloading foo.tar.bz2..."
-os.system("curl -s -o foo.tar.bz2 http://people.mozilla.org/~asasaki/foo.tar.bz2")
-#os.system("curl -v -o foo.tar.bz2 http://people.mozilla.org/~asasaki/foo.tar.bz2")
-
-#os.rename("foo.tar.bz2", "foo3.tar.bz2")
-os.system("tar xjf foo.tar.bz2")
-
-#os.chdir("x")
-os.remove("x/ship2")
-os.remove("foo.tar.bz2")
-os.system("tar cjf foo.tar.bz2 x")
-shutil.rmtree("x")
-#os.system("scp -q foo.tar.bz2 people.mozilla.org:public_html/foo2.tar.bz2")
-os.remove("foo.tar.bz2")
diff --git a/testing/mozharness/examples/venv.py b/testing/mozharness/examples/venv.py
deleted file mode 100755
index 6b3c88f96..000000000
--- a/testing/mozharness/examples/venv.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env python
-"""venv.py
-
-Test virtualenv creation. This installs talos in the local venv; that's it.
-"""
-
-import os
-import sys
-
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.errors import PythonErrorList
-from mozharness.base.python import virtualenv_config_options, VirtualenvMixin
-from mozharness.base.script import BaseScript
-
-# VirtualenvExample {{{1
-class VirtualenvExample(VirtualenvMixin, BaseScript):
- config_options = [[
- ["--talos-url"],
- {"action": "store",
- "dest": "talos_url",
- "default": "https://hg.mozilla.org/build/talos/archive/tip.tar.gz",
- "help": "Specify the talos pip url"
- }
- ]] + virtualenv_config_options
-
- def __init__(self, require_config_file=False):
- super(VirtualenvExample, self).__init__(
- config_options=self.config_options,
- all_actions=['create-virtualenv',
- ],
- default_actions=['create-virtualenv',
- ],
- require_config_file=require_config_file,
- config={"virtualenv_modules": ["talos"]},
- )
-
-# __main__ {{{1
-if __name__ == '__main__':
- venv_example = VirtualenvExample()
- venv_example.run_and_exit()
diff --git a/testing/mozharness/examples/verbose_script.py b/testing/mozharness/examples/verbose_script.py
deleted file mode 100755
index e8afd7567..000000000
--- a/testing/mozharness/examples/verbose_script.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-"""verbose_script.py
-
-Contrast to silent_script.py.
-"""
-
-import os
-import sys
-
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-#from mozharness.base.errors import TarErrorList, SSHErrorList
-from mozharness.base.script import BaseScript
-
-
-# VerboseExample {{{1
-class VerboseExample(BaseScript):
- def __init__(self, require_config_file=False):
- super(VerboseExample, self).__init__(
- all_actions=['verbosity', ],
- require_config_file=require_config_file,
- config={"tarball_name": "bar.tar.bz2"}
- )
-
- def verbosity(self):
- tarball_name = self.config["tarball_name"]
- self.download_file(
- "http://people.mozilla.org/~asasaki/foo.tar.bz2",
- file_name=tarball_name
- )
- # the error_list adds more error checking.
- # the halt_on_failure will kill the script at this point if
- # unsuccessful. Be aware if you need to do any cleanup before you
- # actually fatal(), though. If so, you may want to either use an
- # |if self.run_command(...):| construct, or define a self._post_fatal()
- # for a generic end-of-fatal-run method.
- self.run_command(
- ["tar", "xjvf", tarball_name],
-# error_list=TarErrorList,
-# halt_on_failure=True,
-# fatal_exit_code=3,
- )
- self.rmtree("x/ship2")
- self.rmtree(tarball_name)
- self.run_command(
- ["tar", "cjvf", tarball_name, "x"],
-# error_list=TarErrorList,
-# halt_on_failure=True,
-# fatal_exit_code=3,
- )
- self.rmtree("x")
- if self.run_command(
- ["scp", tarball_name, "people.mozilla.org:public_html/foo2.tar.bz2"],
-# error_list=SSHErrorList,
- ):
- self.error("There's been a problem with the scp. We're going to proceed anyway.")
- self.rmtree(tarball_name)
-
-
-# __main__ {{{1
-if __name__ == '__main__':
- verbose_example = VerboseExample()
- verbose_example.run_and_exit()
diff --git a/testing/mozharness/external_tools/__init__.py b/testing/mozharness/external_tools/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/external_tools/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/external_tools/clobberer.py b/testing/mozharness/external_tools/clobberer.py
deleted file mode 100755
index a58b00402..000000000
--- a/testing/mozharness/external_tools/clobberer.py
+++ /dev/null
@@ -1,280 +0,0 @@
-#!/usr/bin/python
-# vim:sts=2 sw=2
-import sys
-import shutil
-import urllib2
-import urllib
-import os
-import traceback
-import time
-if os.name == 'nt':
- from win32file import RemoveDirectory, DeleteFile, \
- GetFileAttributesW, SetFileAttributesW, \
- FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_DIRECTORY
- from win32api import FindFiles
-
-clobber_suffix = '.deleteme'
-
-
-def ts_to_str(ts):
- if ts is None:
- return None
- return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts))
-
-
-def write_file(ts, fn):
- assert isinstance(ts, int)
- f = open(fn, "w")
- f.write(str(ts))
- f.close()
-
-
-def read_file(fn):
- if not os.path.exists(fn):
- return None
-
- data = open(fn).read().strip()
- try:
- return int(data)
- except ValueError:
- return None
-
-def rmdirRecursiveWindows(dir):
- """Windows-specific version of rmdirRecursive that handles
- path lengths longer than MAX_PATH.
- """
-
- dir = os.path.realpath(dir)
- # Make sure directory is writable
- SetFileAttributesW('\\\\?\\' + dir, FILE_ATTRIBUTE_NORMAL)
-
- for ffrec in FindFiles('\\\\?\\' + dir + '\\*.*'):
- file_attr = ffrec[0]
- name = ffrec[8]
- if name == '.' or name == '..':
- continue
- full_name = os.path.join(dir, name)
-
- if file_attr & FILE_ATTRIBUTE_DIRECTORY:
- rmdirRecursiveWindows(full_name)
- else:
- SetFileAttributesW('\\\\?\\' + full_name, FILE_ATTRIBUTE_NORMAL)
- DeleteFile('\\\\?\\' + full_name)
- RemoveDirectory('\\\\?\\' + dir)
-
-def rmdirRecursive(dir):
- """This is a replacement for shutil.rmtree that works better under
- windows. Thanks to Bear at the OSAF for the code.
- (Borrowed from buildbot.slave.commands)"""
- if os.name == 'nt':
- rmdirRecursiveWindows(dir)
- return
-
- if not os.path.exists(dir):
- # This handles broken links
- if os.path.islink(dir):
- os.remove(dir)
- return
-
- if os.path.islink(dir):
- os.remove(dir)
- return
-
- # Verify the directory is read/write/execute for the current user
- os.chmod(dir, 0700)
-
- for name in os.listdir(dir):
- full_name = os.path.join(dir, name)
- # on Windows, if we don't have write permission we can't remove
- # the file/directory either, so turn that on
- if os.name == 'nt':
- if not os.access(full_name, os.W_OK):
- # I think this is now redundant, but I don't have an NT
- # machine to test on, so I'm going to leave it in place
- # -warner
- os.chmod(full_name, 0600)
-
- if os.path.isdir(full_name):
- rmdirRecursive(full_name)
- else:
- # Don't try to chmod links
- if not os.path.islink(full_name):
- os.chmod(full_name, 0700)
- os.remove(full_name)
- os.rmdir(dir)
-
-
-def do_clobber(dir, dryrun=False, skip=None):
- try:
- for f in os.listdir(dir):
- if skip is not None and f in skip:
- print "Skipping", f
- continue
- clobber_path = f + clobber_suffix
- if os.path.isfile(f):
- print "Removing", f
- if not dryrun:
- if os.path.exists(clobber_path):
- os.unlink(clobber_path)
- # Prevent repeated moving.
- if f.endswith(clobber_suffix):
- os.unlink(f)
- else:
- shutil.move(f, clobber_path)
- os.unlink(clobber_path)
- elif os.path.isdir(f):
- print "Removing %s/" % f
- if not dryrun:
- if os.path.exists(clobber_path):
- rmdirRecursive(clobber_path)
- # Prevent repeated moving.
- if f.endswith(clobber_suffix):
- rmdirRecursive(f)
- else:
- shutil.move(f, clobber_path)
- rmdirRecursive(clobber_path)
- except:
- print "Couldn't clobber properly, bailing out."
- sys.exit(1)
-
-
-def getClobberDates(clobberURL, branch, buildername, builddir, slave, master):
- params = dict(branch=branch, buildername=buildername,
- builddir=builddir, slave=slave, master=master)
- url = "%s?%s" % (clobberURL, urllib.urlencode(params))
- print "Checking clobber URL: %s" % url
- # The timeout arg was added to urlopen() at Python 2.6
- # Deprecate this test when esr17 reaches EOL
- if sys.version_info[:2] < (2, 6):
- data = urllib2.urlopen(url).read().strip()
- else:
- data = urllib2.urlopen(url, timeout=30).read().strip()
-
- retval = {}
- try:
- for line in data.split("\n"):
- line = line.strip()
- if not line:
- continue
- builddir, builder_time, who = line.split(":")
- builder_time = int(builder_time)
- retval[builddir] = (builder_time, who)
- return retval
- except ValueError:
- print "Error parsing response from server"
- print data
- raise
-
-if __name__ == "__main__":
- from optparse import OptionParser
- parser = OptionParser(
- "%prog [options] clobberURL branch buildername builddir slave master")
- parser.add_option("-n", "--dry-run", dest="dryrun", action="store_true",
- default=False, help="don't actually delete anything")
- parser.add_option("-t", "--periodic", dest="period", type="float",
- default=None, help="hours between periodic clobbers")
- parser.add_option('-s', '--skip', help='do not delete this file/directory',
- action='append', dest='skip', default=['last-clobber'])
- parser.add_option('-d', '--dir', help='clobber this directory',
- dest='dir', default='.', type='string')
- parser.add_option('-v', '--verbose', help='be more verbose',
- dest='verbose', action='store_true', default=False)
-
- options, args = parser.parse_args()
- if len(args) != 6:
- parser.error("Incorrect number of arguments")
-
- if options.period:
- periodicClobberTime = options.period * 3600
- else:
- periodicClobberTime = None
-
- clobberURL, branch, builder, my_builddir, slave, master = args
-
- try:
- server_clobber_dates = getClobberDates(
- clobberURL, branch, builder, my_builddir, slave, master)
- except:
- if options.verbose:
- traceback.print_exc()
- print "Error contacting server"
- sys.exit(1)
-
- if options.verbose:
- print "Server gave us", server_clobber_dates
-
- now = int(time.time())
-
- # Add ourself to the server_clobber_dates if it's not set
- # This happens when this slave has never been clobbered
- if my_builddir not in server_clobber_dates:
- server_clobber_dates[my_builddir] = None, ""
-
- root_dir = os.path.abspath(options.dir)
-
- for builddir, (server_clobber_date, who) in server_clobber_dates.items():
- builder_dir = os.path.join(root_dir, builddir)
- if not os.path.isdir(builder_dir):
- print "%s doesn't exist, skipping" % builder_dir
- continue
- os.chdir(builder_dir)
-
- our_clobber_date = read_file("last-clobber")
-
- clobber = False
- clobberType = None
-
- print "%s:Our last clobber date: " % builddir, ts_to_str(our_clobber_date)
- print "%s:Server clobber date: " % builddir, ts_to_str(server_clobber_date)
-
- # If we don't have a last clobber date, then this is probably a fresh build.
- # We should only do a forced server clobber if we know when our last clobber
- # was, and if the server date is more recent than that.
- if server_clobber_date is not None and our_clobber_date is not None:
- # If the server is giving us a clobber date, compare the server's idea of
- # the clobber date to our last clobber date
- if server_clobber_date > our_clobber_date:
- # If the server's clobber date is greater than our last clobber date,
- # then we should clobber.
- clobber = True
- clobberType = "forced"
- # We should also update our clobber date to match the server's
- our_clobber_date = server_clobber_date
- if who:
- print "%s:Server is forcing a clobber, initiated by %s" % (builddir, who)
- else:
- print "%s:Server is forcing a clobber" % builddir
-
- if not clobber:
- # Disable periodic clobbers for builders that aren't my_builddir
- if builddir != my_builddir:
- continue
-
- # Next, check if more than the periodicClobberTime period has passed since
- # our last clobber
- if our_clobber_date is None:
- # We've never been clobbered
- # Set our last clobber time to now, so that we'll clobber
- # properly after periodicClobberTime
- clobberType = "purged"
- our_clobber_date = now
- write_file(our_clobber_date, "last-clobber")
- elif periodicClobberTime and now > our_clobber_date + periodicClobberTime:
- # periodicClobberTime has passed since our last clobber
- clobber = True
- clobberType = "periodic"
- # Update our clobber date to now
- our_clobber_date = now
- print "%s:More than %s seconds have passed since our last clobber" % (builddir, periodicClobberTime)
-
- if clobber:
- # Finally, perform a clobber if we're supposed to
- print "%s:Clobbering..." % builddir
- do_clobber(builder_dir, options.dryrun, options.skip)
- write_file(our_clobber_date, "last-clobber")
-
- # If this is the build dir for the current job, display the clobber type in TBPL.
- # Note in the case of purged clobber, we output the clobber type even though no
- # clobber was performed this time.
- if clobberType and builddir == my_builddir:
- print "TinderboxPrint: %s clobber" % clobberType
diff --git a/testing/mozharness/external_tools/count_and_reboot.py b/testing/mozharness/external_tools/count_and_reboot.py
deleted file mode 100755
index 9e8ae35a6..000000000
--- a/testing/mozharness/external_tools/count_and_reboot.py
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/usr/bin/env python
-# encoding: utf-8
-# Created by Chris AtLee on 2008-11-04
-"""count_and_reboot.py [-n maxcount] -f countfile
-
-Increments the value in countfile, and reboots the machine once the count
-reaches or exceeds maxcount."""
-
-import os, sys, time
-
-if sys.platform in ('darwin', 'linux2'):
- def reboot():
- # -S means to accept password from stdin, which we then redirect from
- # /dev/null
- # This results in sudo not waiting forever for a password. If sudoers
- # isn't set up properly, this will fail immediately
- os.system("sudo -S reboot < /dev/null")
- # After starting the shutdown, we go to sleep since the system can
- # take a few minutes to shut everything down and reboot
- time.sleep(600)
-
-elif sys.platform == "win32":
- # Windows
- def reboot():
- os.system("shutdown -f -r -t 0")
- # After starting the shutdown, we go to sleep since the system can
- # take a few minutes to shut everything down and reboot
- time.sleep(600)
-
-def increment_count(fname):
- try:
- current_count = int(open(fname).read())
- except:
- current_count = 0
- current_count += 1
- open(fname, "w").write("%i\n" % current_count)
- return current_count
-
-if __name__ == '__main__':
- from optparse import OptionParser
-
- parser = OptionParser(__doc__)
- parser.add_option("-n", "--max-count", dest="maxcount", default=10,
- help="reboot after <maxcount> runs", type="int")
- parser.add_option("-f", "--count-file", dest="countfile", default=None,
- help="file to record count in")
- parser.add_option("-z", "--zero-count", dest="zero", default=False,
- action="store_true", help="zero out the counter before rebooting")
-
- options, args = parser.parse_args()
-
- if not options.countfile:
- parser.error("countfile is required")
-
- if increment_count(options.countfile) >= options.maxcount:
- if options.zero:
- open(options.countfile, "w").write("0\n")
- print "************************************************************************************************"
- print "*********** END OF RUN - NOW DOING SCHEDULED REBOOT; FOLLOWING ERROR MESSAGE EXPECTED **********"
- print "************************************************************************************************"
- sys.stdout.flush()
- reboot()
diff --git a/testing/mozharness/external_tools/detect_repo.py b/testing/mozharness/external_tools/detect_repo.py
deleted file mode 100644
index 67466a03e..000000000
--- a/testing/mozharness/external_tools/detect_repo.py
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env python
-# Stolen from taskcluster-vcs
-# https://github.com/taskcluster/taskcluster-vcs/blob/master/src/vcs/detect_remote.js
-
-from urllib2 import Request, urlopen
-from urlparse import urlsplit, urlunsplit
-from os.path import exists, join
-
-def first(seq):
- return next(iter(filter(lambda x: x, seq)), '')
-
-def all_first(*sequences):
- return map(lambda x: first(x), sequences)
-
-# http://codereview.stackexchange.com/questions/13027/joining-url-path-components-intelligently
-# I wonder why this is not a builtin feature in Python
-def urljoin(*parts):
- schemes, netlocs, paths, queries, fragments = zip(*(urlsplit(part) for part in parts))
- scheme, netloc, query, fragment = all_first(schemes, netlocs, queries, fragments)
- path = '/'.join(p.strip('/') for p in paths if p)
- return urlunsplit((scheme, netloc, path, query, fragment))
-
-def _detect_remote(url, content):
- try:
- response = urlopen(url)
- except Exception:
- return False
-
- if response.getcode() != 200:
- return False
-
- content_type = response.headers.get('content-type', '')
- return True if content in content_type else False
-
-def detect_git(url):
- location = urljoin(url, '/info/refs?service=git-upload-pack')
- req = Request(location, headers={'User-Agent':'git/2.0.1'})
- return _detect_remote(req, 'x-git')
-
-def detect_hg(url):
- location = urljoin(url, '?cmd=lookup&key=0')
- return _detect_remote(location, 'mercurial')
-
-def detect_local(url):
- if exists(join(url, '.git')):
- return 'git'
-
- if exists(join(url, '.hg')):
- return 'hg'
-
- return ''
-
diff --git a/testing/mozharness/external_tools/download_file.py b/testing/mozharness/external_tools/download_file.py
deleted file mode 100755
index 91b0a4668..000000000
--- a/testing/mozharness/external_tools/download_file.py
+++ /dev/null
@@ -1,69 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-""" Helper script for download_file()
-
-We lose some mozharness functionality by splitting this out, but we gain output_timeout.
-"""
-
-import os
-import socket
-import sys
-import urllib2
-import urlparse
-
-
-def download_file(url, file_name):
- try:
- f_length = None
- f = urllib2.urlopen(url, timeout=30)
- if f.info().get('content-length') is not None:
- f_length = int(f.info()['content-length'])
- got_length = 0
- local_file = open(file_name, 'wb')
- while True:
- block = f.read(1024 ** 2)
- if not block:
- if f_length is not None and got_length != f_length:
- raise urllib2.URLError("Download incomplete; content-length was %d, but only received %d" % (f_length, got_length))
- break
- local_file.write(block)
- if f_length is not None:
- got_length += len(block)
- local_file.close()
- print "%s downloaded to %s" % (url, file_name)
- except urllib2.HTTPError, e:
- print "Warning: Server returned status %s %s for %s" % (str(e.code), str(e), url)
- raise
- except urllib2.URLError, e:
- print "URL Error: %s" % url
- remote_host = urlparse.urlsplit(url)[1]
- if remote_host:
- os.system("nslookup %s" % remote_host)
- raise
- except socket.timeout, e:
- print "Timed out accessing %s: %s" % (url, str(e))
- raise
- except socket.error, e:
- print "Socket error when accessing %s: %s" % (url, str(e))
- raise
-
-if __name__ == '__main__':
- if len(sys.argv) != 3:
- if len(sys.argv) != 2:
- print "Usage: download_file.py URL [FILENAME]"
- sys.exit(-1)
- parts = urlparse.urlparse(sys.argv[1])
- file_name = parts[2].split('/')[-1]
- else:
- file_name = sys.argv[2]
- if os.path.exists(file_name):
- print "%s exists; removing" % file_name
- os.remove(file_name)
- if os.path.exists(file_name):
- print "%s still exists; exiting"
- sys.exit(-1)
- download_file(sys.argv[1], file_name)
diff --git a/testing/mozharness/external_tools/extract_and_run_command.py b/testing/mozharness/external_tools/extract_and_run_command.py
deleted file mode 100644
index ab48ee1df..000000000
--- a/testing/mozharness/external_tools/extract_and_run_command.py
+++ /dev/null
@@ -1,205 +0,0 @@
-#!/usr/bin/env python
-"""\
-Usage: extract_and_run_command.py [-j N] [command to run] -- [files and/or directories]
- -j is the number of workers to start, defaulting to 1.
- [command to run] must be a command that can accept one or many files
- to process as arguments.
-
-WARNING: This script does NOT respond to SIGINT. You must use SIGQUIT or SIGKILL to
- terminate it early.
- """
-
-### The canonical location for this file is
-### https://hg.mozilla.org/build/tools/file/default/stage/extract_and_run_command.py
-###
-### Please update the copy in puppet to deploy new changes to
-### stage.mozilla.org, see
-# https://wiki.mozilla.org/ReleaseEngineering/How_To/Modify_scripts_on_stage
-
-import logging
-import os
-from os import path
-import sys
-from Queue import Queue
-import shutil
-import subprocess
-import tempfile
-from threading import Thread
-import time
-
-logging.basicConfig(
- stream=sys.stdout, level=logging.INFO, format="%(message)s")
-log = logging.getLogger(__name__)
-
-try:
- # the future - https://github.com/mozilla/build-mar via a venv
- from mardor.marfile import BZ2MarFile
-except:
- # the past - http://hg.mozilla.org/build/tools/file/default/buildfarm/utils/mar.py
- sys.path.append(
- path.join(path.dirname(path.realpath(__file__)), "../buildfarm/utils"))
- from mar import BZ2MarFile
-
-SEVENZIP = "7za"
-
-
-def extractMar(filename, tempdir):
- m = BZ2MarFile(filename)
- m.extractall(path=tempdir)
-
-
-def extractExe(filename, tempdir):
- try:
- # We don't actually care about output, put we redirect to a tempfile
- # to avoid deadlocking in wait() when stdout=PIPE
- fd = tempfile.TemporaryFile()
- proc = subprocess.Popen([SEVENZIP, 'x', '-o%s' % tempdir, filename],
- stdout=fd, stderr=subprocess.STDOUT)
- proc.wait()
- except subprocess.CalledProcessError:
- # Not all EXEs are 7-zip files, so we have to ignore extraction errors
- pass
-
-# The keys here are matched against the last 3 characters of filenames.
-# The values are callables that accept two string arguments.
-EXTRACTORS = {
- '.mar': extractMar,
- '.exe': extractExe,
-}
-
-
-def find_files(d):
- """yields all of the files in `d'"""
- for root, dirs, files in os.walk(d):
- for f in files:
- yield path.abspath(path.join(root, f))
-
-
-def rchmod(d, mode=0755):
- """chmods everything in `d' to `mode', including `d' itself"""
- os.chmod(d, mode)
- for root, dirs, files in os.walk(d):
- for item in dirs:
- os.chmod(path.join(root, item), mode)
- for item in files:
- os.chmod(path.join(root, item), mode)
-
-
-def maybe_extract(filename):
- """If an extractor is found for `filename', extracts it to a temporary
- directory and chmods it. The consumer is responsible for removing
- the extracted files, if desired."""
- ext = path.splitext(filename)[1]
- if ext not in EXTRACTORS.keys():
- return None
- # Append the full filepath to the tempdir
- tempdir_root = tempfile.mkdtemp()
- tempdir = path.join(tempdir_root, filename.lstrip('/'))
- os.makedirs(tempdir)
- EXTRACTORS[ext](filename, tempdir)
- rchmod(tempdir_root)
- return tempdir_root
-
-
-def process(item, command):
- def format_time(t):
- return time.strftime("%H:%M:%S", time.localtime(t))
- # Buffer output to avoid interleaving of multiple workers'
- logs = []
- args = [item]
- proc = None
- start = time.time()
- logs.append("START %s: %s" % (format_time(start), item))
- # If the file was extracted, we need to process all of its files, too.
- tempdir = maybe_extract(item)
- if tempdir:
- for f in find_files(tempdir):
- args.append(f)
-
- try:
- fd = tempfile.TemporaryFile()
- proc = subprocess.Popen(command + args, stdout=fd)
- proc.wait()
- if proc.returncode != 0:
- raise Exception("returned %s" % proc.returncode)
- finally:
- if tempdir:
- shutil.rmtree(tempdir)
- fd.seek(0)
- # rstrip() here to avoid an unnecessary newline, if it exists.
- logs.append(fd.read().rstrip())
- end = time.time()
- elapsed = end - start
- logs.append("END %s (%d seconds elapsed): %s\n" % (
- format_time(end), elapsed, item))
- # Now that we've got all of our output, print it. It's important that
- # the logging module is used for this, because "print" is not
- # thread-safe.
- log.info("\n".join(logs))
-
-
-def worker(command, errors):
- item = q.get()
- while item != None:
- try:
- process(item, command)
- except:
- errors.put(item)
- item = q.get()
-
-if __name__ == '__main__':
- # getopt is used in favour of optparse to enable "--" as a separator
- # between the command and list of files. optparse doesn't allow that.
- from getopt import getopt
- options, args = getopt(sys.argv[1:], 'j:h', ['help'])
-
- concurrency = 1
- for o, a in options:
- if o == '-j':
- concurrency = int(a)
- elif o in ('-h', '--help'):
- log.info(__doc__)
- sys.exit(0)
-
- if len(args) < 3 or '--' not in args:
- log.error(__doc__)
- sys.exit(1)
-
- command = []
- while args[0] != "--":
- command.append(args.pop(0))
- args.pop(0)
-
- q = Queue()
- errors = Queue()
- threads = []
- for i in range(concurrency):
- t = Thread(target=worker, args=(command, errors))
- t.start()
- threads.append(t)
-
- # find_files is a generator, so work will begin prior to it finding
- # all of the files
- for arg in args:
- if path.isfile(arg):
- q.put(arg)
- else:
- for f in find_files(arg):
- q.put(f)
- # Because the workers are started before we start populating the q
- # they can't use .empty() to determine whether or not their done.
- # We also can't use q.join() or j.task_done(), because we need to
- # support Python 2.4. We know that find_files won't yield None,
- # so we can detect doneness by having workers die when they get None
- # as an item.
- for i in range(concurrency):
- q.put(None)
-
- for t in threads:
- t.join()
-
- if not errors.empty():
- log.error("Command failed for the following files:")
- while not errors.empty():
- log.error(" %s" % errors.get())
- sys.exit(1)
diff --git a/testing/mozharness/external_tools/git-ssh-wrapper.sh b/testing/mozharness/external_tools/git-ssh-wrapper.sh
deleted file mode 100755
index 86ea37088..000000000
--- a/testing/mozharness/external_tools/git-ssh-wrapper.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-# From http://www.reddit.com/r/git/comments/hdn1a/howto_using_the_git_ssh_variable_for_private_keys/
-
-# In the example, this was
-# if [ -e "$GIT_SSH_KEY" ]; then
-# However, that broke on tilde expansion.
-# Let's just assume if GIT_SSH_KEY is set, we want to use it.
-if [ "x$GIT_SSH_KEY" != "x" ]; then
- exec ssh -o IdentityFile="$GIT_SSH_KEY" -o ServerAliveInterval=600 "$@"
-else
- exec ssh -o ServerAliveInterval=600 "$@"
-fi
diff --git a/testing/mozharness/external_tools/gittool.py b/testing/mozharness/external_tools/gittool.py
deleted file mode 100755
index 520aeaf38..000000000
--- a/testing/mozharness/external_tools/gittool.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/env python
-### Compressed module sources ###
-module_sources = [('util', 'eJxlkMEKgzAQRO/5isWTQhFaSg8Ff6LnQknM2ixoItmov1+T2FLb3DY7mZkXGkbnAxjJpiclKI+K\nrOSWSAihsQM28sjBk32WXF0FrKe4YZi8hWAwrZMDuC5fJC1wkaQ+K7eIOqpXm1rTEzmU1ZahLuc/\ncwYlGS9nQNs6jfoACwUDQVIf/RdDAXmULYK0Gpo1aXAz6l3sG6VWJ/nIdjHdx45jWTR3W3xVSKTT\n8NuEE9a+DMzomZz9QOencdyDJ7LvH6zEC9SEeBQ=\n'), ('util.file', 'eJzNVk2P2zYQvftXTF0sLC9ctTbaSwAfim2BFCjSIsktCLy0SFnMSqRAUuv1v+8MP0RZ3uTQU3Sw\nJXLmcWbem5GWy+Vb0fbCQD2oykmtLDgNDVO8FVBL/NG4y/zOcrlcyK7XxkGrTyepTulR23Rnm8HJ\nNj01zDatPKZHJ7qeMBe10R08aFXL07/MWDw+Wrxn5+nyAs+BfTqtPAn3N94KUxwOinXicFgvFgsu\naqh01zMjCkLfbnzgu/WbBeCFUcddTK0RaKqcUM6CrsGdtbe1G+iZtYKDVCAkmhlg1rvjhRVQoRah\nLuiK21UlrJXHVKaeucaW8IfGbQYW88E8I4Bi8lmAdQaTiKFKq9UGrAauQWkHg8VKK2iZOREZFBOV\nm7xlDdJKZR1T1ZjhkVkRAGOadPk9rBcFnAxXZrWBj2YQ66+A7b4BtpuC7W7A/BGHsaD7sFAawXiR\nLXZzi93Uwgg3GHUDtZ+5Rp65NKJy2lxQJY5hHsW4gtUc6lq+ZNrhfcB2GDAlTuyfkAmVYbwaCMdv\n9kY/S44qOMuWV8xwjxRgN8SpRH6oPx5bC7XWP98fmXmERFQjHWbI1KX4VJdCcXtGJRUxKrRHXklf\n2pattA5jyMGvP4/0kBoQKROB6i+FMdoUywc9tNxb1FJxuL+zBHhnl3AHRYozg15VGDHHZukvVN3C\nmgrNrdv4pU5zsffkjhV8wGVAK8rZ2/XYRcI8k45xLHQSO4BGBrYFONmh9GU9YqHQvFZSecJoKG9O\nHzNPjjn1iQttzFxmFqhpN7EIudqGbe3QFXVOKqkCCf/w9veftn5K+Wkwmw6+rx/rxw0VuREvRGHH\n3Eg3kh0HXEnHJMn3Y9NQwxxXYfncEBrVI6d3bHX1RE3Rh474bbuDe9j+svs1JxgV4U2zp/dGn6dx\npSmHnjMnCm95zXyJwXN5wh4vxrqwWhwG1Ur15JubxmkuUdiAtAHypLRRxLoXok3d5CvEceSplQPx\ngqpOxXHm8maaA4qeJmQpLel+duI4crBFjNbOa9iGMW5jy5xZmyPdoCB7rs9qqtc5km82D3G7n4mK\ncX3RUhXh7Hr9qvlVxfpbG0QyHSVHKHlbtFZcnz+phi+Z/Vo5IuqcJW8jXirRO/jnw59EyAYmZ/wI\nfxFdApbvNA6vqonvcZMnw3JKjaDpojTN3N11AEE/30jFMGnFVFGz5kbFZVGRQXvxXT7OFDTAVx8J\ni/mvA20YDmWJPWg6wSXqOcyWBoe2ofTpo4PwonOSW81REl3vxbofvzPK7snSPc3Zfao53pNZ4YNb\nvzaZ9PFL4RvYJ+FbeENE1Dy0NZ61OuPijXOeQDGWYEHK8NQVcTlWJhau1YzTew6/euZKCKuY0ey7\nqJjMTFoN4+NT8v68hh/2kB8zaXEivNNKTCdEQInx4FdWCif84atP+G9DrEIf/tGODW0iN8eB8/AQ\njYv4v/YMTvYDRjHDXN8EGV0wnBvbaewxlJvgD6ii7yUBCuV/5XDUuv1ekqBYBLt1eS2R/wBE3uXX\n'), ('util.commands', 'eJzdWW1v2zgS/u5fwXPQs9x1laDFvSBA9pDdJnfBtkkucS9XtIEgS+OYG4n0kVRc76+/GZKSKPkl\n2T3slzPQOhLJ4bw888yQHg6H55XIDJdCs7lUTFVCcPHAMlmWqcj1cDgc8HIplWG6mi2VzEDr+o1s\n/jK8hPrvZZEaFFXWz4V8eECRA/xmJ/VT/ADmA/4JKkoSkZaQJOPBwKj18YDhxy9dcfHu7ZwXsPEy\nXXL77vrz3cXlu7coeKoqGMC3DJaGXdiZZ0pJddybdp4WGgaDQQ5z0iXJyjzCfxP2+vXjKlUPeuxW\nHLBslTOumV5CxtOCccHMIsXtgaXFKl1rtkqFYRwNVlwYQBHwBILxOb4baSak8YLg27LgGTfFmmUL\nqUHY92431Mj9EWdyuY7GztA5G+HuI5JB+7oZTq926Rc75x4lSE3uxCe/Hu2KuZjLaOjDeMxup6c3\n0+HO4Vd6yF4FEY4Lrs1b9EvBBZB/xm4pQeQR1hP2lBYVtLrF3IDCf6WOxq2eWzeym02cFG1UZCWh\neBeSEtQDJCCeIvznRQlY0RtnKP7BlRShu/x4XC3z1IBdaN8rMJUS9bDfAAG+M+YI9ptKMBxiUcrI\nBUzOGU6oShBGj2PGblKuIXTUj2lRQH7tniziMHxWmllAnUYIAW4QMNwsMKbizS+gJAq7mHcmOX0R\ncVVGwuZVUawnoSVHMaWj9+wWKzze7oA5V6B0BHA6x9jUecdmkKUVmoAwzqUYGYdiNIJMJW24WNhQ\n5jV60fNPqdKsrHKCwwMKtxNlZZaVaQCL80b7wErjBNY2wp0Rp3xDAPYBZxOxxPSfj/UOWDldjoft\nJO+yIFLZArLHJENTt7nNM8feyG5B9qhR4ezm5upmNCFBCQ2dECEF+hBwXA5xgZIDO6FIlxryrrXs\nDTP7LD67fM+iV/Hbua7Xj78KzKv6IYD7fyoOZifoc3gSiDTKriWICFPMv5mw0WrUyaQ9ztQmxxic\nNEvxGZRqn1tnt3opKKWBVjEN6gnUhCE8FZWEk0spAF/rxU+wbh9ORvjfaNI4J/j0TEOpyVLBnH9D\n677gqvsarfUWbRDauTF8MyDy6MvoTTFqtblvuNkp9MxSjkvRl8vULPDtEmNGgiK3duyFBSvT5ZJW\nOh80W3HNhTapyMC5aJZqQNLELBx39if78Os+jFbAdLUXvmM95Hc4MVli4sucZ8lS1nHFedQPJFTh\nFFL1ybujowmj8fbVUfz2T1vD4T+1DELLLM0efSh/JfkSt6QBBBlRpoUhI27FxFgWQI2MlVabQpn2\nYtrepGwr67fQdkvZg20uYHPfdaFwzL0ZSrMKub1I+hxdLFdEt40LvIYOOW5z7DPgG2SVFWXSR9DI\nFQK7KpooNqLXYgZBpUxCVNNQBoYV3VHH4v+6zDxbQcgTKCQAzLVlxy2OaD25pVwVqUbmtSA9CWYO\nHCgW2NnavrU1Q9G2tGdsc3A8aEbQeBzktrFklEHHnZQjk3KYVQ/R0KPaQxBBZRsulY07C5y8kxN2\ndLyRu7sqUmBBf8lvKVF9GXXOdAYA+/VNDdXzCR2pbEJ0EvhQyNWOngK9QYNvwoh9vyd/6HOACmsw\n4RIjWfokeY6nhrQs7UHKZ3w3WCEscN+ewbXznUY7nI4a91ll000BKshBpNBOKqLGPHqlx3gS2EPm\nUX/9JFBwvBnTTkcXfvpyop2UtCnUN2tn9otU37oDGQ8WCdZ4a6zFTY61w8vAxRPGH4SkmhrH8XBf\nIfNbb2vv7NBWpJIW3lbUoykuNWljQiNvU2Aa4k7FcK8Swz4sMcvy8TNrJvWeWyDwzNJbCgw5zRBE\nmuDgA+U2HRyjvkbPefH5T4CG/1lWTTgBE1gO0AXAMuo0M3VLhOfpxJUEx/lcZEWVQ+L7WnuLMKHS\nZhIMcP38a1uatn0ISp3rMLobuvKHPQaYurduOgc/M3c3FLUU7D7xQa2IJrlpJmvcGFmqPaASbSps\nI7xQbC4hLWPnqDsXVXfvsZYV0wtZFTmVc6rttuw3jQxSX5Yu0RbANq1AI/G7lJUgm600pxeLvsfx\nOaxwuaw0eWC2NqDHk0bNHNK8kNljc9rlfXeEfYxVu1Oqb6fvrz5N3amuk5LNZCqfg+c6nN/nUOu9\ncMKGbdbtOuju7UL8iSscvLg+a05e7uv53OnaXO+KjMVNoEmjtR10W8eIlLxbQu2oA3Qmc2B/2Ogu\nXlK3e1J8EQ+2oQ6oTr3NLujZq4HORDe8cW8QdJ0vuRlAUmwVOWAfsRPHBQpc6njvufxl0qVpU7za\ne4C4cXOwfeu13+X6YP/tAZ7QnyChQ2xE/7W8NqXcp64f5yyLNANiNHs9qBdYZIpYlcgk3v6VVI8a\n2cfQCaESCEx/rhK5XOmYTbHk4QRkkB8gVVhnrIOubk/PrUR32MrBHaWiHyR6fIUGz5Us2aziRT6T\nBsk8fYK4vrceB0eYugO6IWuIz2w/bO0Z1JmecJ14fbbfYH7StDJxZtVTGXUMLXZ6o85lPWQ1OxKI\n2wsCrA06dLHDkfUyOicv8GA3U/IRz3TYxD3qMBtqIVzTUF8IfXCGi+R+jfYLeomQA/YvPNTN1zZk\nOVeQGanWhBPiisMVHfgOXR8CbWgrpQg8dD8y8Dtli1LmdqMJO/rL0ZEPFC2huxiiZOkuqXGXvqZ0\nAre/KbgbY2vTz5ILL49GxoGTMR/vXMAmtqmuT6wLxBOzKtNtQsm1tud1qpk07JwRyLGndjzRHbaG\nA6cajJwsmS/yxAaiFz2n6gkbCTPqBq6FSWrvFqLGNHu5dJdc/TTe7DgP2AXVZvHoKrQ9Mq5Q3xxT\nD0/hE8wZg1MCK7EdvpxukVOmGcoBykws0aS6teViVLIHaTsDyQogCdz+UGGZYIucN9Qf+uj2gOki\nHdh19Ocm3Bu4pGA3U3uWh1zVzglYst+cH7D31gNYnm3zQor0sqsbgzA5dmmx0yoL4t4sn089bWmg\nbGCNTHwQspPtGfs0RDc/AudZRizlLwtyt9aOxLdQm15rAyWVc/9bXezetL8/+RkY02joswM5c/iR\nZ0pqOTfDwG5fMu0PcJ3lsW3iNd1p4dHn89/vLi6fWbczG8K53qxtZNvUpzql39if7+Y8Y2FBqimV\n1iCAxYNZ6PD8xT6e/ju5Pp3+I24UuJb2DGQ9nBVyNgMFKl6u486FWaqRxEzX5e5CiXZq6QjpsGir\nquM2QoGfNvqKn799/Tpi39mVe2pGs2zDseEi//vncZhWXVRv4dHA7/Vd8iiHgh2es8N/siFW0RGe\n/brVYDPN+hIsttnh7XYZYe/UKSBExOnM/xLc/C4c34I5x+9TYxRHWgN9F/WdNwmmn198OEtOp9Ob\nix8+Tc+Sy6ubj6cf6p1v8ZABjuDxFOLwgp2UvZJNLbUT+5VAHZbeFhLnxf7+m4hv9XkPBRggCzaX\ntSVvPkdHUC7WP33H5wguWqU3luEXvnodvx6FFRGnJin6CLFlhX05um8vxVyldO//et+BSJ2L8YjV\npdc+xr1ClWE3zkXVcv+LanC4VaviH3fH6/3FzdmP06ubz93d+1TwIvp/MYYCFn8RkDY32BHlnprt\nfNuowvsa/lug8V+mJBic\n'), ('util.retry', 'eJytVk2P2zYQvetXDFwsLDuC4C2wORhxsUHQFgWKnHqXaYmyiUqkQ1LxGkX/e2dIivpy0h6qw1oa\nDh9nHt/MjmivSluwouVJrVULdSdLq1RjQPilm2ZX49dKJS1/s4049YvB0jLJzlwnwdqo81nIc4K/\ncOi/8jO3v+Mr12lRSNbyotgkSVLxGjS3+p6y0golM2DW8vZqzeElA9NwfqXgDu93GbTsrRgsL7AF\ntCYQH4dT8LeSPJQ0h/Tn/j3bZFA2nMnuevisJMdj9Bkd0Pznzb3+9fdm77BWq9Un1jRw9AGtgdHB\nou1aUDVaQ3hrR5qBTlrRgLBgurLkvDJDRJgb6xqLyYNV8JLDMUa/BmHAXjjIrj1xTciGI5uVIdcb\nEzainLi9cS4jL9kM9/0OmKygUt2pIRNn5cVT0W/J0C3CTbOZULrOAY5zEl2kDGx3bThuiTiRWsqD\nYfoX1TUVRgsl684Xm8NvNQwwoDBbTa4S/yjDI1AjjOUVCPnobKY5aCYMOjgJ9peSEXl3uAm8qNOA\nFVxF2/JKMMubuwvjGK7e5XLV6quo0ItYK/Gm2QkzwwsksBHrbm0KBqy2mASmELMnxD7hz4pU1bVc\nWhOBQohwZYZCwwsTnpu76nSvSV92BKf5l05o1NUSCUPEwzTKBCOSlIEjHnFckbp1ScH1WxtuTETO\nI86R9L526R+9+D3P/SU7NYnSkkBiFBQ4pQBY8YOY0HjsKVxj4bgFSpR6Q7CHwt6M16SyMXWlB9dg\n876inlY8fBj6wX6QjzrnFT9153Q19X6qwBHgJDc2r+AJ0lHbgOkxo66z8YFI7GLP7u12EUiQhA+H\nWI5DJKjd/QSWQhOyVunKCXsP1FeoRJ8MysJeXA/a41ffhPz7agISn1U4EX4IKfQN01id0u6Nf/VQ\n+CFD+LE4uO00qsNtS7fklcF2G/yjqy+/RTNdphZYj7lREQwVv4dVRl8FMXD4Q3d8Gg3ebrjt/SLf\nsJAuduBNPGL+m4T/Kr4S36QyidwSbWM1Ttih1jE/b5DNT7D7D+f9wlAfVVCQu+kq9vUTrxV1M/LE\nJYzl8T3TMyhw4UPW3K2n3/EaAj+M3rfw48JzluWkFJYZz7En7hNvGg2E7AZjLSTKf1YiEt5RbQ1z\ngHB9YOvV10vUfwWheoD1eg0f8T9hqTSz2EKQ2zBHbHLszqylTtYZHEu8/+sA7tmiA2ulRhrL8zyZ\n+8Zh5Hm3G48jz7sB5cR0utlPYEKESfQpImRRowIVxkmNebTt1Q1a3jqeIMZbyeWKA9S8dveP6tyz\nQXhh2PGbwrjjfxBjxPS39Ti7gmR21DLE5PFqyB3v+3U2OsY5EEsjBP3vIlhwFlEKYb/D0v/M0CN2\n7oLjNNTHkvwDPQB6iA==\n'), ('util.git', 'eJzNW+uT27YR/66/ApF7IymWeEk/Xuam4/iReJrGntiZdMZ2JEoEJcQUIRPgyddM/vfuAyDAh+S7\nNkmrGVsiCSx2F7u/fRA3Ho+f1eXGKl0aketKqNLKKoUb5VYcld2J3XY8Ho/U/qArK7Txv0y9PlR6\nI01zp66KQ1oZGV0Xau2vKjka5ZXei9qqItno/T4tMyP807pcbvbZHIbt9Y1cHlK7m9PdD7WSFp9F\ns3NVSD/TpLlc1mWhyvcjv1aht1vgfwTf4tpfJVtpv4Ofspoul2W6l8vlbDQabYrUGPFE5mld2Fe7\ntJJfp0ZejQR8DvBo1H0EFLu3pkgok7lY7tP3cpmujS5qK6eVPOgZk1K5wKvE2LSyBhU7HaMYV5eX\nYzcEPw/EP4CCcE9QhUZ4cs0gVA5wgfTeFLKMCb1rBuFTGOSfXZixuIDtS3ByAiTxe4r/zWiKLIDD\nMRIRpbZgBUTgqkuuS4AkHPEAW1c8yykD9L3ES1J2rIu1sgZoeXtJUMpDoWxEbaeN5SFgQsmHWoM2\ncVpSSlvozVyMx7NRpIv+QGKzMLZSh+kYVBOmOE69KL9oVU5xvblgdTD3u9QA9zfKgGdMM4mP/aUT\nA9ziByJlxOuqlrzFPELIj8qAkKBGnIoOhDNsdRtpNDbu6ZvJVtnJXEzAWvFrsdAl7Ekp6aL8chKW\nfzcXm2N2jYRn0f6QUMgI7+fHjTzEXpo8TotCZi/56mlV6eqqO/tZWoD7xvLnjeg57uI5yWlAR/DE\nKZyfbdJSrKVIxbpKy81OANrYdCvwWXIfFZmdPi6AKKkmmzTc/TmKUSVYKmtlDf5/Tc+CYp7DY5UW\n6l8SPBcMYX+wt+QVRlld3YrUsmbE85x+eI0BGgplyonlKXOhLOBvUaDGGBQz1ibMW+HCKxhOYs2F\n3ckS1Qp32VH9xE0lUwsTvXZho9C7vekrk6mKZIkgCAwwUWWup2NaFuMAgMdctNUawe40PJGFh078\nYDhBfeF6BQg5sBgNi3CFnJGVm89ao06x1RkGEralyzur8a42QWbamd+WYEhamEDPH4hv/BbloOb3\nQtcWl4ebADqw+1Y7/XNM3ctM4QUwJTdgCjgENORoscxoBLSZ8N8tW0YifmLP2SHhHez5EQccagA8\n0AFodw+hSB0K3nrj6MF9AFe07AIZMRiqMjYOFBu424ElbnRpUxiK4VjTDFnamENH7TtpJ8ZLA0SR\nv7YgqjK278CwFRgRYaSJrYRd8MUrcra5iBQO+pOJrKoSgs21+OsX7a14IL4H602blUFFSCFJEgBL\noXNii4UweEn+xU6Vdgg1JFr3q1ShnztO0J8CAwBBYKgNCCEMMFDjMPr1YcJe8m7AF07NDnNGbSsX\nY3YGmDhzcauFhnjfI5JZAlmKtbF/DaC0Uwio8AYgKhMwjWziPvjQhsTeliOqgqQRvr7UB0hS3oxh\nMfBXcN+bBcV9vFgs4O4CVhlH4D0XgBXgTdcxkecvn85iM8EHyTEFLJ6Jz65Fx1JaTDbWWNtDjWkF\nzeU1ErDpbDpLOFEIK6BCga0Imkpd7QkxBrCKKc9aUQc0DLOnDaFr1j5gYnRrgNY4QUXNehGMSf4+\nMQxTM8fFCYthT4LcCsADf6OlBLdDZOco9gx+NXHHMEAphg02Nmtkkc9pRiW3dZFW7aE07JJkdkYI\nSbesbN+qRwN+BACWK5cwrbUu+BeIxw8rmZB3skeeMk0qPO5mfJHVscOYJUn/SZtSeRiLWTluxjjs\nUTYcA50tDOAJTsAxscY8Ac4oplkr3c3c1hvYeooGlG3POTK4/U8LiFMlYLzpshMbDGXpoF69/gXM\nwTCc5Rq/A4EJL07Ul27kOaLMRkTVRVkqQWmXAm0YdZzMQGqRR8lGcqwUJP/jC/O2xFqntbSHyk0h\n0zKuRR6I10cNNpNDfNvDMyPGNAatZK+zupCYZBx3CvJVir0QNY9SHFOIk0aLPK2SBpxbSSpRIXPM\no/+zicM5p/wTpsbMplm2xFTF+r3iC6qnmotIFnCgR1mG6M7PKLPOxCqatvL+DEUU4JPHf0wXVvhj\nxVYOu0MNABi8itZZeRftScuDyAQyzsiHOY2kn0UG6UZAFXdnSV9JyygFkwhdvNR34BGWXMC0+/G5\nbfjs8ziMn54zxs8bWbopcwwC32PKojhlcduVaYm5ioN4FerGDugFQRY3d4W28/Y2BG3IORaglEp2\nwA3vm2mUFOypHwHJnt3sphX6oHk4ffvq4Uy8neYSbr6d/QWdEsZIs0kPqMOgvTkt1Arv+8F4vk+2\nla4P0y/7xnM/wznvIIM2j6lZJtf1FiHmCs2BXISHIkiE7sX+1jEFWjlrNj40RBOuY667QXzUnwCg\nhCkbmtNQDYesmharUDahjPD/9AgQemFmjvfTypuH9aIK8F5+OxDC2kwCbrR5vDCf5Cswc3eo9N7s\n2k1z0WpwXKMeQ6vFXdaHDOLOEkdeU8UdlOBbgNfdniDoTGEeZhwNigdMotMxwI6fAdeF1ICKshUO\noup+B/uz8rysEDVWjs+V2OzkBiorqjqxM0rUGMMTNpMnmsMV1o20BOw6VmO8yi49AEDMwbs3RU2q\nh6TMqHVxC6zq9VpW2EGlVIMaOU3vwYlFDIINzLkEttjagOq1NpIgzY0Sawk4IhvGnMiNHTf6Q2rD\nTdiWmjmFkOWNqnSJHd3p+Jvnr5evvn30w9Pl149ePV0+ef4D2A3qfDa8St9bmiZl466tpmWbi05V\nQImMCZvezB2y+JgAstBmkB5EDJI+qRkbZcLNyMGODVXouJehFURuFGY1k1pFG7GBfa1moGtuobW3\nGyQgeG0V6CYaytr2I1x18pS+wHDbyyCzx7QqgUvgV9dFhuW5ay3EbYoL8xVUHCZdU58Dn8B3LMsc\nV1qi4ANsxhZDqu497O0D1Sv9FjfXHp3q/DF6H/JFkzr9MVdFnyjL3Yhust7vi7U0BYDo0gOBjgtV\nFHgzNVNDJd/UZ19FLtzr3LHFhwZYJN85a+x2YkKf06UwsGVosAAJgJd0j+j0bazPTqhJXAXWN9d+\nX+6BeAGLVEcFewziUqICOmmKIv+hZ4NY774DUrvvNuAzWvueH72eIazWdcWMopbijJnUobY7Kw5F\nupFnfTx24s37Jb3Y+lSVRIqB2lCVmfyY4Lzx7IxlNYQHzGuooRrGt/coaoEODDmzhU5zEDuOEnJX\n0N4BQg24OVsw6dqpLm0i75wDHMpzlI7CLr1xwat5z5IWmI7eUjfd6HnTPIWaH5UsSknrOAKUiYKV\n3todvhBkr9dLvn0ddYviVzmwW+2deoAFYKbRFYmjwLQwB7lRuZKQdENxiD1azJ7ljax4yVC+h1XD\nmwl8Bdd97dJ648Srx5ylG1unBcRsZCIXbM6wNHDoRMc6iAWPSPhMgAz56PbAO3L+aS7RfD/9gmxI\nWdT1CZtsmi1ym6PsydX9zvj7V4OY1QWJZ0QCnRUkM4wRjeu2xvYiIhN4/eLJiyvxLWAb+CYtzHkq\nYYeByuU9Kc1c2nRrLv8Jnx6R6P1Yz5riD1GP+zIc5jrwNOvNHX5pcXeKPUjsvBO5V7sxaO6V3ksy\ne7CB0oojpGzbzwbGPeZgFSEkBpJKLrgd350QgIu6/2FPaG8hUC7a4W8gmvhPHAfPDQuvBfxn0Fju\nt8/Rfrg3XnjblTHXYw0xRJXj++/23ej+IXseZaLNDpzMQO+5Cffd9n6a0V3sxIj2Zve1Pbj1saOx\n1v8jHzuRNP+P5AcXhmyOsRONh1u6oaHBgk7Yoia+A+JxOkqihmqVH33c51bkRh9uvYquKPn3UeLK\ntwJyX827KBMFGYIahXgcOSAe34HYAhE4NVGUjsNGs0Y7Tf10hCOIagdrp4fLCzOhTlcvFg7owLCD\nIIM+fgO/xkJSgy8wPZHxkNRhS3NXvPYkDENcyhDXO+4Bnp6hnZqeyI6bZkifBZVHfY22oNxpHzyL\nAXQaIxmaHk/1bftTOTw3V9qtFq4iOXHvN29C4+UxUjWhCY5bSim7wZ5J04khu4bbFMgg+8R0jmDB\nv+iifDMR4jWkT0ddUV1I5uyPYdCJjju3ULiYodNu/U4K94NhBC5CY1o9H6TO4nePh6CUUXltGuZq\n8JEwOdIWUXBKJBKQTw+K506ZNM0dt7XnK9wTJSj2NlngIcx4ZC3q0lULkaLcnChaYvua79IZiS7N\nNt3HsUIJbXhC29kGgb9508s2yvM6Vto2wuj3kDN3X/b6j4sQf5e3a51W2XM8U1LVBzvAUi9tult0\nkf7xdAxhfl3IfdvSnDpP6gc/eKJElXVYvh8/g9pfukMs8RaKPIXCMvsKvvnhOoUy0OrQD3aW0n0T\njOp3RyrexW2YwTDk0/ofwYv5BMflYuHkQ2/+WwCjfZZQqzSbThaLUi+oLtW1nQSL9WGrNUl+tDjp\nDb6ZpvNu0UG1TmsyuzqxHD+dBIkbqgEL34XTIc25EEd8UHRnYdzojIKbx9rBYDDYFo967CFdbdCV\n4jtAaQsyXG+b37G4Tja3tV2TOyEYKqVCUPUAiz0lX9kPQxAznTVvN3HlqE2gaSorsa7okJNbHtb7\njvOPXVpuZYDFTJkNuFl0eM61MLpFP8Sbo8Iak9ZOrRv7EyFrM+rnL8SUqxpaFi7XstDHGVW+utpw\n8c0lJfVFHJkMjDGHf+WGMhlEPb3fA5arzPj30nvq7iPAc88EKO35NFrpzj0hHZvC00wYC7pJIFbx\n6Qv5oVaANKgRoD1piOD0xYJnTeYeQJQ/EEY9nAo1vr4VugAuBURFQ6fINb1dGeqj9LteXSf2vuWP\nRvF784bGQzH5+YtJdMg5GH337GcbdxwW9ByVHcLnT5MLc7lPIfuqOINrzPsMmrVnc+437bx96uT7\ndxWaCXuZ7yL0p3y7X6V0Hbzv0Z36cSjh4gHY/+hkWNR8Adv0zkVAfyLfwiMIhA53TpS4O9RLlOgs\nYpwuuQwpfu/UywfukC6cCv+ocVbsYPA/W+/9udG8KRn/D8P5A/FYlzeycraBzeCy+dMHPopGh2sn\nWMpxyRhOVTvjpz9RGPobjKGEgZTR+Bwd+ojThmDTcdbwhDqZbHj4LPQTmSXqAXKnEUq7jWziBebO\n6a1vRTMxKE/1RnHjVUOsoLNOrkFKb8GpGkhxxUNdbSV6CUY2d+TIydTOTpCBySyAbwfvVN7y5k7J\nFoiNH1JL0x1uuPw1nvTb5a+O7m9X7VERfESDxgk41z7F9+29yjLATQsyW4gTX0THIvuW2Od/B3W0\n+aPZnZ0IOL+Doj8/x/HnEad/ih7/O25mztFPhK/4kJWLXPTnOL2TVZzzNClBOJS6wvErn+AVt3R8\nIjom0SRyJ48ohwNW7ogyXnz79NETf2qP/yztPqeoXHw4czr03yOfFDU=\n')]
-
-### Load the compressed module sources ###
-import sys, imp
-for name, source in module_sources:
- source = source.decode("base64").decode("zlib")
- mod = imp.new_module(name)
- exec source in mod.__dict__
- sys.modules[name] = mod
-
-### Original script follows ###
-#!/usr/bin/python
-"""%prog [-p|--props-file] [-r|--rev revision] [-b|--branch branch]
- [-s|--shared-dir shared_dir] repo [dest]
-
-Tool to do safe operations with git.
-
-revision/branch on commandline will override those in props-file"""
-
-# Import snippet to find tools lib
-import os
-import site
-import logging
-site.addsitedir(os.path.join(os.path.dirname(os.path.realpath(__file__)),
- "../../lib/python"))
-
-try:
- import simplejson as json
- assert json
-except ImportError:
- import json
-
-from util.git import git
-
-
-if __name__ == '__main__':
- from optparse import OptionParser
-
- parser = OptionParser(__doc__)
- parser.set_defaults(
- revision=os.environ.get('GIT_REV'),
- branch=os.environ.get('GIT_BRANCH', None),
- propsfile=os.environ.get('PROPERTIES_FILE'),
- loglevel=logging.INFO,
- shared_dir=os.environ.get('GIT_SHARE_BASE_DIR'),
- mirrors=None,
- clean=False,
- )
- parser.add_option(
- "-r", "--rev", dest="revision", help="which revision to update to")
- parser.add_option(
- "-b", "--branch", dest="branch", help="which branch to update to")
- parser.add_option("-p", "--props-file", dest="propsfile",
- help="build json file containing revision information")
- parser.add_option("-s", "--shared-dir", dest="shared_dir",
- help="clone to a shared directory")
- parser.add_option("--mirror", dest="mirrors", action="append",
- help="add a mirror to try cloning/pulling from before repo")
- parser.add_option("--clean", dest="clean", action="store_true", default=False,
- help="run 'git clean' after updating the local repository")
- parser.add_option("-v", "--verbose", dest="loglevel",
- action="store_const", const=logging.DEBUG)
-
- options, args = parser.parse_args()
-
- logging.basicConfig(
- level=options.loglevel, format="%(asctime)s %(message)s")
-
- if len(args) not in (1, 2):
- parser.error("Invalid number of arguments")
-
- repo = args[0]
- if len(args) == 2:
- dest = args[1]
- else:
- dest = os.path.basename(repo)
-
- # Parse propsfile
- if options.propsfile:
- js = json.load(open(options.propsfile))
- if options.revision is None:
- options.revision = js['sourcestamp']['revision']
- if options.branch is None:
- options.branch = js['sourcestamp']['branch']
-
- got_revision = git(repo, dest, options.branch, options.revision,
- shareBase=options.shared_dir,
- mirrors=options.mirrors,
- clean_dest=options.clean,
- )
-
- print "Got revision %s" % got_revision
diff --git a/testing/mozharness/external_tools/machine-configuration.json b/testing/mozharness/external_tools/machine-configuration.json
deleted file mode 100644
index 29118c0fd..000000000
--- a/testing/mozharness/external_tools/machine-configuration.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "win7": {
- "screen_resolution": {
- "x": 1280,
- "y": 1024
- },
- "mouse_position": {
- "x": 1010,
- "y": 10
- }
- }
-}
diff --git a/testing/mozharness/external_tools/mouse_and_screen_resolution.py b/testing/mozharness/external_tools/mouse_and_screen_resolution.py
deleted file mode 100755
index 29e46e1bc..000000000
--- a/testing/mozharness/external_tools/mouse_and_screen_resolution.py
+++ /dev/null
@@ -1,153 +0,0 @@
-#! /usr/bin/env python
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-#
-# Script name: mouse_and_screen_resolution.py
-# Purpose: Sets mouse position and screen resolution for Windows 7 32-bit slaves
-# Author(s): Zambrano Gasparnian, Armen <armenzg@mozilla.com>
-# Target: Python 2.5 or newer
-#
-from optparse import OptionParser
-from ctypes import windll, Structure, c_ulong, byref
-try:
- import json
-except:
- import simplejson as json
-import os
-import sys
-import urllib2
-import socket
-import platform
-import time
-
-default_screen_resolution = {"x": 1024, "y": 768}
-default_mouse_position = {"x": 1010, "y": 10}
-
-def wfetch(url, retries=5):
- while True:
- try:
- return urllib2.urlopen(url, timeout=30).read()
- except urllib2.HTTPError, e:
- print("Failed to fetch '%s': %s" % (url, str(e)))
- except urllib2.URLError, e:
- print("Failed to fetch '%s': %s" % (url, str(e)))
- except socket.timeout, e:
- print("Time out accessing %s: %s" % (url, str(e)))
- except socket.error, e:
- print("Socket error when accessing %s: %s" % (url, str(e)))
- if retries < 0:
- raise Exception("Could not fetch url '%s'" % url)
- retries -= 1
- print("Retrying")
- time.sleep(60)
-
-def main():
-
- if not (platform.version().startswith('6.1.760') and not 'PROGRAMFILES(X86)' in os.environ):
- # We only want to run this for Windows 7 32-bit
- print "INFO: This script was written to be used with Windows 7 32-bit machines."
- return 0
-
- parser = OptionParser()
- parser.add_option(
- "--configuration-url", dest="configuration_url", type="string",
- help="Specifies the url of the configuration file.")
- parser.add_option(
- "--configuration-file", dest="configuration_file", type="string",
- help="Specifies the path to the configuration file.")
- (options, args) = parser.parse_args()
-
- if (options.configuration_url == None and
- options.configuration_file == None):
- print "You must specify --configuration-url or --configuration-file."
- return 1
-
- if options.configuration_file:
- with open(options.configuration_file) as f:
- conf_dict = json.load(f)
- new_screen_resolution = conf_dict["win7"]["screen_resolution"]
- new_mouse_position = conf_dict["win7"]["mouse_position"]
- else:
- try:
- conf_dict = json.loads(wfetch(options.configuration_url))
- new_screen_resolution = conf_dict["win7"]["screen_resolution"]
- new_mouse_position = conf_dict["win7"]["mouse_position"]
- except urllib2.HTTPError, e:
- print "This branch does not seem to have the configuration file %s" % str(e)
- print "Let's fail over to 1024x768."
- new_screen_resolution = default_screen_resolution
- new_mouse_position = default_mouse_position
- except urllib2.URLError, e:
- print "INFRA-ERROR: We couldn't reach hg.mozilla.org: %s" % str(e)
- return 1
- except Exception, e:
- print "ERROR: We were not expecting any more exceptions: %s" % str(e)
- return 1
-
- current_screen_resolution = queryScreenResolution()
- print "Screen resolution (current): (%(x)s, %(y)s)" % (current_screen_resolution)
-
- if current_screen_resolution == new_screen_resolution:
- print "No need to change the screen resolution."
- else:
- print "Changing the screen resolution..."
- try:
- changeScreenResolution(new_screen_resolution["x"], new_screen_resolution["y"])
- except Exception, e:
- print "INFRA-ERROR: We have attempted to change the screen resolution but " + \
- "something went wrong: %s" % str(e)
- return 1
- time.sleep(3) # just in case
- current_screen_resolution = queryScreenResolution()
- print "Screen resolution (new): (%(x)s, %(y)s)" % current_screen_resolution
-
- print "Mouse position (current): (%(x)s, %(y)s)" % (queryMousePosition())
- setCursorPos(new_mouse_position["x"], new_mouse_position["y"])
- current_mouse_position = queryMousePosition()
- print "Mouse position (new): (%(x)s, %(y)s)" % (current_mouse_position)
-
- if current_screen_resolution != new_screen_resolution or current_mouse_position != new_mouse_position:
- print "INFRA-ERROR: The new screen resolution or mouse positions are not what we expected"
- return 1
- else:
- return 0
-
-class POINT(Structure):
- _fields_ = [("x", c_ulong), ("y", c_ulong)]
-
-def queryMousePosition():
- pt = POINT()
- windll.user32.GetCursorPos(byref(pt))
- return { "x": pt.x, "y": pt.y}
-
-def setCursorPos(x, y):
- windll.user32.SetCursorPos(x, y)
-
-def queryScreenResolution():
- return {"x": windll.user32.GetSystemMetrics(0),
- "y": windll.user32.GetSystemMetrics(1)}
-
-def changeScreenResolution(xres = None, yres = None, BitsPerPixel = None):
- import struct
-
- DM_BITSPERPEL = 0x00040000
- DM_PELSWIDTH = 0x00080000
- DM_PELSHEIGHT = 0x00100000
- CDS_FULLSCREEN = 0x00000004
- SIZEOF_DEVMODE = 148
-
- DevModeData = struct.calcsize("32BHH") * '\x00'
- DevModeData += struct.pack("H", SIZEOF_DEVMODE)
- DevModeData += struct.calcsize("H") * '\x00'
- dwFields = (xres and DM_PELSWIDTH or 0) | (yres and DM_PELSHEIGHT or 0) | (BitsPerPixel and DM_BITSPERPEL or 0)
- DevModeData += struct.pack("L", dwFields)
- DevModeData += struct.calcsize("l9h32BHL") * '\x00'
- DevModeData += struct.pack("LLL", BitsPerPixel or 0, xres or 0, yres or 0)
- DevModeData += struct.calcsize("8L") * '\x00'
-
- return windll.user32.ChangeDisplaySettingsA(DevModeData, 0)
-
-if __name__ == '__main__':
- sys.exit(main())
diff --git a/testing/mozharness/external_tools/performance-artifact-schema.json b/testing/mozharness/external_tools/performance-artifact-schema.json
deleted file mode 100644
index f79a0419b..000000000
--- a/testing/mozharness/external_tools/performance-artifact-schema.json
+++ /dev/null
@@ -1,164 +0,0 @@
-{
- "definitions": {
- "framework_schema": {
- "properties": {
- "name": {
- "title": "Framework name",
- "type": "string"
- }
- },
- "type": "object"
- },
- "subtest_schema": {
- "properties": {
- "name": {
- "title": "Subtest name",
- "type": "string"
- },
- "value": {
- "description": "Summary value for subtest",
- "title": "Subtest value",
- "type": "number",
- "minimum": -1000000000000.0,
- "maximum": 1000000000000.0
- },
- "lowerIsBetter": {
- "description": "Whether lower values are better for subtest",
- "title": "Lower is better",
- "type": "boolean"
- },
- "shouldAlert": {
- "description": "Whether we should alert",
- "title": "Should alert",
- "type": "boolean"
- },
- "alertThreshold": {
- "description": "% change threshold before alerting",
- "title": "Alert threshold",
- "type": "number",
- "minimum": 0.0,
- "maximum": 1000.0
- },
- "minBackWindow": {
- "description": "Minimum back window to use for alerting",
- "title": "Minimum back window",
- "type": "number",
- "minimum": 1,
- "maximum": 255
- },
- "maxBackWindow": {
- "description": "Maximum back window to use for alerting",
- "title": "Maximum back window",
- "type": "number",
- "minimum": 1,
- "maximum": 255
- },
- "foreWindow": {
- "description": "Fore window to use for alerting",
- "title": "Fore window",
- "type": "number",
- "minimum": 1,
- "maximum": 255
- }
- },
- "required": [
- "name",
- "value"
- ],
- "type": "object"
- },
- "suite_schema": {
- "properties": {
- "name": {
- "title": "Suite name",
- "type": "string"
- },
- "extraOptions": {
- "type": "array",
- "title": "Extra options used in running suite",
- "items": {
- "type": "string"
- },
- "uniqueItems": true
- },
- "subtests": {
- "items": {
- "$ref": "#/definitions/subtest_schema"
- },
- "title": "Subtests",
- "type": "array"
- },
- "value": {
- "title": "Suite value",
- "type": "number",
- "minimum": -1000000000000.0,
- "maximum": 1000000000000.0
- },
- "lowerIsBetter": {
- "description": "Whether lower values are better for suite",
- "title": "Lower is better",
- "type": "boolean"
- },
- "shouldAlert": {
- "description": "Whether we should alert on this suite (overrides default behaviour)",
- "title": "Should alert",
- "type": "boolean"
- },
- "alertThreshold": {
- "description": "% change threshold before alerting",
- "title": "Alert threshold",
- "type": "number",
- "minimum": 0.0,
- "maximum": 1000.0
- },
- "minBackWindow": {
- "description": "Minimum back window to use for alerting",
- "title": "Minimum back window",
- "type": "integer",
- "minimum": 1,
- "maximum": 255
- },
- "maxBackWindow": {
- "description": "Maximum back window to use for alerting",
- "title": "Maximum back window",
- "type": "integer",
- "minimum": 1,
- "maximum": 255
- },
- "foreWindow": {
- "description": "Fore window to use for alerting",
- "title": "Fore window",
- "type": "integer",
- "minimum": 1,
- "maximum": 255
- }
- },
- "required": [
- "name",
- "subtests"
- ],
- "type": "object"
- }
- },
- "description": "Structure for submitting performance data as part of a job",
- "id": "https://treeherder.mozilla.org/schemas/v1/performance-artifact.json#",
- "properties": {
- "framework": {
- "$ref": "#/definitions/framework_schema"
- },
- "suites": {
- "description": "List of suite-level data submitted as part of this structure",
- "items": {
- "$ref": "#/definitions/suite_schema"
- },
- "title": "Performance suites",
- "type": "array"
- }
- },
- "required": [
- "framework",
- "suites"
- ],
- "title": "Perfherder Schema",
- "type": "object"
-}
diff --git a/testing/mozharness/external_tools/robustcheckout.py b/testing/mozharness/external_tools/robustcheckout.py
deleted file mode 100644
index e801724c1..000000000
--- a/testing/mozharness/external_tools/robustcheckout.py
+++ /dev/null
@@ -1,451 +0,0 @@
-# This software may be used and distributed according to the terms of the
-# GNU General Public License version 2 or any later version.
-
-"""Robustly perform a checkout.
-
-This extension provides the ``hg robustcheckout`` command for
-ensuring a working directory is updated to the specified revision
-from a source repo using best practices to ensure optimal clone
-times and storage efficiency.
-"""
-
-from __future__ import absolute_import
-
-import contextlib
-import errno
-import functools
-import os
-import random
-import re
-import socket
-import ssl
-import time
-import urllib2
-
-from mercurial.i18n import _
-from mercurial.node import hex
-from mercurial import (
- commands,
- error,
- exchange,
- extensions,
- cmdutil,
- hg,
- registrar,
- scmutil,
- util,
-)
-
-testedwith = '3.7 3.8 3.9 4.0 4.1 4.2 4.3'
-minimumhgversion = '3.7'
-
-cmdtable = {}
-
-# Mercurial 4.3 introduced registrar.command as a replacement for
-# cmdutil.command.
-if util.safehasattr(registrar, 'command'):
- command = registrar.command(cmdtable)
-else:
- command = cmdutil.command(cmdtable)
-
-# Mercurial 4.2 introduced the vfs module and deprecated the symbol in
-# scmutil.
-def getvfs():
- try:
- from mercurial.vfs import vfs
- return vfs
- except ImportError:
- return scmutil.vfs
-
-
-if os.name == 'nt':
- import ctypes
-
- # Get a reference to the DeleteFileW function
- # DeleteFileW accepts filenames encoded as a null terminated sequence of
- # wide chars (UTF-16). Python's ctypes.c_wchar_p correctly encodes unicode
- # strings to null terminated UTF-16 strings.
- # However, we receive (byte) strings from mercurial. When these are passed
- # to DeleteFileW via the c_wchar_p type, they are implicitly decoded via
- # the 'mbcs' encoding on windows.
- kernel32 = ctypes.windll.kernel32
- DeleteFile = kernel32.DeleteFileW
- DeleteFile.argtypes = [ctypes.c_wchar_p]
- DeleteFile.restype = ctypes.c_bool
-
- def unlinklong(fn):
- normalized_path = '\\\\?\\' + os.path.normpath(fn)
- if not DeleteFile(normalized_path):
- raise OSError(errno.EPERM, "couldn't remove long path", fn)
-
-# Not needed on other platforms, but is handy for testing
-else:
- def unlinklong(fn):
- os.unlink(fn)
-
-
-def unlinkwrapper(unlinkorig, fn, ui):
- '''Calls unlink_long if original unlink function fails.'''
- try:
- ui.debug('calling unlink_orig %s\n' % fn)
- return unlinkorig(fn)
- except WindowsError as e:
- # Windows error 3 corresponds to ERROR_PATH_NOT_FOUND
- # only handle this case; re-raise the exception for other kinds of
- # failures.
- if e.winerror != 3:
- raise
- ui.debug('caught WindowsError ERROR_PATH_NOT_FOUND; '
- 'calling unlink_long %s\n' % fn)
- return unlinklong(fn)
-
-
-@contextlib.contextmanager
-def wrapunlink(ui):
- '''Context manager that temporarily monkeypatches unlink functions.'''
- purgemod = extensions.find('purge')
- to_wrap = [(purgemod.util, 'unlink')]
-
- # Pass along the ui object to the unlink_wrapper so we can get logging out
- # of it.
- wrapped = functools.partial(unlinkwrapper, ui=ui)
-
- # Wrap the original function(s) with our unlink wrapper.
- originals = {}
- for mod, func in to_wrap:
- ui.debug('wrapping %s %s\n' % (mod, func))
- originals[mod, func] = extensions.wrapfunction(mod, func, wrapped)
-
- try:
- yield
- finally:
- # Restore the originals.
- for mod, func in to_wrap:
- ui.debug('restoring %s %s\n' % (mod, func))
- setattr(mod, func, originals[mod, func])
-
-
-def purgewrapper(orig, ui, *args, **kwargs):
- '''Runs original purge() command with unlink monkeypatched.'''
- with wrapunlink(ui):
- return orig(ui, *args, **kwargs)
-
-
-@command('robustcheckout', [
- ('', 'upstream', '', 'URL of upstream repo to clone from'),
- ('r', 'revision', '', 'Revision to check out'),
- ('b', 'branch', '', 'Branch to check out'),
- ('', 'purge', False, 'Whether to purge the working directory'),
- ('', 'sharebase', '', 'Directory where shared repos should be placed'),
- ('', 'networkattempts', 3, 'Maximum number of attempts for network '
- 'operations'),
- ],
- '[OPTION]... URL DEST',
- norepo=True)
-def robustcheckout(ui, url, dest, upstream=None, revision=None, branch=None,
- purge=False, sharebase=None, networkattempts=None):
- """Ensure a working copy has the specified revision checked out."""
- if not revision and not branch:
- raise error.Abort('must specify one of --revision or --branch')
-
- if revision and branch:
- raise error.Abort('cannot specify both --revision and --branch')
-
- # Require revision to look like a SHA-1.
- if revision:
- if len(revision) < 12 or len(revision) > 40 or not re.match('^[a-f0-9]+$', revision):
- raise error.Abort('--revision must be a SHA-1 fragment 12-40 '
- 'characters long')
-
- sharebase = sharebase or ui.config('share', 'pool')
- if not sharebase:
- raise error.Abort('share base directory not defined; refusing to operate',
- hint='define share.pool config option or pass --sharebase')
-
- # worker.backgroundclose only makes things faster if running anti-virus,
- # which our automation doesn't. Disable it.
- ui.setconfig('worker', 'backgroundclose', False)
-
- # By default the progress bar starts after 3s and updates every 0.1s. We
- # change this so it shows and updates every 1.0s.
- # We also tell progress to assume a TTY is present so updates are printed
- # even if there is no known TTY.
- # We make the config change here instead of in a config file because
- # otherwise we're at the whim of whatever configs are used in automation.
- ui.setconfig('progress', 'delay', 1.0)
- ui.setconfig('progress', 'refresh', 1.0)
- ui.setconfig('progress', 'assume-tty', True)
-
- sharebase = os.path.realpath(sharebase)
-
- return _docheckout(ui, url, dest, upstream, revision, branch, purge,
- sharebase, networkattempts)
-
-def _docheckout(ui, url, dest, upstream, revision, branch, purge, sharebase,
- networkattemptlimit, networkattempts=None):
- if not networkattempts:
- networkattempts = [1]
-
- def callself():
- return _docheckout(ui, url, dest, upstream, revision, branch, purge,
- sharebase, networkattemptlimit, networkattempts)
-
- ui.write('ensuring %s@%s is available at %s\n' % (url, revision or branch,
- dest))
-
- # We assume that we're the only process on the machine touching the
- # repository paths that we were told to use. This means our recovery
- # scenario when things aren't "right" is to just nuke things and start
- # from scratch. This is easier to implement than verifying the state
- # of the data and attempting recovery. And in some scenarios (such as
- # potential repo corruption), it is probably faster, since verifying
- # repos can take a while.
-
- destvfs = getvfs()(dest, audit=False, realpath=True)
-
- def deletesharedstore(path=None):
- storepath = path or destvfs.read('.hg/sharedpath').strip()
- if storepath.endswith('.hg'):
- storepath = os.path.dirname(storepath)
-
- storevfs = getvfs()(storepath, audit=False)
- storevfs.rmtree(forcibly=True)
-
- if destvfs.exists() and not destvfs.exists('.hg'):
- raise error.Abort('destination exists but no .hg directory')
-
- # Require checkouts to be tied to shared storage because efficiency.
- if destvfs.exists('.hg') and not destvfs.exists('.hg/sharedpath'):
- ui.warn('(destination is not shared; deleting)\n')
- destvfs.rmtree(forcibly=True)
-
- # Verify the shared path exists and is using modern pooled storage.
- if destvfs.exists('.hg/sharedpath'):
- storepath = destvfs.read('.hg/sharedpath').strip()
-
- ui.write('(existing repository shared store: %s)\n' % storepath)
-
- if not os.path.exists(storepath):
- ui.warn('(shared store does not exist; deleting destination)\n')
- destvfs.rmtree(forcibly=True)
- elif not re.search('[a-f0-9]{40}/\.hg$', storepath.replace('\\', '/')):
- ui.warn('(shared store does not belong to pooled storage; '
- 'deleting destination to improve efficiency)\n')
- destvfs.rmtree(forcibly=True)
-
- storevfs = getvfs()(storepath, audit=False)
- if storevfs.isfileorlink('store/lock'):
- ui.warn('(shared store has an active lock; assuming it is left '
- 'over from a previous process and that the store is '
- 'corrupt; deleting store and destination just to be '
- 'sure)\n')
- destvfs.rmtree(forcibly=True)
- deletesharedstore(storepath)
-
- # FUTURE when we require generaldelta, this is where we can check
- # for that.
-
- if destvfs.isfileorlink('.hg/wlock'):
- ui.warn('(dest has an active working directory lock; assuming it is '
- 'left over from a previous process and that the destination '
- 'is corrupt; deleting it just to be sure)\n')
- destvfs.rmtree(forcibly=True)
-
- def handlerepoerror(e):
- if e.message == _('abandoned transaction found'):
- ui.warn('(abandoned transaction found; trying to recover)\n')
- repo = hg.repository(ui, dest)
- if not repo.recover():
- ui.warn('(could not recover repo state; '
- 'deleting shared store)\n')
- deletesharedstore()
-
- ui.warn('(attempting checkout from beginning)\n')
- return callself()
-
- raise
-
- # At this point we either have an existing working directory using
- # shared, pooled storage or we have nothing.
-
- def handlenetworkfailure():
- if networkattempts[0] >= networkattemptlimit:
- raise error.Abort('reached maximum number of network attempts; '
- 'giving up\n')
-
- ui.warn('(retrying after network failure on attempt %d of %d)\n' %
- (networkattempts[0], networkattemptlimit))
-
- # Do a backoff on retries to mitigate the thundering herd
- # problem. This is an exponential backoff with a multipler
- # plus random jitter thrown in for good measure.
- # With the default settings, backoffs will be:
- # 1) 2.5 - 6.5
- # 2) 5.5 - 9.5
- # 3) 11.5 - 15.5
- backoff = (2 ** networkattempts[0] - 1) * 1.5
- jittermin = ui.configint('robustcheckout', 'retryjittermin', 1000)
- jittermax = ui.configint('robustcheckout', 'retryjittermax', 5000)
- backoff += float(random.randint(jittermin, jittermax)) / 1000.0
- ui.warn('(waiting %.2fs before retry)\n' % backoff)
- time.sleep(backoff)
-
- networkattempts[0] += 1
-
- def handlepullerror(e):
- """Handle an exception raised during a pull.
-
- Returns True if caller should call ``callself()`` to retry.
- """
- if isinstance(e, error.Abort):
- if e.args[0] == _('repository is unrelated'):
- ui.warn('(repository is unrelated; deleting)\n')
- destvfs.rmtree(forcibly=True)
- return True
- elif e.args[0].startswith(_('stream ended unexpectedly')):
- ui.warn('%s\n' % e.args[0])
- # Will raise if failure limit reached.
- handlenetworkfailure()
- return True
- elif isinstance(e, ssl.SSLError):
- # Assume all SSL errors are due to the network, as Mercurial
- # should convert non-transport errors like cert validation failures
- # to error.Abort.
- ui.warn('ssl error: %s\n' % e)
- handlenetworkfailure()
- return True
- elif isinstance(e, urllib2.URLError):
- if isinstance(e.reason, socket.error):
- ui.warn('socket error: %s\n' % e.reason)
- handlenetworkfailure()
- return True
-
- return False
-
- created = False
-
- if not destvfs.exists():
- # Ensure parent directories of destination exist.
- # Mercurial 3.8 removed ensuredirs and made makedirs race safe.
- if util.safehasattr(util, 'ensuredirs'):
- makedirs = util.ensuredirs
- else:
- makedirs = util.makedirs
-
- makedirs(os.path.dirname(destvfs.base), notindexed=True)
- makedirs(sharebase, notindexed=True)
-
- if upstream:
- ui.write('(cloning from upstream repo %s)\n' % upstream)
- cloneurl = upstream or url
-
- try:
- res = hg.clone(ui, {}, cloneurl, dest=dest, update=False,
- shareopts={'pool': sharebase, 'mode': 'identity'})
- except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
- if handlepullerror(e):
- return callself()
- raise
- except error.RepoError as e:
- return handlerepoerror(e)
- except error.RevlogError as e:
- ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
- deletesharedstore()
- return callself()
-
- # TODO retry here.
- if res is None:
- raise error.Abort('clone failed')
-
- # Verify it is using shared pool storage.
- if not destvfs.exists('.hg/sharedpath'):
- raise error.Abort('clone did not create a shared repo')
-
- created = True
-
- # The destination .hg directory should exist. Now make sure we have the
- # wanted revision.
-
- repo = hg.repository(ui, dest)
-
- # We only pull if we are using symbolic names or the requested revision
- # doesn't exist.
- havewantedrev = False
- if revision and revision in repo:
- ctx = repo[revision]
-
- if not ctx.hex().startswith(revision):
- raise error.Abort('--revision argument is ambiguous',
- hint='must be the first 12+ characters of a '
- 'SHA-1 fragment')
-
- checkoutrevision = ctx.hex()
- havewantedrev = True
-
- if not havewantedrev:
- ui.write('(pulling to obtain %s)\n' % (revision or branch,))
-
- remote = None
- try:
- remote = hg.peer(repo, {}, url)
- pullrevs = [remote.lookup(revision or branch)]
- checkoutrevision = hex(pullrevs[0])
- if branch:
- ui.warn('(remote resolved %s to %s; '
- 'result is not deterministic)\n' %
- (branch, checkoutrevision))
-
- if checkoutrevision in repo:
- ui.warn('(revision already present locally; not pulling)\n')
- else:
- pullop = exchange.pull(repo, remote, heads=pullrevs)
- if not pullop.rheads:
- raise error.Abort('unable to pull requested revision')
- except (error.Abort, ssl.SSLError, urllib2.URLError) as e:
- if handlepullerror(e):
- return callself()
- raise
- except error.RepoError as e:
- return handlerepoerror(e)
- except error.RevlogError as e:
- ui.warn('(repo corruption: %s; deleting shared store)\n' % e.message)
- deletesharedstore()
- return callself()
- finally:
- if remote:
- remote.close()
-
- # Now we should have the wanted revision in the store. Perform
- # working directory manipulation.
-
- # Purge if requested. We purge before update because this way we're
- # guaranteed to not have conflicts on `hg update`.
- if purge and not created:
- ui.write('(purging working directory)\n')
- purgeext = extensions.find('purge')
-
- if purgeext.purge(ui, repo, all=True, abort_on_err=True,
- # The function expects all arguments to be
- # defined.
- **{'print': None, 'print0': None, 'dirs': None,
- 'files': None}):
- raise error.Abort('error purging')
-
- # Update the working directory.
- if commands.update(ui, repo, rev=checkoutrevision, clean=True):
- raise error.Abort('error updating')
-
- ui.write('updated to %s\n' % checkoutrevision)
- return None
-
-
-def extsetup(ui):
- # Ensure required extensions are loaded.
- for ext in ('purge', 'share'):
- try:
- extensions.find(ext)
- except KeyError:
- extensions.load(ui, ext, None)
-
- purgemod = extensions.find('purge')
- extensions.wrapcommand(purgemod.cmdtable, 'purge', purgewrapper)
diff --git a/testing/mozharness/external_tools/virtualenv/AUTHORS.txt b/testing/mozharness/external_tools/virtualenv/AUTHORS.txt
deleted file mode 100644
index 272494163..000000000
--- a/testing/mozharness/external_tools/virtualenv/AUTHORS.txt
+++ /dev/null
@@ -1,91 +0,0 @@
-Author
-------
-
-Ian Bicking
-
-Maintainers
------------
-
-Brian Rosner
-Carl Meyer
-Jannis Leidel
-Paul Moore
-Paul Nasrat
-Marcus Smith
-
-Contributors
-------------
-
-Alex Grönholm
-Anatoly Techtonik
-Antonio Cuni
-Antonio Valentino
-Armin Ronacher
-Barry Warsaw
-Benjamin Root
-Bradley Ayers
-Branden Rolston
-Brandon Carl
-Brian Kearns
-Cap Petschulat
-CBWhiz
-Chris Adams
-Chris McDonough
-Christos Kontas
-Christian Hudon
-Christian Stefanescu
-Christopher Nilsson
-Cliff Xuan
-Curt Micol
-Damien Nozay
-Dan Sully
-Daniel Hahler
-Daniel Holth
-David Schoonover
-Denis Costa
-Doug Hellmann
-Doug Napoleone
-Douglas Creager
-Eduard-Cristian Stefan
-Erik M. Bray
-Ethan Jucovy
-Gabriel de Perthuis
-Gunnlaugur Thor Briem
-Graham Dennis
-Greg Haskins
-Jason Penney
-Jason R. Coombs
-Jeff Hammel
-Jeremy Orem
-Jason Penney
-Jason R. Coombs
-John Kleint
-Jonathan Griffin
-Jonathan Hitchcock
-Jorge Vargas
-Josh Bronson
-Kamil Kisiel
-Kyle Gibson
-Konstantin Zemlyak
-Kumar McMillan
-Lars Francke
-Marc Abramowitz
-Mika Laitio
-Mike Hommey
-Miki Tebeka
-Philip Jenvey
-Philippe Ombredanne
-Piotr Dobrogost
-Preston Holmes
-Ralf Schmitt
-Raul Leal
-Ronny Pfannschmidt
-Satrajit Ghosh
-Sergio de Carvalho
-Stefano Rivera
-Tarek Ziadé
-Thomas Aglassinger
-Vinay Sajip
-Vitaly Babiy
-Vladimir Rutsky
-Wang Xuerui \ No newline at end of file
diff --git a/testing/mozharness/external_tools/virtualenv/LICENSE.txt b/testing/mozharness/external_tools/virtualenv/LICENSE.txt
deleted file mode 100644
index ab145001f..000000000
--- a/testing/mozharness/external_tools/virtualenv/LICENSE.txt
+++ /dev/null
@@ -1,22 +0,0 @@
-Copyright (c) 2007 Ian Bicking and Contributors
-Copyright (c) 2009 Ian Bicking, The Open Planning Project
-Copyright (c) 2011-2016 The virtualenv developers
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/testing/mozharness/external_tools/virtualenv/MANIFEST.in b/testing/mozharness/external_tools/virtualenv/MANIFEST.in
deleted file mode 100644
index 49037ada6..000000000
--- a/testing/mozharness/external_tools/virtualenv/MANIFEST.in
+++ /dev/null
@@ -1,12 +0,0 @@
-recursive-include docs *
-recursive-include tests *.py *.sh *.expected
-recursive-include virtualenv_support *.whl
-recursive-include virtualenv_embedded *
-recursive-exclude docs/_templates *
-recursive-exclude docs/_build *
-include virtualenv_support/__init__.py
-include bin/*
-include scripts/*
-include *.py
-include AUTHORS.txt
-include LICENSE.txt
diff --git a/testing/mozharness/external_tools/virtualenv/PKG-INFO b/testing/mozharness/external_tools/virtualenv/PKG-INFO
deleted file mode 100644
index dbfda645d..000000000
--- a/testing/mozharness/external_tools/virtualenv/PKG-INFO
+++ /dev/null
@@ -1,87 +0,0 @@
-Metadata-Version: 1.1
-Name: virtualenv
-Version: 15.0.1
-Summary: Virtual Python Environment builder
-Home-page: https://virtualenv.pypa.io/
-Author: Jannis Leidel, Carl Meyer and Brian Rosner
-Author-email: python-virtualenv@groups.google.com
-License: MIT
-Description: Virtualenv
- ==========
-
- `Mailing list <http://groups.google.com/group/python-virtualenv>`_ |
- `Issues <https://github.com/pypa/virtualenv/issues>`_ |
- `Github <https://github.com/pypa/virtualenv>`_ |
- `PyPI <https://pypi.python.org/pypi/virtualenv/>`_ |
- User IRC: #pypa
- Dev IRC: #pypa-dev
-
- Introduction
- ------------
-
- ``virtualenv`` is a tool to create isolated Python environments.
-
- The basic problem being addressed is one of dependencies and versions,
- and indirectly permissions. Imagine you have an application that
- needs version 1 of LibFoo, but another application requires version
- 2. How can you use both these applications? If you install
- everything into ``/usr/lib/python2.7/site-packages`` (or whatever your
- platform's standard location is), it's easy to end up in a situation
- where you unintentionally upgrade an application that shouldn't be
- upgraded.
-
- Or more generally, what if you want to install an application *and
- leave it be*? If an application works, any change in its libraries or
- the versions of those libraries can break the application.
-
- Also, what if you can't install packages into the global
- ``site-packages`` directory? For instance, on a shared host.
-
- In all these cases, ``virtualenv`` can help you. It creates an
- environment that has its own installation directories, that doesn't
- share libraries with other virtualenv environments (and optionally
- doesn't access the globally installed libraries either).
-
- .. comment:
-
- Release History
- ===============
-
- 15.0.1 (2016-03-17)
- -------------------
-
- * Print error message when DEST_DIR exists and is a file
-
- * Upgrade setuptools to 20.3
-
- * Upgrade pip to 8.1.1.
-
-
- 15.0.0 (2016-03-05)
- -------------------
-
- * Remove the `virtualenv-N.N` script from the package; this can no longer be
- correctly created from a wheel installation.
- Resolves #851, #692
-
- * Remove accidental runtime dependency on pip by extracting certificate in the
- subprocess.
-
- * Upgrade setuptools 20.2.2.
-
- * Upgrade pip to 8.1.0.
-
-
- `Full Changelog <https://virtualenv.pypa.io/en/latest/changes.html>`_.
-Keywords: setuptools deployment installation distutils
-Platform: UNKNOWN
-Classifier: Development Status :: 5 - Production/Stable
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 2
-Classifier: Programming Language :: Python :: 2.6
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.3
-Classifier: Programming Language :: Python :: 3.4
-Classifier: Programming Language :: Python :: 3.5
diff --git a/testing/mozharness/external_tools/virtualenv/README.rst b/testing/mozharness/external_tools/virtualenv/README.rst
deleted file mode 100644
index 0d5984dce..000000000
--- a/testing/mozharness/external_tools/virtualenv/README.rst
+++ /dev/null
@@ -1,31 +0,0 @@
-virtualenv
-==========
-
-A tool for creating isolated 'virtual' python environments.
-
-.. image:: https://img.shields.io/pypi/v/virtualenv.svg
- :target: https://pypi.python.org/pypi/virtualenv
-
-.. image:: https://img.shields.io/travis/pypa/virtualenv/develop.svg
- :target: http://travis-ci.org/pypa/virtualenv
-
-* `Installation <https://virtualenv.pypa.io/en/latest/installation.html>`_
-* `Documentation <https://virtualenv.pypa.io/>`_
-* `Changelog <https://virtualenv.pypa.io/en/latest/changes.html>`_
-* `Issues <https://github.com/pypa/virtualenv/issues>`_
-* `PyPI <https://pypi.python.org/pypi/virtualenv/>`_
-* `Github <https://github.com/pypa/virtualenv>`_
-* `User mailing list <http://groups.google.com/group/python-virtualenv>`_
-* `Dev mailing list <http://groups.google.com/group/pypa-dev>`_
-* User IRC: #pypa on Freenode.
-* Dev IRC: #pypa-dev on Freenode.
-
-
-Code of Conduct
----------------
-
-Everyone interacting in the virtualenv project's codebases, issue trackers,
-chat rooms, and mailing lists is expected to follow the
-`PyPA Code of Conduct`_.
-
-.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/
diff --git a/testing/mozharness/external_tools/virtualenv/bin/rebuild-script.py b/testing/mozharness/external_tools/virtualenv/bin/rebuild-script.py
deleted file mode 100755
index a816af3eb..000000000
--- a/testing/mozharness/external_tools/virtualenv/bin/rebuild-script.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env python
-"""
-Helper script to rebuild virtualenv.py from virtualenv_support
-"""
-from __future__ import print_function
-
-import os
-import re
-import codecs
-from zlib import crc32
-
-here = os.path.dirname(__file__)
-script = os.path.join(here, '..', 'virtualenv.py')
-
-gzip = codecs.lookup('zlib')
-b64 = codecs.lookup('base64')
-
-file_regex = re.compile(
- br'##file (.*?)\n([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*convert\("""\n(.*?)"""\)',
- re.S)
-file_template = b'##file %(filename)s\n%(varname)s = convert("""\n%(data)s""")'
-
-def rebuild(script_path):
- with open(script_path, 'rb') as f:
- script_content = f.read()
- parts = []
- last_pos = 0
- match = None
- for match in file_regex.finditer(script_content):
- parts += [script_content[last_pos:match.start()]]
- last_pos = match.end()
- filename, fn_decoded = match.group(1), match.group(1).decode()
- varname = match.group(2)
- data = match.group(3)
-
- print('Found file %s' % fn_decoded)
- pathname = os.path.join(here, '..', 'virtualenv_embedded', fn_decoded)
-
- with open(pathname, 'rb') as f:
- embedded = f.read()
- new_crc = crc32(embedded)
- new_data = b64.encode(gzip.encode(embedded)[0])[0]
-
- if new_data == data:
- print(' File up to date (crc: %s)' % new_crc)
- parts += [match.group(0)]
- continue
- # Else: content has changed
- crc = crc32(gzip.decode(b64.decode(data)[0])[0])
- print(' Content changed (crc: %s -> %s)' %
- (crc, new_crc))
- new_match = file_template % {
- b'filename': filename,
- b'varname': varname,
- b'data': new_data
- }
- parts += [new_match]
-
- parts += [script_content[last_pos:]]
- new_content = b''.join(parts)
-
- if new_content != script_content:
- print('Content updated; overwriting... ', end='')
- with open(script_path, 'wb') as f:
- f.write(new_content)
- print('done.')
- else:
- print('No changes in content')
- if match is None:
- print('No variables were matched/found')
-
-if __name__ == '__main__':
- rebuild(script)
diff --git a/testing/mozharness/external_tools/virtualenv/docs/Makefile b/testing/mozharness/external_tools/virtualenv/docs/Makefile
deleted file mode 100644
index e4de9f847..000000000
--- a/testing/mozharness/external_tools/virtualenv/docs/Makefile
+++ /dev/null
@@ -1,130 +0,0 @@
-# Makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line.
-SPHINXOPTS =
-SPHINXBUILD = sphinx-build
-PAPER =
-BUILDDIR = _build
-
-# Internal variables.
-PAPEROPT_a4 = -D latex_paper_size=a4
-PAPEROPT_letter = -D latex_paper_size=letter
-ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
-
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest
-
-help:
- @echo "Please use \`make <target>' where <target> is one of"
- @echo " html to make standalone HTML files"
- @echo " dirhtml to make HTML files named index.html in directories"
- @echo " singlehtml to make a single large HTML file"
- @echo " pickle to make pickle files"
- @echo " json to make JSON files"
- @echo " htmlhelp to make HTML files and a HTML help project"
- @echo " qthelp to make HTML files and a qthelp project"
- @echo " devhelp to make HTML files and a Devhelp project"
- @echo " epub to make an epub"
- @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
- @echo " latexpdf to make LaTeX files and run them through pdflatex"
- @echo " text to make text files"
- @echo " man to make manual pages"
- @echo " changes to make an overview of all changed/added/deprecated items"
- @echo " linkcheck to check all external links for integrity"
- @echo " doctest to run all doctests embedded in the documentation (if enabled)"
-
-clean:
- -rm -rf $(BUILDDIR)/*
-
-html:
- $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
- @echo
- @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
-
-dirhtml:
- $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
- @echo
- @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
-
-singlehtml:
- $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
- @echo
- @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
-
-pickle:
- $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
- @echo
- @echo "Build finished; now you can process the pickle files."
-
-json:
- $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
- @echo
- @echo "Build finished; now you can process the JSON files."
-
-htmlhelp:
- $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
- @echo
- @echo "Build finished; now you can run HTML Help Workshop with the" \
- ".hhp project file in $(BUILDDIR)/htmlhelp."
-
-qthelp:
- $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
- @echo
- @echo "Build finished; now you can run "qcollectiongenerator" with the" \
- ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
- @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/django-compressor.qhcp"
- @echo "To view the help file:"
- @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/django-compressor.qhc"
-
-devhelp:
- $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
- @echo
- @echo "Build finished."
- @echo "To view the help file:"
- @echo "# mkdir -p $$HOME/.local/share/devhelp/django-compressor"
- @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/django-compressor"
- @echo "# devhelp"
-
-epub:
- $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
- @echo
- @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
-
-latex:
- $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
- @echo
- @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
- @echo "Run \`make' in that directory to run these through (pdf)latex" \
- "(use \`make latexpdf' here to do that automatically)."
-
-latexpdf:
- $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
- @echo "Running LaTeX files through pdflatex..."
- make -C $(BUILDDIR)/latex all-pdf
- @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
-
-text:
- $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
- @echo
- @echo "Build finished. The text files are in $(BUILDDIR)/text."
-
-man:
- $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
- @echo
- @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
-
-changes:
- $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
- @echo
- @echo "The overview file is in $(BUILDDIR)/changes."
-
-linkcheck:
- $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
- @echo
- @echo "Link check complete; look for any errors in the above output " \
- "or in $(BUILDDIR)/linkcheck/output.txt."
-
-doctest:
- $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
- @echo "Testing of doctests in the sources finished, look at the " \
- "results in $(BUILDDIR)/doctest/output.txt."
diff --git a/testing/mozharness/external_tools/virtualenv/docs/changes.rst b/testing/mozharness/external_tools/virtualenv/docs/changes.rst
deleted file mode 100644
index 2df19f666..000000000
--- a/testing/mozharness/external_tools/virtualenv/docs/changes.rst
+++ /dev/null
@@ -1,985 +0,0 @@
-Release History
-===============
-
-15.0.1 (2016-03-17)
--------------------
-
-* Print error message when DEST_DIR exists and is a file
-
-* Upgrade setuptools to 20.3
-
-* Upgrade pip to 8.1.1.
-
-
-15.0.0 (2016-03-05)
--------------------
-
-* Remove the `virtualenv-N.N` script from the package; this can no longer be
- correctly created from a wheel installation.
- Resolves :issue:`851`, :issue:`692`
-
-* Remove accidental runtime dependency on pip by extracting certificate in the
- subprocess.
-
-* Upgrade setuptools 20.2.2.
-
-* Upgrade pip to 8.1.0.
-
-
-14.0.6 (2016-02-07)
--------------------
-
-* Upgrade setuptools to 20.0
-
-* Upgrade wheel to 0.29.0
-
-* Fix an error where virtualenv didn't pass in a working ssl certificate for
- pip, causing "weird" errors related to ssl.
-
-
-14.0.5 (2016-02-01)
--------------------
-
-* Homogenize drive letter casing for both prefixes and filenames. :issue:`858`
-
-
-14.0.4 (2016-01-31)
--------------------
-
-* Upgrade setuptools to 19.6.2
-
-* Revert ac4ea65; only correct drive letter case.
- Fixes :issue:`856`, :issue:`815`
-
-
-14.0.3 (2016-01-28)
--------------------
-
-* Upgrade setuptools to 19.6.1
-
-
-14.0.2 (2016-01-28)
--------------------
-
-* Upgrade setuptools to 19.6
-
-* Supress any errors from `unset` on different shells (:pull:`843`)
-
-* Normalize letter case for prefix path checking. Fixes :issue:`837`
-
-
-14.0.1 (2016-01-21)
--------------------
-
-* Upgrade from pip 8.0.0 to 8.0.2.
-
-* Fix the default of ``--(no-)download`` to default to downloading.
-
-
-14.0.0 (2016-01-19)
--------------------
-
-* **BACKWARDS INCOMPATIBLE** Drop support for Python 3.2.
-
-* Upgrade setuptools to 19.4
-
-* Upgrade wheel to 0.26.0
-
-* Upgrade pip to 8.0.0
-
-* Upgrade argparse to 1.4.0
-
-* Added support for ``python-config`` script (:pull:`798`)
-
-* Updated activate.fish (:pull:`589`) (:pull:`799`)
-
-* Account for a ``site.pyo`` correctly in some python implementations (:pull:`759`)
-
-* Properly restore an empty PS1 (:issue:`407`)
-
-* Properly remove ``pydoc`` when deactivating
-
-* Remove workaround for very old Mageia / Mandriva linuxes (:pull:`472`)
-
-* Added a space after virtualenv name in the prompt: ``(env) $PS1``
-
-* Make sure not to run a --user install when creating the virtualenv (:pull:`803`)
-
-* Remove virtualenv.py's path from sys.path when executing with a new
- python. Fixes issue :issue:`779`, :issue:`763` (:pull:`805`)
-
-* Remove use of () in .bat files so ``Program Files (x86)`` works :issue:`35`
-
-* Download new releases of the preinstalled software from PyPI when there are
- new releases available. This behavior can be disabled using
- ``--no-download``.
-
-* Make ``--no-setuptools``, ``--no-pip``, and ``--no-wheel`` independent of
- each other.
-
-
-13.1.2 (2015-08-23)
--------------------
-
-* Upgrade pip to 7.1.2.
-
-
-13.1.1 (2015-08-20)
--------------------
-
-* Upgrade pip to 7.1.1.
-
-* Upgrade setuptools to 18.2.
-
-* Make the activate script safe to use when bash is running with ``-u``.
-
-
-13.1.0 (2015-06-30)
--------------------
-
-* Upgrade pip to 7.1.0
-
-* Upgrade setuptools to 18.0.1
-
-
-13.0.3 (2015-06-01)
--------------------
-
-* Upgrade pip to 7.0.3
-
-
-13.0.2 (2015-06-01)
--------------------
-
-* Upgrade pip to 7.0.2
-
-* Upgrade setuptools to 17.0
-
-
-13.0.1 (2015-05-22)
--------------------
-
-* Upgrade pip to 7.0.1
-
-
-13.0.0 (2015-05-21)
--------------------
-
-* Automatically install wheel when creating a new virutalenv. This can be
- disabled by using the ``--no-wheel`` option.
-
-* Don't trust the current directory as a location to discover files to install
- packages from.
-
-* Upgrade setuptools to 16.0.
-
-* Upgrade pip to 7.0.0.
-
-
-12.1.1 (2015-04-07)
--------------------
-
-* Upgrade pip to 6.1.1
-
-
-12.1.0 (2015-04-07)
--------------------
-
-* Upgrade setuptools to 15.0
-
-* Upgrade pip to 6.1.0
-
-
-12.0.7 (2015-02-04)
--------------------
-
-* Upgrade pip to 6.0.8
-
-
-12.0.6 (2015-01-28)
--------------------
-
-* Upgrade pip to 6.0.7
-
-* Upgrade setuptools to 12.0.5
-
-
-12.0.5 (2015-01-03)
--------------------
-
-* Upgrade pip to 6.0.6
-
-* Upgrade setuptools to 11.0
-
-
-12.0.4 (2014-12-23)
--------------------
-
-* Revert the fix to ``-p`` on Debian based pythons as it was broken in other
- situations.
-
-* Revert several sys.path changes new in 12.0 which were breaking virtualenv.
-
-12.0.3 (2014-12-23)
--------------------
-
-* Fix an issue where Debian based Pythons would fail when using -p with the
- host Python.
-
-* Upgrade pip to 6.0.3
-
-12.0.2 (2014-12-23)
--------------------
-
-* Upgraded pip to 6.0.2
-
-12.0.1 (2014-12-22)
--------------------
-
-* Upgraded pip to 6.0.1
-
-
-12.0 (2014-12-22)
------------------
-
-* **PROCESS** Version numbers are now simply ``X.Y`` where the leading ``1``
- has been dropped.
-* Split up documentation into structured pages
-* Now using pytest framework
-* Correct sys.path ordering for debian, issue #461
-* Correctly throws error on older Pythons, issue #619
-* Allow for empty $PATH, pull #601
-* Don't set prompt if $env:VIRTUAL_ENV_DISABLE_PROMPT is set for Powershell
-* Updated setuptools to 7.0
-
-1.11.6 (2014-05-16)
--------------------
-
-* Updated setuptools to 3.6
-* Updated pip to 1.5.6
-
-1.11.5 (2014-05-03)
--------------------
-
-* Updated setuptools to 3.4.4
-* Updated documentation to use https://virtualenv.pypa.io/
-* Updated pip to 1.5.5
-
-1.11.4 (2014-02-21)
--------------------
-
-* Updated pip to 1.5.4
-
-
-1.11.3 (2014-02-20)
--------------------
-
-* Updated setuptools to 2.2
-* Updated pip to 1.5.3
-
-
-1.11.2 (2014-01-26)
--------------------
-
-* Fixed easy_install installed virtualenvs by updated pip to 1.5.2
-
-1.11.1 (2014-01-20)
--------------------
-
-* Fixed an issue where pip and setuptools were not getting installed when using
- the ``--system-site-packages`` flag.
-* Updated setuptools to fix an issue when installed with easy_install
-* Fixed an issue with Python 3.4 and sys.stdout encoding being set to ascii
-* Upgraded pip to v1.5.1
-* Upgraded setuptools to v2.1
-
-1.11 (2014-01-02)
------------------
-
-* **BACKWARDS INCOMPATIBLE** Switched to using wheels for the bundled copies of
- setuptools and pip. Using sdists is no longer supported - users supplying
- their own versions of pip/setuptools will need to provide wheels.
-* **BACKWARDS INCOMPATIBLE** Modified the handling of ``--extra-search-dirs``.
- This option now works like pip's ``--find-links`` option, in that it adds
- extra directories to search for compatible wheels for pip and setuptools.
- The actual wheel selected is chosen based on version and compatibility, using
- the same algorithm as ``pip install setuptools``.
-* Fixed #495, --always-copy was failing (#PR 511)
-* Upgraded pip to v1.5
-* Upgraded setuptools to v1.4
-
-1.10.1 (2013-08-07)
--------------------
-
-* **New Signing Key** Release 1.10.1 is using a different key than normal with
- fingerprint: 7C6B 7C5D 5E2B 6356 A926 F04F 6E3C BCE9 3372 DCFA
-* Upgraded pip to v1.4.1
-* Upgraded setuptools to v0.9.8
-
-
-1.10 (2013-07-23)
------------------
-
-* **BACKWARDS INCOMPATIBLE** Dropped support for Python 2.5. The minimum
- supported Python version is now Python 2.6.
-
-* **BACKWARDS INCOMPATIBLE** Using ``virtualenv.py`` as an isolated script
- (i.e. without an associated ``virtualenv_support`` directory) is no longer
- supported for security reasons and will fail with an error.
-
- Along with this, ``--never-download`` is now always pinned to ``True``, and
- is only being maintained in the short term for backward compatibility
- (Pull #412).
-
-* **IMPORTANT** Switched to the new setuptools (v0.9.7) which has been merged
- with Distribute_ again and works for Python 2 and 3 with one codebase.
- The ``--distribute`` and ``--setuptools`` options are now no-op.
-
-* Updated to pip 1.4.
-
-* Added support for PyPy3k
-
-* Added the option to use a version number with the ``-p`` option to get the
- system copy of that Python version (Windows only)
-
-* Removed embedded ``ez_setup.py``, ``distribute_setup.py`` and
- ``distribute_from_egg.py`` files as part of switching to merged setuptools.
-
-* Fixed ``--relocatable`` to work better on Windows.
-
-* Fixed issue with readline on Windows.
-
-.. _Distribute: https://pypi.python.org/pypi/distribute
-
-1.9.1 (2013-03-08)
-------------------
-
-* Updated to pip 1.3.1 that fixed a major backward incompatible change of
- parsing URLs to externally hosted packages that got accidentily included
- in pip 1.3.
-
-1.9 (2013-03-07)
-----------------
-
-* Unset VIRTUAL_ENV environment variable in deactivate.bat (Pull #364)
-* Upgraded distribute to 0.6.34.
-* Added ``--no-setuptools`` and ``--no-pip`` options (Pull #336).
-* Fixed Issue #373. virtualenv-1.8.4 was failing in cygwin (Pull #382).
-* Fixed Issue #378. virtualenv is now "multiarch" aware on debian/ubuntu (Pull #379).
-* Fixed issue with readline module path on pypy and OSX (Pull #374).
-* Made 64bit detection compatible with Python 2.5 (Pull #393).
-
-
-1.8.4 (2012-11-25)
-------------------
-
-* Updated distribute to 0.6.31. This fixes #359 (numpy install regression) on
- UTF-8 platforms, and provides a workaround on other platforms:
- ``PYTHONIOENCODING=utf8 pip install numpy``.
-
-* When installing virtualenv via curl, don't forget to filter out arguments
- the distribute setup script won't understand. Fixes #358.
-
-* Added some more integration tests.
-
-* Removed the unsupported embedded setuptools egg for Python 2.4 to reduce
- file size.
-
-1.8.3 (2012-11-21)
-------------------
-
-* Fixed readline on OS X. Thanks minrk
-
-* Updated distribute to 0.6.30 (improves our error reporting, plus new
- distribute features and fixes). Thanks Gabriel (g2p)
-
-* Added compatibility with multiarch Python (Python 3.3 for example). Added an
- integration test. Thanks Gabriel (g2p)
-
-* Added ability to install distribute from a user-provided egg, rather than the
- bundled sdist, for better speed. Thanks Paul Moore.
-
-* Make the creation of lib64 symlink smarter about already-existing symlink,
- and more explicit about full paths. Fixes #334 and #330. Thanks Jeremy Orem.
-
-* Give lib64 site-dir preference over lib on 64-bit systems, to avoid wrong
- 32-bit compiles in the venv. Fixes #328. Thanks Damien Nozay.
-
-* Fix a bug with prompt-handling in ``activate.csh`` in non-interactive csh
- shells. Fixes #332. Thanks Benjamin Root for report and patch.
-
-* Make it possible to create a virtualenv from within a Python
- 3.3. pyvenv. Thanks Chris McDonough for the report.
-
-* Add optional --setuptools option to be able to switch to it in case
- distribute is the default (like in Debian).
-
-1.8.2 (2012-09-06)
-------------------
-
-* Updated the included pip version to 1.2.1 to fix regressions introduced
- there in 1.2.
-
-
-1.8.1 (2012-09-03)
-------------------
-
-* Fixed distribute version used with `--never-download`. Thanks michr for
- report and patch.
-
-* Fix creating Python 3.3 based virtualenvs by unsetting the
- ``__PYVENV_LAUNCHER__`` environment variable in subprocesses.
-
-
-1.8 (2012-09-01)
-----------------
-
-* **Dropped support for Python 2.4** The minimum supported Python version is
- now Python 2.5.
-
-* Fix `--relocatable` on systems that use lib64. Fixes #78. Thanks Branden
- Rolston.
-
-* Symlink some additional modules under Python 3. Fixes #194. Thanks Vinay
- Sajip, Ian Clelland, and Stefan Holek for the report.
-
-* Fix ``--relocatable`` when a script uses ``__future__`` imports. Thanks
- Branden Rolston.
-
-* Fix a bug in the config option parser that prevented setting negative
- options with environment variables. Thanks Ralf Schmitt.
-
-* Allow setting ``--no-site-packages`` from the config file.
-
-* Use ``/usr/bin/multiarch-platform`` if available to figure out the include
- directory. Thanks for the patch, Mika Laitio.
-
-* Fix ``install_name_tool`` replacement to work on Python 3.X.
-
-* Handle paths of users' site-packages on Mac OS X correctly when changing
- the prefix.
-
-* Updated the embedded version of distribute to 0.6.28 and pip to 1.2.
-
-
-1.7.2 (2012-06-22)
-------------------
-
-* Updated to distribute 0.6.27.
-
-* Fix activate.fish on OS X. Fixes #8. Thanks David Schoonover.
-
-* Create a virtualenv-x.x script with the Python version when installing, so
- virtualenv for multiple Python versions can be installed to the same
- script location. Thanks Miki Tebeka.
-
-* Restored ability to create a virtualenv with a path longer than 78
- characters, without breaking creation of virtualenvs with non-ASCII paths.
- Thanks, Bradley Ayers.
-
-* Added ability to create virtualenvs without having installed Apple's
- developers tools (using an own implementation of ``install_name_tool``).
- Thanks Mike Hommey.
-
-* Fixed PyPy and Jython support on Windows. Thanks Konstantin Zemlyak.
-
-* Added pydoc script to ease use. Thanks Marc Abramowitz. Fixes #149.
-
-* Fixed creating a bootstrap script on Python 3. Thanks Raul Leal. Fixes #280.
-
-* Fixed inconsistency when having set the ``PYTHONDONTWRITEBYTECODE`` env var
- with the --distribute option or the ``VIRTUALENV_USE_DISTRIBUTE`` env var.
- ``VIRTUALENV_USE_DISTRIBUTE`` is now considered again as a legacy alias.
-
-
-1.7.1.2 (2012-02-17)
---------------------
-
-* Fixed minor issue in `--relocatable`. Thanks, Cap Petschulat.
-
-
-1.7.1.1 (2012-02-16)
---------------------
-
-* Bumped the version string in ``virtualenv.py`` up, too.
-
-* Fixed rST rendering bug of long description.
-
-
-1.7.1 (2012-02-16)
-------------------
-
-* Update embedded pip to version 1.1.
-
-* Fix `--relocatable` under Python 3. Thanks Doug Hellmann.
-
-* Added environ PATH modification to activate_this.py. Thanks Doug
- Napoleone. Fixes #14.
-
-* Support creating virtualenvs directly from a Python build directory on
- Windows. Thanks CBWhiz. Fixes #139.
-
-* Use non-recursive symlinks to fix things up for posix_local install
- scheme. Thanks michr.
-
-* Made activate script available for use with msys and cygwin on Windows.
- Thanks Greg Haskins, Cliff Xuan, Jonathan Griffin and Doug Napoleone.
- Fixes #176.
-
-* Fixed creation of virtualenvs on Windows when Python is not installed for
- all users. Thanks Anatoly Techtonik for report and patch and Doug
- Napoleone for testing and confirmation. Fixes #87.
-
-* Fixed creation of virtualenvs using -p in installs where some modules
- that ought to be in the standard library (e.g. `readline`) are actually
- installed in `site-packages` next to `virtualenv.py`. Thanks Greg Haskins
- for report and fix. Fixes #167.
-
-* Added activation script for Powershell (signed by Jannis Leidel). Many
- thanks to Jason R. Coombs.
-
-
-1.7 (2011-11-30)
-----------------
-
-* Gave user-provided ``--extra-search-dir`` priority over default dirs for
- finding setuptools/distribute (it already had priority for finding pip).
- Thanks Ethan Jucovy.
-
-* Updated embedded Distribute release to 0.6.24. Thanks Alex Gronholm.
-
-* Made ``--no-site-packages`` behavior the default behavior. The
- ``--no-site-packages`` flag is still permitted, but displays a warning when
- used. Thanks Chris McDonough.
-
-* New flag: ``--system-site-packages``; this flag should be passed to get the
- previous default global-site-package-including behavior back.
-
-* Added ability to set command options as environment variables and options
- in a ``virtualenv.ini`` file.
-
-* Fixed various encoding related issues with paths. Thanks Gunnlaugur Thor Briem.
-
-* Made ``virtualenv.py`` script executable.
-
-
-1.6.4 (2011-07-21)
-------------------
-
-* Restored ability to run on Python 2.4, too.
-
-
-1.6.3 (2011-07-16)
-------------------
-
-* Restored ability to run on Python < 2.7.
-
-
-1.6.2 (2011-07-16)
-------------------
-
-* Updated embedded distribute release to 0.6.19.
-
-* Updated embedded pip release to 1.0.2.
-
-* Fixed #141 - Be smarter about finding pkg_resources when using the
- non-default Python interpreter (by using the ``-p`` option).
-
-* Fixed #112 - Fixed path in docs.
-
-* Fixed #109 - Corrected doctests of a Logger method.
-
-* Fixed #118 - Fixed creating virtualenvs on platforms that use the
- "posix_local" install scheme, such as Ubuntu with Python 2.7.
-
-* Add missing library to Python 3 virtualenvs (``_dummy_thread``).
-
-
-1.6.1 (2011-04-30)
-------------------
-
-* Start to use git-flow.
-
-* Added support for PyPy 1.5
-
-* Fixed #121 -- added sanity-checking of the -p argument. Thanks Paul Nasrat.
-
-* Added progress meter for pip installation as well as setuptools. Thanks Ethan
- Jucovy.
-
-* Added --never-download and --search-dir options. Thanks Ethan Jucovy.
-
-
-1.6
----
-
-* Added Python 3 support! Huge thanks to Vinay Sajip and Vitaly Babiy.
-
-* Fixed creation of virtualenvs on Mac OS X when standard library modules
- (readline) are installed outside the standard library.
-
-* Updated bundled pip to 1.0.
-
-
-1.5.2
------
-
-* Moved main repository to Github: https://github.com/pypa/virtualenv
-
-* Transferred primary maintenance from Ian to Jannis Leidel, Carl Meyer and Brian Rosner
-
-* Fixed a few more pypy related bugs.
-
-* Updated bundled pip to 0.8.2.
-
-* Handed project over to new team of maintainers.
-
-* Moved virtualenv to Github at https://github.com/pypa/virtualenv
-
-
-1.5.1
------
-
-* Added ``_weakrefset`` requirement for Python 2.7.1.
-
-* Fixed Windows regression in 1.5
-
-
-1.5
----
-
-* Include pip 0.8.1.
-
-* Add support for PyPy.
-
-* Uses a proper temporary dir when installing environment requirements.
-
-* Add ``--prompt`` option to be able to override the default prompt prefix.
-
-* Fix an issue with ``--relocatable`` on Windows.
-
-* Fix issue with installing the wrong version of distribute.
-
-* Add fish and csh activate scripts.
-
-
-1.4.9
------
-
-* Include pip 0.7.2
-
-
-1.4.8
------
-
-* Fix for Mac OS X Framework builds that use
- ``--universal-archs=intel``
-
-* Fix ``activate_this.py`` on Windows.
-
-* Allow ``$PYTHONHOME`` to be set, so long as you use ``source
- bin/activate`` it will get unset; if you leave it set and do not
- activate the environment it will still break the environment.
-
-* Include pip 0.7.1
-
-
-1.4.7
------
-
-* Include pip 0.7
-
-
-1.4.6
------
-
-* Allow ``activate.sh`` to skip updating the prompt (by setting
- ``$VIRTUAL_ENV_DISABLE_PROMPT``).
-
-
-1.4.5
------
-
-* Include pip 0.6.3
-
-* Fix ``activate.bat`` and ``deactivate.bat`` under Windows when
- ``PATH`` contained a parenthesis
-
-
-1.4.4
------
-
-* Include pip 0.6.2 and Distribute 0.6.10
-
-* Create the ``virtualenv`` script even when Setuptools isn't
- installed
-
-* Fix problem with ``virtualenv --relocate`` when ``bin/`` has
- subdirectories (e.g., ``bin/.svn/``); from Alan Franzoni.
-
-* If you set ``$VIRTUALENV_DISTRIBUTE`` then virtualenv will use
- Distribute by default (so you don't have to remember to use
- ``--distribute``).
-
-
-1.4.3
------
-
-* Include pip 0.6.1
-
-
-1.4.2
------
-
-* Fix pip installation on Windows
-
-* Fix use of stand-alone ``virtualenv.py`` (and boot scripts)
-
-* Exclude ~/.local (user site-packages) from environments when using
- ``--no-site-packages``
-
-
-1.4.1
------
-
-* Include pip 0.6
-
-
-1.4
----
-
-* Updated setuptools to 0.6c11
-
-* Added the --distribute option
-
-* Fixed packaging problem of support-files
-
-
-1.3.4
------
-
-* Virtualenv now copies the actual embedded Python binary on
- Mac OS X to fix a hang on Snow Leopard (10.6).
-
-* Fail more gracefully on Windows when ``win32api`` is not installed.
-
-* Fix site-packages taking precedent over Jython's ``__classpath__``
- and also specially handle the new ``__pyclasspath__`` entry in
- ``sys.path``.
-
-* Now copies Jython's ``registry`` file to the virtualenv if it exists.
-
-* Better find libraries when compiling extensions on Windows.
-
-* Create ``Scripts\pythonw.exe`` on Windows.
-
-* Added support for the Debian/Ubuntu
- ``/usr/lib/pythonX.Y/dist-packages`` directory.
-
-* Set ``distutils.sysconfig.get_config_vars()['LIBDIR']`` (based on
- ``sys.real_prefix``) which is reported to help building on Windows.
-
-* Make ``deactivate`` work on ksh
-
-* Fixes for ``--python``: make it work with ``--relocatable`` and the
- symlink created to the exact Python version.
-
-
-1.3.3
------
-
-* Use Windows newlines in ``activate.bat``, which has been reported to help
- when using non-ASCII directory names.
-
-* Fixed compatibility with Jython 2.5b1.
-
-* Added a function ``virtualenv.install_python`` for more fine-grained
- access to what ``virtualenv.create_environment`` does.
-
-* Fix `a problem <https://bugs.launchpad.net/virtualenv/+bug/241581>`_
- with Windows and paths that contain spaces.
-
-* If ``/path/to/env/.pydistutils.cfg`` exists (or
- ``/path/to/env/pydistutils.cfg`` on Windows systems) then ignore
- ``~/.pydistutils.cfg`` and use that other file instead.
-
-* Fix ` a problem
- <https://bugs.launchpad.net/virtualenv/+bug/340050>`_ picking up
- some ``.so`` libraries in ``/usr/local``.
-
-
-1.3.2
------
-
-* Remove the ``[install] prefix = ...`` setting from the virtualenv
- ``distutils.cfg`` -- this has been causing problems for a lot of
- people, in rather obscure ways.
-
-* If you use a boot script it will attempt to import ``virtualenv``
- and find a pre-downloaded Setuptools egg using that.
-
-* Added platform-specific paths, like ``/usr/lib/pythonX.Y/plat-linux2``
-
-
-1.3.1
------
-
-* Real Python 2.6 compatibility. Backported the Python 2.6 updates to
- ``site.py``, including `user directories
- <http://docs.python.org/dev/whatsnew/2.6.html#pep-370-per-user-site-packages-directory>`_
- (this means older versions of Python will support user directories,
- whether intended or not).
-
-* Always set ``[install] prefix`` in ``distutils.cfg`` -- previously
- on some platforms where a system-wide ``distutils.cfg`` was present
- with a ``prefix`` setting, packages would be installed globally
- (usually in ``/usr/local/lib/pythonX.Y/site-packages``).
-
-* Sometimes Cygwin seems to leave ``.exe`` off ``sys.executable``; a
- workaround is added.
-
-* Fix ``--python`` option.
-
-* Fixed handling of Jython environments that use a
- jython-complete.jar.
-
-
-1.3
----
-
-* Update to Setuptools 0.6c9
-* Added an option ``virtualenv --relocatable EXISTING_ENV``, which
- will make an existing environment "relocatable" -- the paths will
- not be absolute in scripts, ``.egg-info`` and ``.pth`` files. This
- may assist in building environments that can be moved and copied.
- You have to run this *after* any new packages installed.
-* Added ``bin/activate_this.py``, a file you can use like
- ``execfile("path_to/activate_this.py",
- dict(__file__="path_to/activate_this.py"))`` -- this will activate
- the environment in place, similar to what `the mod_wsgi example
- does <http://code.google.com/p/modwsgi/wiki/VirtualEnvironments>`_.
-* For Mac framework builds of Python, the site-packages directory
- ``/Library/Python/X.Y/site-packages`` is added to ``sys.path``, from
- Andrea Rech.
-* Some platform-specific modules in Macs are added to the path now
- (``plat-darwin/``, ``plat-mac/``, ``plat-mac/lib-scriptpackages``),
- from Andrea Rech.
-* Fixed a small Bashism in the ``bin/activate`` shell script.
-* Added ``__future__`` to the list of required modules, for Python
- 2.3. You'll still need to backport your own ``subprocess`` module.
-* Fixed the ``__classpath__`` entry in Jython's ``sys.path`` taking
- precedent over virtualenv's libs.
-
-
-1.2
----
-
-* Added a ``--python`` option to select the Python interpreter.
-* Add ``warnings`` to the modules copied over, for Python 2.6 support.
-* Add ``sets`` to the module copied over for Python 2.3 (though Python
- 2.3 still probably doesn't work).
-
-
-1.1.1
------
-
-* Added support for Jython 2.5.
-
-
-1.1
----
-
-* Added support for Python 2.6.
-* Fix a problem with missing ``DLLs/zlib.pyd`` on Windows. Create
-* ``bin/python`` (or ``bin/python.exe``) even when you run virtualenv
- with an interpreter named, e.g., ``python2.4``
-* Fix MacPorts Python
-* Added --unzip-setuptools option
-* Update to Setuptools 0.6c8
-* If the current directory is not writable, run ez_setup.py in ``/tmp``
-* Copy or symlink over the ``include`` directory so that packages will
- more consistently compile.
-
-
-1.0
----
-
-* Fix build on systems that use ``/usr/lib64``, distinct from
- ``/usr/lib`` (specifically CentOS x64).
-* Fixed bug in ``--clear``.
-* Fixed typos in ``deactivate.bat``.
-* Preserve ``$PYTHONPATH`` when calling subprocesses.
-
-
-0.9.2
------
-
-* Fix include dir copying on Windows (makes compiling possible).
-* Include the main ``lib-tk`` in the path.
-* Patch ``distutils.sysconfig``: ``get_python_inc`` and
- ``get_python_lib`` to point to the global locations.
-* Install ``distutils.cfg`` before Setuptools, so that system
- customizations of ``distutils.cfg`` won't effect the installation.
-* Add ``bin/pythonX.Y`` to the virtualenv (in addition to
- ``bin/python``).
-* Fixed an issue with Mac Framework Python builds, and absolute paths
- (from Ronald Oussoren).
-
-
-0.9.1
------
-
-* Improve ability to create a virtualenv from inside a virtualenv.
-* Fix a little bug in ``bin/activate``.
-* Actually get ``distutils.cfg`` to work reliably.
-
-
-0.9
----
-
-* Added ``lib-dynload`` and ``config`` to things that need to be
- copied over in an environment.
-* Copy over or symlink the ``include`` directory, so that you can
- build packages that need the C headers.
-* Include a ``distutils`` package, so you can locally update
- ``distutils.cfg`` (in ``lib/pythonX.Y/distutils/distutils.cfg``).
-* Better avoid downloading Setuptools, and hitting PyPI on environment
- creation.
-* Fix a problem creating a ``lib64/`` directory.
-* Should work on MacOSX Framework builds (the default Python
- installations on Mac). Thanks to Ronald Oussoren.
-
-
-0.8.4
------
-
-* Windows installs would sometimes give errors about ``sys.prefix`` that
- were inaccurate.
-* Slightly prettier output.
-
-
-0.8.3
------
-
-* Added support for Windows.
-
-
-0.8.2
------
-
-* Give a better warning if you are on an unsupported platform (Mac
- Framework Pythons, and Windows).
-* Give error about running while inside a workingenv.
-* Give better error message about Python 2.3.
-
-
-0.8.1
------
-
-Fixed packaging of the library.
-
-
-0.8
----
-
-Initial release. Everything is changed and new!
diff --git a/testing/mozharness/external_tools/virtualenv/docs/conf.py b/testing/mozharness/external_tools/virtualenv/docs/conf.py
deleted file mode 100644
index 9332aa1bc..000000000
--- a/testing/mozharness/external_tools/virtualenv/docs/conf.py
+++ /dev/null
@@ -1,153 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Paste documentation build configuration file, created by
-# sphinx-quickstart on Tue Apr 22 22:08:49 2008.
-#
-# This file is execfile()d with the current directory set to its containing dir.
-#
-# The contents of this file are pickled, so don't put values in the namespace
-# that aren't pickleable (module imports are okay, they're removed automatically).
-#
-# All configuration values have a default value; values that are commented out
-# serve to show the default value.
-
-import os
-import sys
-
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-
-# If your extensions are in another directory, add it here.
-sys.path.insert(0, os.path.abspath(os.pardir))
-
-# General configuration
-# ---------------------
-
-# Add any Sphinx extension module names here, as strings. They can be extensions
-# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.extlinks']
-
-# Add any paths that contain templates here, relative to this directory.
-#templates_path = ['_templates']
-
-# The suffix of source filenames.
-source_suffix = '.rst'
-
-# The master toctree document.
-master_doc = 'index'
-
-# General substitutions.
-project = 'virtualenv'
-copyright = '2007-2014, Ian Bicking, The Open Planning Project, PyPA'
-
-# The default replacements for |version| and |release|, also used in various
-# other places throughout the built documents.
-try:
- from virtualenv import __version__
- # The short X.Y version.
- version = '.'.join(__version__.split('.')[:2])
- # The full version, including alpha/beta/rc tags.
- release = __version__
-except ImportError:
- version = release = 'dev'
-
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
-#today = ''
-# Else, today_fmt is used as the format for a strftime call.
-today_fmt = '%B %d, %Y'
-
-# List of documents that shouldn't be included in the build.
-unused_docs = []
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
-
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-#add_module_names = True
-
-# If true, sectionauthor and moduleauthor directives will be shown in the
-# output. They are ignored by default.
-#show_authors = False
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-extlinks = {
- 'issue': ('https://github.com/pypa/virtualenv/issues/%s', '#'),
- 'pull': ('https://github.com/pypa/virtualenv/pull/%s', 'PR #'),
-}
-
-
-# Options for HTML output
-# -----------------------
-
-# The style sheet to use for HTML and HTML Help pages. A file of that name
-# must exist either in Sphinx' static/ path, or in one of the custom paths
-# given in html_static_path.
-#html_style = 'default.css'
-
-html_theme = 'default'
-if not on_rtd:
- try:
- import sphinx_rtd_theme
- html_theme = 'sphinx_rtd_theme'
- html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
- except ImportError:
- pass
-
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-# html_static_path = ['_static']
-
-# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
-# using the given strftime format.
-html_last_updated_fmt = '%b %d, %Y'
-
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-#html_use_smartypants = True
-
-# Content template for the index page.
-#html_index = ''
-
-# Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
-
-# Additional templates that should be rendered to pages, maps page names to
-# template names.
-#html_additional_pages = {}
-
-# If false, no module index is generated.
-#html_use_modindex = True
-
-# If true, the reST sources are included in the HTML build as _sources/<name>.
-#html_copy_source = True
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'Pastedoc'
-
-
-# Options for LaTeX output
-# ------------------------
-
-# The paper size ('letter' or 'a4').
-#latex_paper_size = 'letter'
-
-# The font size ('10pt', '11pt' or '12pt').
-#latex_font_size = '10pt'
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title, author, document class [howto/manual]).
-#latex_documents = []
-
-# Additional stuff for the LaTeX preamble.
-#latex_preamble = ''
-
-# Documents to append as an appendix to all manuals.
-#latex_appendices = []
-
-# If false, no module index is generated.
-#latex_use_modindex = True
diff --git a/testing/mozharness/external_tools/virtualenv/docs/development.rst b/testing/mozharness/external_tools/virtualenv/docs/development.rst
deleted file mode 100644
index aba2785a3..000000000
--- a/testing/mozharness/external_tools/virtualenv/docs/development.rst
+++ /dev/null
@@ -1,61 +0,0 @@
-Development
-===========
-
-Contributing
-------------
-
-Refer to the `pip development`_ documentation - it applies equally to
-virtualenv, except that virtualenv issues should filed on the `virtualenv
-repo`_ at GitHub.
-
-Virtualenv's release schedule is tied to pip's -- each time there's a new pip
-release, there will be a new virtualenv release that bundles the new version of
-pip.
-
-Files in the `virtualenv_embedded/` subdirectory are embedded into
-`virtualenv.py` itself as base64-encoded strings (in order to support
-single-file use of `virtualenv.py` without installing it). If your patch
-changes any file in `virtualenv_embedded/`, run `bin/rebuild-script.py` to
-update the embedded version of that file in `virtualenv.py`; commit that and
-submit it as part of your patch / pull request.
-
-.. _pip development: http://www.pip-installer.org/en/latest/development.html
-.. _virtualenv repo: https://github.com/pypa/virtualenv/
-
-Running the tests
------------------
-
-Virtualenv's test suite is small and not yet at all comprehensive, but we aim
-to grow it.
-
-The easy way to run tests (handles test dependencies automatically)::
-
- $ python setup.py test
-
-If you want to run only a selection of the tests, you'll need to run them
-directly with pytest instead. Create a virtualenv, and install required
-packages::
-
- $ pip install pytest mock
-
-Run pytest::
-
- $ pytest
-
-Or select just a single test file to run::
-
- $ pytest tests/test_virtualenv
-
-Status and License
-------------------
-
-``virtualenv`` is a successor to `workingenv
-<http://cheeseshop.python.org/pypi/workingenv.py>`_, and an extension
-of `virtual-python
-<http://peak.telecommunity.com/DevCenter/EasyInstall#creating-a-virtual-python>`_.
-
-It was written by Ian Bicking, sponsored by the `Open Planning
-Project <http://openplans.org>`_ and is now maintained by a
-`group of developers <https://github.com/pypa/virtualenv/raw/master/AUTHORS.txt>`_.
-It is licensed under an
-`MIT-style permissive license <https://github.com/pypa/virtualenv/raw/master/LICENSE.txt>`_.
diff --git a/testing/mozharness/external_tools/virtualenv/docs/index.rst b/testing/mozharness/external_tools/virtualenv/docs/index.rst
deleted file mode 100644
index e745a87b7..000000000
--- a/testing/mozharness/external_tools/virtualenv/docs/index.rst
+++ /dev/null
@@ -1,137 +0,0 @@
-Virtualenv
-==========
-
-`Mailing list <http://groups.google.com/group/python-virtualenv>`_ |
-`Issues <https://github.com/pypa/virtualenv/issues>`_ |
-`Github <https://github.com/pypa/virtualenv>`_ |
-`PyPI <https://pypi.python.org/pypi/virtualenv/>`_ |
-User IRC: #pypa
-Dev IRC: #pypa-dev
-
-Introduction
-------------
-
-``virtualenv`` is a tool to create isolated Python environments.
-
-The basic problem being addressed is one of dependencies and versions,
-and indirectly permissions. Imagine you have an application that
-needs version 1 of LibFoo, but another application requires version
-2. How can you use both these applications? If you install
-everything into ``/usr/lib/python2.7/site-packages`` (or whatever your
-platform's standard location is), it's easy to end up in a situation
-where you unintentionally upgrade an application that shouldn't be
-upgraded.
-
-Or more generally, what if you want to install an application *and
-leave it be*? If an application works, any change in its libraries or
-the versions of those libraries can break the application.
-
-Also, what if you can't install packages into the global
-``site-packages`` directory? For instance, on a shared host.
-
-In all these cases, ``virtualenv`` can help you. It creates an
-environment that has its own installation directories, that doesn't
-share libraries with other virtualenv environments (and optionally
-doesn't access the globally installed libraries either).
-
-.. comment: split here
-
-.. toctree::
- :maxdepth: 2
-
- installation
- userguide
- reference
- development
- changes
-
-.. warning::
-
- Python bugfix releases 2.6.8, 2.7.3, 3.1.5 and 3.2.3 include a change that
- will cause "import random" to fail with "cannot import name urandom" on any
- virtualenv created on a Unix host with an earlier release of Python
- 2.6/2.7/3.1/3.2, if the underlying system Python is upgraded. This is due to
- the fact that a virtualenv uses the system Python's standard library but
- contains its own copy of the Python interpreter, so an upgrade to the system
- Python results in a mismatch between the version of the Python interpreter
- and the version of the standard library. It can be fixed by removing
- ``$ENV/bin/python`` and re-running virtualenv on the same target directory
- with the upgraded Python.
-
-Other Documentation and Links
------------------------------
-
-* `Blog announcement of virtualenv`__.
-
- .. __: http://blog.ianbicking.org/2007/10/10/workingenv-is-dead-long-live-virtualenv/
-
-* James Gardner has written a tutorial on using `virtualenv with
- Pylons
- <http://wiki.pylonshq.com/display/pylonscookbook/Using+a+Virtualenv+Sandbox>`_.
-
-* Chris Perkins created a `showmedo video including virtualenv
- <http://showmedo.com/videos/video?name=2910000&fromSeriesID=291>`_.
-
-* Doug Hellmann's `virtualenvwrapper`_ is a useful set of scripts to make
- your workflow with many virtualenvs even easier. `His initial blog post on it`__.
- He also wrote `an example of using virtualenv to try IPython`__.
-
- .. _virtualenvwrapper: https://pypi.python.org/pypi/virtualenvwrapper/
- .. __: https://doughellmann.com/blog/2008/05/01/virtualenvwrapper/
- .. __: https://doughellmann.com/blog/2008/02/01/ipython-and-virtualenv/
-
-* `Pew`_ is another wrapper for virtualenv that makes use of a different
- activation technique.
-
- .. _Pew: https://pypi.python.org/pypi/pew/
-
-* `Using virtualenv with mod_wsgi
- <http://code.google.com/p/modwsgi/wiki/VirtualEnvironments>`_.
-
-* `virtualenv commands
- <https://github.com/thisismedium/virtualenv-commands>`_ for some more
- workflow-related tools around virtualenv.
-
-* PyCon US 2011 talk: `Reverse-engineering Ian Bicking's brain: inside pip and virtualenv
- <http://pyvideo.org/video/568/reverse-engineering-ian-bicking--39-s-brain--insi>`_.
- By the end of the talk, you'll have a good idea exactly how pip
- and virtualenv do their magic, and where to go looking in the source
- for particular behaviors or bug fixes.
-
-Compare & Contrast with Alternatives
-------------------------------------
-
-There are several alternatives that create isolated environments:
-
-* ``workingenv`` (which I do not suggest you use anymore) is the
- predecessor to this library. It used the main Python interpreter,
- but relied on setting ``$PYTHONPATH`` to activate the environment.
- This causes problems when running Python scripts that aren't part of
- the environment (e.g., a globally installed ``hg`` or ``bzr``). It
- also conflicted a lot with Setuptools.
-
-* `virtual-python
- <http://peak.telecommunity.com/DevCenter/EasyInstall#creating-a-virtual-python>`_
- is also a predecessor to this library. It uses only symlinks, so it
- couldn't work on Windows. It also symlinks over the *entire*
- standard library and global ``site-packages``. As a result, it
- won't see new additions to the global ``site-packages``.
-
- This script only symlinks a small portion of the standard library
- into the environment, and so on Windows it is feasible to simply
- copy these files over. Also, it creates a new/empty
- ``site-packages`` and also adds the global ``site-packages`` to the
- path, so updates are tracked separately. This script also installs
- Setuptools automatically, saving a step and avoiding the need for
- network access.
-
-* `zc.buildout <http://pypi.python.org/pypi/zc.buildout>`_ doesn't
- create an isolated Python environment in the same style, but
- achieves similar results through a declarative config file that sets
- up scripts with very particular packages. As a declarative system,
- it is somewhat easier to repeat and manage, but more difficult to
- experiment with. ``zc.buildout`` includes the ability to setup
- non-Python systems (e.g., a database server or an Apache instance).
-
-I *strongly* recommend anyone doing application development or
-deployment use one of these tools.
diff --git a/testing/mozharness/external_tools/virtualenv/docs/installation.rst b/testing/mozharness/external_tools/virtualenv/docs/installation.rst
deleted file mode 100644
index 3006d7617..000000000
--- a/testing/mozharness/external_tools/virtualenv/docs/installation.rst
+++ /dev/null
@@ -1,58 +0,0 @@
-Installation
-============
-
-.. warning::
-
- We advise installing virtualenv-1.9 or greater. Prior to version 1.9, the
- pip included in virtualenv did not download from PyPI over SSL.
-
-.. warning::
-
- When using pip to install virtualenv, we advise using pip 1.3 or greater.
- Prior to version 1.3, pip did not download from PyPI over SSL.
-
-.. warning::
-
- We advise against using easy_install to install virtualenv when using
- setuptools < 0.9.7, because easy_install didn't download from PyPI over SSL
- and was broken in some subtle ways.
-
-To install globally with `pip` (if you have pip 1.3 or greater installed globally):
-
-::
-
- $ [sudo] pip install virtualenv
-
-Or to get the latest unreleased dev version:
-
-::
-
- $ [sudo] pip install https://github.com/pypa/virtualenv/tarball/develop
-
-
-To install version X.X globally from source:
-
-::
-
- $ curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-X.X.tar.gz
- $ tar xvfz virtualenv-X.X.tar.gz
- $ cd virtualenv-X.X
- $ [sudo] python setup.py install
-
-
-To *use* locally from source:
-
-::
-
- $ curl -O https://pypi.python.org/packages/source/v/virtualenv/virtualenv-X.X.tar.gz
- $ tar xvfz virtualenv-X.X.tar.gz
- $ cd virtualenv-X.X
- $ python virtualenv.py myVE
-
-.. note::
-
- The ``virtualenv.py`` script is *not* supported if run without the
- necessary pip/setuptools/virtualenv distributions available locally. All
- of the installation methods above include a ``virtualenv_support``
- directory alongside ``virtualenv.py`` which contains a complete set of
- pip and setuptools distributions, and so are fully supported.
diff --git a/testing/mozharness/external_tools/virtualenv/docs/make.bat b/testing/mozharness/external_tools/virtualenv/docs/make.bat
deleted file mode 100644
index aa5c189fc..000000000
--- a/testing/mozharness/external_tools/virtualenv/docs/make.bat
+++ /dev/null
@@ -1,170 +0,0 @@
-@ECHO OFF
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
- set SPHINXBUILD=sphinx-build
-)
-set BUILDDIR=_build
-set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
-if NOT "%PAPER%" == "" (
- set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
-)
-
-if "%1" == "" goto help
-
-if "%1" == "help" (
- :help
- echo.Please use `make ^<target^>` where ^<target^> is one of
- echo. html to make standalone HTML files
- echo. dirhtml to make HTML files named index.html in directories
- echo. singlehtml to make a single large HTML file
- echo. pickle to make pickle files
- echo. json to make JSON files
- echo. htmlhelp to make HTML files and a HTML help project
- echo. qthelp to make HTML files and a qthelp project
- echo. devhelp to make HTML files and a Devhelp project
- echo. epub to make an epub
- echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
- echo. text to make text files
- echo. man to make manual pages
- echo. changes to make an overview over all changed/added/deprecated items
- echo. linkcheck to check all external links for integrity
- echo. doctest to run all doctests embedded in the documentation if enabled
- goto end
-)
-
-if "%1" == "clean" (
- for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
- del /q /s %BUILDDIR%\*
- goto end
-)
-
-if "%1" == "html" (
- %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
- if errorlevel 1 exit /b 1
- echo.
- echo.Build finished. The HTML pages are in %BUILDDIR%/html.
- goto end
-)
-
-if "%1" == "dirhtml" (
- %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
- if errorlevel 1 exit /b 1
- echo.
- echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
- goto end
-)
-
-if "%1" == "singlehtml" (
- %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
- if errorlevel 1 exit /b 1
- echo.
- echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
- goto end
-)
-
-if "%1" == "pickle" (
- %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
- if errorlevel 1 exit /b 1
- echo.
- echo.Build finished; now you can process the pickle files.
- goto end
-)
-
-if "%1" == "json" (
- %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
- if errorlevel 1 exit /b 1
- echo.
- echo.Build finished; now you can process the JSON files.
- goto end
-)
-
-if "%1" == "htmlhelp" (
- %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
- if errorlevel 1 exit /b 1
- echo.
- echo.Build finished; now you can run HTML Help Workshop with the ^
-.hhp project file in %BUILDDIR%/htmlhelp.
- goto end
-)
-
-if "%1" == "qthelp" (
- %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
- if errorlevel 1 exit /b 1
- echo.
- echo.Build finished; now you can run "qcollectiongenerator" with the ^
-.qhcp project file in %BUILDDIR%/qthelp, like this:
- echo.^> qcollectiongenerator %BUILDDIR%\qthelp\django-compressor.qhcp
- echo.To view the help file:
- echo.^> assistant -collectionFile %BUILDDIR%\qthelp\django-compressor.ghc
- goto end
-)
-
-if "%1" == "devhelp" (
- %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
- if errorlevel 1 exit /b 1
- echo.
- echo.Build finished.
- goto end
-)
-
-if "%1" == "epub" (
- %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
- if errorlevel 1 exit /b 1
- echo.
- echo.Build finished. The epub file is in %BUILDDIR%/epub.
- goto end
-)
-
-if "%1" == "latex" (
- %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
- if errorlevel 1 exit /b 1
- echo.
- echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
- goto end
-)
-
-if "%1" == "text" (
- %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
- if errorlevel 1 exit /b 1
- echo.
- echo.Build finished. The text files are in %BUILDDIR%/text.
- goto end
-)
-
-if "%1" == "man" (
- %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
- if errorlevel 1 exit /b 1
- echo.
- echo.Build finished. The manual pages are in %BUILDDIR%/man.
- goto end
-)
-
-if "%1" == "changes" (
- %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
- if errorlevel 1 exit /b 1
- echo.
- echo.The overview file is in %BUILDDIR%/changes.
- goto end
-)
-
-if "%1" == "linkcheck" (
- %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
- if errorlevel 1 exit /b 1
- echo.
- echo.Link check complete; look for any errors in the above output ^
-or in %BUILDDIR%/linkcheck/output.txt.
- goto end
-)
-
-if "%1" == "doctest" (
- %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
- if errorlevel 1 exit /b 1
- echo.
- echo.Testing of doctests in the sources finished, look at the ^
-results in %BUILDDIR%/doctest/output.txt.
- goto end
-)
-
-:end
diff --git a/testing/mozharness/external_tools/virtualenv/docs/reference.rst b/testing/mozharness/external_tools/virtualenv/docs/reference.rst
deleted file mode 100644
index 9249473c9..000000000
--- a/testing/mozharness/external_tools/virtualenv/docs/reference.rst
+++ /dev/null
@@ -1,261 +0,0 @@
-Reference Guide
-===============
-
-``virtualenv`` Command
-----------------------
-
-.. _usage:
-
-Usage
-~~~~~
-
-:command:`virtualenv [OPTIONS] ENV_DIR`
-
- Where ``ENV_DIR`` is an absolute or relative path to a directory to create
- the virtual environment in.
-
-.. _options:
-
-Options
-~~~~~~~
-
-.. program: virtualenv
-
-.. option:: --version
-
- show program's version number and exit
-
-.. option:: -h, --help
-
- show this help message and exit
-
-.. option:: -v, --verbose
-
- Increase verbosity.
-
-.. option:: -q, --quiet
-
- Decrease verbosity.
-
-.. option:: -p PYTHON_EXE, --python=PYTHON_EXE
-
- The Python interpreter to use, e.g.,
- --python=python2.5 will use the python2.5 interpreter
- to create the new environment. The default is the
- interpreter that virtualenv was installed with
- (like ``/usr/bin/python``)
-
-.. option:: --clear
-
- Clear out the non-root install and start from scratch.
-
-.. option:: --system-site-packages
-
- Give the virtual environment access to the global
- site-packages.
-
-.. option:: --always-copy
-
- Always copy files rather than symlinking.
-
-.. option:: --relocatable
-
- Make an EXISTING virtualenv environment relocatable.
- This fixes up scripts and makes all .pth files relative.
-
-.. option:: --unzip-setuptools
-
- Unzip Setuptools when installing it.
-
-.. option:: --no-setuptools
-
- Do not install setuptools in the new virtualenv.
-
-.. option:: --no-pip
-
- Do not install pip in the new virtualenv.
-
-.. option:: --no-wheel
-
- Do not install wheel in the new virtualenv.
-
-.. option:: --extra-search-dir=DIR
-
- Directory to look for setuptools/pip distributions in.
- This option can be specified multiple times.
-
-.. option:: --prompt=PROMPT
-
- Provides an alternative prompt prefix for this
- environment.
-
-.. option:: --download
-
- Download preinstalled packages from PyPI.
-
-.. option:: --no-download
-
- Do not download preinstalled packages from PyPI.
-
-.. option:: --no-site-packages
-
- DEPRECATED. Retained only for backward compatibility.
- Not having access to global site-packages is now the
- default behavior.
-
-.. option:: --distribute
-.. option:: --setuptools
-
- Legacy; now have no effect. Before version 1.10 these could be used
- to choose whether to install Distribute_ or Setuptools_ into the created
- virtualenv. Distribute has now been merged into Setuptools, and the
- latter is always installed.
-
-.. _Distribute: https://pypi.python.org/pypi/distribute
-.. _Setuptools: https://pypi.python.org/pypi/setuptools
-
-
-Configuration
--------------
-
-Environment Variables
-~~~~~~~~~~~~~~~~~~~~~
-
-Each command line option is automatically used to look for environment
-variables with the name format ``VIRTUALENV_<UPPER_NAME>``. That means
-the name of the command line options are capitalized and have dashes
-(``'-'``) replaced with underscores (``'_'``).
-
-For example, to automatically use a custom Python binary instead of the
-one virtualenv is run with you can also set an environment variable::
-
- $ export VIRTUALENV_PYTHON=/opt/python-3.3/bin/python
- $ virtualenv ENV
-
-It's the same as passing the option to virtualenv directly::
-
- $ virtualenv --python=/opt/python-3.3/bin/python ENV
-
-This also works for appending command line options, like ``--find-links``.
-Just leave an empty space between the passed values, e.g.::
-
- $ export VIRTUALENV_EXTRA_SEARCH_DIR="/path/to/dists /path/to/other/dists"
- $ virtualenv ENV
-
-is the same as calling::
-
- $ virtualenv --extra-search-dir=/path/to/dists --extra-search-dir=/path/to/other/dists ENV
-
-.. envvar:: VIRTUAL_ENV_DISABLE_PROMPT
-
- Any virtualenv created when this is set to a non-empty value will not have
- it's :ref:`activate` modify the shell prompt.
-
-
-Configuration File
-~~~~~~~~~~~~~~~~~~
-
-virtualenv also looks for a standard ini config file. On Unix and Mac OS X
-that's ``$HOME/.virtualenv/virtualenv.ini`` and on Windows, it's
-``%APPDATA%\virtualenv\virtualenv.ini``.
-
-The names of the settings are derived from the long command line option,
-e.g. the option :option:`--python <-p>` would look like this::
-
- [virtualenv]
- python = /opt/python-3.3/bin/python
-
-Appending options like :option:`--extra-search-dir` can be written on multiple
-lines::
-
- [virtualenv]
- extra-search-dir =
- /path/to/dists
- /path/to/other/dists
-
-Please have a look at the output of :option:`--help <-h>` for a full list
-of supported options.
-
-
-Extending Virtualenv
---------------------
-
-
-Creating Your Own Bootstrap Scripts
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-While this creates an environment, it doesn't put anything into the
-environment. Developers may find it useful to distribute a script
-that sets up a particular environment, for example a script that
-installs a particular web application.
-
-To create a script like this, call
-:py:func:`virtualenv.create_bootstrap_script`, and write the
-result to your new bootstrapping script.
-
-.. py:function:: create_bootstrap_script(extra_text)
-
- Creates a bootstrap script from ``extra_text``, which is like
- this script but with extend_parser, adjust_options, and after_install hooks.
-
-This returns a string that (written to disk of course) can be used
-as a bootstrap script with your own customizations. The script
-will be the standard virtualenv.py script, with your extra text
-added (your extra text should be Python code).
-
-If you include these functions, they will be called:
-
-.. py:function:: extend_parser(optparse_parser)
-
- You can add or remove options from the parser here.
-
-.. py:function:: adjust_options(options, args)
-
- You can change options here, or change the args (if you accept
- different kinds of arguments, be sure you modify ``args`` so it is
- only ``[DEST_DIR]``).
-
-.. py:function:: after_install(options, home_dir)
-
- After everything is installed, this function is called. This
- is probably the function you are most likely to use. An
- example would be::
-
- def after_install(options, home_dir):
- if sys.platform == 'win32':
- bin = 'Scripts'
- else:
- bin = 'bin'
- subprocess.call([join(home_dir, bin, 'easy_install'),
- 'MyPackage'])
- subprocess.call([join(home_dir, bin, 'my-package-script'),
- 'setup', home_dir])
-
- This example immediately installs a package, and runs a setup
- script from that package.
-
-Bootstrap Example
-~~~~~~~~~~~~~~~~~
-
-Here's a more concrete example of how you could use this::
-
- import virtualenv, textwrap
- output = virtualenv.create_bootstrap_script(textwrap.dedent("""
- import os, subprocess
- def after_install(options, home_dir):
- etc = join(home_dir, 'etc')
- if not os.path.exists(etc):
- os.makedirs(etc)
- subprocess.call([join(home_dir, 'bin', 'easy_install'),
- 'BlogApplication'])
- subprocess.call([join(home_dir, 'bin', 'paster'),
- 'make-config', 'BlogApplication',
- join(etc, 'blog.ini')])
- subprocess.call([join(home_dir, 'bin', 'paster'),
- 'setup-app', join(etc, 'blog.ini')])
- """))
- f = open('blog-bootstrap.py', 'w').write(output)
-
-Another example is available `here`__.
-
-.. __: https://github.com/socialplanning/fassembler/blob/master/fassembler/create-venv-script.py
diff --git a/testing/mozharness/external_tools/virtualenv/docs/userguide.rst b/testing/mozharness/external_tools/virtualenv/docs/userguide.rst
deleted file mode 100644
index 35f0dc950..000000000
--- a/testing/mozharness/external_tools/virtualenv/docs/userguide.rst
+++ /dev/null
@@ -1,258 +0,0 @@
-User Guide
-==========
-
-
-Usage
------
-
-Virtualenv has one basic command::
-
- $ virtualenv ENV
-
-Where ``ENV`` is a directory to place the new virtual environment. It has
-a number of usual effects (modifiable by many :ref:`options`):
-
- - :file:`ENV/lib/` and :file:`ENV/include/` are created, containing supporting
- library files for a new virtualenv python. Packages installed in this
- environment will live under :file:`ENV/lib/pythonX.X/site-packages/`.
-
- - :file:`ENV/bin` is created, where executables live - noticeably a new
- :command:`python`. Thus running a script with ``#! /path/to/ENV/bin/python``
- would run that script under this virtualenv's python.
-
- - The crucial packages pip_ and setuptools_ are installed, which allow other
- packages to be easily installed to the environment. This associated pip
- can be run from :file:`ENV/bin/pip`.
-
-The python in your new virtualenv is effectively isolated from the python that
-was used to create it.
-
-.. _pip: https://pypi.python.org/pypi/pip
-.. _setuptools: https://pypi.python.org/pypi/setuptools
-
-
-.. _activate:
-
-activate script
-~~~~~~~~~~~~~~~
-
-In a newly created virtualenv there will also be a :command:`activate` shell
-script. For Windows systems, activation scripts are provided for
-the Command Prompt and Powershell.
-
-On Posix systems, this resides in :file:`/ENV/bin/`, so you can run::
-
- $ source bin/activate
-
-For some shells (e.g. the original Bourne Shell) you may need to use the
-:command:`.` command, when :command:`source` does not exist. There are also
-separate activate files for some other shells, like csh and fish.
-:file:`bin/activate` should work for bash/zsh/dash.
-
-This will change your ``$PATH`` so its first entry is the virtualenv's
-``bin/`` directory. (You have to use ``source`` because it changes your
-shell environment in-place.) This is all it does; it's purely a
-convenience. If you directly run a script or the python interpreter
-from the virtualenv's ``bin/`` directory (e.g. ``path/to/ENV/bin/pip``
-or ``/path/to/ENV/bin/python-script.py``) there's no need for
-activation.
-
-The ``activate`` script will also modify your shell prompt to indicate
-which environment is currently active. To disable this behaviour, see
-:envvar:`VIRTUAL_ENV_DISABLE_PROMPT`.
-
-To undo these changes to your path (and prompt), just run::
-
- $ deactivate
-
-On Windows, the equivalent `activate` script is in the ``Scripts`` folder::
-
- > \path\to\env\Scripts\activate
-
-And type ``deactivate`` to undo the changes.
-
-Based on your active shell (CMD.exe or Powershell.exe), Windows will use
-either activate.bat or activate.ps1 (as appropriate) to activate the
-virtual environment. If using Powershell, see the notes about code signing
-below.
-
-.. note::
-
- If using Powershell, the ``activate`` script is subject to the
- `execution policies`_ on the system. By default on Windows 7, the system's
- excution policy is set to ``Restricted``, meaning no scripts like the
- ``activate`` script are allowed to be executed. But that can't stop us
- from changing that slightly to allow it to be executed.
-
- In order to use the script, you can relax your system's execution
- policy to ``AllSigned``, meaning all scripts on the system must be
- digitally signed to be executed. Since the virtualenv activation
- script is signed by one of the authors (Jannis Leidel) this level of
- the execution policy suffices. As an administrator run::
-
- PS C:\> Set-ExecutionPolicy AllSigned
-
- Then you'll be asked to trust the signer, when executing the script.
- You will be prompted with the following::
-
- PS C:\> virtualenv .\foo
- New python executable in C:\foo\Scripts\python.exe
- Installing setuptools................done.
- Installing pip...................done.
- PS C:\> .\foo\scripts\activate
-
- Do you want to run software from this untrusted publisher?
- File C:\foo\scripts\activate.ps1 is published by E=jannis@leidel.info,
- CN=Jannis Leidel, L=Berlin, S=Berlin, C=DE, Description=581796-Gh7xfJxkxQSIO4E0
- and is not trusted on your system. Only run scripts from trusted publishers.
- [V] Never run [D] Do not run [R] Run once [A] Always run [?] Help
- (default is "D"):A
- (foo) PS C:\>
-
- If you select ``[A] Always Run``, the certificate will be added to the
- Trusted Publishers of your user account, and will be trusted in this
- user's context henceforth. If you select ``[R] Run Once``, the script will
- be run, but you will be prometed on a subsequent invocation. Advanced users
- can add the signer's certificate to the Trusted Publishers of the Computer
- account to apply to all users (though this technique is out of scope of this
- document).
-
- Alternatively, you may relax the system execution policy to allow running
- of local scripts without verifying the code signature using the following::
-
- PS C:\> Set-ExecutionPolicy RemoteSigned
-
- Since the ``activate.ps1`` script is generated locally for each virtualenv,
- it is not considered a remote script and can then be executed.
-
-.. _`execution policies`: http://technet.microsoft.com/en-us/library/dd347641.aspx
-
-Removing an Environment
-~~~~~~~~~~~~~~~~~~~~~~~
-
-Removing a virtual environment is simply done by deactivating it and deleting the
-environment folder with all its contents::
-
- (ENV)$ deactivate
- $ rm -r /path/to/ENV
-
-The :option:`--system-site-packages` Option
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If you build with ``virtualenv --system-site-packages ENV``, your virtual
-environment will inherit packages from ``/usr/lib/python2.7/site-packages``
-(or wherever your global site-packages directory is).
-
-This can be used if you have control over the global site-packages directory,
-and you want to depend on the packages there. If you want isolation from the
-global system, do not use this flag.
-
-Windows Notes
-~~~~~~~~~~~~~
-
-Some paths within the virtualenv are slightly different on Windows: scripts and
-executables on Windows go in ``ENV\Scripts\`` instead of ``ENV/bin/`` and
-libraries go in ``ENV\Lib\`` rather than ``ENV/lib/``.
-
-To create a virtualenv under a path with spaces in it on Windows, you'll need
-the `win32api <http://sourceforge.net/projects/pywin32/>`_ library installed.
-
-
-Using Virtualenv without ``bin/python``
----------------------------------------
-
-Sometimes you can't or don't want to use the Python interpreter
-created by the virtualenv. For instance, in a `mod_python
-<http://www.modpython.org/>`_ or `mod_wsgi <http://www.modwsgi.org/>`_
-environment, there is only one interpreter.
-
-Luckily, it's easy. You must use the custom Python interpreter to
-*install* libraries. But to *use* libraries, you just have to be sure
-the path is correct. A script is available to correct the path. You
-can setup the environment like::
-
- activate_this = '/path/to/env/bin/activate_this.py'
- execfile(activate_this, dict(__file__=activate_this))
-
-This will change ``sys.path`` and even change ``sys.prefix``, but also allow
-you to use an existing interpreter. Items in your environment will show up
-first on ``sys.path``, before global items. However, global items will
-always be accessible (as if the :option:`--system-site-packages` flag had been
-used in creating the environment, whether it was or not). Also, this cannot undo
-the activation of other environments, or modules that have been imported.
-You shouldn't try to, for instance, activate an environment before a web
-request; you should activate *one* environment as early as possible, and not
-do it again in that process.
-
-Making Environments Relocatable
--------------------------------
-
-**Note:** this option is somewhat experimental, and there are probably
-caveats that have not yet been identified.
-
-.. warning::
-
- The ``--relocatable`` option currently has a number of issues,
- and is not guaranteed to work in all circumstances. It is possible
- that the option will be deprecated in a future version of ``virtualenv``.
-
-Normally environments are tied to a specific path. That means that
-you cannot move an environment around or copy it to another computer.
-You can fix up an environment to make it relocatable with the
-command::
-
- $ virtualenv --relocatable ENV
-
-This will make some of the files created by setuptools use relative paths,
-and will change all the scripts to use ``activate_this.py`` instead of using
-the location of the Python interpreter to select the environment.
-
-**Note:** scripts which have been made relocatable will only work if
-the virtualenv is activated, specifically the python executable from
-the virtualenv must be the first one on the system PATH. Also note that
-the activate scripts are not currently made relocatable by
-``virtualenv --relocatable``.
-
-**Note:** you must run this after you've installed *any* packages into
-the environment. If you make an environment relocatable, then
-install a new package, you must run ``virtualenv --relocatable``
-again.
-
-Also, this **does not make your packages cross-platform**. You can
-move the directory around, but it can only be used on other similar
-computers. Some known environmental differences that can cause
-incompatibilities: a different version of Python, when one platform
-uses UCS2 for its internal unicode representation and another uses
-UCS4 (a compile-time option), obvious platform changes like Windows
-vs. Linux, or Intel vs. ARM, and if you have libraries that bind to C
-libraries on the system, if those C libraries are located somewhere
-different (either different versions, or a different filesystem
-layout).
-
-If you use this flag to create an environment, currently, the
-:option:`--system-site-packages` option will be implied.
-
-The :option:`--extra-search-dir` option
----------------------------------------
-
-This option allows you to provide your own versions of setuptools and/or
-pip to use instead of the embedded versions that come with virtualenv.
-
-To use this feature, pass one or more ``--extra-search-dir`` options to
-virtualenv like this::
-
- $ virtualenv --extra-search-dir=/path/to/distributions ENV
-
-The ``/path/to/distributions`` path should point to a directory that contains
-setuptools and/or pip wheels.
-
-virtualenv will look for wheels in the specified directories, but will use
-pip's standard algorithm for selecting the wheel to install, which looks for
-the latest compatible wheel.
-
-As well as the extra directories, the search order includes:
-
-#. The ``virtualenv_support`` directory relative to virtualenv.py
-#. The directory where virtualenv.py is located.
-#. The current directory.
-
diff --git a/testing/mozharness/external_tools/virtualenv/scripts/virtualenv b/testing/mozharness/external_tools/virtualenv/scripts/virtualenv
deleted file mode 100644
index c961dd7db..000000000
--- a/testing/mozharness/external_tools/virtualenv/scripts/virtualenv
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env python
-import virtualenv
-virtualenv.main()
diff --git a/testing/mozharness/external_tools/virtualenv/setup.cfg b/testing/mozharness/external_tools/virtualenv/setup.cfg
deleted file mode 100644
index 6662fa569..000000000
--- a/testing/mozharness/external_tools/virtualenv/setup.cfg
+++ /dev/null
@@ -1,8 +0,0 @@
-[bdist_wheel]
-universal = 1
-
-[egg_info]
-tag_date = 0
-tag_build =
-tag_svn_revision = 0
-
diff --git a/testing/mozharness/external_tools/virtualenv/setup.py b/testing/mozharness/external_tools/virtualenv/setup.py
deleted file mode 100644
index ee03bc531..000000000
--- a/testing/mozharness/external_tools/virtualenv/setup.py
+++ /dev/null
@@ -1,123 +0,0 @@
-import os
-import re
-import shutil
-import sys
-
-if sys.version_info[:2] < (2, 6):
- sys.exit('virtualenv requires Python 2.6 or higher.')
-
-try:
- from setuptools import setup
- from setuptools.command.test import test as TestCommand
-
- class PyTest(TestCommand):
- user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")]
-
- def initialize_options(self):
- TestCommand.initialize_options(self)
- self.pytest_args = []
-
- def finalize_options(self):
- TestCommand.finalize_options(self)
- #self.test_args = []
- #self.test_suite = True
-
- def run_tests(self):
- # import here, because outside the eggs aren't loaded
- import pytest
- sys.exit(pytest.main(self.pytest_args))
-
- setup_params = {
- 'entry_points': {
- 'console_scripts': ['virtualenv=virtualenv:main'],
- },
- 'zip_safe': False,
- 'cmdclass': {'test': PyTest},
- 'tests_require': ['pytest', 'mock'],
- }
-except ImportError:
- from distutils.core import setup
- if sys.platform == 'win32':
- print('Note: without Setuptools installed you will '
- 'have to use "python -m virtualenv ENV"')
- setup_params = {}
- else:
- script = 'scripts/virtualenv'
- setup_params = {'scripts': [script]}
-
-
-def read_file(*paths):
- here = os.path.dirname(os.path.abspath(__file__))
- with open(os.path.join(here, *paths)) as f:
- return f.read()
-
-# Get long_description from index.rst:
-long_description = read_file('docs', 'index.rst')
-long_description = long_description.strip().split('split here', 1)[0]
-# Add release history
-changes = read_file('docs', 'changes.rst')
-# Only report last two releases for brevity
-releases_found = 0
-change_lines = []
-for line in changes.splitlines():
- change_lines.append(line)
- if line.startswith('--------------'):
- releases_found += 1
- if releases_found > 2:
- break
-
-changes = '\n'.join(change_lines[:-2]) + '\n'
-changes += '`Full Changelog <https://virtualenv.pypa.io/en/latest/changes.html>`_.'
-# Replace issue/pull directives
-changes = re.sub(r':pull:`(\d+)`', r'PR #\1', changes)
-changes = re.sub(r':issue:`(\d+)`', r'#\1', changes)
-
-long_description += '\n\n' + changes
-
-
-def get_version():
- version_file = read_file('virtualenv.py')
- version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
- version_file, re.M)
- if version_match:
- return version_match.group(1)
- raise RuntimeError("Unable to find version string.")
-
-
-# Hack to prevent stupid TypeError: 'NoneType' object is not callable error on
-# exit of python setup.py test # in multiprocessing/util.py _exit_function when
-# running python setup.py test (see
-# http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html)
-try:
- import multiprocessing # noqa
-except ImportError:
- pass
-
-setup(
- name='virtualenv',
- version=get_version(),
- description="Virtual Python Environment builder",
- long_description=long_description,
- classifiers=[
- 'Development Status :: 5 - Production/Stable',
- 'Intended Audience :: Developers',
- 'License :: OSI Approved :: MIT License',
- 'Programming Language :: Python :: 2',
- 'Programming Language :: Python :: 2.6',
- 'Programming Language :: Python :: 2.7',
- 'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 3.3',
- 'Programming Language :: Python :: 3.4',
- 'Programming Language :: Python :: 3.5',
- ],
- keywords='setuptools deployment installation distutils',
- author='Ian Bicking',
- author_email='ianb@colorstudy.com',
- maintainer='Jannis Leidel, Carl Meyer and Brian Rosner',
- maintainer_email='python-virtualenv@groups.google.com',
- url='https://virtualenv.pypa.io/',
- license='MIT',
- py_modules=['virtualenv'],
- packages=['virtualenv_support'],
- package_data={'virtualenv_support': ['*.whl']},
- **setup_params)
diff --git a/testing/mozharness/external_tools/virtualenv/site.py b/testing/mozharness/external_tools/virtualenv/site.py
deleted file mode 100644
index 4e426cdb6..000000000
--- a/testing/mozharness/external_tools/virtualenv/site.py
+++ /dev/null
@@ -1,760 +0,0 @@
-"""Append module search paths for third-party packages to sys.path.
-
-****************************************************************
-* This module is automatically imported during initialization. *
-****************************************************************
-
-In earlier versions of Python (up to 1.5a3), scripts or modules that
-needed to use site-specific modules would place ``import site''
-somewhere near the top of their code. Because of the automatic
-import, this is no longer necessary (but code that does it still
-works).
-
-This will append site-specific paths to the module search path. On
-Unix, it starts with sys.prefix and sys.exec_prefix (if different) and
-appends lib/python<version>/site-packages as well as lib/site-python.
-It also supports the Debian convention of
-lib/python<version>/dist-packages. On other platforms (mainly Mac and
-Windows), it uses just sys.prefix (and sys.exec_prefix, if different,
-but this is unlikely). The resulting directories, if they exist, are
-appended to sys.path, and also inspected for path configuration files.
-
-FOR DEBIAN, this sys.path is augmented with directories in /usr/local.
-Local addons go into /usr/local/lib/python<version>/site-packages
-(resp. /usr/local/lib/site-python), Debian addons install into
-/usr/{lib,share}/python<version>/dist-packages.
-
-A path configuration file is a file whose name has the form
-<package>.pth; its contents are additional directories (one per line)
-to be added to sys.path. Non-existing directories (or
-non-directories) are never added to sys.path; no directory is added to
-sys.path more than once. Blank lines and lines beginning with
-'#' are skipped. Lines starting with 'import' are executed.
-
-For example, suppose sys.prefix and sys.exec_prefix are set to
-/usr/local and there is a directory /usr/local/lib/python2.X/site-packages
-with three subdirectories, foo, bar and spam, and two path
-configuration files, foo.pth and bar.pth. Assume foo.pth contains the
-following:
-
- # foo package configuration
- foo
- bar
- bletch
-
-and bar.pth contains:
-
- # bar package configuration
- bar
-
-Then the following directories are added to sys.path, in this order:
-
- /usr/local/lib/python2.X/site-packages/bar
- /usr/local/lib/python2.X/site-packages/foo
-
-Note that bletch is omitted because it doesn't exist; bar precedes foo
-because bar.pth comes alphabetically before foo.pth; and spam is
-omitted because it is not mentioned in either path configuration file.
-
-After these path manipulations, an attempt is made to import a module
-named sitecustomize, which can perform arbitrary additional
-site-specific customizations. If this import fails with an
-ImportError exception, it is silently ignored.
-
-"""
-
-import sys
-import os
-
-try:
- import __builtin__ as builtins
-except ImportError:
- import builtins
-try:
- set
-except NameError:
- from sets import Set as set
-
-# Prefixes for site-packages; add additional prefixes like /usr/local here
-PREFIXES = [sys.prefix, sys.exec_prefix]
-# Enable per user site-packages directory
-# set it to False to disable the feature or True to force the feature
-ENABLE_USER_SITE = None
-# for distutils.commands.install
-USER_SITE = None
-USER_BASE = None
-
-_is_64bit = (getattr(sys, 'maxsize', None) or getattr(sys, 'maxint')) > 2**32
-_is_pypy = hasattr(sys, 'pypy_version_info')
-_is_jython = sys.platform[:4] == 'java'
-if _is_jython:
- ModuleType = type(os)
-
-def makepath(*paths):
- dir = os.path.join(*paths)
- if _is_jython and (dir == '__classpath__' or
- dir.startswith('__pyclasspath__')):
- return dir, dir
- dir = os.path.abspath(dir)
- return dir, os.path.normcase(dir)
-
-def abs__file__():
- """Set all module' __file__ attribute to an absolute path"""
- for m in sys.modules.values():
- if ((_is_jython and not isinstance(m, ModuleType)) or
- hasattr(m, '__loader__')):
- # only modules need the abspath in Jython. and don't mess
- # with a PEP 302-supplied __file__
- continue
- f = getattr(m, '__file__', None)
- if f is None:
- continue
- m.__file__ = os.path.abspath(f)
-
-def removeduppaths():
- """ Remove duplicate entries from sys.path along with making them
- absolute"""
- # This ensures that the initial path provided by the interpreter contains
- # only absolute pathnames, even if we're running from the build directory.
- L = []
- known_paths = set()
- for dir in sys.path:
- # Filter out duplicate paths (on case-insensitive file systems also
- # if they only differ in case); turn relative paths into absolute
- # paths.
- dir, dircase = makepath(dir)
- if not dircase in known_paths:
- L.append(dir)
- known_paths.add(dircase)
- sys.path[:] = L
- return known_paths
-
-# XXX This should not be part of site.py, since it is needed even when
-# using the -S option for Python. See http://www.python.org/sf/586680
-def addbuilddir():
- """Append ./build/lib.<platform> in case we're running in the build dir
- (especially for Guido :-)"""
- from distutils.util import get_platform
- s = "build/lib.%s-%.3s" % (get_platform(), sys.version)
- if hasattr(sys, 'gettotalrefcount'):
- s += '-pydebug'
- s = os.path.join(os.path.dirname(sys.path[-1]), s)
- sys.path.append(s)
-
-def _init_pathinfo():
- """Return a set containing all existing directory entries from sys.path"""
- d = set()
- for dir in sys.path:
- try:
- if os.path.isdir(dir):
- dir, dircase = makepath(dir)
- d.add(dircase)
- except TypeError:
- continue
- return d
-
-def addpackage(sitedir, name, known_paths):
- """Add a new path to known_paths by combining sitedir and 'name' or execute
- sitedir if it starts with 'import'"""
- if known_paths is None:
- _init_pathinfo()
- reset = 1
- else:
- reset = 0
- fullname = os.path.join(sitedir, name)
- try:
- f = open(fullname, "rU")
- except IOError:
- return
- try:
- for line in f:
- if line.startswith("#"):
- continue
- if line.startswith("import"):
- exec(line)
- continue
- line = line.rstrip()
- dir, dircase = makepath(sitedir, line)
- if not dircase in known_paths and os.path.exists(dir):
- sys.path.append(dir)
- known_paths.add(dircase)
- finally:
- f.close()
- if reset:
- known_paths = None
- return known_paths
-
-def addsitedir(sitedir, known_paths=None):
- """Add 'sitedir' argument to sys.path if missing and handle .pth files in
- 'sitedir'"""
- if known_paths is None:
- known_paths = _init_pathinfo()
- reset = 1
- else:
- reset = 0
- sitedir, sitedircase = makepath(sitedir)
- if not sitedircase in known_paths:
- sys.path.append(sitedir) # Add path component
- try:
- names = os.listdir(sitedir)
- except os.error:
- return
- names.sort()
- for name in names:
- if name.endswith(os.extsep + "pth"):
- addpackage(sitedir, name, known_paths)
- if reset:
- known_paths = None
- return known_paths
-
-def addsitepackages(known_paths, sys_prefix=sys.prefix, exec_prefix=sys.exec_prefix):
- """Add site-packages (and possibly site-python) to sys.path"""
- prefixes = [os.path.join(sys_prefix, "local"), sys_prefix]
- if exec_prefix != sys_prefix:
- prefixes.append(os.path.join(exec_prefix, "local"))
-
- for prefix in prefixes:
- if prefix:
- if sys.platform in ('os2emx', 'riscos') or _is_jython:
- sitedirs = [os.path.join(prefix, "Lib", "site-packages")]
- elif _is_pypy:
- sitedirs = [os.path.join(prefix, 'site-packages')]
- elif sys.platform == 'darwin' and prefix == sys_prefix:
-
- if prefix.startswith("/System/Library/Frameworks/"): # Apple's Python
-
- sitedirs = [os.path.join("/Library/Python", sys.version[:3], "site-packages"),
- os.path.join(prefix, "Extras", "lib", "python")]
-
- else: # any other Python distros on OSX work this way
- sitedirs = [os.path.join(prefix, "lib",
- "python" + sys.version[:3], "site-packages")]
-
- elif os.sep == '/':
- sitedirs = [os.path.join(prefix,
- "lib",
- "python" + sys.version[:3],
- "site-packages"),
- os.path.join(prefix, "lib", "site-python"),
- os.path.join(prefix, "python" + sys.version[:3], "lib-dynload")]
- lib64_dir = os.path.join(prefix, "lib64", "python" + sys.version[:3], "site-packages")
- if (os.path.exists(lib64_dir) and
- os.path.realpath(lib64_dir) not in [os.path.realpath(p) for p in sitedirs]):
- if _is_64bit:
- sitedirs.insert(0, lib64_dir)
- else:
- sitedirs.append(lib64_dir)
- try:
- # sys.getobjects only available in --with-pydebug build
- sys.getobjects
- sitedirs.insert(0, os.path.join(sitedirs[0], 'debug'))
- except AttributeError:
- pass
- # Debian-specific dist-packages directories:
- sitedirs.append(os.path.join(prefix, "local/lib",
- "python" + sys.version[:3],
- "dist-packages"))
- if sys.version[0] == '2':
- sitedirs.append(os.path.join(prefix, "lib",
- "python" + sys.version[:3],
- "dist-packages"))
- else:
- sitedirs.append(os.path.join(prefix, "lib",
- "python" + sys.version[0],
- "dist-packages"))
- sitedirs.append(os.path.join(prefix, "lib", "dist-python"))
- else:
- sitedirs = [prefix, os.path.join(prefix, "lib", "site-packages")]
- if sys.platform == 'darwin':
- # for framework builds *only* we add the standard Apple
- # locations. Currently only per-user, but /Library and
- # /Network/Library could be added too
- if 'Python.framework' in prefix:
- home = os.environ.get('HOME')
- if home:
- sitedirs.append(
- os.path.join(home,
- 'Library',
- 'Python',
- sys.version[:3],
- 'site-packages'))
- for sitedir in sitedirs:
- if os.path.isdir(sitedir):
- addsitedir(sitedir, known_paths)
- return None
-
-def check_enableusersite():
- """Check if user site directory is safe for inclusion
-
- The function tests for the command line flag (including environment var),
- process uid/gid equal to effective uid/gid.
-
- None: Disabled for security reasons
- False: Disabled by user (command line option)
- True: Safe and enabled
- """
- if hasattr(sys, 'flags') and getattr(sys.flags, 'no_user_site', False):
- return False
-
- if hasattr(os, "getuid") and hasattr(os, "geteuid"):
- # check process uid == effective uid
- if os.geteuid() != os.getuid():
- return None
- if hasattr(os, "getgid") and hasattr(os, "getegid"):
- # check process gid == effective gid
- if os.getegid() != os.getgid():
- return None
-
- return True
-
-def addusersitepackages(known_paths):
- """Add a per user site-package to sys.path
-
- Each user has its own python directory with site-packages in the
- home directory.
-
- USER_BASE is the root directory for all Python versions
-
- USER_SITE is the user specific site-packages directory
-
- USER_SITE/.. can be used for data.
- """
- global USER_BASE, USER_SITE, ENABLE_USER_SITE
- env_base = os.environ.get("PYTHONUSERBASE", None)
-
- def joinuser(*args):
- return os.path.expanduser(os.path.join(*args))
-
- #if sys.platform in ('os2emx', 'riscos'):
- # # Don't know what to put here
- # USER_BASE = ''
- # USER_SITE = ''
- if os.name == "nt":
- base = os.environ.get("APPDATA") or "~"
- if env_base:
- USER_BASE = env_base
- else:
- USER_BASE = joinuser(base, "Python")
- USER_SITE = os.path.join(USER_BASE,
- "Python" + sys.version[0] + sys.version[2],
- "site-packages")
- else:
- if env_base:
- USER_BASE = env_base
- else:
- USER_BASE = joinuser("~", ".local")
- USER_SITE = os.path.join(USER_BASE, "lib",
- "python" + sys.version[:3],
- "site-packages")
-
- if ENABLE_USER_SITE and os.path.isdir(USER_SITE):
- addsitedir(USER_SITE, known_paths)
- if ENABLE_USER_SITE:
- for dist_libdir in ("lib", "local/lib"):
- user_site = os.path.join(USER_BASE, dist_libdir,
- "python" + sys.version[:3],
- "dist-packages")
- if os.path.isdir(user_site):
- addsitedir(user_site, known_paths)
- return known_paths
-
-
-
-def setBEGINLIBPATH():
- """The OS/2 EMX port has optional extension modules that do double duty
- as DLLs (and must use the .DLL file extension) for other extensions.
- The library search path needs to be amended so these will be found
- during module import. Use BEGINLIBPATH so that these are at the start
- of the library search path.
-
- """
- dllpath = os.path.join(sys.prefix, "Lib", "lib-dynload")
- libpath = os.environ['BEGINLIBPATH'].split(';')
- if libpath[-1]:
- libpath.append(dllpath)
- else:
- libpath[-1] = dllpath
- os.environ['BEGINLIBPATH'] = ';'.join(libpath)
-
-
-def setquit():
- """Define new built-ins 'quit' and 'exit'.
- These are simply strings that display a hint on how to exit.
-
- """
- if os.sep == ':':
- eof = 'Cmd-Q'
- elif os.sep == '\\':
- eof = 'Ctrl-Z plus Return'
- else:
- eof = 'Ctrl-D (i.e. EOF)'
-
- class Quitter(object):
- def __init__(self, name):
- self.name = name
- def __repr__(self):
- return 'Use %s() or %s to exit' % (self.name, eof)
- def __call__(self, code=None):
- # Shells like IDLE catch the SystemExit, but listen when their
- # stdin wrapper is closed.
- try:
- sys.stdin.close()
- except:
- pass
- raise SystemExit(code)
- builtins.quit = Quitter('quit')
- builtins.exit = Quitter('exit')
-
-
-class _Printer(object):
- """interactive prompt objects for printing the license text, a list of
- contributors and the copyright notice."""
-
- MAXLINES = 23
-
- def __init__(self, name, data, files=(), dirs=()):
- self.__name = name
- self.__data = data
- self.__files = files
- self.__dirs = dirs
- self.__lines = None
-
- def __setup(self):
- if self.__lines:
- return
- data = None
- for dir in self.__dirs:
- for filename in self.__files:
- filename = os.path.join(dir, filename)
- try:
- fp = open(filename, "rU")
- data = fp.read()
- fp.close()
- break
- except IOError:
- pass
- if data:
- break
- if not data:
- data = self.__data
- self.__lines = data.split('\n')
- self.__linecnt = len(self.__lines)
-
- def __repr__(self):
- self.__setup()
- if len(self.__lines) <= self.MAXLINES:
- return "\n".join(self.__lines)
- else:
- return "Type %s() to see the full %s text" % ((self.__name,)*2)
-
- def __call__(self):
- self.__setup()
- prompt = 'Hit Return for more, or q (and Return) to quit: '
- lineno = 0
- while 1:
- try:
- for i in range(lineno, lineno + self.MAXLINES):
- print(self.__lines[i])
- except IndexError:
- break
- else:
- lineno += self.MAXLINES
- key = None
- while key is None:
- try:
- key = raw_input(prompt)
- except NameError:
- key = input(prompt)
- if key not in ('', 'q'):
- key = None
- if key == 'q':
- break
-
-def setcopyright():
- """Set 'copyright' and 'credits' in __builtin__"""
- builtins.copyright = _Printer("copyright", sys.copyright)
- if _is_jython:
- builtins.credits = _Printer(
- "credits",
- "Jython is maintained by the Jython developers (www.jython.org).")
- elif _is_pypy:
- builtins.credits = _Printer(
- "credits",
- "PyPy is maintained by the PyPy developers: http://pypy.org/")
- else:
- builtins.credits = _Printer("credits", """\
- Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
- for supporting Python development. See www.python.org for more information.""")
- here = os.path.dirname(os.__file__)
- builtins.license = _Printer(
- "license", "See http://www.python.org/%.3s/license.html" % sys.version,
- ["LICENSE.txt", "LICENSE"],
- [os.path.join(here, os.pardir), here, os.curdir])
-
-
-class _Helper(object):
- """Define the built-in 'help'.
- This is a wrapper around pydoc.help (with a twist).
-
- """
-
- def __repr__(self):
- return "Type help() for interactive help, " \
- "or help(object) for help about object."
- def __call__(self, *args, **kwds):
- import pydoc
- return pydoc.help(*args, **kwds)
-
-def sethelper():
- builtins.help = _Helper()
-
-def aliasmbcs():
- """On Windows, some default encodings are not provided by Python,
- while they are always available as "mbcs" in each locale. Make
- them usable by aliasing to "mbcs" in such a case."""
- if sys.platform == 'win32':
- import locale, codecs
- enc = locale.getdefaultlocale()[1]
- if enc.startswith('cp'): # "cp***" ?
- try:
- codecs.lookup(enc)
- except LookupError:
- import encodings
- encodings._cache[enc] = encodings._unknown
- encodings.aliases.aliases[enc] = 'mbcs'
-
-def setencoding():
- """Set the string encoding used by the Unicode implementation. The
- default is 'ascii', but if you're willing to experiment, you can
- change this."""
- encoding = "ascii" # Default value set by _PyUnicode_Init()
- if 0:
- # Enable to support locale aware default string encodings.
- import locale
- loc = locale.getdefaultlocale()
- if loc[1]:
- encoding = loc[1]
- if 0:
- # Enable to switch off string to Unicode coercion and implicit
- # Unicode to string conversion.
- encoding = "undefined"
- if encoding != "ascii":
- # On Non-Unicode builds this will raise an AttributeError...
- sys.setdefaultencoding(encoding) # Needs Python Unicode build !
-
-
-def execsitecustomize():
- """Run custom site specific code, if available."""
- try:
- import sitecustomize
- except ImportError:
- pass
-
-def virtual_install_main_packages():
- f = open(os.path.join(os.path.dirname(__file__), 'orig-prefix.txt'))
- sys.real_prefix = f.read().strip()
- f.close()
- pos = 2
- hardcoded_relative_dirs = []
- if sys.path[0] == '':
- pos += 1
- if _is_jython:
- paths = [os.path.join(sys.real_prefix, 'Lib')]
- elif _is_pypy:
- if sys.version_info > (3, 2):
- cpyver = '%d' % sys.version_info[0]
- elif sys.pypy_version_info >= (1, 5):
- cpyver = '%d.%d' % sys.version_info[:2]
- else:
- cpyver = '%d.%d.%d' % sys.version_info[:3]
- paths = [os.path.join(sys.real_prefix, 'lib_pypy'),
- os.path.join(sys.real_prefix, 'lib-python', cpyver)]
- if sys.pypy_version_info < (1, 9):
- paths.insert(1, os.path.join(sys.real_prefix,
- 'lib-python', 'modified-%s' % cpyver))
- hardcoded_relative_dirs = paths[:] # for the special 'darwin' case below
- #
- # This is hardcoded in the Python executable, but relative to sys.prefix:
- for path in paths[:]:
- plat_path = os.path.join(path, 'plat-%s' % sys.platform)
- if os.path.exists(plat_path):
- paths.append(plat_path)
- # MOZ: The MSYS2 and MinGW versions of Python have their main packages in the UNIX directory this checks specifically for the native win32 python
- elif sys.platform == 'win32' and os.sep == '\\':
- paths = [os.path.join(sys.real_prefix, 'Lib'), os.path.join(sys.real_prefix, 'DLLs')]
- else:
- paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3])]
- hardcoded_relative_dirs = paths[:] # for the special 'darwin' case below
- lib64_path = os.path.join(sys.real_prefix, 'lib64', 'python'+sys.version[:3])
- if os.path.exists(lib64_path):
- if _is_64bit:
- paths.insert(0, lib64_path)
- else:
- paths.append(lib64_path)
- # This is hardcoded in the Python executable, but relative to
- # sys.prefix. Debian change: we need to add the multiarch triplet
- # here, which is where the real stuff lives. As per PEP 421, in
- # Python 3.3+, this lives in sys.implementation, while in Python 2.7
- # it lives in sys.
- try:
- arch = getattr(sys, 'implementation', sys)._multiarch
- except AttributeError:
- # This is a non-multiarch aware Python. Fallback to the old way.
- arch = sys.platform
- plat_path = os.path.join(sys.real_prefix, 'lib',
- 'python'+sys.version[:3],
- 'plat-%s' % arch)
- if os.path.exists(plat_path):
- paths.append(plat_path)
- # This is hardcoded in the Python executable, but
- # relative to sys.prefix, so we have to fix up:
- for path in list(paths):
- tk_dir = os.path.join(path, 'lib-tk')
- if os.path.exists(tk_dir):
- paths.append(tk_dir)
-
- # These are hardcoded in the Apple's Python executable,
- # but relative to sys.prefix, so we have to fix them up:
- if sys.platform == 'darwin':
- hardcoded_paths = [os.path.join(relative_dir, module)
- for relative_dir in hardcoded_relative_dirs
- for module in ('plat-darwin', 'plat-mac', 'plat-mac/lib-scriptpackages')]
-
- for path in hardcoded_paths:
- if os.path.exists(path):
- paths.append(path)
-
- sys.path.extend(paths)
-
-def force_global_eggs_after_local_site_packages():
- """
- Force easy_installed eggs in the global environment to get placed
- in sys.path after all packages inside the virtualenv. This
- maintains the "least surprise" result that packages in the
- virtualenv always mask global packages, never the other way
- around.
-
- """
- egginsert = getattr(sys, '__egginsert', 0)
- for i, path in enumerate(sys.path):
- if i > egginsert and path.startswith(sys.prefix):
- egginsert = i
- sys.__egginsert = egginsert + 1
-
-def virtual_addsitepackages(known_paths):
- force_global_eggs_after_local_site_packages()
- return addsitepackages(known_paths, sys_prefix=sys.real_prefix)
-
-def fixclasspath():
- """Adjust the special classpath sys.path entries for Jython. These
- entries should follow the base virtualenv lib directories.
- """
- paths = []
- classpaths = []
- for path in sys.path:
- if path == '__classpath__' or path.startswith('__pyclasspath__'):
- classpaths.append(path)
- else:
- paths.append(path)
- sys.path = paths
- sys.path.extend(classpaths)
-
-def execusercustomize():
- """Run custom user specific code, if available."""
- try:
- import usercustomize
- except ImportError:
- pass
-
-
-def main():
- global ENABLE_USER_SITE
- virtual_install_main_packages()
- abs__file__()
- paths_in_sys = removeduppaths()
- if (os.name == "posix" and sys.path and
- os.path.basename(sys.path[-1]) == "Modules"):
- addbuilddir()
- if _is_jython:
- fixclasspath()
- GLOBAL_SITE_PACKAGES = not os.path.exists(os.path.join(os.path.dirname(__file__), 'no-global-site-packages.txt'))
- if not GLOBAL_SITE_PACKAGES:
- ENABLE_USER_SITE = False
- if ENABLE_USER_SITE is None:
- ENABLE_USER_SITE = check_enableusersite()
- paths_in_sys = addsitepackages(paths_in_sys)
- paths_in_sys = addusersitepackages(paths_in_sys)
- if GLOBAL_SITE_PACKAGES:
- paths_in_sys = virtual_addsitepackages(paths_in_sys)
- if sys.platform == 'os2emx':
- setBEGINLIBPATH()
- setquit()
- setcopyright()
- sethelper()
- aliasmbcs()
- setencoding()
- execsitecustomize()
- if ENABLE_USER_SITE:
- execusercustomize()
- # Remove sys.setdefaultencoding() so that users cannot change the
- # encoding after initialization. The test for presence is needed when
- # this module is run as a script, because this code is executed twice.
- if hasattr(sys, "setdefaultencoding"):
- del sys.setdefaultencoding
-
-main()
-
-def _script():
- help = """\
- %s [--user-base] [--user-site]
-
- Without arguments print some useful information
- With arguments print the value of USER_BASE and/or USER_SITE separated
- by '%s'.
-
- Exit codes with --user-base or --user-site:
- 0 - user site directory is enabled
- 1 - user site directory is disabled by user
- 2 - uses site directory is disabled by super user
- or for security reasons
- >2 - unknown error
- """
- args = sys.argv[1:]
- if not args:
- print("sys.path = [")
- for dir in sys.path:
- print(" %r," % (dir,))
- print("]")
- def exists(path):
- if os.path.isdir(path):
- return "exists"
- else:
- return "doesn't exist"
- print("USER_BASE: %r (%s)" % (USER_BASE, exists(USER_BASE)))
- print("USER_SITE: %r (%s)" % (USER_SITE, exists(USER_BASE)))
- print("ENABLE_USER_SITE: %r" % ENABLE_USER_SITE)
- sys.exit(0)
-
- buffer = []
- if '--user-base' in args:
- buffer.append(USER_BASE)
- if '--user-site' in args:
- buffer.append(USER_SITE)
-
- if buffer:
- print(os.pathsep.join(buffer))
- if ENABLE_USER_SITE:
- sys.exit(0)
- elif ENABLE_USER_SITE is False:
- sys.exit(1)
- elif ENABLE_USER_SITE is None:
- sys.exit(2)
- else:
- sys.exit(3)
- else:
- import textwrap
- print(textwrap.dedent(help % (sys.argv[0], os.pathsep)))
- sys.exit(10)
-
-if __name__ == '__main__':
- _script()
diff --git a/testing/mozharness/external_tools/virtualenv/tests/__init__.py b/testing/mozharness/external_tools/virtualenv/tests/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/external_tools/virtualenv/tests/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/external_tools/virtualenv/tests/test_activate.sh b/testing/mozharness/external_tools/virtualenv/tests/test_activate.sh
deleted file mode 100755
index e27727386..000000000
--- a/testing/mozharness/external_tools/virtualenv/tests/test_activate.sh
+++ /dev/null
@@ -1,96 +0,0 @@
-#!/bin/sh
-
-set -u
-
-ROOT="$(dirname $0)/.."
-VIRTUALENV="${ROOT}/virtualenv.py"
-TESTENV="/tmp/test_virtualenv_activate.venv"
-
-rm -rf ${TESTENV}
-
-echo "$0: Creating virtualenv ${TESTENV}..." 1>&2
-
-${VIRTUALENV} ${TESTENV} | tee ${ROOT}/tests/test_activate_output.actual
-if ! diff ${ROOT}/tests/test_activate_output.expected ${ROOT}/tests/test_activate_output.actual; then
- echo "$0: Failed to get expected output from ${VIRTUALENV}!" 1>&2
- exit 1
-fi
-
-echo "$0: Created virtualenv ${TESTENV}." 1>&2
-
-echo "$0: Activating ${TESTENV}..." 1>&2
-. ${TESTENV}/bin/activate
-echo "$0: Activated ${TESTENV}." 1>&2
-
-echo "$0: Checking value of \$VIRTUAL_ENV..." 1>&2
-
-if [ "$VIRTUAL_ENV" != "${TESTENV}" ]; then
- echo "$0: Expected \$VIRTUAL_ENV to be set to \"${TESTENV}\"; actual value: \"${VIRTUAL_ENV}\"!" 1>&2
- exit 2
-fi
-
-echo "$0: \$VIRTUAL_ENV = \"${VIRTUAL_ENV}\" -- OK." 1>&2
-
-echo "$0: Checking output of \$(which python)..." 1>&2
-
-if [ "$(which python)" != "${TESTENV}/bin/python" ]; then
- echo "$0: Expected \$(which python) to return \"${TESTENV}/bin/python\"; actual value: \"$(which python)\"!" 1>&2
- exit 3
-fi
-
-echo "$0: Output of \$(which python) is OK." 1>&2
-
-echo "$0: Checking output of \$(which pip)..." 1>&2
-
-if [ "$(which pip)" != "${TESTENV}/bin/pip" ]; then
- echo "$0: Expected \$(which pip) to return \"${TESTENV}/bin/pip\"; actual value: \"$(which pip)\"!" 1>&2
- exit 4
-fi
-
-echo "$0: Output of \$(which pip) is OK." 1>&2
-
-echo "$0: Checking output of \$(which easy_install)..." 1>&2
-
-if [ "$(which easy_install)" != "${TESTENV}/bin/easy_install" ]; then
- echo "$0: Expected \$(which easy_install) to return \"${TESTENV}/bin/easy_install\"; actual value: \"$(which easy_install)\"!" 1>&2
- exit 5
-fi
-
-echo "$0: Output of \$(which easy_install) is OK." 1>&2
-
-echo "$0: Executing a simple Python program..." 1>&2
-
-TESTENV=${TESTENV} python <<__END__
-import os, sys
-
-expected_site_packages = os.path.join(os.environ['TESTENV'], 'lib','python%s' % sys.version[:3], 'site-packages')
-site_packages = os.path.join(os.environ['VIRTUAL_ENV'], 'lib', 'python%s' % sys.version[:3], 'site-packages')
-
-assert site_packages == expected_site_packages, 'site_packages did not have expected value; actual value: %r' % site_packages
-
-open(os.path.join(site_packages, 'pydoc_test.py'), 'w').write('"""This is pydoc_test.py"""\n')
-__END__
-
-if [ $? -ne 0 ]; then
- echo "$0: Python script failed!" 1>&2
- exit 6
-fi
-
-echo "$0: Execution of a simple Python program -- OK." 1>&2
-
-echo "$0: Testing pydoc..." 1>&2
-
-if ! PAGER=cat pydoc pydoc_test | grep 'This is pydoc_test.py' > /dev/null; then
- echo "$0: pydoc test failed!" 1>&2
- exit 7
-fi
-
-echo "$0: pydoc is OK." 1>&2
-
-echo "$0: Deactivating ${TESTENV}..." 1>&2
-deactivate
-echo "$0: Deactivated ${TESTENV}." 1>&2
-echo "$0: OK!" 1>&2
-
-rm -rf ${TESTENV}
-
diff --git a/testing/mozharness/external_tools/virtualenv/tests/test_activate_output.expected b/testing/mozharness/external_tools/virtualenv/tests/test_activate_output.expected
deleted file mode 100644
index d49469feb..000000000
--- a/testing/mozharness/external_tools/virtualenv/tests/test_activate_output.expected
+++ /dev/null
@@ -1,2 +0,0 @@
-New python executable in /tmp/test_virtualenv_activate.venv/bin/python
-Installing setuptools, pip, wheel...done.
diff --git a/testing/mozharness/external_tools/virtualenv/tests/test_cmdline.py b/testing/mozharness/external_tools/virtualenv/tests/test_cmdline.py
deleted file mode 100644
index 9682ef003..000000000
--- a/testing/mozharness/external_tools/virtualenv/tests/test_cmdline.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import sys
-import subprocess
-import virtualenv
-import pytest
-
-VIRTUALENV_SCRIPT = virtualenv.__file__
-
-def test_commandline_basic(tmpdir):
- """Simple command line usage should work"""
- subprocess.check_call([
- sys.executable,
- VIRTUALENV_SCRIPT,
- str(tmpdir.join('venv'))
- ])
-
-def test_commandline_explicit_interp(tmpdir):
- """Specifying the Python interpreter should work"""
- subprocess.check_call([
- sys.executable,
- VIRTUALENV_SCRIPT,
- '-p', sys.executable,
- str(tmpdir.join('venv'))
- ])
-
-# The registry lookups to support the abbreviated "-p 3.5" form of specifying
-# a Python interpreter on Windows don't seem to work with Python 3.5. The
-# registry layout is not well documented, and it's not clear that the feature
-# is sufficiently widely used to be worth fixing.
-# See https://github.com/pypa/virtualenv/issues/864
-@pytest.mark.skipif("sys.platform == 'win32' and sys.version_info[:2] >= (3,5)")
-def test_commandline_abbrev_interp(tmpdir):
- """Specifying abbreviated forms of the Python interpreter should work"""
- if sys.platform == 'win32':
- fmt = '%s.%s'
- else:
- fmt = 'python%s.%s'
- abbrev = fmt % (sys.version_info[0], sys.version_info[1])
- subprocess.check_call([
- sys.executable,
- VIRTUALENV_SCRIPT,
- '-p', abbrev,
- str(tmpdir.join('venv'))
- ])
-
diff --git a/testing/mozharness/external_tools/virtualenv/tests/test_virtualenv.py b/testing/mozharness/external_tools/virtualenv/tests/test_virtualenv.py
deleted file mode 100644
index 756cde936..000000000
--- a/testing/mozharness/external_tools/virtualenv/tests/test_virtualenv.py
+++ /dev/null
@@ -1,139 +0,0 @@
-import virtualenv
-import optparse
-import os
-import shutil
-import sys
-import tempfile
-import pytest
-import platform # noqa
-
-from mock import patch, Mock
-
-
-def test_version():
- """Should have a version string"""
- assert virtualenv.virtualenv_version, "Should have version"
-
-
-@patch('os.path.exists')
-def test_resolve_interpreter_with_absolute_path(mock_exists):
- """Should return absolute path if given and exists"""
- mock_exists.return_value = True
- virtualenv.is_executable = Mock(return_value=True)
- test_abs_path = os.path.abspath("/usr/bin/python53")
-
- exe = virtualenv.resolve_interpreter(test_abs_path)
-
- assert exe == test_abs_path, "Absolute path should return as is"
- mock_exists.assert_called_with(test_abs_path)
- virtualenv.is_executable.assert_called_with(test_abs_path)
-
-
-@patch('os.path.exists')
-def test_resolve_interpreter_with_nonexistent_interpreter(mock_exists):
- """Should SystemExit with an nonexistent python interpreter path"""
- mock_exists.return_value = False
-
- with pytest.raises(SystemExit):
- virtualenv.resolve_interpreter("/usr/bin/python53")
-
- mock_exists.assert_called_with("/usr/bin/python53")
-
-
-@patch('os.path.exists')
-def test_resolve_interpreter_with_invalid_interpreter(mock_exists):
- """Should exit when with absolute path if not exists"""
- mock_exists.return_value = True
- virtualenv.is_executable = Mock(return_value=False)
- invalid = os.path.abspath("/usr/bin/pyt_hon53")
-
- with pytest.raises(SystemExit):
- virtualenv.resolve_interpreter(invalid)
-
- mock_exists.assert_called_with(invalid)
- virtualenv.is_executable.assert_called_with(invalid)
-
-
-def test_activate_after_future_statements():
- """Should insert activation line after last future statement"""
- script = [
- '#!/usr/bin/env python',
- 'from __future__ import with_statement',
- 'from __future__ import print_function',
- 'print("Hello, world!")'
- ]
- assert virtualenv.relative_script(script) == [
- '#!/usr/bin/env python',
- 'from __future__ import with_statement',
- 'from __future__ import print_function',
- '',
- "import os; activate_this=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'activate_this.py'); exec(compile(open(activate_this).read(), activate_this, 'exec'), dict(__file__=activate_this)); del os, activate_this",
- '',
- 'print("Hello, world!")'
- ]
-
-
-def test_cop_update_defaults_with_store_false():
- """store_false options need reverted logic"""
- class MyConfigOptionParser(virtualenv.ConfigOptionParser):
- def __init__(self, *args, **kwargs):
- self.config = virtualenv.ConfigParser.RawConfigParser()
- self.files = []
- optparse.OptionParser.__init__(self, *args, **kwargs)
-
- def get_environ_vars(self, prefix='VIRTUALENV_'):
- yield ("no_site_packages", "1")
-
- cop = MyConfigOptionParser()
- cop.add_option(
- '--no-site-packages',
- dest='system_site_packages',
- action='store_false',
- help="Don't give access to the global site-packages dir to the "
- "virtual environment (default)")
-
- defaults = {}
- cop.update_defaults(defaults)
- assert defaults == {'system_site_packages': 0}
-
-def test_install_python_bin():
- """Should create the right python executables and links"""
- tmp_virtualenv = tempfile.mkdtemp()
- try:
- home_dir, lib_dir, inc_dir, bin_dir = \
- virtualenv.path_locations(tmp_virtualenv)
- virtualenv.install_python(home_dir, lib_dir, inc_dir, bin_dir, False,
- False)
-
- if virtualenv.is_win:
- required_executables = [ 'python.exe', 'pythonw.exe']
- else:
- py_exe_no_version = 'python'
- py_exe_version_major = 'python%s' % sys.version_info[0]
- py_exe_version_major_minor = 'python%s.%s' % (
- sys.version_info[0], sys.version_info[1])
- required_executables = [ py_exe_no_version, py_exe_version_major,
- py_exe_version_major_minor ]
-
- for pth in required_executables:
- assert os.path.exists(os.path.join(bin_dir, pth)), ("%s should "
- "exist in bin_dir" % pth)
- finally:
- shutil.rmtree(tmp_virtualenv)
-
-
-@pytest.mark.skipif("platform.python_implementation() == 'PyPy'")
-def test_always_copy_option():
- """Should be no symlinks in directory tree"""
- tmp_virtualenv = tempfile.mkdtemp()
- ve_path = os.path.join(tmp_virtualenv, 'venv')
- try:
- virtualenv.create_environment(ve_path, symlink=False)
-
- for root, dirs, files in os.walk(tmp_virtualenv):
- for f in files + dirs:
- full_name = os.path.join(root, f)
- assert not os.path.islink(full_name), "%s should not be a" \
- " symlink (to %s)" % (full_name, os.readlink(full_name))
- finally:
- shutil.rmtree(tmp_virtualenv)
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv.py b/testing/mozharness/external_tools/virtualenv/virtualenv.py
deleted file mode 100755
index e363021cc..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv.py
+++ /dev/null
@@ -1,2329 +0,0 @@
-#!/usr/bin/env python
-"""Create a "virtual" Python installation"""
-
-import os
-import sys
-
-# If we are running in a new interpreter to create a virtualenv,
-# we do NOT want paths from our existing location interfering with anything,
-# So we remove this file's directory from sys.path - most likely to be
-# the previous interpreter's site-packages. Solves #705, #763, #779
-if os.environ.get('VIRTUALENV_INTERPRETER_RUNNING'):
- for path in sys.path[:]:
- if os.path.realpath(os.path.dirname(__file__)) == os.path.realpath(path):
- sys.path.remove(path)
-
-import base64
-import codecs
-import optparse
-import re
-import shutil
-import logging
-import zlib
-import errno
-import glob
-import distutils.sysconfig
-import struct
-import subprocess
-import pkgutil
-import tempfile
-import textwrap
-from distutils.util import strtobool
-from os.path import join
-
-try:
- import ConfigParser
-except ImportError:
- import configparser as ConfigParser
-
-__version__ = "15.0.1"
-virtualenv_version = __version__ # legacy
-
-if sys.version_info < (2, 6):
- print('ERROR: %s' % sys.exc_info()[1])
- print('ERROR: this script requires Python 2.6 or greater.')
- sys.exit(101)
-
-try:
- basestring
-except NameError:
- basestring = str
-
-py_version = 'python%s.%s' % (sys.version_info[0], sys.version_info[1])
-
-is_jython = sys.platform.startswith('java')
-is_pypy = hasattr(sys, 'pypy_version_info')
-is_win = (sys.platform == 'win32' and os.sep == '\\')
-is_cygwin = (sys.platform == 'cygwin')
-is_msys2 = (sys.platform == 'win32' and os.sep == '/')
-is_darwin = (sys.platform == 'darwin')
-abiflags = getattr(sys, 'abiflags', '')
-
-user_dir = os.path.expanduser('~')
-if is_win:
- default_storage_dir = os.path.join(user_dir, 'virtualenv')
-else:
- default_storage_dir = os.path.join(user_dir, '.virtualenv')
-default_config_file = os.path.join(default_storage_dir, 'virtualenv.ini')
-
-if is_pypy:
- expected_exe = 'pypy'
-elif is_jython:
- expected_exe = 'jython'
-else:
- expected_exe = 'python'
-
-# Return a mapping of version -> Python executable
-# Only provided for Windows, where the information in the registry is used
-if not is_win:
- def get_installed_pythons():
- return {}
-else:
- try:
- import winreg
- except ImportError:
- import _winreg as winreg
-
- def get_installed_pythons():
- try:
- python_core = winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE,
- "Software\\Python\\PythonCore")
- except WindowsError:
- # No registered Python installations
- return {}
- i = 0
- versions = []
- while True:
- try:
- versions.append(winreg.EnumKey(python_core, i))
- i = i + 1
- except WindowsError:
- break
- exes = dict()
- for ver in versions:
- try:
- path = winreg.QueryValue(python_core, "%s\\InstallPath" % ver)
- except WindowsError:
- continue
- exes[ver] = join(path, "python.exe")
-
- winreg.CloseKey(python_core)
-
- # Add the major versions
- # Sort the keys, then repeatedly update the major version entry
- # Last executable (i.e., highest version) wins with this approach
- for ver in sorted(exes):
- exes[ver[0]] = exes[ver]
-
- return exes
-
-REQUIRED_MODULES = ['os', 'posix', 'posixpath', 'nt', 'ntpath', 'genericpath',
- 'fnmatch', 'locale', 'encodings', 'codecs',
- 'stat', 'UserDict', 'readline', 'copy_reg', 'types',
- 're', 'sre', 'sre_parse', 'sre_constants', 'sre_compile',
- 'zlib']
-
-REQUIRED_FILES = ['lib-dynload', 'config']
-
-majver, minver = sys.version_info[:2]
-if majver == 2:
- if minver >= 6:
- REQUIRED_MODULES.extend(['warnings', 'linecache', '_abcoll', 'abc'])
- if minver >= 7:
- REQUIRED_MODULES.extend(['_weakrefset'])
- if is_msys2:
- REQUIRED_MODULES.extend(['functools'])
-elif majver == 3:
- # Some extra modules are needed for Python 3, but different ones
- # for different versions.
- REQUIRED_MODULES.extend([
- '_abcoll', 'warnings', 'linecache', 'abc', 'io', '_weakrefset',
- 'copyreg', 'tempfile', 'random', '__future__', 'collections',
- 'keyword', 'tarfile', 'shutil', 'struct', 'copy', 'tokenize',
- 'token', 'functools', 'heapq', 'bisect', 'weakref', 'reprlib'
- ])
- if minver >= 2:
- REQUIRED_FILES[-1] = 'config-%s' % majver
- if minver >= 3:
- import sysconfig
- platdir = sysconfig.get_config_var('PLATDIR')
- REQUIRED_FILES.append(platdir)
- REQUIRED_MODULES.extend([
- 'base64', '_dummy_thread', 'hashlib', 'hmac',
- 'imp', 'importlib', 'rlcompleter'
- ])
- if minver >= 4:
- REQUIRED_MODULES.extend([
- 'operator',
- '_collections_abc',
- '_bootlocale',
- ])
-
-if is_pypy:
- # these are needed to correctly display the exceptions that may happen
- # during the bootstrap
- REQUIRED_MODULES.extend(['traceback', 'linecache'])
-
-
-class Logger(object):
-
- """
- Logging object for use in command-line script. Allows ranges of
- levels, to avoid some redundancy of displayed information.
- """
-
- DEBUG = logging.DEBUG
- INFO = logging.INFO
- NOTIFY = (logging.INFO+logging.WARN)/2
- WARN = WARNING = logging.WARN
- ERROR = logging.ERROR
- FATAL = logging.FATAL
-
- LEVELS = [DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL]
-
- def __init__(self, consumers):
- self.consumers = consumers
- self.indent = 0
- self.in_progress = None
- self.in_progress_hanging = False
-
- def debug(self, msg, *args, **kw):
- self.log(self.DEBUG, msg, *args, **kw)
-
- def info(self, msg, *args, **kw):
- self.log(self.INFO, msg, *args, **kw)
-
- def notify(self, msg, *args, **kw):
- self.log(self.NOTIFY, msg, *args, **kw)
-
- def warn(self, msg, *args, **kw):
- self.log(self.WARN, msg, *args, **kw)
-
- def error(self, msg, *args, **kw):
- self.log(self.ERROR, msg, *args, **kw)
-
- def fatal(self, msg, *args, **kw):
- self.log(self.FATAL, msg, *args, **kw)
-
- def log(self, level, msg, *args, **kw):
- if args:
- if kw:
- raise TypeError(
- "You may give positional or keyword arguments, not both")
- args = args or kw
- rendered = None
- for consumer_level, consumer in self.consumers:
- if self.level_matches(level, consumer_level):
- if (self.in_progress_hanging
- and consumer in (sys.stdout, sys.stderr)):
- self.in_progress_hanging = False
- sys.stdout.write('\n')
- sys.stdout.flush()
- if rendered is None:
- if args:
- rendered = msg % args
- else:
- rendered = msg
- rendered = ' '*self.indent + rendered
- if hasattr(consumer, 'write'):
- consumer.write(rendered+'\n')
- else:
- consumer(rendered)
-
- def start_progress(self, msg):
- assert not self.in_progress, (
- "Tried to start_progress(%r) while in_progress %r"
- % (msg, self.in_progress))
- if self.level_matches(self.NOTIFY, self._stdout_level()):
- sys.stdout.write(msg)
- sys.stdout.flush()
- self.in_progress_hanging = True
- else:
- self.in_progress_hanging = False
- self.in_progress = msg
-
- def end_progress(self, msg='done.'):
- assert self.in_progress, (
- "Tried to end_progress without start_progress")
- if self.stdout_level_matches(self.NOTIFY):
- if not self.in_progress_hanging:
- # Some message has been printed out since start_progress
- sys.stdout.write('...' + self.in_progress + msg + '\n')
- sys.stdout.flush()
- else:
- sys.stdout.write(msg + '\n')
- sys.stdout.flush()
- self.in_progress = None
- self.in_progress_hanging = False
-
- def show_progress(self):
- """If we are in a progress scope, and no log messages have been
- shown, write out another '.'"""
- if self.in_progress_hanging:
- sys.stdout.write('.')
- sys.stdout.flush()
-
- def stdout_level_matches(self, level):
- """Returns true if a message at this level will go to stdout"""
- return self.level_matches(level, self._stdout_level())
-
- def _stdout_level(self):
- """Returns the level that stdout runs at"""
- for level, consumer in self.consumers:
- if consumer is sys.stdout:
- return level
- return self.FATAL
-
- def level_matches(self, level, consumer_level):
- """
- >>> l = Logger([])
- >>> l.level_matches(3, 4)
- False
- >>> l.level_matches(3, 2)
- True
- >>> l.level_matches(slice(None, 3), 3)
- False
- >>> l.level_matches(slice(None, 3), 2)
- True
- >>> l.level_matches(slice(1, 3), 1)
- True
- >>> l.level_matches(slice(2, 3), 1)
- False
- """
- if isinstance(level, slice):
- start, stop = level.start, level.stop
- if start is not None and start > consumer_level:
- return False
- if stop is not None and stop <= consumer_level:
- return False
- return True
- else:
- return level >= consumer_level
-
- #@classmethod
- def level_for_integer(cls, level):
- levels = cls.LEVELS
- if level < 0:
- return levels[0]
- if level >= len(levels):
- return levels[-1]
- return levels[level]
-
- level_for_integer = classmethod(level_for_integer)
-
-# create a silent logger just to prevent this from being undefined
-# will be overridden with requested verbosity main() is called.
-logger = Logger([(Logger.LEVELS[-1], sys.stdout)])
-
-def mkdir(path):
- if not os.path.exists(path):
- logger.info('Creating %s', path)
- os.makedirs(path)
- else:
- logger.info('Directory %s already exists', path)
-
-def copyfileordir(src, dest, symlink=True):
- if os.path.isdir(src):
- shutil.copytree(src, dest, symlink)
- else:
- shutil.copy2(src, dest)
-
-def copyfile(src, dest, symlink=True):
- if not os.path.exists(src):
- # Some bad symlink in the src
- logger.warn('Cannot find file %s (bad symlink)', src)
- return
- if os.path.exists(dest):
- logger.debug('File %s already exists', dest)
- return
- if not os.path.exists(os.path.dirname(dest)):
- logger.info('Creating parent directories for %s', os.path.dirname(dest))
- os.makedirs(os.path.dirname(dest))
- if not os.path.islink(src):
- srcpath = os.path.abspath(src)
- else:
- srcpath = os.readlink(src)
- if symlink and hasattr(os, 'symlink') and not is_win:
- logger.info('Symlinking %s', dest)
- try:
- os.symlink(srcpath, dest)
- except (OSError, NotImplementedError):
- logger.info('Symlinking failed, copying to %s', dest)
- copyfileordir(src, dest, symlink)
- else:
- logger.info('Copying to %s', dest)
- copyfileordir(src, dest, symlink)
-
-def writefile(dest, content, overwrite=True):
- if not os.path.exists(dest):
- logger.info('Writing %s', dest)
- with open(dest, 'wb') as f:
- f.write(content.encode('utf-8'))
- return
- else:
- with open(dest, 'rb') as f:
- c = f.read()
- if c != content.encode("utf-8"):
- if not overwrite:
- logger.notify('File %s exists with different content; not overwriting', dest)
- return
- logger.notify('Overwriting %s with new content', dest)
- with open(dest, 'wb') as f:
- f.write(content.encode('utf-8'))
- else:
- logger.info('Content %s already in place', dest)
-
-def rmtree(dir):
- if os.path.exists(dir):
- logger.notify('Deleting tree %s', dir)
- shutil.rmtree(dir)
- else:
- logger.info('Do not need to delete %s; already gone', dir)
-
-def make_exe(fn):
- if hasattr(os, 'chmod'):
- oldmode = os.stat(fn).st_mode & 0xFFF # 0o7777
- newmode = (oldmode | 0x16D) & 0xFFF # 0o555, 0o7777
- os.chmod(fn, newmode)
- logger.info('Changed mode of %s to %s', fn, oct(newmode))
-
-def _find_file(filename, dirs):
- for dir in reversed(dirs):
- files = glob.glob(os.path.join(dir, filename))
- if files and os.path.isfile(files[0]):
- return True, files[0]
- return False, filename
-
-def file_search_dirs():
- here = os.path.dirname(os.path.abspath(__file__))
- dirs = [here, join(here, 'virtualenv_support')]
- if os.path.splitext(os.path.dirname(__file__))[0] != 'virtualenv':
- # Probably some boot script; just in case virtualenv is installed...
- try:
- import virtualenv
- except ImportError:
- pass
- else:
- dirs.append(os.path.join(
- os.path.dirname(virtualenv.__file__), 'virtualenv_support'))
- return [d for d in dirs if os.path.isdir(d)]
-
-
-class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter):
- """
- Custom help formatter for use in ConfigOptionParser that updates
- the defaults before expanding them, allowing them to show up correctly
- in the help listing
- """
- def expand_default(self, option):
- if self.parser is not None:
- self.parser.update_defaults(self.parser.defaults)
- return optparse.IndentedHelpFormatter.expand_default(self, option)
-
-
-class ConfigOptionParser(optparse.OptionParser):
- """
- Custom option parser which updates its defaults by checking the
- configuration files and environmental variables
- """
- def __init__(self, *args, **kwargs):
- self.config = ConfigParser.RawConfigParser()
- self.files = self.get_config_files()
- self.config.read(self.files)
- optparse.OptionParser.__init__(self, *args, **kwargs)
-
- def get_config_files(self):
- config_file = os.environ.get('VIRTUALENV_CONFIG_FILE', False)
- if config_file and os.path.exists(config_file):
- return [config_file]
- return [default_config_file]
-
- def update_defaults(self, defaults):
- """
- Updates the given defaults with values from the config files and
- the environ. Does a little special handling for certain types of
- options (lists).
- """
- # Then go and look for the other sources of configuration:
- config = {}
- # 1. config files
- config.update(dict(self.get_config_section('virtualenv')))
- # 2. environmental variables
- config.update(dict(self.get_environ_vars()))
- # Then set the options with those values
- for key, val in config.items():
- key = key.replace('_', '-')
- if not key.startswith('--'):
- key = '--%s' % key # only prefer long opts
- option = self.get_option(key)
- if option is not None:
- # ignore empty values
- if not val:
- continue
- # handle multiline configs
- if option.action == 'append':
- val = val.split()
- else:
- option.nargs = 1
- if option.action == 'store_false':
- val = not strtobool(val)
- elif option.action in ('store_true', 'count'):
- val = strtobool(val)
- try:
- val = option.convert_value(key, val)
- except optparse.OptionValueError:
- e = sys.exc_info()[1]
- print("An error occurred during configuration: %s" % e)
- sys.exit(3)
- defaults[option.dest] = val
- return defaults
-
- def get_config_section(self, name):
- """
- Get a section of a configuration
- """
- if self.config.has_section(name):
- return self.config.items(name)
- return []
-
- def get_environ_vars(self, prefix='VIRTUALENV_'):
- """
- Returns a generator with all environmental vars with prefix VIRTUALENV
- """
- for key, val in os.environ.items():
- if key.startswith(prefix):
- yield (key.replace(prefix, '').lower(), val)
-
- def get_default_values(self):
- """
- Overridding to make updating the defaults after instantiation of
- the option parser possible, update_defaults() does the dirty work.
- """
- if not self.process_default_values:
- # Old, pre-Optik 1.5 behaviour.
- return optparse.Values(self.defaults)
-
- defaults = self.update_defaults(self.defaults.copy()) # ours
- for option in self._get_all_options():
- default = defaults.get(option.dest)
- if isinstance(default, basestring):
- opt_str = option.get_opt_string()
- defaults[option.dest] = option.check_value(opt_str, default)
- return optparse.Values(defaults)
-
-
-def main():
- parser = ConfigOptionParser(
- version=virtualenv_version,
- usage="%prog [OPTIONS] DEST_DIR",
- formatter=UpdatingDefaultsHelpFormatter())
-
- parser.add_option(
- '-v', '--verbose',
- action='count',
- dest='verbose',
- default=0,
- help="Increase verbosity.")
-
- parser.add_option(
- '-q', '--quiet',
- action='count',
- dest='quiet',
- default=0,
- help='Decrease verbosity.')
-
- parser.add_option(
- '-p', '--python',
- dest='python',
- metavar='PYTHON_EXE',
- help='The Python interpreter to use, e.g., --python=python2.5 will use the python2.5 '
- 'interpreter to create the new environment. The default is the interpreter that '
- 'virtualenv was installed with (%s)' % sys.executable)
-
- parser.add_option(
- '--clear',
- dest='clear',
- action='store_true',
- help="Clear out the non-root install and start from scratch.")
-
- parser.set_defaults(system_site_packages=False)
- parser.add_option(
- '--no-site-packages',
- dest='system_site_packages',
- action='store_false',
- help="DEPRECATED. Retained only for backward compatibility. "
- "Not having access to global site-packages is now the default behavior.")
-
- parser.add_option(
- '--system-site-packages',
- dest='system_site_packages',
- action='store_true',
- help="Give the virtual environment access to the global site-packages.")
-
- parser.add_option(
- '--always-copy',
- dest='symlink',
- action='store_false',
- default=True,
- help="Always copy files rather than symlinking.")
-
- parser.add_option(
- '--unzip-setuptools',
- dest='unzip_setuptools',
- action='store_true',
- help="Unzip Setuptools when installing it.")
-
- parser.add_option(
- '--relocatable',
- dest='relocatable',
- action='store_true',
- help='Make an EXISTING virtualenv environment relocatable. '
- 'This fixes up scripts and makes all .pth files relative.')
-
- parser.add_option(
- '--no-setuptools',
- dest='no_setuptools',
- action='store_true',
- help='Do not install setuptools in the new virtualenv.')
-
- parser.add_option(
- '--no-pip',
- dest='no_pip',
- action='store_true',
- help='Do not install pip in the new virtualenv.')
-
- parser.add_option(
- '--no-wheel',
- dest='no_wheel',
- action='store_true',
- help='Do not install wheel in the new virtualenv.')
-
- default_search_dirs = file_search_dirs()
- parser.add_option(
- '--extra-search-dir',
- dest="search_dirs",
- action="append",
- metavar='DIR',
- default=default_search_dirs,
- help="Directory to look for setuptools/pip distributions in. "
- "This option can be used multiple times.")
-
- parser.add_option(
- "--download",
- dest="download",
- default=True,
- action="store_true",
- help="Download preinstalled packages from PyPI.",
- )
-
- parser.add_option(
- "--no-download",
- '--never-download',
- dest="download",
- action="store_false",
- help="Do not download preinstalled packages from PyPI.",
- )
-
- parser.add_option(
- '--prompt',
- dest='prompt',
- help='Provides an alternative prompt prefix for this environment.')
-
- parser.add_option(
- '--setuptools',
- dest='setuptools',
- action='store_true',
- help="DEPRECATED. Retained only for backward compatibility. This option has no effect.")
-
- parser.add_option(
- '--distribute',
- dest='distribute',
- action='store_true',
- help="DEPRECATED. Retained only for backward compatibility. This option has no effect.")
-
- if 'extend_parser' in globals():
- extend_parser(parser)
-
- options, args = parser.parse_args()
-
- global logger
-
- if 'adjust_options' in globals():
- adjust_options(options, args)
-
- verbosity = options.verbose - options.quiet
- logger = Logger([(Logger.level_for_integer(2 - verbosity), sys.stdout)])
-
- if options.python and not os.environ.get('VIRTUALENV_INTERPRETER_RUNNING'):
- env = os.environ.copy()
- interpreter = resolve_interpreter(options.python)
- if interpreter == sys.executable:
- logger.warn('Already using interpreter %s' % interpreter)
- else:
- logger.notify('Running virtualenv with interpreter %s' % interpreter)
- env['VIRTUALENV_INTERPRETER_RUNNING'] = 'true'
- file = __file__
- if file.endswith('.pyc'):
- file = file[:-1]
- popen = subprocess.Popen([interpreter, file] + sys.argv[1:], env=env)
- raise SystemExit(popen.wait())
-
- if not args:
- print('You must provide a DEST_DIR')
- parser.print_help()
- sys.exit(2)
- if len(args) > 1:
- print('There must be only one argument: DEST_DIR (you gave %s)' % (
- ' '.join(args)))
- parser.print_help()
- sys.exit(2)
-
- home_dir = args[0]
-
- if os.path.exists(home_dir) and os.path.isfile(home_dir):
- logger.fatal('ERROR: File already exists and is not a directory.')
- logger.fatal('Please provide a different path or delete the file.')
- sys.exit(3)
-
- if os.environ.get('WORKING_ENV'):
- logger.fatal('ERROR: you cannot run virtualenv while in a workingenv')
- logger.fatal('Please deactivate your workingenv, then re-run this script')
- sys.exit(3)
-
- if 'PYTHONHOME' in os.environ:
- logger.warn('PYTHONHOME is set. You *must* activate the virtualenv before using it')
- del os.environ['PYTHONHOME']
-
- if options.relocatable:
- make_environment_relocatable(home_dir)
- return
-
- create_environment(home_dir,
- site_packages=options.system_site_packages,
- clear=options.clear,
- unzip_setuptools=options.unzip_setuptools,
- prompt=options.prompt,
- search_dirs=options.search_dirs,
- download=options.download,
- no_setuptools=options.no_setuptools,
- no_pip=options.no_pip,
- no_wheel=options.no_wheel,
- symlink=options.symlink and hasattr(os, 'symlink')) # MOZ: Make sure we don't use symlink when we don't have it
- if 'after_install' in globals():
- after_install(options, home_dir)
-
-def call_subprocess(cmd, show_stdout=True,
- filter_stdout=None, cwd=None,
- raise_on_returncode=True, extra_env=None,
- remove_from_env=None, stdin=None):
- cmd_parts = []
- for part in cmd:
- if len(part) > 45:
- part = part[:20]+"..."+part[-20:]
- if ' ' in part or '\n' in part or '"' in part or "'" in part:
- part = '"%s"' % part.replace('"', '\\"')
- if hasattr(part, 'decode'):
- try:
- part = part.decode(sys.getdefaultencoding())
- except UnicodeDecodeError:
- part = part.decode(sys.getfilesystemencoding())
- cmd_parts.append(part)
- cmd_desc = ' '.join(cmd_parts)
- if show_stdout:
- stdout = None
- else:
- stdout = subprocess.PIPE
- logger.debug("Running command %s" % cmd_desc)
- if extra_env or remove_from_env:
- env = os.environ.copy()
- if extra_env:
- env.update(extra_env)
- if remove_from_env:
- for varname in remove_from_env:
- env.pop(varname, None)
- else:
- env = None
- try:
- proc = subprocess.Popen(
- cmd, stderr=subprocess.STDOUT,
- stdin=None if stdin is None else subprocess.PIPE,
- stdout=stdout,
- cwd=cwd, env=env)
- except Exception:
- e = sys.exc_info()[1]
- logger.fatal(
- "Error %s while executing command %s" % (e, cmd_desc))
- raise
- all_output = []
- if stdout is not None:
- if stdin is not None:
- proc.stdin.write(stdin)
- proc.stdin.close()
-
- stdout = proc.stdout
- encoding = sys.getdefaultencoding()
- fs_encoding = sys.getfilesystemencoding()
- while 1:
- line = stdout.readline()
- try:
- line = line.decode(encoding)
- except UnicodeDecodeError:
- line = line.decode(fs_encoding)
- if not line:
- break
- line = line.rstrip()
- all_output.append(line)
- if filter_stdout:
- level = filter_stdout(line)
- if isinstance(level, tuple):
- level, line = level
- logger.log(level, line)
- if not logger.stdout_level_matches(level):
- logger.show_progress()
- else:
- logger.info(line)
- else:
- proc.communicate(stdin)
- proc.wait()
- if proc.returncode:
- if raise_on_returncode:
- if all_output:
- logger.notify('Complete output from command %s:' % cmd_desc)
- logger.notify('\n'.join(all_output) + '\n----------------------------------------')
- raise OSError(
- "Command %s failed with error code %s"
- % (cmd_desc, proc.returncode))
- else:
- logger.warn(
- "Command %s had error code %s"
- % (cmd_desc, proc.returncode))
-
-def filter_install_output(line):
- if line.strip().startswith('running'):
- return Logger.INFO
- return Logger.DEBUG
-
-def find_wheels(projects, search_dirs):
- """Find wheels from which we can import PROJECTS.
-
- Scan through SEARCH_DIRS for a wheel for each PROJECT in turn. Return
- a list of the first wheel found for each PROJECT
- """
-
- wheels = []
-
- # Look through SEARCH_DIRS for the first suitable wheel. Don't bother
- # about version checking here, as this is simply to get something we can
- # then use to install the correct version.
- for project in projects:
- for dirname in search_dirs:
- # This relies on only having "universal" wheels available.
- # The pattern could be tightened to require -py2.py3-none-any.whl.
- files = glob.glob(os.path.join(dirname, project + '-*.whl'))
- if files:
- wheels.append(os.path.abspath(files[0]))
- break
- else:
- # We're out of luck, so quit with a suitable error
- logger.fatal('Cannot find a wheel for %s' % (project,))
-
- return wheels
-
-def install_wheel(project_names, py_executable, search_dirs=None,
- download=False):
- if search_dirs is None:
- search_dirs = file_search_dirs()
-
- wheels = find_wheels(['setuptools', 'pip'], search_dirs)
- pythonpath = os.pathsep.join(wheels)
-
- # PIP_FIND_LINKS uses space as the path separator and thus cannot have paths
- # with spaces in them. Convert any of those to local file:// URL form.
- try:
- from urlparse import urljoin
- from urllib import pathname2url
- except ImportError:
- from urllib.parse import urljoin
- from urllib.request import pathname2url
- def space_path2url(p):
- if ' ' not in p:
- return p
- return urljoin('file:', pathname2url(os.path.abspath(p)))
- findlinks = ' '.join(space_path2url(d) for d in search_dirs)
-
- SCRIPT = textwrap.dedent("""
- import sys
- import pkgutil
- import tempfile
- import os
-
- import pip
-
- cert_data = pkgutil.get_data("pip._vendor.requests", "cacert.pem")
- if cert_data is not None:
- cert_file = tempfile.NamedTemporaryFile(delete=False)
- cert_file.write(cert_data)
- cert_file.close()
- else:
- cert_file = None
-
- try:
- args = ["install", "--ignore-installed"]
- if cert_file is not None:
- args += ["--cert", cert_file.name]
- args += sys.argv[1:]
-
- sys.exit(pip.main(args))
- finally:
- if cert_file is not None:
- os.remove(cert_file.name)
- """).encode("utf8")
-
- cmd = [py_executable, '-'] + project_names
- logger.start_progress('Installing %s...' % (', '.join(project_names)))
- logger.indent += 2
-
- env = {
- "PYTHONPATH": pythonpath,
- "JYTHONPATH": pythonpath, # for Jython < 3.x
- "PIP_FIND_LINKS": findlinks,
- "PIP_USE_WHEEL": "1",
- "PIP_ONLY_BINARY": ":all:",
- "PIP_PRE": "1",
- "PIP_USER": "0",
- }
-
- if not download:
- env["PIP_NO_INDEX"] = "1"
-
- try:
- call_subprocess(cmd, show_stdout=False, extra_env=env, stdin=SCRIPT)
- finally:
- logger.indent -= 2
- logger.end_progress()
-
-
-def create_environment(home_dir, site_packages=False, clear=False,
- unzip_setuptools=False,
- prompt=None, search_dirs=None, download=False,
- no_setuptools=False, no_pip=False, no_wheel=False,
- symlink=True):
- """
- Creates a new environment in ``home_dir``.
-
- If ``site_packages`` is true, then the global ``site-packages/``
- directory will be on the path.
-
- If ``clear`` is true (default False) then the environment will
- first be cleared.
- """
- home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir)
-
- py_executable = os.path.abspath(install_python(
- home_dir, lib_dir, inc_dir, bin_dir,
- site_packages=site_packages, clear=clear, symlink=symlink))
-
- install_distutils(home_dir)
-
- to_install = []
-
- if not no_setuptools:
- to_install.append('setuptools')
-
- if not no_pip:
- to_install.append('pip')
-
- if not no_wheel:
- to_install.append('wheel')
-
- if to_install:
- install_wheel(
- to_install,
- py_executable,
- search_dirs,
- download=download,
- )
-
- install_activate(home_dir, bin_dir, prompt)
-
- install_python_config(home_dir, bin_dir, prompt)
-
-def is_executable_file(fpath):
- return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
-
-def path_locations(home_dir):
- """Return the path locations for the environment (where libraries are,
- where scripts go, etc)"""
- home_dir = os.path.abspath(home_dir)
- # XXX: We'd use distutils.sysconfig.get_python_inc/lib but its
- # prefix arg is broken: http://bugs.python.org/issue3386
- if is_win:
- # Windows has lots of problems with executables with spaces in
- # the name; this function will remove them (using the ~1
- # format):
- mkdir(home_dir)
- if ' ' in home_dir:
- import ctypes
- GetShortPathName = ctypes.windll.kernel32.GetShortPathNameW
- size = max(len(home_dir)+1, 256)
- buf = ctypes.create_unicode_buffer(size)
- try:
- u = unicode
- except NameError:
- u = str
- ret = GetShortPathName(u(home_dir), buf, size)
- if not ret:
- print('Error: the path "%s" has a space in it' % home_dir)
- print('We could not determine the short pathname for it.')
- print('Exiting.')
- sys.exit(3)
- home_dir = str(buf.value)
- lib_dir = join(home_dir, 'Lib')
- inc_dir = join(home_dir, 'Include')
- bin_dir = join(home_dir, 'Scripts')
- if is_jython:
- lib_dir = join(home_dir, 'Lib')
- inc_dir = join(home_dir, 'Include')
- bin_dir = join(home_dir, 'bin')
- elif is_pypy:
- lib_dir = home_dir
- inc_dir = join(home_dir, 'include')
- bin_dir = join(home_dir, 'bin')
- elif not is_win:
- lib_dir = join(home_dir, 'lib', py_version)
- inc_dir = join(home_dir, 'include', py_version + abiflags)
- bin_dir = join(home_dir, 'bin')
- return home_dir, lib_dir, inc_dir, bin_dir
-
-
-def change_prefix(filename, dst_prefix):
- prefixes = [sys.prefix]
-
- if is_darwin:
- prefixes.extend((
- os.path.join("/Library/Python", sys.version[:3], "site-packages"),
- os.path.join(sys.prefix, "Extras", "lib", "python"),
- os.path.join("~", "Library", "Python", sys.version[:3], "site-packages"),
- # Python 2.6 no-frameworks
- os.path.join("~", ".local", "lib","python", sys.version[:3], "site-packages"),
- # System Python 2.7 on OSX Mountain Lion
- os.path.join("~", "Library", "Python", sys.version[:3], "lib", "python", "site-packages")))
-
- if hasattr(sys, 'real_prefix'):
- prefixes.append(sys.real_prefix)
- if hasattr(sys, 'base_prefix'):
- prefixes.append(sys.base_prefix)
- prefixes = list(map(os.path.expanduser, prefixes))
- prefixes = list(map(os.path.abspath, prefixes))
- # Check longer prefixes first so we don't split in the middle of a filename
- prefixes = sorted(prefixes, key=len, reverse=True)
- filename = os.path.abspath(filename)
- # On Windows, make sure drive letter is uppercase
- if is_win and filename[0] in 'abcdefghijklmnopqrstuvwxyz':
- filename = filename[0].upper() + filename[1:]
- for i, prefix in enumerate(prefixes):
- if is_win and prefix[0] in 'abcdefghijklmnopqrstuvwxyz':
- prefixes[i] = prefix[0].upper() + prefix[1:]
- for src_prefix in prefixes:
- if filename.startswith(src_prefix):
- _, relpath = filename.split(src_prefix, 1)
- if src_prefix != os.sep: # sys.prefix == "/"
- assert relpath[0] == os.sep
- relpath = relpath[1:]
- return join(dst_prefix, relpath)
- assert False, "Filename %s does not start with any of these prefixes: %s" % \
- (filename, prefixes)
-
-def copy_required_modules(dst_prefix, symlink):
- import imp
-
- for modname in REQUIRED_MODULES:
- if modname in sys.builtin_module_names:
- logger.info("Ignoring built-in bootstrap module: %s" % modname)
- continue
- try:
- f, filename, _ = imp.find_module(modname)
- except ImportError:
- logger.info("Cannot import bootstrap module: %s" % modname)
- else:
- if f is not None:
- f.close()
- # special-case custom readline.so on OS X, but not for pypy:
- if modname == 'readline' and sys.platform == 'darwin' and not (
- is_pypy or filename.endswith(join('lib-dynload', 'readline.so'))):
- dst_filename = join(dst_prefix, 'lib', 'python%s' % sys.version[:3], 'readline.so')
- elif modname == 'readline' and sys.platform == 'win32':
- # special-case for Windows, where readline is not a
- # standard module, though it may have been installed in
- # site-packages by a third-party package
- pass
- else:
- dst_filename = change_prefix(filename, dst_prefix)
- copyfile(filename, dst_filename, symlink)
- if filename.endswith('.pyc'):
- pyfile = filename[:-1]
- if os.path.exists(pyfile):
- copyfile(pyfile, dst_filename[:-1], symlink)
-
-
-def subst_path(prefix_path, prefix, home_dir):
- prefix_path = os.path.normpath(prefix_path)
- prefix = os.path.normpath(prefix)
- home_dir = os.path.normpath(home_dir)
- if not prefix_path.startswith(prefix):
- logger.warn('Path not in prefix %r %r', prefix_path, prefix)
- return
- return prefix_path.replace(prefix, home_dir, 1)
-
-
-def install_python(home_dir, lib_dir, inc_dir, bin_dir, site_packages, clear, symlink=True):
- """Install just the base environment, no distutils patches etc"""
- if sys.executable.startswith(bin_dir):
- print('Please use the *system* python to run this script')
- return
-
- if clear:
- rmtree(lib_dir)
- ## FIXME: why not delete it?
- ## Maybe it should delete everything with #!/path/to/venv/python in it
- logger.notify('Not deleting %s', bin_dir)
-
- if hasattr(sys, 'real_prefix'):
- logger.notify('Using real prefix %r' % sys.real_prefix)
- prefix = sys.real_prefix
- elif hasattr(sys, 'base_prefix'):
- logger.notify('Using base prefix %r' % sys.base_prefix)
- prefix = sys.base_prefix
- else:
- prefix = sys.prefix
- mkdir(lib_dir)
- fix_lib64(lib_dir, symlink)
- stdlib_dirs = [os.path.dirname(os.__file__)]
- if is_win:
- stdlib_dirs.append(join(os.path.dirname(stdlib_dirs[0]), 'DLLs'))
- elif is_darwin:
- stdlib_dirs.append(join(stdlib_dirs[0], 'site-packages'))
- if hasattr(os, 'symlink'):
- logger.info('Symlinking Python bootstrap modules')
- else:
- logger.info('Copying Python bootstrap modules')
- logger.indent += 2
- try:
- # copy required files...
- for stdlib_dir in stdlib_dirs:
- if not os.path.isdir(stdlib_dir):
- continue
- for fn in os.listdir(stdlib_dir):
- bn = os.path.splitext(fn)[0]
- if fn != 'site-packages' and bn in REQUIRED_FILES:
- copyfile(join(stdlib_dir, fn), join(lib_dir, fn), symlink)
- # ...and modules
- copy_required_modules(home_dir, symlink)
- finally:
- logger.indent -= 2
- mkdir(join(lib_dir, 'site-packages'))
- import site
- site_filename = site.__file__
- if site_filename.endswith('.pyc') or site_filename.endswith('.pyo'):
- site_filename = site_filename[:-1]
- elif site_filename.endswith('$py.class'):
- site_filename = site_filename.replace('$py.class', '.py')
- site_filename_dst = change_prefix(site_filename, home_dir)
- site_dir = os.path.dirname(site_filename_dst)
- # MOZ: Copies a site.py if it exists instead of using the one hex encoded in
- # this file. Necessary for some site.py fixes for MinGW64 version of python
- site_py_src_path = os.path.join(os.path.dirname(__file__), 'site.py')
- if os.path.isfile(site_py_src_path):
- shutil.copy(site_py_src_path, site_filename_dst)
- else:
- writefile(site_filename_dst, SITE_PY)
- writefile(join(site_dir, 'orig-prefix.txt'), prefix)
- site_packages_filename = join(site_dir, 'no-global-site-packages.txt')
- if not site_packages:
- writefile(site_packages_filename, '')
-
- if is_pypy or is_win:
- stdinc_dir = join(prefix, 'include')
- else:
- stdinc_dir = join(prefix, 'include', py_version + abiflags)
- if os.path.exists(stdinc_dir):
- copyfile(stdinc_dir, inc_dir, symlink)
- else:
- logger.debug('No include dir %s' % stdinc_dir)
-
- platinc_dir = distutils.sysconfig.get_python_inc(plat_specific=1)
- if platinc_dir != stdinc_dir:
- platinc_dest = distutils.sysconfig.get_python_inc(
- plat_specific=1, prefix=home_dir)
- if platinc_dir == platinc_dest:
- # Do platinc_dest manually due to a CPython bug;
- # not http://bugs.python.org/issue3386 but a close cousin
- platinc_dest = subst_path(platinc_dir, prefix, home_dir)
- if platinc_dest:
- # PyPy's stdinc_dir and prefix are relative to the original binary
- # (traversing virtualenvs), whereas the platinc_dir is relative to
- # the inner virtualenv and ignores the prefix argument.
- # This seems more evolved than designed.
- copyfile(platinc_dir, platinc_dest, symlink)
-
- # pypy never uses exec_prefix, just ignore it
- if sys.exec_prefix != prefix and not is_pypy:
- if is_win:
- exec_dir = join(sys.exec_prefix, 'lib')
- elif is_jython:
- exec_dir = join(sys.exec_prefix, 'Lib')
- else:
- exec_dir = join(sys.exec_prefix, 'lib', py_version)
- for fn in os.listdir(exec_dir):
- copyfile(join(exec_dir, fn), join(lib_dir, fn), symlink)
-
- if is_jython:
- # Jython has either jython-dev.jar and javalib/ dir, or just
- # jython.jar
- for name in 'jython-dev.jar', 'javalib', 'jython.jar':
- src = join(prefix, name)
- if os.path.exists(src):
- copyfile(src, join(home_dir, name), symlink)
- # XXX: registry should always exist after Jython 2.5rc1
- src = join(prefix, 'registry')
- if os.path.exists(src):
- copyfile(src, join(home_dir, 'registry'), symlink=False)
- copyfile(join(prefix, 'cachedir'), join(home_dir, 'cachedir'),
- symlink=False)
-
- mkdir(bin_dir)
- py_executable = join(bin_dir, os.path.basename(sys.executable))
- if 'Python.framework' in prefix:
- # OS X framework builds cause validation to break
- # https://github.com/pypa/virtualenv/issues/322
- if os.environ.get('__PYVENV_LAUNCHER__'):
- del os.environ["__PYVENV_LAUNCHER__"]
- if re.search(r'/Python(?:-32|-64)*$', py_executable):
- # The name of the python executable is not quite what
- # we want, rename it.
- py_executable = os.path.join(
- os.path.dirname(py_executable), 'python')
-
- logger.notify('New %s executable in %s', expected_exe, py_executable)
- pcbuild_dir = os.path.dirname(sys.executable)
- pyd_pth = os.path.join(lib_dir, 'site-packages', 'virtualenv_builddir_pyd.pth')
- if is_win and os.path.exists(os.path.join(pcbuild_dir, 'build.bat')):
- logger.notify('Detected python running from build directory %s', pcbuild_dir)
- logger.notify('Writing .pth file linking to build directory for *.pyd files')
- writefile(pyd_pth, pcbuild_dir)
- else:
- pcbuild_dir = None
- if os.path.exists(pyd_pth):
- logger.info('Deleting %s (not Windows env or not build directory python)' % pyd_pth)
- os.unlink(pyd_pth)
-
- if sys.executable != py_executable:
- ## FIXME: could I just hard link?
- executable = sys.executable
- shutil.copyfile(executable, py_executable)
- make_exe(py_executable)
- if is_win or is_cygwin:
- pythonw = os.path.join(os.path.dirname(sys.executable), 'pythonw.exe')
- if os.path.exists(pythonw):
- logger.info('Also created pythonw.exe')
- shutil.copyfile(pythonw, os.path.join(os.path.dirname(py_executable), 'pythonw.exe'))
- python_d = os.path.join(os.path.dirname(sys.executable), 'python_d.exe')
- python_d_dest = os.path.join(os.path.dirname(py_executable), 'python_d.exe')
- if os.path.exists(python_d):
- logger.info('Also created python_d.exe')
- shutil.copyfile(python_d, python_d_dest)
- elif os.path.exists(python_d_dest):
- logger.info('Removed python_d.exe as it is no longer at the source')
- os.unlink(python_d_dest)
- # we need to copy the DLL to enforce that windows will load the correct one.
- # may not exist if we are cygwin.
- py_executable_dll = 'python%s%s.dll' % (
- sys.version_info[0], sys.version_info[1])
- py_executable_dll_d = 'python%s%s_d.dll' % (
- sys.version_info[0], sys.version_info[1])
- pythondll = os.path.join(os.path.dirname(sys.executable), py_executable_dll)
- pythondll_d = os.path.join(os.path.dirname(sys.executable), py_executable_dll_d)
- pythondll_d_dest = os.path.join(os.path.dirname(py_executable), py_executable_dll_d)
- if os.path.exists(pythondll):
- logger.info('Also created %s' % py_executable_dll)
- shutil.copyfile(pythondll, os.path.join(os.path.dirname(py_executable), py_executable_dll))
- if os.path.exists(pythondll_d):
- logger.info('Also created %s' % py_executable_dll_d)
- shutil.copyfile(pythondll_d, pythondll_d_dest)
- elif os.path.exists(pythondll_d_dest):
- logger.info('Removed %s as the source does not exist' % pythondll_d_dest)
- os.unlink(pythondll_d_dest)
- if is_pypy:
- # make a symlink python --> pypy-c
- python_executable = os.path.join(os.path.dirname(py_executable), 'python')
- if sys.platform in ('win32', 'cygwin'):
- python_executable += '.exe'
- logger.info('Also created executable %s' % python_executable)
- copyfile(py_executable, python_executable, symlink)
-
- if is_win:
- for name in ['libexpat.dll', 'libpypy.dll', 'libpypy-c.dll',
- 'libeay32.dll', 'ssleay32.dll', 'sqlite3.dll',
- 'tcl85.dll', 'tk85.dll']:
- src = join(prefix, name)
- if os.path.exists(src):
- copyfile(src, join(bin_dir, name), symlink)
-
- for d in sys.path:
- if d.endswith('lib_pypy'):
- break
- else:
- logger.fatal('Could not find lib_pypy in sys.path')
- raise SystemExit(3)
- logger.info('Copying lib_pypy')
- copyfile(d, os.path.join(home_dir, 'lib_pypy'), symlink)
-
- if os.path.splitext(os.path.basename(py_executable))[0] != expected_exe:
- secondary_exe = os.path.join(os.path.dirname(py_executable),
- expected_exe)
- py_executable_ext = os.path.splitext(py_executable)[1]
- if py_executable_ext.lower() == '.exe':
- # python2.4 gives an extension of '.4' :P
- secondary_exe += py_executable_ext
- if os.path.exists(secondary_exe):
- logger.warn('Not overwriting existing %s script %s (you must use %s)'
- % (expected_exe, secondary_exe, py_executable))
- else:
- logger.notify('Also creating executable in %s' % secondary_exe)
- shutil.copyfile(sys.executable, secondary_exe)
- make_exe(secondary_exe)
-
- if '.framework' in prefix:
- if 'Python.framework' in prefix:
- logger.debug('MacOSX Python framework detected')
- # Make sure we use the embedded interpreter inside
- # the framework, even if sys.executable points to
- # the stub executable in ${sys.prefix}/bin
- # See http://groups.google.com/group/python-virtualenv/
- # browse_thread/thread/17cab2f85da75951
- original_python = os.path.join(
- prefix, 'Resources/Python.app/Contents/MacOS/Python')
- if 'EPD' in prefix:
- logger.debug('EPD framework detected')
- original_python = os.path.join(prefix, 'bin/python')
- shutil.copy(original_python, py_executable)
-
- # Copy the framework's dylib into the virtual
- # environment
- virtual_lib = os.path.join(home_dir, '.Python')
-
- if os.path.exists(virtual_lib):
- os.unlink(virtual_lib)
- copyfile(
- os.path.join(prefix, 'Python'),
- virtual_lib,
- symlink)
-
- # And then change the install_name of the copied python executable
- try:
- mach_o_change(py_executable,
- os.path.join(prefix, 'Python'),
- '@executable_path/../.Python')
- except:
- e = sys.exc_info()[1]
- logger.warn("Could not call mach_o_change: %s. "
- "Trying to call install_name_tool instead." % e)
- try:
- call_subprocess(
- ["install_name_tool", "-change",
- os.path.join(prefix, 'Python'),
- '@executable_path/../.Python',
- py_executable])
- except:
- logger.fatal("Could not call install_name_tool -- you must "
- "have Apple's development tools installed")
- raise
-
- if not is_win:
- # Ensure that 'python', 'pythonX' and 'pythonX.Y' all exist
- py_exe_version_major = 'python%s' % sys.version_info[0]
- py_exe_version_major_minor = 'python%s.%s' % (
- sys.version_info[0], sys.version_info[1])
- py_exe_no_version = 'python'
- required_symlinks = [ py_exe_no_version, py_exe_version_major,
- py_exe_version_major_minor ]
-
- py_executable_base = os.path.basename(py_executable)
-
- if py_executable_base in required_symlinks:
- # Don't try to symlink to yourself.
- required_symlinks.remove(py_executable_base)
-
- for pth in required_symlinks:
- full_pth = join(bin_dir, pth)
- if os.path.exists(full_pth):
- os.unlink(full_pth)
- if symlink:
- os.symlink(py_executable_base, full_pth)
- else:
- copyfile(py_executable, full_pth, symlink)
-
- if is_win and ' ' in py_executable:
- # There's a bug with subprocess on Windows when using a first
- # argument that has a space in it. Instead we have to quote
- # the value:
- py_executable = '"%s"' % py_executable
- # NOTE: keep this check as one line, cmd.exe doesn't cope with line breaks
- cmd = [py_executable, '-c', 'import sys;out=sys.stdout;'
- 'getattr(out, "buffer", out).write(sys.prefix.encode("utf-8"))']
- logger.info('Testing executable with %s %s "%s"' % tuple(cmd))
- try:
- proc = subprocess.Popen(cmd,
- stdout=subprocess.PIPE)
- proc_stdout, proc_stderr = proc.communicate()
- except OSError:
- e = sys.exc_info()[1]
- if e.errno == errno.EACCES:
- logger.fatal('ERROR: The executable %s could not be run: %s' % (py_executable, e))
- sys.exit(100)
- else:
- raise e
-
- proc_stdout = proc_stdout.strip().decode("utf-8")
- proc_stdout = os.path.normcase(os.path.abspath(proc_stdout))
- norm_home_dir = os.path.normcase(os.path.abspath(home_dir))
- if hasattr(norm_home_dir, 'decode'):
- norm_home_dir = norm_home_dir.decode(sys.getfilesystemencoding())
- if proc_stdout != norm_home_dir:
- logger.fatal(
- 'ERROR: The executable %s is not functioning' % py_executable)
- logger.fatal(
- 'ERROR: It thinks sys.prefix is %r (should be %r)'
- % (proc_stdout, norm_home_dir))
- logger.fatal(
- 'ERROR: virtualenv is not compatible with this system or executable')
- if is_win:
- logger.fatal(
- 'Note: some Windows users have reported this error when they '
- 'installed Python for "Only this user" or have multiple '
- 'versions of Python installed. Copying the appropriate '
- 'PythonXX.dll to the virtualenv Scripts/ directory may fix '
- 'this problem.')
- sys.exit(100)
- else:
- logger.info('Got sys.prefix result: %r' % proc_stdout)
-
- pydistutils = os.path.expanduser('~/.pydistutils.cfg')
- if os.path.exists(pydistutils):
- logger.notify('Please make sure you remove any previous custom paths from '
- 'your %s file.' % pydistutils)
- ## FIXME: really this should be calculated earlier
-
- fix_local_scheme(home_dir, symlink)
-
- if site_packages:
- if os.path.exists(site_packages_filename):
- logger.info('Deleting %s' % site_packages_filename)
- os.unlink(site_packages_filename)
-
- return py_executable
-
-
-def install_activate(home_dir, bin_dir, prompt=None):
- if is_win or is_jython and os._name == 'nt':
- files = {
- 'activate.bat': ACTIVATE_BAT,
- 'deactivate.bat': DEACTIVATE_BAT,
- 'activate.ps1': ACTIVATE_PS,
- }
-
- # MSYS needs paths of the form /c/path/to/file
- drive, tail = os.path.splitdrive(home_dir.replace(os.sep, '/'))
- home_dir_msys = (drive and "/%s%s" or "%s%s") % (drive[:1], tail)
-
- # Run-time conditional enables (basic) Cygwin compatibility
- home_dir_sh = ("""$(if [ "$OSTYPE" "==" "cygwin" ]; then cygpath -u '%s'; else echo '%s'; fi;)""" %
- (home_dir, home_dir_msys))
- files['activate'] = ACTIVATE_SH.replace('__VIRTUAL_ENV__', home_dir_sh)
-
- else:
- files = {'activate': ACTIVATE_SH}
-
- # suppling activate.fish in addition to, not instead of, the
- # bash script support.
- files['activate.fish'] = ACTIVATE_FISH
-
- # same for csh/tcsh support...
- files['activate.csh'] = ACTIVATE_CSH
-
- files['activate_this.py'] = ACTIVATE_THIS
-
- install_files(home_dir, bin_dir, prompt, files)
-
-def install_files(home_dir, bin_dir, prompt, files):
- if hasattr(home_dir, 'decode'):
- home_dir = home_dir.decode(sys.getfilesystemencoding())
- vname = os.path.basename(home_dir)
- for name, content in files.items():
- content = content.replace('__VIRTUAL_PROMPT__', prompt or '')
- content = content.replace('__VIRTUAL_WINPROMPT__', prompt or '(%s)' % vname)
- content = content.replace('__VIRTUAL_ENV__', home_dir)
- content = content.replace('__VIRTUAL_NAME__', vname)
- content = content.replace('__BIN_NAME__', os.path.basename(bin_dir))
- writefile(os.path.join(bin_dir, name), content)
-
-def install_python_config(home_dir, bin_dir, prompt=None):
- if sys.platform == 'win32' or is_jython and os._name == 'nt':
- files = {}
- else:
- files = {'python-config': PYTHON_CONFIG}
- install_files(home_dir, bin_dir, prompt, files)
- for name, content in files.items():
- make_exe(os.path.join(bin_dir, name))
-
-def install_distutils(home_dir):
- distutils_path = change_prefix(distutils.__path__[0], home_dir)
- mkdir(distutils_path)
- ## FIXME: maybe this prefix setting should only be put in place if
- ## there's a local distutils.cfg with a prefix setting?
- home_dir = os.path.abspath(home_dir)
- ## FIXME: this is breaking things, removing for now:
- #distutils_cfg = DISTUTILS_CFG + "\n[install]\nprefix=%s\n" % home_dir
- writefile(os.path.join(distutils_path, '__init__.py'), DISTUTILS_INIT)
- writefile(os.path.join(distutils_path, 'distutils.cfg'), DISTUTILS_CFG, overwrite=False)
-
-def fix_local_scheme(home_dir, symlink=True):
- """
- Platforms that use the "posix_local" install scheme (like Ubuntu with
- Python 2.7) need to be given an additional "local" location, sigh.
- """
- try:
- import sysconfig
- except ImportError:
- pass
- else:
- if sysconfig._get_default_scheme() == 'posix_local':
- local_path = os.path.join(home_dir, 'local')
- if not os.path.exists(local_path):
- os.mkdir(local_path)
- for subdir_name in os.listdir(home_dir):
- if subdir_name == 'local':
- continue
- copyfile(os.path.abspath(os.path.join(home_dir, subdir_name)), \
- os.path.join(local_path, subdir_name), symlink)
-
-def fix_lib64(lib_dir, symlink=True):
- """
- Some platforms (particularly Gentoo on x64) put things in lib64/pythonX.Y
- instead of lib/pythonX.Y. If this is such a platform we'll just create a
- symlink so lib64 points to lib
- """
- # PyPy's library path scheme is not affected by this.
- # Return early or we will die on the following assert.
- if is_pypy:
- logger.debug('PyPy detected, skipping lib64 symlinking')
- return
- # Check we have a lib64 library path
- if not [p for p in distutils.sysconfig.get_config_vars().values()
- if isinstance(p, basestring) and 'lib64' in p]:
- return
-
- logger.debug('This system uses lib64; symlinking lib64 to lib')
-
- assert os.path.basename(lib_dir) == 'python%s' % sys.version[:3], (
- "Unexpected python lib dir: %r" % lib_dir)
- lib_parent = os.path.dirname(lib_dir)
- top_level = os.path.dirname(lib_parent)
- lib_dir = os.path.join(top_level, 'lib')
- lib64_link = os.path.join(top_level, 'lib64')
- assert os.path.basename(lib_parent) == 'lib', (
- "Unexpected parent dir: %r" % lib_parent)
- if os.path.lexists(lib64_link):
- return
- if symlink:
- os.symlink('lib', lib64_link)
- else:
- copyfile('lib', lib64_link)
-
-def resolve_interpreter(exe):
- """
- If the executable given isn't an absolute path, search $PATH for the interpreter
- """
- # If the "executable" is a version number, get the installed executable for
- # that version
- python_versions = get_installed_pythons()
- if exe in python_versions:
- exe = python_versions[exe]
-
- if os.path.abspath(exe) != exe:
- paths = os.environ.get('PATH', '').split(os.pathsep)
- for path in paths:
- if os.path.exists(join(path, exe)):
- exe = join(path, exe)
- break
- if not os.path.exists(exe):
- logger.fatal('The executable %s (from --python=%s) does not exist' % (exe, exe))
- raise SystemExit(3)
- if not is_executable(exe):
- logger.fatal('The executable %s (from --python=%s) is not executable' % (exe, exe))
- raise SystemExit(3)
- return exe
-
-def is_executable(exe):
- """Checks a file is executable"""
- return os.access(exe, os.X_OK)
-
-############################################################
-## Relocating the environment:
-
-def make_environment_relocatable(home_dir):
- """
- Makes the already-existing environment use relative paths, and takes out
- the #!-based environment selection in scripts.
- """
- home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir)
- activate_this = os.path.join(bin_dir, 'activate_this.py')
- if not os.path.exists(activate_this):
- logger.fatal(
- 'The environment doesn\'t have a file %s -- please re-run virtualenv '
- 'on this environment to update it' % activate_this)
- fixup_scripts(home_dir, bin_dir)
- fixup_pth_and_egg_link(home_dir)
- ## FIXME: need to fix up distutils.cfg
-
-OK_ABS_SCRIPTS = ['python', 'python%s' % sys.version[:3],
- 'activate', 'activate.bat', 'activate_this.py',
- 'activate.fish', 'activate.csh']
-
-def fixup_scripts(home_dir, bin_dir):
- if is_win:
- new_shebang_args = (
- '%s /c' % os.path.normcase(os.environ.get('COMSPEC', 'cmd.exe')),
- '', '.exe')
- else:
- new_shebang_args = ('/usr/bin/env', sys.version[:3], '')
-
- # This is what we expect at the top of scripts:
- shebang = '#!%s' % os.path.normcase(os.path.join(
- os.path.abspath(bin_dir), 'python%s' % new_shebang_args[2]))
- # This is what we'll put:
- new_shebang = '#!%s python%s%s' % new_shebang_args
-
- for filename in os.listdir(bin_dir):
- filename = os.path.join(bin_dir, filename)
- if not os.path.isfile(filename):
- # ignore subdirs, e.g. .svn ones.
- continue
- lines = None
- with open(filename, 'rb') as f:
- try:
- lines = f.read().decode('utf-8').splitlines()
- except UnicodeDecodeError:
- # This is probably a binary program instead
- # of a script, so just ignore it.
- continue
- if not lines:
- logger.warn('Script %s is an empty file' % filename)
- continue
-
- old_shebang = lines[0].strip()
- old_shebang = old_shebang[0:2] + os.path.normcase(old_shebang[2:])
-
- if not old_shebang.startswith(shebang):
- if os.path.basename(filename) in OK_ABS_SCRIPTS:
- logger.debug('Cannot make script %s relative' % filename)
- elif lines[0].strip() == new_shebang:
- logger.info('Script %s has already been made relative' % filename)
- else:
- logger.warn('Script %s cannot be made relative (it\'s not a normal script that starts with %s)'
- % (filename, shebang))
- continue
- logger.notify('Making script %s relative' % filename)
- script = relative_script([new_shebang] + lines[1:])
- with open(filename, 'wb') as f:
- f.write('\n'.join(script).encode('utf-8'))
-
-
-def relative_script(lines):
- "Return a script that'll work in a relocatable environment."
- activate = "import os; activate_this=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'activate_this.py'); exec(compile(open(activate_this).read(), activate_this, 'exec'), dict(__file__=activate_this)); del os, activate_this"
- # Find the last future statement in the script. If we insert the activation
- # line before a future statement, Python will raise a SyntaxError.
- activate_at = None
- for idx, line in reversed(list(enumerate(lines))):
- if line.split()[:3] == ['from', '__future__', 'import']:
- activate_at = idx + 1
- break
- if activate_at is None:
- # Activate after the shebang.
- activate_at = 1
- return lines[:activate_at] + ['', activate, ''] + lines[activate_at:]
-
-def fixup_pth_and_egg_link(home_dir, sys_path=None):
- """Makes .pth and .egg-link files use relative paths"""
- home_dir = os.path.normcase(os.path.abspath(home_dir))
- if sys_path is None:
- sys_path = sys.path
- for path in sys_path:
- if not path:
- path = '.'
- if not os.path.isdir(path):
- continue
- path = os.path.normcase(os.path.abspath(path))
- if not path.startswith(home_dir):
- logger.debug('Skipping system (non-environment) directory %s' % path)
- continue
- for filename in os.listdir(path):
- filename = os.path.join(path, filename)
- if filename.endswith('.pth'):
- if not os.access(filename, os.W_OK):
- logger.warn('Cannot write .pth file %s, skipping' % filename)
- else:
- fixup_pth_file(filename)
- if filename.endswith('.egg-link'):
- if not os.access(filename, os.W_OK):
- logger.warn('Cannot write .egg-link file %s, skipping' % filename)
- else:
- fixup_egg_link(filename)
-
-def fixup_pth_file(filename):
- lines = []
- prev_lines = []
- with open(filename) as f:
- prev_lines = f.readlines()
- for line in prev_lines:
- line = line.strip()
- if (not line or line.startswith('#') or line.startswith('import ')
- or os.path.abspath(line) != line):
- lines.append(line)
- else:
- new_value = make_relative_path(filename, line)
- if line != new_value:
- logger.debug('Rewriting path %s as %s (in %s)' % (line, new_value, filename))
- lines.append(new_value)
- if lines == prev_lines:
- logger.info('No changes to .pth file %s' % filename)
- return
- logger.notify('Making paths in .pth file %s relative' % filename)
- with open(filename, 'w') as f:
- f.write('\n'.join(lines) + '\n')
-
-def fixup_egg_link(filename):
- with open(filename) as f:
- link = f.readline().strip()
- if os.path.abspath(link) != link:
- logger.debug('Link in %s already relative' % filename)
- return
- new_link = make_relative_path(filename, link)
- logger.notify('Rewriting link %s in %s as %s' % (link, filename, new_link))
- with open(filename, 'w') as f:
- f.write(new_link)
-
-def make_relative_path(source, dest, dest_is_directory=True):
- """
- Make a filename relative, where the filename is dest, and it is
- being referred to from the filename source.
-
- >>> make_relative_path('/usr/share/something/a-file.pth',
- ... '/usr/share/another-place/src/Directory')
- '../another-place/src/Directory'
- >>> make_relative_path('/usr/share/something/a-file.pth',
- ... '/home/user/src/Directory')
- '../../../home/user/src/Directory'
- >>> make_relative_path('/usr/share/a-file.pth', '/usr/share/')
- './'
- """
- source = os.path.dirname(source)
- if not dest_is_directory:
- dest_filename = os.path.basename(dest)
- dest = os.path.dirname(dest)
- dest = os.path.normpath(os.path.abspath(dest))
- source = os.path.normpath(os.path.abspath(source))
- dest_parts = dest.strip(os.path.sep).split(os.path.sep)
- source_parts = source.strip(os.path.sep).split(os.path.sep)
- while dest_parts and source_parts and dest_parts[0] == source_parts[0]:
- dest_parts.pop(0)
- source_parts.pop(0)
- full_parts = ['..']*len(source_parts) + dest_parts
- if not dest_is_directory:
- full_parts.append(dest_filename)
- if not full_parts:
- # Special case for the current directory (otherwise it'd be '')
- return './'
- return os.path.sep.join(full_parts)
-
-
-
-############################################################
-## Bootstrap script creation:
-
-def create_bootstrap_script(extra_text, python_version=''):
- """
- Creates a bootstrap script, which is like this script but with
- extend_parser, adjust_options, and after_install hooks.
-
- This returns a string that (written to disk of course) can be used
- as a bootstrap script with your own customizations. The script
- will be the standard virtualenv.py script, with your extra text
- added (your extra text should be Python code).
-
- If you include these functions, they will be called:
-
- ``extend_parser(optparse_parser)``:
- You can add or remove options from the parser here.
-
- ``adjust_options(options, args)``:
- You can change options here, or change the args (if you accept
- different kinds of arguments, be sure you modify ``args`` so it is
- only ``[DEST_DIR]``).
-
- ``after_install(options, home_dir)``:
-
- After everything is installed, this function is called. This
- is probably the function you are most likely to use. An
- example would be::
-
- def after_install(options, home_dir):
- subprocess.call([join(home_dir, 'bin', 'easy_install'),
- 'MyPackage'])
- subprocess.call([join(home_dir, 'bin', 'my-package-script'),
- 'setup', home_dir])
-
- This example immediately installs a package, and runs a setup
- script from that package.
-
- If you provide something like ``python_version='2.5'`` then the
- script will start with ``#!/usr/bin/env python2.5`` instead of
- ``#!/usr/bin/env python``. You can use this when the script must
- be run with a particular Python version.
- """
- filename = __file__
- if filename.endswith('.pyc'):
- filename = filename[:-1]
- with codecs.open(filename, 'r', encoding='utf-8') as f:
- content = f.read()
- py_exe = 'python%s' % python_version
- content = (('#!/usr/bin/env %s\n' % py_exe)
- + '## WARNING: This file is generated\n'
- + content)
- return content.replace('##EXT' 'END##', extra_text)
-
-##EXTEND##
-
-def convert(s):
- b = base64.b64decode(s.encode('ascii'))
- return zlib.decompress(b).decode('utf-8')
-
-##file site.py
-SITE_PY = convert("""
-eJzFPf1z2zaWv/OvwMqToZTKdOJ0e3tO3RsncVrfuYm3yc7m1vXoKAmyWFMkS5C2tTd3f/u9DwAE
-+CHb2+6cphNLJPDw8PC+8PAeOhqNTopCZkuxyZd1KoWScblYiyKu1kqs8lJU66Rc7hdxWW3h6eIm
-vpZKVLlQWxVhqygInv/GT/BcfF4nyqAA3+K6yjdxlSziNN2KZFPkZSWXYlmXSXYtkiypkjhN/g4t
-8iwSz387BsFZJmDmaSJLcStLBXCVyFfiYlut80yM6wLn/DL6Y/xqMhVqUSZFBQ1KjTNQZB1XQSbl
-EtCElrUCUiaV3FeFXCSrZGEb3uV1uhRFGi+k+K//4qlR0zAMVL6Rd2tZSpEBMgBTAqwC8YCvSSkW
-+VJGQryRixgH4OcNsQKGNsU1U0jGLBdpnl3DnDK5kErF5VaM53VFgAhlscwBpwQwqJI0De7y8kZN
-YElpPe7gkYiZPfzJMHvAPHH8LucAjh+z4C9Zcj9l2MA9CK5aM9uUcpXcixjBwk95Lxcz/WycrMQy
-Wa2ABlk1wSYBI6BEmswPClqOb/UKfXdAWFmujGEMiShzY35JPaLgrBJxqoBt6wJppAjzd3KexBlQ
-I7uF4QAikDToG2eZqMqOQ7MTOQAocR0rkJKNEuNNnGTArD/GC0L7r0m2zO/UhCgAq6XEL7Wq3PmP
-ewgArR0CTANcLLOadZYmNzLdTgCBz4B9KVWdVigQy6SUiyovE6kIAKC2FfIekJ6KuJSahMyZRm6n
-RH+iSZLhwqKAocDjSyTJKrmuS5IwsUqAc4Er3n/8Sbw7fXN28kHzmAHGMnu9AZwBCi20gxMMIA5q
-VR6kOQh0FJzjHxEvlyhk1zg+4NU0OHhwpYMxzL2I2n2cBQey68XVw8AcK1AmNFZA/f4bukzVGujz
-Pw+sdxCcDFGFJs7f7tY5yGQWb6RYx8xfyBnBtxrOd1FRrV8DNyiEUwGpFC4OIpggPCCJS7NxnklR
-AIulSSYnAVBoTm39VQRW+JBn+7TWLU4ACGWQwUvn2YRGzCRMtAvrNeoL03hLM9NNArvOm7wkxQH8
-ny1IF6VxdkM4KmIo/jaX10mWIULIC0G4F9LA6iYBTlxG4pxakV4wjUTI2otbokjUwEvIdMCT8j7e
-FKmcsviibt2tRmgwWQmz1ilzHLSsSL3SqjVT7eW9w+hLi+sIzWpdSgBezz2hW+X5VMxBZxM2Rbxh
-8arucuKcoEeeqBPyBLWEvvgdKHqiVL2R9iXyCmgWYqhgladpfgckOwoCIfawkTHKPnPCW3gH/wJc
-/DeV1WIdBM5IFrAGhcgPgUIgYBJkprlaI+Fxm2bltpJJMtYUebmUJQ31OGIfMOKPbIxzDT7klTZq
-PF1c5XyTVKiS5tpkJmzxsrBi/fia5w3TAMutiGamaUOnDU4vLdbxXBqXZC5XKAl6kV7bZYcxg54x
-yRZXYsNWBt4BWWTCFqRfsaDSWVWSnACAwcIXZ0lRp9RIIYOJGAbaFAR/E6NJz7WzBOzNZjlAhcTm
-ewH2B3D7O4jR3ToB+iwAAmgY1FKwfPOkKtFBaPRR4Bt905/HB049W2nbxEOu4iTVVj7OgjN6eFqW
-JL4LWWCvqSaGghlmFbp21xnQEcV8NBoFgXGHtsp8zVVQldsjYAVhxpnN5nWChm82Q1Ovf6iARxHO
-wF43287CAw1hOn0AKjldVmW+wdd2bp9AmcBY2CPYExekZSQ7yB4nvkbyuSq9ME3RdjvsLFAPBRc/
-nb4/+3L6SRyLy0alTdv67ArGPM1iYGuyCMBUrWEbXQYtUfElqPvEezDvxBRgz6g3ia+Mqxp4F1D/
-XNb0Gqax8F4Gpx9O3pyfzv7y6fSn2aezz6eAINgZGezRlNE81uAwqgiEA7hyqSJtX4NOD3rw5uST
-fRDMEjX75mtgN3gyvpYVMHE5hhlPRbiJ7xUwaDilphPEsdMALHg4mYjvxOHz568OCVqxLbYADMyu
-0xQfzrRFnyXZKg8n1PgXdumPWUlp/+3y6OsrcXwswl/i2zgMwIdqmjJL/Eji9HlbSOhawZ9xriZB
-sJQrEL0biQI6fk5+8YQ7wJJAy1zb6V/yJDPvmSvdIUh/jKkH4DCbLdJYKWw8m4VABOrQ84EOETvX
-KHVj6Fhs3a4TjQp+SgkLm2GXKf7Tg2I8p36IBqPodjGNQFw3i1hJbkXTh36zGeqs2WysBwRhJokB
-h4vVUChME9RZZQJ+LXEe6rC5ylP8ifBRC5AA4tYKtSQukt46RbdxWks1diYFRByPW2RERZso4kdw
-UcZgiZulm0za1DQ8A82AfGkOWrRsUQ4/e+DvgLoymzjc6PHei2mGmP477zQIB3A5Q1T3SrWgsHYU
-F6cX4tWLw310Z2DPubTU8ZqjhU6yWtqHK1gtIw+MMPcy8uLSZYV6Fp8e7Ya5iezKdFlhpZe4lJv8
-Vi4BW2RgZ5XFT/QGduYwj0UMqwh6nfwBVqHGb4xxH8qzB2lB3wGotyEoZv3N0u9xMEBmChQRb6yJ
-1HrXz6awKPPbBJ2N+Va/BFsJyhItpnFsAmfhPCZDkwgaArzgDCl1J0NQh2XNDivhjSDRXiwbxRoR
-uHPU1Ff09SbL77IZ74SPUemOJ5Z1UbA082KDZgn2xHuwQoBkDhu7hmgMBVx+gbK1D8jD9GG6QFna
-WwAgMPSKtmsOLLPVoynyrhGHRRiT14KEt5ToL9yaIWirZYjhQKK3kX1gtARCgslZBWdVg2YylDXT
-DAZ2SOJz3XnEW1AfQIuKEZjNsYbGjQz9Lo9AOYtzVyk5/dAif/nyhdlGrSm+gojNcdLoQqzIWEbF
-FgxrAjrBeGQcrSE2uAPnFsDUSrOm2P8k8oK9MVjPCy3b4AfA7q6qiqODg7u7u0hHF/Ly+kCtDv74
-p2+++dML1onLJfEPTMeRFh1qiw7oHXq00bfGAn1nVq7Fj0nmcyPBGkvyysgVRfy+r5NlLo72J1Z/
-Ihc3Zhr/Na4MKJCZGZSpDLQdNRg9U/vPoldqJJ6RdbZtxxP2S7RJtVbMt7rQo8rBEwC/ZZHXaKob
-TlDiK7BusENfynl9HdrBPRtpfsBUUU7Hlgf2X14hBj5nGL4ypniGWoLYAi2+Q/qfmG1i8o60hkDy
-oonq7J63/VrMEHf5eHm3vqYjNGaGiULuQInwmzxaAG3jruTgR7u2aPcc19Z8PENgLH1gmFc7lmMU
-HMIF12LqSp3D1ejxgjTdsWoGBeOqRlDQ4CTOmdoaHNnIEEGid2M2+7ywugXQqRU5NPEBswrQwh2n
-Y+3arOB4QsgDx+IlPZHgIh913r3gpa3TlAI6LR71qMKAvYVGO50DX44NgKkYlX8ZcUuzTfnYWhRe
-gx5gOceAkMFWHWbCN64PONob9bBTx+oP9WYa94HARRpzLOpR0AnlYx6hVCBNxdjvOcTilrjdwXZa
-HGIqs0wk0mpAuNrKo1eodhqmVZKh7nUWKVqkOXjFVisSIzXvfWeB9kH4uM+YaQnUZGjI4TQ6Jm/P
-E8BQt8Pw2XWNgQY3DoMYbRJF1g3JtIZ/wK2g+AYFo4CWBM2CeayU+RP7HWTOzld/GWAPS2hkCLfp
-kBvSsRgajnm/J5CMOhoDUpABCbvCSK4jq4MUOMxZIE+44bUclG6CESmQM8eCkJoB3Omlt8HBJxGe
-gJCEIuT7SslCfCVGsHxtUX2c7v5dudQEIcZOA3IVdPTi2I1sOFGN41aUw2doP75BZyVFDhw8B5fH
-DfS7bG6Y1gZdwFn3FbdFCjQyxWFGExfVK0MYN5j8h2OnRUMsM4hhKG8g70jHjDQJ7HJr0LDgBoy3
-5u2x9GM3YoF9x2GuDuXmHvZ/YZmoRa5Cipm0YxfuR3NFlzYW2/NkPoI/3gKMJlceJJnq+AVGWf6B
-QUIPetgH3ZsshkWWcXmXZCEpME2/Y39pOnhYUnpG7uATbacOYKIY8Tx4X4KA0NHnAYgTagLYlctQ
-abe/C3bnFEcWLncfeW7z5dGrqy5xp0MRHvvpX6rT+6qMFa5WyovGQoGr1TXgqHRhcnG21YeX+nAb
-twllrmAXKT5++iKQEBzXvYu3T5t6w/CIzYNz8j4GddBrD5KrNTtiF0AEtSIyykH4dI58PLJPndyO
-iT0ByJMYZseiGEiaT/4ROLsWCsbYX24zjKO1VQZ+4PU3X896IqMukt98PXpglBYx+sR+3PIE7cic
-VLBrtqWMU3I1nD4UVMwa1rFtignrc9r+aR676vE5NVo29t3fAj8GCobUJfgIL6YN2bpTxY/vTg3C
-03ZqB7DObtV89mgRYG+fz3+BHbLSQbXbOEnpXAEmv7+PytVs7jle0a89PEg7FYxDgr79l7p8AdwQ
-cjRh0p2OdsZOTMC5ZxdsPkWsuqjs6RyC5gjMywtwjz+HFU6ve+B7Bge/r7p8IiBvTqMeMmpbbIZ4
-wQclhz1K9gnzfvqMf9dZP27mw4L1/zHLF/+cST5hKgaaNh4+rH5iuXbXAHuEeRpwO3e4hd2h+axy
-ZZw7VklKPEfd9VzcUboCxVbxpAigLNnv64GDUqoPvd/WZclH16QCC1nu43HsVGCmlvH8ek3Mnjj4
-ICvExDZbUKzayevJ+4Qv1NFnO5Ow2Tf0c+c6NzErmd0mJfQFhTsOf/j442nYb0IwjgudHm9FHu83
-INwnMG6oiRM+pQ9T6Cld/nH10d66+AQ1GQEmIqzJ1iVsJxBs4gj9a/BARMg7sOVjdtyhL9ZycTOT
-lDqAbIpdnaD4W3yNmNiMAj//S8UrSmKDmSzSGmnFjjdmH67qbEHnI5UE/0qnCmPqECUEcPhvlcbX
-Ykydlxh60txI0anbuNTeZ1HmmJwq6mR5cJ0shfy1jlPc1svVCnDBwyv9KuLhKQIl3nFOAyctKrmo
-y6TaAglileuzP0p/cBrOtzzRsYckH/MwATEh4kh8wmnjeybc0pDLBAf8Ew+cJO67sYOTrBDRc3if
-5TMcdUY5vlNGqnsuT4+D9gg5ABgBUJj/aKIjd/4bSa/cA0Zac5eoqCU9UrqRhpycMYQynmCkg3/T
-T58RXd4awPJ6GMvr3Vhet7G87sXy2sfyejeWrkjgwtqglZGEvsBV+1ijN9/GjTnxMKfxYs3tMPcT
-czwBoijMBtvIFKdAe5EtPt8jIKS2nQNnetjkzyScVFrmHALXIJH78RBLb+ZN8rrTmbJxdGeeinFn
-h3KI/L4HUUSpYnPqzvK2jKs48uTiOs3nILYW3WkDYCra6UQcK81uZ3OO7rYs1ejiPz//8PEDNkdQ
-I5PeQN1wEdGw4FTGz+PyWnWlqdn8FcCO1NJPxKFuGuDeIyNrPMoe//OOMjyQccQdZSjkogAPgLK6
-bDM39ykMW891kpR+zkzOh03HYpRVo2ZSA0Q6ubh4d/L5ZEQhv9H/jlyBMbT1pcPFx7SwDbr+m9vc
-Uhz7gFDr2FZj/Nw5ebRuOOJhG2vAdjzf1oPDxxjs3jCBP8t/KqVgSYBQkQ7+PoVQj945/Kb9UIc+
-hhE7yX/uyRo7K/adI3uOi+KIft+xQ3sA/7AT9xgzIIB2ocZmZ9DslVtK35rXHRR1gD7S1/vNe832
-1qu9k/EpaifR4wA6lLXNht0/75yGjZ6S1ZvT788+nJ+9uTj5/IPjAqIr9/HTwaE4/fGLoPwQNGDs
-E8WYGlFhJhIYFrfQSSxz+K/GyM+yrjhIDL3enZ/rk5oNlrpg7jPanAiecxqThcZBM45C24c6/wgx
-SvUGyakponQdqjnC/dKG61lUrvOjqVRpjs5qrbdeulbM1JTRuXYE0geNXVIwCE4xg1eUxV6ZXWHJ
-J4C6zqoHKW2jbWJISkHBTrqAc/5lTle8QCl1hidNZ63oL0MX1/AqUkWawE7udWhlSXfD9JiGcfRD
-e8DNePVpQKc7jKwb8qwHsUCr9Trkuen+k4bRfq0Bw4bB3sG8M0npIZSBjcltIsRGfJITynv4apde
-r4GCBcODvgoX0TBdArOPYXMt1glsIIAn12B9cZ8AEFor4R8IHDnRAZljdkb4drPc/3OoCeK3/vnn
-nuZVme7/TRSwCxKcShT2ENNt/A42PpGMxOnH95OQkaPUXPHnGssDwCGhAKgj7ZS/xCfos7GS6Urn
-l/j6AF9oP4Fet7qXsih1937XOEQJeKbG5DU8U4Z+IaZ7WdhTnMqkBRorHyxmWEHopiGYz574tJZp
-qvPdz96dn4LviMUYKEF87nYKw3G8BI/QdfIdVzi2QOEBO7wukY1LdGEpyWIZec16g9YoctTby8uw
-60SB4W6vThS4jBPloj3GaTMsU04QISvDWphlZdZutUEKu22I4igzzBKzi5ISWH2eAF6mpzFviWCv
-hKUeJgLPp8hJVpmMxTRZgB4FlQsKdQpCgsTFekbivDzjGHheKlMGBQ+LbZlcrys83YDOEZVgYPMf
-T76cn32gsoTDV43X3cOcU9oJTDmJ5BhTBDHaAV/ctD/kqtmsj2f1K4SB2gf+tF9xdsoxD9Dpx4FF
-/NN+xXVox85OkGcACqou2uKBGwCnW5/cNLLAuNp9MH7cFMAGMx8MxSKx7EUnerjz63KibdkyJRT3
-MS+fcICzKmxKmu7spqS1P3qOqwLPuZbj/kbwtk+2zGcOXW86b4aS39xPRwqxJBYw6rb2xzDZYZ2m
-ejoOsw1xC21rtY39OXNipU67RYaiDEQcu50nLpP1K2HdnDnQS6PuABPfanSNJPaq8tHP2Uh7GB4m
-ltidfYrpSGUsZAQwkiF17U8NPhRaBFAglP07diR3Onl+6M3RsQYPz1HrLrCNP4Ai1Lm4VOORl8CJ
-8OVXdhz5FaGFevRIhI6nkskst3li+Llbo1f50p9jrwxQEBPFroyzazlmWFMD8yuf2AMhWNK2Hqkv
-k6s+wyLOwDm9H+Dwrlz0H5wY1FqM0Gl3I7dtdeSTBxv0loLsJJgPvozvQPcXdTXmlRw4h+6tpRuG
-+jBEzD6Epvr0fRxiOObXcGB9GsC91NCw0MP7deDsktfGOLLWPraqmkL7QnuwixK2ZpWiYxmnONH4
-otYLaAzucWPyR/apThSyv3vqxJyYkAXKg7sgvbmNdINWOGHE5UpcOZpQOnxTTaPfLeWtTMFogJEd
-Y7XDL7baYRLZcEpvHthvxu5ie7Htx43eNJgdmXIMRIAKMXoDPbsQanDAFf5Z70Ti7Iac47d/PZuK
-tx9+gn/fyI9gQbHmcSr+BqOLt3kJ20ou2qXbFLCAo+L9Yl4rLIwkaHRCwRdPoLd24ZEXT0N0ZYlf
-UmIVpMBk2nLDt50AijxBKmRv3ANTLwG/TUFXywk1DmLfWoz0S6TBcI0L1oUc6JbRutqkaCac4Eiz
-iJej87O3px8+nUbVPTK2+Tlygid+HhZORx8Nl3gMNhX2yaLGJ1eOv/yDTIsed1nvNU29DO41RQjb
-kcLuL/kmjdjuKeISAwai2C7zRYQtgdO5RK+6A/954mwrH7TvnnFFWOOJPjxrnHh8DNQQP7f1zwga
-Uh89J+pJCMVzrBXjx9Go3wJPBUW04c/zm7ulGxDXRT80wTamzazHfnerAtdMZw3PchLhdWyXwdSB
-pkmsNvOFWx/4MRP6IhRQbnS8IVdxnVZCZrCVor093UgBCt4t6WMJYVZhK0Z1bhSdSe/irXJyj2Il
-RjjqiIrq8RyGAoWw9f4xvmEzgLWGouYSaIBOiNK2KXe6qnqxZgnmnRBRryff4C7JXrnJL5rCPChv
-jBeN/wrzRG+RMbqWlZ4/PxhPLl82CQ4UjF54Bb2LAoydyyZ7oDGL58+fj8S/Pez0MCpRmuc34I0B
-7F5n5ZxeDxhsPTm7Wl2H3ryJgB8Xa3kJD64oaG6f1xlFJHd0pQWR9q+BEeLahJYZTfuWOeZYXcnn
-y9yCz6m0wfhLltB1RxhRkqhs9a1RGG0y0kQsCYohjNUiSUKOTsB6bPMaa/Ewuqj5Rd4DxycIZopv
-8WCMd9hrdCwpb9Zyj0XnWIwI8IhSyng0KmamajTAc3ax1WjOzrKkaspIXrhnpvoKgMreYqT5SsR3
-KBlmHi1iOGWdHqs2jnW+k0W9jUq+uHTjjK1Z8uuHcAfWBknLVyuDKTw0i7TIZbkw5hRXLFkklQPG
-tEM43JkubyLrEwU9KI1AvZNVWFqJtm//YNfFxfQjHR/vm5F01lBlL8TimFCctfIKo6gZn6JPlpCW
-b82XCYzygaLZ2hPwxhJ/0LFUrCHw7u1wyxnrTN/HwWkbzSUdAIfugLIK0rKjpyOci8csfGbagVs0
-8EM7c8LtNimrOk5n+tqHGfppM3uervG0ZXA7CzyttwK+fQ6O777O2AfHwSTXID0x49ZUZByLlY5M
-RG5lmV+EVeTo5R2yrwQ+BVJmOTP10CZ2dGnZ1Raa6gRHR8UjqK9M8dKAQ26qZjoFJy7mU0pvMuUO
-A86zn29JV1eI78T41VQctnY+i2KLNzkBss+Woe+KUTeYihMMMHNs34shvjsW45dT8ccd0KOBAY4O
-3RHa+9gWhEEgr66eTMY0mRPZwr4U9of76hxG0PSM4+SqTf4umb4lKv1ri0pcIagTlV+2E5VbYw/u
-WzsfH8lwA4pjlcjl/jOFJNRIN7p5mMEJPyyg37M5Wrp2vKmoocK5OWxG7ho96GhE4zbbQUxRulZf
-XL+LuoYNp71zwKTJtFIV7S1zmMao0WsRFQDM+o7S8Bve7QLvNSlc/2zwiFUXAViwPREEXenJB2ZN
-w0ZQH3QEn6QBHmAUEeJhaqMoXMl6goiEdA8OMdFXrUNsh+N/d+bhEoOho9AOlt98vQtPVzB7izp6
-FnR3pYUnsra8ollu8+kPzHmM0tf1NwmMA6URHXBWzVWV5GYeYfYy30GT2yzmDV4GSSfTaBJT6bpN
-vJXmW7/Qj6HYASWTwVqAJ1Wv8CD5lu62PFGU9IZX1Hx9+HJqKoMZkJ7Aq+jVV/oKSOpmLj/wfeyp
-3rvBS93vMPoXB1hS+b3tq85uhqZ13LoLyh8spOjZJJpZOjSG6eE6kGbNYoF3JjbEZN/aXgDyHryd
-Ofg55vLTHBw22JBGfei6GqOR3iHVNiDAD5uMIcl5VNdGkSLSu4RtSHnuUpxPFgXdq9+CYAgBOX8d
-8xt0BeviyIbYjE3Bk8+xm82Jn+qmt+6M7Qka2+om3DV97r9r7rpFYGdukhk6c/frS10a6L7DVrSP
-Bhze0IR4VIlEo/H7jYlrB6Y6h6Y/Qq8/SH63E850wKw8BMZk7GC8n9hTY2/M/iZeuN8xIWyfL2R2
-y4l7nY3WtDs2o83xj/EUOPkFn9sbBiijaak5kPdLdMPejHNkZ/L6Ws1ivN1xRptsyufq7J7Mtu09
-Xc4nY7U1uy28tAhAGG7Smbducj0wBuhKvmWa06Gc22kEDU1Jw04WskqWbBL01g7ARRwxpf4mEM9p
-xKNUYqBb1WVRwm54pO8i5jydvtTmBqgJ4G1idWNQNz2m+mpaUqyUHGZKkDlO20ryASKwEe+YhtnM
-vgNeedFcs5BMLTPIrN7IMq6aK4b8jIAENl3NCFR0jovrhOcaqWxxiYtYYnnDQQoDZPb7V7Cx9DbV
-O+5VmFht93h2oh465PuUKxscY2S4OLm31wu611ot6Wpr1zu0zRqus1cqwTKYu/JIR+pYGb/V93fx
-HbMcyUf/0uEfkHe38tLPQrfqjL1bi4bzzFUI3Qub8MYAMs599zB2OKB742JrA2zH9/WFZZSOhznQ
-2FJR++S9CqcZbdJEkDBh9IEIkl8U8MQIkgf/kREkfWsmGBqNj9YDvWUCD4SaWD24V1A2jAB9ZkAk
-PMBuXWBoTOXYTbovcpXcj+yF0qwrnUo+Yx6QI7t3kxEIvmpSuRnK3lVwuyJIvnTR4+/PP745OSda
-zC5O3v7HyfeUlIXHJS1b9egQW5bvM7X3vfRvN9ymE2n6Bm+w7bkhlmuYNITO+04OQg+E/nq1vgVt
-KzL39VCHTt1PtxMgvnvaLahDKrsXcscv0zUmbvpMK0870E85qdb8cjITzCNzUsfi0JzEmffN4YmW
-0U5seWjhnPTWrjrR/qq+BXQg7j2xSda0Anhmgvxlj0xMxYwNzLOD0v7ffFBmOFYbmht0QAoX0rnJ
-kS5xZFCV//8TKUHZxbi3Y0dxau/mpnZ8PKTspfN49ruQkSGIV+436s7PFfalTAeoEASs8PQ9hYyI
-0X/6QNWmHzxT4nKfCov3Udlc2V+4Ztq5/WuCSQaVve9LcYISH7NC41WduokDtk+nAzl9dBqVr5xK
-FtB8B0DnRjwVsDf6S6wQ51sRwsZRu2SYHEt01Jf1Ocij3XSwN7R6IfaHyk7dskshXg43XLYqO3WP
-Q+6hHuihalPc51hgzNIcqicV3xFkPs4UdMGX53zgGbre9sPX28uXR/ZwAfkdXzuKhLLJRo5hv3Sy
-MXdeKul0J2Ypp5Suh3s1JySsW1w5UNknGNrbdEpSBvY/Js+BIY289/0hM9PDu3p/1MbUst4RTEmM
-n6kJTcsp4tG42yeT7nQbtdUFwgVJjwDSUYEAC8F0dKOTILrlLO/xC70bnNd0Ha97whQ6UkHJYj5H
-cA/j+zX4tbtTIfGjujOKpj83aHOgXnIQbvYduNXEC4UMm4T21Bs+GHABuCa7v//LR/TvpjHa7oe7
-/Grb6lVvHSD7spj5iplBLRKZxxEYGdCbY9LWWC5hBB2voWno6DJUMzfkC3T8KJsWL9umDQY5szPt
-AVijEPwfucjncQ==
-""")
-
-##file activate.sh
-ACTIVATE_SH = convert("""
-eJytVd9v2kAMfs9fYQLq2m4MscdNVKMqEkgtVIQxbeuUHolpTgsXdHehpT/+9/mSEBJS2MOaB0ji
-z77P9menDpOAK5jzEGERKw0zhFihD/dcB2CrKJYewoyLFvM0XzGNNpzOZbSAGVPBqVWHdRSDx4SI
-NMhYANfgc4meDteW5ePGC45P4MkCumKhUENzDsu1H3lw1vJx1RJxGMKns6O2lWDqINGgotAHFCsu
-I7FAoWHFJGezEFWGqsEvaD5C42naHb93X+A3+elYCgVaxgh8DmQAys9HL2SS0mIaWBgm7mTN/O3G
-kzu6vHCng/HkW/fSve5O+hTOpnhfQAcoEry5jKVjNypoO0fgwzKSOgHm79KUK06Jfc7/RebHpD8a
-9kdXvT2UcnuFWG6p0stNB0mWUUQ1q3uiGRVEMfXHR03dTuQATPjwqIIPcB9wL4CArRAY/ZHJixYL
-Y9YBtcAoLQtFevOoI9QaHcEdMSAB0d08kuZhyUiSmav6CPCdVBnFOjNrLu6yMCWgKRA0TInBC5i4
-QwX3JG/mm581GKnSsSSxJTFHf9MAKr8w5T/vOv1mUurn5/zlT6fvTntjZzAaNl9rQ5JkU5KIc0GX
-inagwU57T2eddqWlTrvaS6d9sImZeUMkhWysveF0m37NcGub9Dpgi0j4qGiOzATjDr06OBjOYQOo
-7RBoGtNm9Denv1i0LVI7lxJDXLHSSBeWRflsyyqw7diuW3h0XdvK6lBMyaoMG1UyHdTsoYBuue75
-YOgOu1c91/2cwYpznPPeDoQpGL2xSm09NKp7BsvQ2hnT3aMs07lUnskpxewvBk73/LLnXo9HV9eT
-ijB3hWBO2ygoiWg/bKuZxqCCQq0DD3vkWIVvI2KosIw+vqW1gIItEG5KJb+xb09g65ktwYKgTc51
-uGJ/EFQs0ayEWLCQM5V9N4g+1+8UbXOJzF8bqhKtIqIwicWvzNFROZJlpfD8A7Vc044R0FxkcezG
-VzsV75usvTdYef+57v5n1b225qhXfwEmxHEs
-""")
-
-##file activate.fish
-ACTIVATE_FISH = convert("""
-eJyFVVFv0zAQfs+vONJO3RDNxCsSQoMVrdK2Vl03CSHkesllMXLsYDvZivjx2GmTOG0YfWhV+7u7
-73z33Y1gnTENKeMIeakNPCKUGhP7xcQTbCJ4ZOKcxoZV1GCUMp1t4O0zMxkTQEGVQjicO4dTyIwp
-Ppyfu386Q86jWOZwBhq1ZlK8jYIRXEoQ0jhDYAYSpjA2fBsFQVoKG0UKSLAJB9MEJrMXi6uYMiXl
-KCrIZYJARQIKTakEGAkmQ+tU5ZSDRTAlRY7CRJMA7GdkgRoNSJ74t1BRxegjR12jWAoGbfpTAeGY
-LK4vycN8tb6/uCbLi/VVWGPcx3maPr2AO4VjYB+HMAxAkQT/i/ptfbW4vVrczAZit3eHDNqL13n0
-Ya+w+Tq/uyLL1eJmuSaLh9lqNb/0+IzgznqnAjAvzBa4jG0BNmNXfdJUkxTU2I6xRaKcy+e6VApz
-WVmoTGFTgwslrYdN03ONrbbMN1E/FQ7H7gOP0UxRjV67TPRBjF3naCMV1mSkYk9MUN7F8cODZzsE
-iIHYviIe6n8WeGQxWKuhl+9Xa49uijq7fehXMRxT9VR9f/8jhDcfYSKkSOyxKp22cNIrIk+nzd2b
-Yc7FNpHx8FUn15ZfzXEE98JxZEohx4r6kosCT+R9ZkHQtLmXGYSEeH8JCTvYkcRgXAutp9Rw7Jmf
-E/J5fktuL25m1tMe3vLdjDt9bNxr2sMo2P3C9BccqGeYhqfQITz6XurXaqdf99LF1mT2YJrvzqCu
-5w7dKvV3PzNyOb+7+Hw923dOuB+AX2SxrZs9Lm0xbCH6kmhjUyuWw+7cC7DX8367H3VzDz6oBtty
-tMIeobE21JT6HaRS+TbaoqhbE7rgdGs3xtE4cOF3xo0TfxwsdyRlhUoxuzes18r+Jp88zDx1G+kd
-/HTrr1BY2CeuyfnbQtAcu9j+pOw6cy9X0k3IuoyKCZPC5ESf6MkgHE5tLiSW3Oa+W2NnrQfkGv/h
-7tR5PNFnMBlw4B9NJTxnzKA9fLTT0aXSb5vw7FUKzcTZPddqYHi2T9/axJmEEN3qHncVCuEPaFmq
-uEtpcBj2Z1wjrqGReJBHrY6/go21NA==
-""")
-
-##file activate.csh
-ACTIVATE_CSH = convert("""
-eJx1U2FP2zAQ/e5f8TAV3Soo+0zXbYUiDQkKQgVp2ibjJNfFUuIg22nVf885SVFLO3+I7Lt3fr6X
-d8eY58ZjYQpCWfuAhFB7yrAyIYf0Ve1SQmLsuU6DWepAw9TnEoOFq0rwdjAUx/hV1Ui1tVWAqy1M
-QGYcpaFYx+yVI67LkKwx1UuTEaYGl4X2Bl+zJpAlP/6V2hTDtCq/DYXQhdEeGW040Q/Eb+t9V/e3
-U/V88zh/mtyqh8n8J47G+IKTE3gKZJdoYrK3h5MRU1tGYS83gqNc+3yEgyyP93cP820evHLvr2H8
-kaYB/peoyY7aVHzpJnE9e+6I5Z+ji4GMTNJWNuOQq6MA1N25p8pW9HWdVWlfsNpPDbdxjgpaahuw
-1M7opCA/FFu1uwxC7L8KUqmto1KyQe3rx0I0Eovdf7BVe67U5c1MzSZ310pddGheZoFPWyytRkzU
-aCA/I+RkBXhFXr5aWV0SxjhUI6jwdAj8kmhPzX7nTfJFkM3MImp2VdVFFq1vLHSU5szYQK4Ri+Jd
-xlW2JBtOGcyYVW7SnB3v6RS91g3gKapZ0oWxbHVteYIIq3iv7QeuSrUj6KSqQ+yqsxDj1ivNQxKF
-YON10Q+NH/ARS95i5Tuqq2Vxfvc23f/FO6zrtXXmJr+ZtMY9/A15ZXFWtmch2rEQ4g1ryVHH
-""")
-
-##file activate.bat
-ACTIVATE_BAT = convert("""
-eJx9Ul9LhEAQfxf8DoOclI/dYyFkaCmcq4gZQTBUrincuZFbff12T133TM+nnd35/Zvxlr7XDFhV
-mUZHOVhFlOWP3g4DUriIWoVomYZpNBWUtGpaWgImO191pFkSpzlcmgaI70jVX7n2Qp8tuByg+46O
-CMHbMq64T+nmlJt082D1T44muCDk2prgEHF4mdI9RaS/QwSt3zSyIAaftRccvqVTBziD1x/WlPD5
-xd729NDBb8Nr4DU9QNMKsJeH9pkhPedhQsIkDuCDCa6A+NF9IevVFAohkqizdHetg/tkWvPoftWJ
-MCqnOxv7/x7Np6yv9P2Ker5dmX8yNyCkkWnbZy3N5LarczlqL8htx2EM9rQ/2H5BvIsIEi8OEG8U
-+g8CsNTr
-""")
-
-##file deactivate.bat
-DEACTIVATE_BAT = convert("""
-eJyFkN0KgkAUhO8F32EQpHqFQEjQUPAPMaErqVxzId3IrV6/XST/UDx3c86c4WMO5FYysKJQFVVp
-CEfqxsnJ9DI7SA25i20fFqs3HO+GYLsDZ7h8GM3xfLHrg1QNvpSX4CWpQGvokZk4uqrQAjXjyElB
-a5IjCz0r+2dHcehHCe5MZNmB5R7TdqMqECMptHZh6DN/utb7Zs6Cej8OXYE5J04YOKFvD4GkHuJ0
-pilSd1jG6n87tDZ+BUwUOepI6CGSkFMYWf0ihvT33Qj1A+tCkSI=
-""")
-
-##file activate.ps1
-ACTIVATE_PS = convert("""
-eJylWdmO41hyfW+g/0FTU7C7IXeJIqmtB/3AnZRIStxF2kaBm7gv4ipyMF/mB3+Sf8GXVGVl1tLT
-43ECSqR4b5wbETeWE8z/+a///vNCDaN6cYtSf5G1dbNw/IVXNIu6aCvX9xa3qsgWl0IJ/7IYinbh
-2nkOVqs2X0TNjz/8eeFFle826fBhQRaLBkD9uviw+LCy3Sbq7Mb/UNbrH3+YNtLcVaB+Xbipb+eL
-tly0eVsD/M6u6g8//vC+dquobH5VWU75eMFUdvHb4n02RHlXuHYTFfmHbHCLLLNz70NpN+GrBI4p
-1EeSk4FAXaZR88u0vPip8usi7fznt3fvP+OuPnx49/Pil4td+XnzigIAPoqYQH2J8v4z+C+8b98m
-Q25t7k76LIK0cOz0V89/MXXx0+Lf6z5q3PA/F+/FIif9uqnaadFf/PzXSXYBfqIb2NeApecJwPzI
-dlL/149nnvyoc7KqYfzTAT8v/voUmX7e+3n364tffl/oVaDyswKY/7J18e6bve8Wv9RuUfqfLHmK
-/u139Hwx+9ePRep97KKqae30YwmCo2y+0vTz1k+rv7159B3pb1SOGj97Pe8/flfkC1Vn/7xYR4n6
-lypNEGDDV5f7lcjil3S+4++p881Wv6qKyn5GQg1yJwcp4BZ5E+Wt/z1P/umbiHir4J8Xip/eFt6n
-9T/9gU9eY+7zUX97Jlmb136ziKrKT/3OzpvP8VX/+MObSP0lL3LvVZlJ9v1b8357jXyw8rXxYPXN
-11n4UzJ8G8S/vUbuJ6RPj999DbtS5kys//JusXwrNLnvT99cFlBNwXCe+niRz8JF/ezNr9Pze+H6
-18W7d5PPvozW7+387Zto/v4pL8BvbxTzvIW9KCv/Fj0WzVQb/YXbVlPZWTz3/9vCaRtQbPN/Bb+j
-2rUrDxTVD68gfQXu/ZewAFX53U/vf/rD2P3558W7+W79Po1y/xXoX/6RFHyNIoVjgAG4H0RTcAe5
-3bSVv3DSwk2mZYHjFB8zj6fC4sLOFTHJJQrwzFYJgso0ApOoBzFiRzzQKjIQCCbQMIFJGCKqGUyS
-8AkjiF2wTwmMEbcEUvq8Nj+X0f4YcCQmYRiOY7eRbAJDqzm1chOoNstbJ8oTBhZQ2NcfgaB6QjLp
-U4+SWFjQGCZpyqby8V4JkPGs9eH1BscXIrTG24QxXLIgCLYNsIlxSYLA6SjAeg7HAg4/kpiIB8k9
-TCLm0EM4gKIxEj8IUj2dQeqSxEwYVH88qiRlCLjEYGuNIkJB1BA5dHOZdGAoUFk54WOqEojkuf4Q
-Ig3WY+96TDlKLicMC04h0+gDCdYHj0kz2xBDj9ECDU5zJ0tba6RKgXBneewhBG/xJ5m5FX+WSzsn
-wnHvKhcOciw9NunZ0BUF0n0IJAcJMdcLqgQb0zP19dl8t9PzmMBjkuIF7KkvHgqEovUPOsY0PBB1
-HCtUUhch83qEJPjQcNQDsgj0cRqx2ZbnnlrlUjE1EX2wFJyyDa/0GLrmKDEFepdWlsbmVU45Wiwt
-eFM6mfs4kxg8yc4YmKDy67dniLV5FUeO5AKNPZaOQQ++gh+dXE7dbJ1aTDr7S4WPd8sQoQkDyODg
-XnEu/voeKRAXZxB/e2xaJ4LTFLPYEJ15Ltb87I45l+P6OGFA5F5Ix8A4ORV6M1NH1uMuZMnmFtLi
-VpYed+gSq9JDBoHc05J4OhKetrk1p0LYiKipxLMe3tYS7c5V7O1KcPU8BJGdLfcswhoFCSGQqJ8f
-ThyQKy5EWFtHVuNhvTnkeTc8JMpN5li3buURh0+3ZGuzdwM55kon+8urbintjdQJf9U1D0ah+hNh
-i1XNu4fSKbTC5AikGEaj0CYM1dpuli7EoqUt7929f1plxGGNZnixFSFP2qzhlZMonu2bB9OWSqYx
-VuHKWNGJI8kqUhMTRtk0vJ5ycZ60JlodlmN3D9XiEj/cG2lSt+WV3OtMgt1Tf4/Z+1BaCus740kx
-Nvj78+jMd9tq537Xz/mNFyiHb0HdwHytJ3uQUzKkYhK7wjGtx3oKX43YeYoJVtqDSrCnQFzMemCS
-2bPSvP+M4yZFi/iZhAjL4UOeMfa7Ex8HKBqw4umOCPh+imOP6yVTwG2MplB+wtg97olEtykNZ6wg
-FJBNXSTJ3g0CCTEEMdUjjcaBDjhJ9fyINXgQVHhA0bjk9lhhhhOGzcqQSxYdj3iIN2xGEOODx4qj
-Q2xikJudC1ujCVOtiRwhga5nPdhe1gSa649bLJ0wCuLMcEYIeSy25YcDQHJb95nfowv3rQnin0fE
-zIXFkM/EwSGxvCCMgEPNcDp/wph1gMEa8Xd1qAWOwWZ/KhjlqzgisBpDDDXz9Cmov46GYBKHC4zZ
-84HJnXoTxyWNBbXV4LK/r+OEwSN45zBp7Cub3gIYIvYlxon5BzDgtPUYfXAMPbENGrI+YVGSeTQ5
-i8NMB5UCcC+YRGIBhgs0xhAGwSgYwywpbu4vpCSTdEKrsy8osXMUnHQYenQHbOBofLCNNTg3CRRj
-A1nXY2MZcjnXI+oQ2Zk+561H4CqoW61tbPKv65Y7fqc3TDUF9CA3F3gM0e0JQ0TPADJFJXVzphpr
-2FzwAY8apGCju1QGOiUVO5KV6/hKbtgVN6hRVwpRYtu+/OC6w2bCcGzZQ8NCc4WejNEjFxOIgR3o
-QqR1ZK0IaUxZ9nbL7GWJIjxBARUhAMnYrq/S0tVOjzlOSYRqeIZxaSaOBX5HSR3MFekOXVdUPbjX
-nru61fDwI8HRYPUS7a6Inzq9JLjokU6P6OzT4UCH+Nha+JrU4VqEo4rRHQJhVuulAnvFhYz5NWFT
-aS/bKxW6J3e46y4PLagGrCDKcq5B9EmP+s1QMCaxHNeM7deGEV3WPn3CeKjndlygdPyoIcNaL3dd
-bdqPs47frcZ3aNWQ2Tk+rjFR01Ul4XnQQB6CSKA+cZusD0CP3F2Ph0e78baybgioepG12luSpFXi
-bHbI6rGLDsGEodMObDG7uyxfCeU+1OiyXYk8fnGu0SpbpRoEuWdSUlNi5bd9nBxYqZGrq7Qa7zV+
-VLazLcelzzP9+n6+xUtWx9OVJZW3gk92XGGkstTJ/LreFVFF2feLpXGGuQqq6/1QbWPyhJXIXIMs
-7ySVlzMYqoPmnmrobbeauMIxrCr3sM+qs5HpwmmFt7SM3aRNQWpCrmeAXY28EJ9uc966urGKBL9H
-18MtDE5OX97GDOHxam11y5LCAzcwtkUu8wqWI1dWgHyxGZdY8mC3lXzbzncLZ2bIUxTD2yW7l9eY
-gBUo7uj02ZI3ydUViL7oAVFag37JsjYG8o4Csc5R7SeONGF8yZP+7xxi9scnHvHPcogJ44VH/LMc
-Yu6Vn3jEzCFw9Eqq1ENQAW8aqbUwSiAqi+nZ+OkZJKpBL66Bj8z+ATqb/8qDIJUeNRTwrI0YrVmb
-9FArKVEbCWUNSi8ipfVv+STgkpSsUhcBg541eeKLoBpLGaiHTNoK0r4nn3tZqrcIULtq20Df+FVQ
-Sa0MnWxTugMuzD410sQygF4qdntbswiJMqjs014Irz/tm+pd5oygJ0fcdNbMg165Pqi7EkYGAXcB
-dwxioCDA3+BY9+JjuOmJu/xyX2GJtaKSQcOZxyqFzTaa6/ot21sez0BtKjirROKRm2zuai02L0N+
-ULaX8H5P6VwsGPbYOY7sAy5FHBROMrMzFVPYhFHZ7M3ZCZa2hsT4jGow6TGtG8Nje9405uMUjdF4
-PtKQjw6yZOmPUmO8LjFWS4aPCfE011N+l3EdYq09O3iQJ9a01B3KXiMF1WmtZ+l1gmyJ/ibAHZil
-vQzdOl6g9PoSJ4TM4ghTnTndEVMOmsSSu+SCVlGCOLQRaw9oLzamSWP62VuxPZ77mZYdfTRGuNBi
-KyhZL32S2YckO/tU7y4Bf+QKKibQSKCTDWPUwWaE8yCBeL5FjpbQuAlb53mGX1jptLeRotREbx96
-gnicYz0496dYauCjpTCA4VA0cdLJewzRmZeTwuXWD0talJsSF9J1Pe72nkaHSpULgNeK1+o+9yi0
-YpYwXZyvaZatK2eL0U0ZY6ekZkFPdC8JTF4Yo1ytawNfepqUKEhwznp6HO6+2l7L2R9Q3N49JMIe
-Z+ax1mVaWussz98QbNTRPo1xu4W33LJpd9H14dd66ype7UktfEDi3oUTccJ4nODjwBKFxS7lYWiq
-XoHu/b7ZVcK5TbRD0F/2GShg2ywwUl07k4LLqhofKxFBNd1grWY+Zt/cPtacBpV9ys2z1moMLrT3
-W0Elrjtt5y/dvDQYtObYS97pqj0eqmwvD3jCPRqamGthLiF0XkgB6IdHLBBwDGPiIDh7oPaRmTrN
-tYA/yQKFxRiok+jM6ciJq/ZgiOi5+W4DEmufPEubeSuYJaM3/JHEevM08yJAXUQwb9LS2+8FOfds
-FfOe3Bel6EDSjIEIKs4o9tyt67L1ylQlzhe0Q+7ue/bJnWMcD3q6wDSIQi8ThnRM65aqLWesi/ZM
-xhHmQvfKBbWcC194IPjbBLYR9JTPITbzwRcu+OSFHDHNSYCLt29sAHO6Gf0h/2UO9Xwvhrjhczyx
-Ygz6CqP4IwxQj5694Q1Pe2IR+KF/yy+5PvCL/vgwv5mPp9n4kx7fnY/nmV++410qF/ZVCMyv5nAP
-pkeOSce53yJ6ahF4aMJi52by1HcCj9mDT5i+7TF6RoPaLL+cN1hXem2DmX/mdIbeeqwQOLD5lKO/
-6FM4x77w6D5wMx3g0IAfa2D/pgY9a7bFQbinLDPz5dZi9ATIrd0cB5xfC0BfCCZO7TKP0jQ2Meih
-nRXhkA3smTAnDN9IW2vA++lsgNuZ2QP0UhqyjUPrDmgfWP2bWWiKA+YiEK7xou8cY0+d3/bk0oHR
-QLrq4KzDYF/ljQDmNhBHtkVNuoDey6TTeaD3SHO/Bf4d3IwGdqQp6FuhmwFbmbQBssDXVKDBYOpk
-Jy7wxOaSRwr0rDmGbsFdCM+7XU/84JPu3D/gW7QXgzlvbjixn99/8CpWFUQWHFEz/RyXvzNXTTOd
-OXLNNFc957Jn/YikNzEpUdRNxXcC6b76ccTwMGoKj5X7c7TvHFgc3Tf4892+5A+iR+D8OaaE6ACe
-gdgHcyCoPm/xiDCWP+OZRjpzfj5/2u0i4qQfmIEOsTV9Hw6jZ3Agnh6hiwjDtGYxWvt5TiWEuabN
-77YCyRXwO8P8wdzG/8489KwfFBZWI6Vvx76gmlOc03JI1HEfXYZEL4sNFQ3+bqf7e2hdSWQknwKF
-ICJjGyDs3fdmnnxubKXebpQYLjPgEt9GTzKkUgTvOoQa1J7N3nv4sR6uvYFLhkXZ+pbCoU3K9bfq
-gF7W82tNutRRZExad+k4GYYsCfmEbvizS4jsRr3fdzqjEthpEwm7pmN7OgVzRbrktjrFw1lc0vM8
-V7dyTJ71qlsd7v3KhmHzeJB35pqEOk2pEe5uPeCToNkmedmxcKbIj+MZzjFSsvCmimaMQB1uJJKa
-+hoWUi7aEFLvIxKxJavqpggXBIk2hr0608dIgnfG5ZEprqmH0b0YSy6jVXTCuIB+WER4d5BPVy9Q
-M4taX0RIlDYxQ2CjBuq78AAcHQf5qoKP8BXHnDnd/+ed5fS+csL4g3eWqECaL+8suy9r8hx7c+4L
-EegEWdqAWN1w1NezP34xsxLkvRRI0DRzKOg0U+BKfQY128YlYsbwSczEg2LqKxRmcgiwHdhc9MQJ
-IwKQHlgBejWeMGDYYxTOQUiJOmIjJbzIzHH6lAMP+y/fR0v1g4wx4St8fcqTt3gz5wc+xXFZZ3qI
-JpXI5iJk7xmNL2tYsDpcqu0375Snd5EKsIvg8u5szTOyZ4v06Ny2TZXRpHUSinh4IFp8Eoi7GINJ
-02lPJnS/9jSxolJwp2slPMIEbjleWw3eec4XaetyEnSSqTPRZ9fVA0cPXMqzrPYQQyrRux3LaAh1
-wujbgcObg1nt4iiJ5IMbc/WNPc280I2T4nTkdwG8H6iS5xO2WfsFsruBwf2QkgZlb6w7om2G65Lr
-r2Gl4dk63F8rCEHoUJ3fW+pU2Srjlmcbp+JXY3DMifEI22HcHAvT7zzXiMTr7VbUR5a2lZtJkk4k
-1heZZFdru8ucCWMTr3Z4eNnjLm7LW7rcN7QjMpxrsCzjxndeyFUX7deIs3PQkgyH8k6luI0uUyLr
-va47TBjM4JmNHFzGPcP6BV6cYgQy8VQYZe5GmzZHMxyBYhGiUdekZQ/qwyxC3WGylQGdUpSf9ZCP
-a7qPdJd31fPRC0TOgzupO7nLuBGr2A02yuUQwt2KQG31sW8Gd9tQiHq+hPDt4OzJuY4pS8XRsepY
-tsd7dVEfJFmc15IYqwHverrpWyS1rFZibDPW1hUUb+85CGUzSBSTK8hpvee/ZxonW51TUXekMy3L
-uy25tMTg4mqbSLQQJ+skiQu2toIfBFYrOWql+EQipgfT15P1aq6FDK3xgSjIGWde0BPftYchDTdM
-i4QdudHFkN0u6fSKiT09QLv2mtSblt5nNzBR6UReePNs+khE4rHcXuoK21igUKHl1c3MXMgPu7y8
-rKQDxR6N/rffXv+lROXet/9Q+l9I4D1U
-""")
-
-##file distutils-init.py
-DISTUTILS_INIT = convert("""
-eJytV1uL4zYUfvevOE0ottuMW9q3gVDa3aUMXXbLMlDKMBiNrSTqOJKRlMxkf33PkXyRbGe7Dw2E
-UXTu37lpxLFV2oIyifAncxmOL0xLIfcG+gv80x9VW6maw7o/CANSWWBwFtqeWMPlGY6qPjV8A0bB
-C4eKSTgZ5LRgFeyErMEeOBhbN+Ipgeizhjtnhkn7DdyjuNLPoCS0l/ayQTG0djwZC08cLXozeMss
-aG5EzQ0IScpnWtHSTXuxByV/QCmxE7y+eS0uxWeoheaVVfqSJHiU7Mhhi6gULbOHorshkrEnKxpT
-0n3A8Y8SMpuwZx6aoix3ouFlmW8gHRSkeSJ2g7hU+kiHLDaQw3bmRDaTGfTnty7gPm0FHbIBg9U9
-oh1kZzAFLaue2R6htPCtAda2nGlDSUJ4PZBgCJBGVcwKTAMz/vJiLD+Oin5Z5QlvDPdulC6EsiyE
-NFzb7McNTKJzbJqzphx92VKRFY1idenzmq3K0emRcbWBD0ryqc4NZGmKOOOX9Pz5x+/l27tP797c
-f/z0d+4NruGNai8uAM0bfsYaw8itFk8ny41jsfpyO+BWlpqfhcG4yxLdi/0tQqoT4a8Vby382mt8
-p7XSo7aWGdPBc+b6utaBmCQ7rQKQoWtAuthQCiold2KfJIPTT8xwg9blPumc+YDZC/wYGdAyHpJk
-vUbHbHWAp5No6pK/WhhLEWrFjUwtPEv1Agf8YmnsuXUQYkeZoHm8ogP16gt2uHoxcEMdf2C6pmbw
-hUMsWGhanboh4IzzmsIpWs134jVPqD/c74bZHdY69UKKSn/+KfVhxLgUlToemayLMYQOqfEC61bh
-cbhwaqoGUzIyZRFHPmau5juaWqwRn3mpWmoEA5nhzS5gog/5jbcFQqOZvmBasZtwYlG93k5GEiyw
-buHhMWLjDarEGpMGB2LFs5nIJkhp/nUmZneFaRth++lieJtHepIvKgx6PJqIlD9X2j6pG1i9x3pZ
-5bHuCPFiirGHeO7McvoXkz786GaKVzC9DSpnOxJdc4xm6NSVq7lNEnKdVlnpu9BNYoKX2Iq3wvgh
-gGEUM66kK6j4NiyoneuPLSwaCWDxczgaolEWpiMyDVDb7dNuLAbriL8ig8mmeju31oNvQdpnvEPC
-1vAXbWacGRVrGt/uXN/gU0CDDwgooKRrHfTBb1/s9lYZ8ZqOBU0yLvpuP6+K9hLFsvIjeNhBi0KL
-MlOuWRn3FRwx5oHXjl0YImUx0+gLzjGchrgzca026ETmYJzPD+IpuKzNi8AFn048Thd63OdD86M6
-84zE8yQm0VqXdbbgvub2pKVnS76icBGdeTHHXTKspUmr4NYo/furFLKiMdQzFjHJNcdAnMhltBJK
-0/IKX3DVFqvPJ2dLE7bDBkH0l/PJ29074+F0CsGYOxsb7U3myTUncYfXqnLLfa6sJybX4g+hmcjO
-kMRBfA1JellfRRKJcyRpxdS4rIl6FdmQCWjo/o9Qz7yKffoP4JHjOvABcRn4CZIT2RH4jnxmfpVG
-qgLaAvQBNfuO6X0/Ux02nb4FKx3vgP+XnkX0QW9pLy/NsXgdN24dD3LxO2Nwil7Zlc1dqtP3d7/h
-kzp1/+7hGBuY4pk0XD/0Ao/oTe/XGrfyM773aB7iUhgkpy+dwAMalxMP0DrBcsVw/6p25+/hobP9
-GBknrWExDhLJ1bwt1NcCNblaFbMKCyvmX0PeRaQ=
-""")
-
-##file distutils.cfg
-DISTUTILS_CFG = convert("""
-eJxNj00KwkAMhfc9xYNuxe4Ft57AjYiUtDO1wXSmNJnK3N5pdSEEAu8nH6lxHVlRhtDHMPATA4uH
-xJ4EFmGbvfJiicSHFRzUSISMY6hq3GLCRLnIvSTnEefN0FIjw5tF0Hkk9Q5dRunBsVoyFi24aaLg
-9FDOlL0FPGluf4QjcInLlxd6f6rqkgPu/5nHLg0cXCscXoozRrP51DRT3j9QNl99AP53T2Q=
-""")
-
-##file activate_this.py
-ACTIVATE_THIS = convert("""
-eJyNU01v2zAMvetXEB4K21jnDOstQA4dMGCHbeihlyEIDMWmE62yJEiKE//7kXKdpEWLzYBt8evx
-kRSzLPs6wiEoswM8YdMpjUXcq1Dz6RZa1cSiTkJdr86GsoTRHuCotBayiWqQEYGtMCgfD1KjGYBe
-5a3p0cRKiEe2NtLAFikftnDco0ko/SFEVgEZ8aRCZDIPY9xbA8pE9M4jfW/B2CjiHq9zbJVZuOQq
-siwTIvpxKYCembPAU4Muwi/Z4zfvrZ/MXipKeB8C+qisSZYiWfjJfs+0/MFMdWn1hJcO5U7G/SLa
-xVx8zU6VG/PXLXvfsyyzUqjeWR8hjGE+2iCE1W1tQ82hsCJN9dzKaoexyB/uH79TnjwvxcW0ntSb
-yZ8jq1Z5Q1UXsyy3gf9nbjTEj7NzQMfCJa/YSmrQ+2D/BqfiOi6sclrGzvoeVivIj8rcfcmnIQRF
-7XCyeZI7DFe5/lhlCs5PRf5QW66VXT/NrlQ46oD/D6InkOmi3IQcbhKxAX2g4a+Xd5s3UtCtG2py
-m8eg6WYWqR6SL5OjKMGfSrYt/6kxxQtOpeAgj1LXBNmpE2ElmCSIy5H0zFd8gJ924HWijWhb2hRC
-6wNEm1QdDZtuSZcEprIUBo/XRNcbQe1OUbQ/r3hPTaPJJDNtFLu8KHV5XoNr3Eo6h6YtOKw8e8yw
-VF5PnJ+ts3a9/Mz38RpG/AUSzYUW
-""")
-
-##file python-config
-PYTHON_CONFIG = convert("""
-eJyNVV1P2zAUfc+v8ODBiSABxlulTipbO6p1LWqBgVhlhcZpPYUkctzSivHfd6+dpGloGH2Ja/ue
-e+65Hz78xNhtf3x90xmw7vCWsRPGLvpDNuz87MKfdKMWSWxZ4ilNpCLZJiuWc66SVFUOZkkcirll
-rfxIBAzOMtImDzSVPBRrekwoX/OZu/0r4lm0DHiG60g86u8sjPw5rCyy86NRkB8QuuBRSqfAKESn
-3orLTCQxE3GYkC9tYp8fk89OSwNsmXgizrhUtnumeSgeo5GbLUMk49Rv+2nK48Cm/qMwfp333J2/
-dVcAGE0CIQHBsgIeEr4Wij0LtWDLzJ9ze5YEvH2WI6CHTAVcSu9ZCsXtgxu81CIvp6/k4eXsdfo7
-PvDCRD75yi41QitfzlcPp1OI7i/1/iQitqnr0iMgQ+A6wa+IKwwdxyk9IiXNAzgquTFU8NIxAVjM
-osm1Zz526e+shQ4hKRVci69nPC3Kw4NQEmkQ65E7OodxorSvxjvpBjQHDmWFIQ1mlmzlS5vedseT
-/mgIEsMJ7Lxz2bLAF9M5xeLEhdbHxpWOw0GdkJApMVBRF1y+a0z3c9WZPAXGFcFrJgCIB+024uad
-0CrzmEoRa3Ub4swNIHPGf7QDV+2uj2OiFWsChgCwjKqN6rp5izpbH6Wc1O1TclQTP/XVwi6anTr1
-1sbubjZLI1+VptPSdCfwnFBrB1jvebrTA9uUhU2/9gad7xPqeFkaQcnnLbCViZK8d7R1kxzFrIJV
-8EaLYmKYpvGVkig+3C5HCXbM1jGCGekiM2pRCVPyRyXYdPf6kcbWEQ36F5V4Gq9N7icNNw+JHwRE
-LTgxRXACpvnQv/PuT0xCCAywY/K4hE6Now2qDwaSE5FB+1agsoUveYDepS83qFcF1NufvULD3fTl
-g6Hgf7WBt6lzMeiyyWVn3P1WVbwaczHmTzE9A5SyItTVgFYyvs/L/fXlaNgbw8v3azT+0eikVlWD
-/vBHbzQumP23uBCjsYdrL9OWARwxs/nuLOzeXbPJTa/Xv6sUmQir5pC1YRLz3eA+CD8Z0XpcW8v9
-MZWF36ryyXXf3yBIz6nzqz8Muyz0m5Qj7OexfYo/Ph3LqvkHUg7AuA==
-""")
-
-MH_MAGIC = 0xfeedface
-MH_CIGAM = 0xcefaedfe
-MH_MAGIC_64 = 0xfeedfacf
-MH_CIGAM_64 = 0xcffaedfe
-FAT_MAGIC = 0xcafebabe
-BIG_ENDIAN = '>'
-LITTLE_ENDIAN = '<'
-LC_LOAD_DYLIB = 0xc
-maxint = majver == 3 and getattr(sys, 'maxsize') or getattr(sys, 'maxint')
-
-
-class fileview(object):
- """
- A proxy for file-like objects that exposes a given view of a file.
- Modified from macholib.
- """
-
- def __init__(self, fileobj, start=0, size=maxint):
- if isinstance(fileobj, fileview):
- self._fileobj = fileobj._fileobj
- else:
- self._fileobj = fileobj
- self._start = start
- self._end = start + size
- self._pos = 0
-
- def __repr__(self):
- return '<fileview [%d, %d] %r>' % (
- self._start, self._end, self._fileobj)
-
- def tell(self):
- return self._pos
-
- def _checkwindow(self, seekto, op):
- if not (self._start <= seekto <= self._end):
- raise IOError("%s to offset %d is outside window [%d, %d]" % (
- op, seekto, self._start, self._end))
-
- def seek(self, offset, whence=0):
- seekto = offset
- if whence == os.SEEK_SET:
- seekto += self._start
- elif whence == os.SEEK_CUR:
- seekto += self._start + self._pos
- elif whence == os.SEEK_END:
- seekto += self._end
- else:
- raise IOError("Invalid whence argument to seek: %r" % (whence,))
- self._checkwindow(seekto, 'seek')
- self._fileobj.seek(seekto)
- self._pos = seekto - self._start
-
- def write(self, bytes):
- here = self._start + self._pos
- self._checkwindow(here, 'write')
- self._checkwindow(here + len(bytes), 'write')
- self._fileobj.seek(here, os.SEEK_SET)
- self._fileobj.write(bytes)
- self._pos += len(bytes)
-
- def read(self, size=maxint):
- assert size >= 0
- here = self._start + self._pos
- self._checkwindow(here, 'read')
- size = min(size, self._end - here)
- self._fileobj.seek(here, os.SEEK_SET)
- bytes = self._fileobj.read(size)
- self._pos += len(bytes)
- return bytes
-
-
-def read_data(file, endian, num=1):
- """
- Read a given number of 32-bits unsigned integers from the given file
- with the given endianness.
- """
- res = struct.unpack(endian + 'L' * num, file.read(num * 4))
- if len(res) == 1:
- return res[0]
- return res
-
-
-def mach_o_change(path, what, value):
- """
- Replace a given name (what) in any LC_LOAD_DYLIB command found in
- the given binary with a new name (value), provided it's shorter.
- """
-
- def do_macho(file, bits, endian):
- # Read Mach-O header (the magic number is assumed read by the caller)
- cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags = read_data(file, endian, 6)
- # 64-bits header has one more field.
- if bits == 64:
- read_data(file, endian)
- # The header is followed by ncmds commands
- for n in range(ncmds):
- where = file.tell()
- # Read command header
- cmd, cmdsize = read_data(file, endian, 2)
- if cmd == LC_LOAD_DYLIB:
- # The first data field in LC_LOAD_DYLIB commands is the
- # offset of the name, starting from the beginning of the
- # command.
- name_offset = read_data(file, endian)
- file.seek(where + name_offset, os.SEEK_SET)
- # Read the NUL terminated string
- load = file.read(cmdsize - name_offset).decode()
- load = load[:load.index('\0')]
- # If the string is what is being replaced, overwrite it.
- if load == what:
- file.seek(where + name_offset, os.SEEK_SET)
- file.write(value.encode() + '\0'.encode())
- # Seek to the next command
- file.seek(where + cmdsize, os.SEEK_SET)
-
- def do_file(file, offset=0, size=maxint):
- file = fileview(file, offset, size)
- # Read magic number
- magic = read_data(file, BIG_ENDIAN)
- if magic == FAT_MAGIC:
- # Fat binaries contain nfat_arch Mach-O binaries
- nfat_arch = read_data(file, BIG_ENDIAN)
- for n in range(nfat_arch):
- # Read arch header
- cputype, cpusubtype, offset, size, align = read_data(file, BIG_ENDIAN, 5)
- do_file(file, offset, size)
- elif magic == MH_MAGIC:
- do_macho(file, 32, BIG_ENDIAN)
- elif magic == MH_CIGAM:
- do_macho(file, 32, LITTLE_ENDIAN)
- elif magic == MH_MAGIC_64:
- do_macho(file, 64, BIG_ENDIAN)
- elif magic == MH_CIGAM_64:
- do_macho(file, 64, LITTLE_ENDIAN)
-
- assert(len(what) >= len(value))
-
- with open(path, 'r+b') as f:
- do_file(f)
-
-
-if __name__ == '__main__':
- main()
-
-# TODO:
-# Copy python.exe.manifest
-# Monkeypatch distutils.sysconfig
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.bat b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.bat
deleted file mode 100644
index 529b9733c..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.bat
+++ /dev/null
@@ -1,30 +0,0 @@
-@echo off
-set "VIRTUAL_ENV=__VIRTUAL_ENV__"
-
-if defined _OLD_VIRTUAL_PROMPT (
- set "PROMPT=%_OLD_VIRTUAL_PROMPT%"
-) else (
- if not defined PROMPT (
- set "PROMPT=$P$G"
- )
- set "_OLD_VIRTUAL_PROMPT=%PROMPT%"
-)
-set "PROMPT=__VIRTUAL_WINPROMPT__ %PROMPT%"
-
-REM Don't use () to avoid problems with them in %PATH%
-if defined _OLD_VIRTUAL_PYTHONHOME goto ENDIFVHOME
- set "_OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%"
-:ENDIFVHOME
-
-set PYTHONHOME=
-
-REM if defined _OLD_VIRTUAL_PATH (
-if not defined _OLD_VIRTUAL_PATH goto ENDIFVPATH1
- set "PATH=%_OLD_VIRTUAL_PATH%"
-:ENDIFVPATH1
-REM ) else (
-if defined _OLD_VIRTUAL_PATH goto ENDIFVPATH2
- set "_OLD_VIRTUAL_PATH=%PATH%"
-:ENDIFVPATH2
-
-set "PATH=%VIRTUAL_ENV%\__BIN_NAME__;%PATH%"
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.csh b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.csh
deleted file mode 100644
index 864865b17..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.csh
+++ /dev/null
@@ -1,36 +0,0 @@
-# This file must be used with "source bin/activate.csh" *from csh*.
-# You cannot run it directly.
-# Created by Davide Di Blasi <davidedb@gmail.com>.
-
-alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc'
-
-# Unset irrelevant variables.
-deactivate nondestructive
-
-setenv VIRTUAL_ENV "__VIRTUAL_ENV__"
-
-set _OLD_VIRTUAL_PATH="$PATH"
-setenv PATH "$VIRTUAL_ENV/__BIN_NAME__:$PATH"
-
-
-
-if ("__VIRTUAL_PROMPT__" != "") then
- set env_name = "__VIRTUAL_PROMPT__"
-else
- set env_name = `basename "$VIRTUAL_ENV"`
-endif
-
-# Could be in a non-interactive environment,
-# in which case, $prompt is undefined and we wouldn't
-# care about the prompt anyway.
-if ( $?prompt ) then
- set _OLD_VIRTUAL_PROMPT="$prompt"
- set prompt = "[$env_name] $prompt"
-endif
-
-unset env_name
-
-alias pydoc python -m pydoc
-
-rehash
-
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.fish b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.fish
deleted file mode 100644
index f3d1797a3..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.fish
+++ /dev/null
@@ -1,76 +0,0 @@
-# This file must be used using `. bin/activate.fish` *within a running fish ( http://fishshell.com ) session*.
-# Do not run it directly.
-
-function deactivate -d 'Exit virtualenv mode and return to the normal environment.'
- # reset old environment variables
- if test -n "$_OLD_VIRTUAL_PATH"
- set -gx PATH $_OLD_VIRTUAL_PATH
- set -e _OLD_VIRTUAL_PATH
- end
-
- if test -n "$_OLD_VIRTUAL_PYTHONHOME"
- set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
- set -e _OLD_VIRTUAL_PYTHONHOME
- end
-
- if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
- # Set an empty local `$fish_function_path` to allow the removal of `fish_prompt` using `functions -e`.
- set -l fish_function_path
-
- # Erase virtualenv's `fish_prompt` and restore the original.
- functions -e fish_prompt
- functions -c _old_fish_prompt fish_prompt
- functions -e _old_fish_prompt
- set -e _OLD_FISH_PROMPT_OVERRIDE
- end
-
- set -e VIRTUAL_ENV
-
- if test "$argv[1]" != 'nondestructive'
- # Self-destruct!
- functions -e pydoc
- functions -e deactivate
- end
-end
-
-# Unset irrelevant variables.
-deactivate nondestructive
-
-set -gx VIRTUAL_ENV "__VIRTUAL_ENV__"
-
-set -gx _OLD_VIRTUAL_PATH $PATH
-set -gx PATH "$VIRTUAL_ENV/__BIN_NAME__" $PATH
-
-# Unset `$PYTHONHOME` if set.
-if set -q PYTHONHOME
- set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
- set -e PYTHONHOME
-end
-
-function pydoc
- python -m pydoc $argv
-end
-
-if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
- # Copy the current `fish_prompt` function as `_old_fish_prompt`.
- functions -c fish_prompt _old_fish_prompt
-
- function fish_prompt
- # Save the current $status, for fish_prompts that display it.
- set -l old_status $status
-
- # Prompt override provided?
- # If not, just prepend the environment name.
- if test -n "__VIRTUAL_PROMPT__"
- printf '%s%s' "__VIRTUAL_PROMPT__" (set_color normal)
- else
- printf '%s(%s%s%s) ' (set_color normal) (set_color -o white) (basename "$VIRTUAL_ENV") (set_color normal)
- end
-
- # Restore the original $status
- echo "exit $old_status" | source
- _old_fish_prompt
- end
-
- set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
-end
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.ps1 b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.ps1
deleted file mode 100644
index 0f4adf19f..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.ps1
+++ /dev/null
@@ -1,150 +0,0 @@
-# This file must be dot sourced from PoSh; you cannot run it
-# directly. Do this: . ./activate.ps1
-
-# FIXME: clean up unused vars.
-$script:THIS_PATH = $myinvocation.mycommand.path
-$script:BASE_DIR = split-path (resolve-path "$THIS_PATH/..") -Parent
-$script:DIR_NAME = split-path $BASE_DIR -Leaf
-
-function global:deactivate ( [switch] $NonDestructive ){
-
- if ( test-path variable:_OLD_VIRTUAL_PATH ) {
- $env:PATH = $variable:_OLD_VIRTUAL_PATH
- remove-variable "_OLD_VIRTUAL_PATH" -scope global
- }
-
- if ( test-path function:_old_virtual_prompt ) {
- $function:prompt = $function:_old_virtual_prompt
- remove-item function:\_old_virtual_prompt
- }
-
- if ($env:VIRTUAL_ENV) {
- $old_env = split-path $env:VIRTUAL_ENV -leaf
- remove-item env:VIRTUAL_ENV -erroraction silentlycontinue
- }
-
- if ( !$NonDestructive ) {
- # Self destruct!
- remove-item function:deactivate
- }
-}
-
-# unset irrelevant variables
-deactivate -nondestructive
-
-$VIRTUAL_ENV = $BASE_DIR
-$env:VIRTUAL_ENV = $VIRTUAL_ENV
-
-$global:_OLD_VIRTUAL_PATH = $env:PATH
-$env:PATH = "$env:VIRTUAL_ENV/Scripts;" + $env:PATH
-if (! $env:VIRTUAL_ENV_DISABLE_PROMPT) {
- function global:_old_virtual_prompt { "" }
- $function:_old_virtual_prompt = $function:prompt
- function global:prompt {
- # Add a prefix to the current prompt, but don't discard it.
- write-host "($(split-path $env:VIRTUAL_ENV -leaf)) " -nonewline
- & $function:_old_virtual_prompt
- }
-}
-
-# SIG # Begin signature block
-# MIISeAYJKoZIhvcNAQcCoIISaTCCEmUCAQExCzAJBgUrDgMCGgUAMGkGCisGAQQB
-# gjcCAQSgWzBZMDQGCisGAQQBgjcCAR4wJgIDAQAABBAfzDtgWUsITrck0sYpfvNR
-# AgEAAgEAAgEAAgEAAgEAMCEwCQYFKw4DAhoFAAQUS5reBwSg3zOUwhXf2jPChZzf
-# yPmggg6tMIIGcDCCBFigAwIBAgIBJDANBgkqhkiG9w0BAQUFADB9MQswCQYDVQQG
-# EwJJTDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERp
-# Z2l0YWwgQ2VydGlmaWNhdGUgU2lnbmluZzEpMCcGA1UEAxMgU3RhcnRDb20gQ2Vy
-# dGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDcxMDI0MjIwMTQ2WhcNMTcxMDI0MjIw
-# MTQ2WjCBjDELMAkGA1UEBhMCSUwxFjAUBgNVBAoTDVN0YXJ0Q29tIEx0ZC4xKzAp
-# BgNVBAsTIlNlY3VyZSBEaWdpdGFsIENlcnRpZmljYXRlIFNpZ25pbmcxODA2BgNV
-# BAMTL1N0YXJ0Q29tIENsYXNzIDIgUHJpbWFyeSBJbnRlcm1lZGlhdGUgT2JqZWN0
-# IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyiOLIjUemqAbPJ1J
-# 0D8MlzgWKbr4fYlbRVjvhHDtfhFN6RQxq0PjTQxRgWzwFQNKJCdU5ftKoM5N4YSj
-# Id6ZNavcSa6/McVnhDAQm+8H3HWoD030NVOxbjgD/Ih3HaV3/z9159nnvyxQEckR
-# ZfpJB2Kfk6aHqW3JnSvRe+XVZSufDVCe/vtxGSEwKCaNrsLc9pboUoYIC3oyzWoU
-# TZ65+c0H4paR8c8eK/mC914mBo6N0dQ512/bkSdaeY9YaQpGtW/h/W/FkbQRT3sC
-# pttLVlIjnkuY4r9+zvqhToPjxcfDYEf+XD8VGkAqle8Aa8hQ+M1qGdQjAye8OzbV
-# uUOw7wIDAQABo4IB6TCCAeUwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMC
-# AQYwHQYDVR0OBBYEFNBOD0CZbLhLGW87KLjg44gHNKq3MB8GA1UdIwQYMBaAFE4L
-# 7xqkQFulF2mHMMo0aEPQQa7yMD0GCCsGAQUFBwEBBDEwLzAtBggrBgEFBQcwAoYh
-# aHR0cDovL3d3dy5zdGFydHNzbC5jb20vc2ZzY2EuY3J0MFsGA1UdHwRUMFIwJ6Al
-# oCOGIWh0dHA6Ly93d3cuc3RhcnRzc2wuY29tL3Nmc2NhLmNybDAnoCWgI4YhaHR0
-# cDovL2NybC5zdGFydHNzbC5jb20vc2ZzY2EuY3JsMIGABgNVHSAEeTB3MHUGCysG
-# AQQBgbU3AQIBMGYwLgYIKwYBBQUHAgEWImh0dHA6Ly93d3cuc3RhcnRzc2wuY29t
-# L3BvbGljeS5wZGYwNAYIKwYBBQUHAgEWKGh0dHA6Ly93d3cuc3RhcnRzc2wuY29t
-# L2ludGVybWVkaWF0ZS5wZGYwEQYJYIZIAYb4QgEBBAQDAgABMFAGCWCGSAGG+EIB
-# DQRDFkFTdGFydENvbSBDbGFzcyAyIFByaW1hcnkgSW50ZXJtZWRpYXRlIE9iamVj
-# dCBTaWduaW5nIENlcnRpZmljYXRlczANBgkqhkiG9w0BAQUFAAOCAgEAcnMLA3Va
-# N4OIE9l4QT5OEtZy5PByBit3oHiqQpgVEQo7DHRsjXD5H/IyTivpMikaaeRxIv95
-# baRd4hoUcMwDj4JIjC3WA9FoNFV31SMljEZa66G8RQECdMSSufgfDYu1XQ+cUKxh
-# D3EtLGGcFGjjML7EQv2Iol741rEsycXwIXcryxeiMbU2TPi7X3elbwQMc4JFlJ4B
-# y9FhBzuZB1DV2sN2irGVbC3G/1+S2doPDjL1CaElwRa/T0qkq2vvPxUgryAoCppU
-# FKViw5yoGYC+z1GaesWWiP1eFKAL0wI7IgSvLzU3y1Vp7vsYaxOVBqZtebFTWRHt
-# XjCsFrrQBngt0d33QbQRI5mwgzEp7XJ9xu5d6RVWM4TPRUsd+DDZpBHm9mszvi9g
-# VFb2ZG7qRRXCSqys4+u/NLBPbXi/m/lU00cODQTlC/euwjk9HQtRrXQ/zqsBJS6U
-# J+eLGw1qOfj+HVBl/ZQpfoLk7IoWlRQvRL1s7oirEaqPZUIWY/grXq9r6jDKAp3L
-# ZdKQpPOnnogtqlU4f7/kLjEJhrrc98mrOWmVMK/BuFRAfQ5oDUMnVmCzAzLMjKfG
-# cVW/iMew41yfhgKbwpfzm3LBr1Zv+pEBgcgW6onRLSAn3XHM0eNtz+AkxH6rRf6B
-# 2mYhLEEGLapH8R1AMAo4BbVFOZR5kXcMCwowggg1MIIHHaADAgECAgIEuDANBgkq
-# hkiG9w0BAQUFADCBjDELMAkGA1UEBhMCSUwxFjAUBgNVBAoTDVN0YXJ0Q29tIEx0
-# ZC4xKzApBgNVBAsTIlNlY3VyZSBEaWdpdGFsIENlcnRpZmljYXRlIFNpZ25pbmcx
-# ODA2BgNVBAMTL1N0YXJ0Q29tIENsYXNzIDIgUHJpbWFyeSBJbnRlcm1lZGlhdGUg
-# T2JqZWN0IENBMB4XDTExMTIwMzE1MzQxOVoXDTEzMTIwMzE0NTgwN1owgYwxIDAe
-# BgNVBA0TFzU4MTc5Ni1HaDd4Zkp4a3hRU0lPNEUwMQswCQYDVQQGEwJERTEPMA0G
-# A1UECBMGQmVybGluMQ8wDQYDVQQHEwZCZXJsaW4xFjAUBgNVBAMTDUphbm5pcyBM
-# ZWlkZWwxITAfBgkqhkiG9w0BCQEWEmphbm5pc0BsZWlkZWwuaW5mbzCCAiIwDQYJ
-# KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMcPeABYdN7nPq/AkZ/EkyUBGx/l2Yui
-# Lfm8ZdLG0ulMb/kQL3fRY7sUjYPyn9S6PhqqlFnNoGHJvbbReCdUC9SIQYmOEjEA
-# raHfb7MZU10NjO4U2DdGucj2zuO5tYxKizizOJF0e4yRQZVxpUGdvkW/+GLjCNK5
-# L7mIv3Z1dagxDKHYZT74HXiS4VFUwHF1k36CwfM2vsetdm46bdgSwV+BCMmZICYT
-# IJAS9UQHD7kP4rik3bFWjUx08NtYYFAVOd/HwBnemUmJe4j3IhZHr0k1+eDG8hDH
-# KVvPgLJIoEjC4iMFk5GWsg5z2ngk0LLu3JZMtckHsnnmBPHQK8a3opUNd8hdMNJx
-# gOwKjQt2JZSGUdIEFCKVDqj0FmdnDMPfwy+FNRtpBMl1sz78dUFhSrnM0D8NXrqa
-# 4rG+2FoOXlmm1rb6AFtpjAKksHRpYcPk2DPGWp/1sWB+dUQkS3gOmwFzyqeTuXpT
-# 0juqd3iAxOGx1VRFQ1VHLLf3AzV4wljBau26I+tu7iXxesVucSdsdQu293jwc2kN
-# xK2JyHCoZH+RyytrwS0qw8t7rMOukU9gwP8mn3X6mgWlVUODMcHTULjSiCEtvyZ/
-# aafcwjUbt4ReEcnmuZtWIha86MTCX7U7e+cnpWG4sIHPnvVTaz9rm8RyBkIxtFCB
-# nQ3FnoQgyxeJAgMBAAGjggOdMIIDmTAJBgNVHRMEAjAAMA4GA1UdDwEB/wQEAwIH
-# gDAuBgNVHSUBAf8EJDAiBggrBgEFBQcDAwYKKwYBBAGCNwIBFQYKKwYBBAGCNwoD
-# DTAdBgNVHQ4EFgQUWyCgrIWo8Ifvvm1/YTQIeMU9nc8wHwYDVR0jBBgwFoAU0E4P
-# QJlsuEsZbzsouODjiAc0qrcwggIhBgNVHSAEggIYMIICFDCCAhAGCysGAQQBgbU3
-# AQICMIIB/zAuBggrBgEFBQcCARYiaHR0cDovL3d3dy5zdGFydHNzbC5jb20vcG9s
-# aWN5LnBkZjA0BggrBgEFBQcCARYoaHR0cDovL3d3dy5zdGFydHNzbC5jb20vaW50
-# ZXJtZWRpYXRlLnBkZjCB9wYIKwYBBQUHAgIwgeowJxYgU3RhcnRDb20gQ2VydGlm
-# aWNhdGlvbiBBdXRob3JpdHkwAwIBARqBvlRoaXMgY2VydGlmaWNhdGUgd2FzIGlz
-# c3VlZCBhY2NvcmRpbmcgdG8gdGhlIENsYXNzIDIgVmFsaWRhdGlvbiByZXF1aXJl
-# bWVudHMgb2YgdGhlIFN0YXJ0Q29tIENBIHBvbGljeSwgcmVsaWFuY2Ugb25seSBm
-# b3IgdGhlIGludGVuZGVkIHB1cnBvc2UgaW4gY29tcGxpYW5jZSBvZiB0aGUgcmVs
-# eWluZyBwYXJ0eSBvYmxpZ2F0aW9ucy4wgZwGCCsGAQUFBwICMIGPMCcWIFN0YXJ0
-# Q29tIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MAMCAQIaZExpYWJpbGl0eSBhbmQg
-# d2FycmFudGllcyBhcmUgbGltaXRlZCEgU2VlIHNlY3Rpb24gIkxlZ2FsIGFuZCBM
-# aW1pdGF0aW9ucyIgb2YgdGhlIFN0YXJ0Q29tIENBIHBvbGljeS4wNgYDVR0fBC8w
-# LTAroCmgJ4YlaHR0cDovL2NybC5zdGFydHNzbC5jb20vY3J0YzItY3JsLmNybDCB
-# iQYIKwYBBQUHAQEEfTB7MDcGCCsGAQUFBzABhitodHRwOi8vb2NzcC5zdGFydHNz
-# bC5jb20vc3ViL2NsYXNzMi9jb2RlL2NhMEAGCCsGAQUFBzAChjRodHRwOi8vYWlh
-# LnN0YXJ0c3NsLmNvbS9jZXJ0cy9zdWIuY2xhc3MyLmNvZGUuY2EuY3J0MCMGA1Ud
-# EgQcMBqGGGh0dHA6Ly93d3cuc3RhcnRzc2wuY29tLzANBgkqhkiG9w0BAQUFAAOC
-# AQEAhrzEV6zwoEtKjnFRhCsjwiPykVpo5Eiye77Ve801rQDiRKgSCCiW6g3HqedL
-# OtaSs65Sj2pm3Viea4KR0TECLcbCTgsdaHqw2x1yXwWBQWZEaV6EB05lIwfr94P1
-# SFpV43zkuc+bbmA3+CRK45LOcCNH5Tqq7VGTCAK5iM7tvHwFlbQRl+I6VEL2mjpF
-# NsuRjDOVrv/9qw/a22YJ9R7Y1D0vUSs3IqZx2KMUaYDP7H2mSRxJO2nADQZBtriF
-# gTyfD3lYV12MlIi5CQwe3QC6DrrfSMP33i5Wa/OFJiQ27WPxmScYVhiqozpImFT4
-# PU9goiBv9RKXdgTmZE1PN0NQ5jGCAzUwggMxAgEBMIGTMIGMMQswCQYDVQQGEwJJ
-# TDEWMBQGA1UEChMNU3RhcnRDb20gTHRkLjErMCkGA1UECxMiU2VjdXJlIERpZ2l0
-# YWwgQ2VydGlmaWNhdGUgU2lnbmluZzE4MDYGA1UEAxMvU3RhcnRDb20gQ2xhc3Mg
-# MiBQcmltYXJ5IEludGVybWVkaWF0ZSBPYmplY3QgQ0ECAgS4MAkGBSsOAwIaBQCg
-# eDAYBgorBgEEAYI3AgEMMQowCKACgAChAoAAMBkGCSqGSIb3DQEJAzEMBgorBgEE
-# AYI3AgEEMBwGCisGAQQBgjcCAQsxDjAMBgorBgEEAYI3AgEVMCMGCSqGSIb3DQEJ
-# BDEWBBRVGw0FDSiaIi38dWteRUAg/9Pr6DANBgkqhkiG9w0BAQEFAASCAgCInvOZ
-# FdaNFzbf6trmFDZKMojyx3UjKMCqNjHVBbuKY0qXwFC/ElYDV1ShJ2CBZbdurydO
-# OQ6cIQ0KREOCwmX/xB49IlLHHUxNhEkVv7HGU3EKAFf9IBt9Yr7jikiR9cjIsfHK
-# 4cjkoKJL7g28yEpLLkHt1eo37f1Ga9lDWEa5Zq3U5yX+IwXhrUBm1h8Xr033FhTR
-# VEpuSz6LHtbrL/zgJnCzJ2ahjtJoYevdcWiNXffosJHFaSfYDDbiNsPRDH/1avmb
-# 5j/7BhP8BcBaR6Fp8tFbNGIcWHHGcjqLMnTc4w13b7b4pDhypqElBa4+lCmwdvv9
-# GydYtRgPz8GHeoBoKj30YBlMzRIfFYaIFGIC4Ai3UEXkuH9TxYohVbGm/W0Kl4Lb
-# RJ1FwiVcLcTOJdgNId2vQvKc+jtNrjcg5SP9h2v/C4aTx8tyc6tE3TOPh2f9b8DL
-# S+SbVArJpuJqrPTxDDoO1QNjTgLcdVYeZDE+r/NjaGZ6cMSd8db3EaG3ijD/0bud
-# SItbm/OlNVbQOFRR76D+ZNgPcU5iNZ3bmvQQIg6aSB9MHUpIE/SeCkNl9YeVk1/1
-# GFULgNMRmIYP4KLvu9ylh5Gu3hvD5VNhH6+FlXANwFy07uXks5uF8mfZVxVCnodG
-# xkNCx+6PsrA5Z7WP4pXcmYnMn97npP/Q9EHJWw==
-# SIG # End signature block
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.sh b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.sh
deleted file mode 100644
index 477b7eca2..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate.sh
+++ /dev/null
@@ -1,78 +0,0 @@
-# This file must be used with "source bin/activate" *from bash*
-# you cannot run it directly
-
-deactivate () {
- unset -f pydoc >/dev/null 2>&1
-
- # reset old environment variables
- # ! [ -z ${VAR+_} ] returns true if VAR is declared at all
- if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
- PATH="$_OLD_VIRTUAL_PATH"
- export PATH
- unset _OLD_VIRTUAL_PATH
- fi
- if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
- PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
- export PYTHONHOME
- unset _OLD_VIRTUAL_PYTHONHOME
- fi
-
- # This should detect bash and zsh, which have a hash command that must
- # be called to get it to forget past commands. Without forgetting
- # past commands the $PATH changes we made may not be respected
- if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
- hash -r 2>/dev/null
- fi
-
- if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
- PS1="$_OLD_VIRTUAL_PS1"
- export PS1
- unset _OLD_VIRTUAL_PS1
- fi
-
- unset VIRTUAL_ENV
- if [ ! "${1-}" = "nondestructive" ] ; then
- # Self destruct!
- unset -f deactivate
- fi
-}
-
-# unset irrelevant variables
-deactivate nondestructive
-
-VIRTUAL_ENV="__VIRTUAL_ENV__"
-export VIRTUAL_ENV
-
-_OLD_VIRTUAL_PATH="$PATH"
-PATH="$VIRTUAL_ENV/__BIN_NAME__:$PATH"
-export PATH
-
-# unset PYTHONHOME if set
-if ! [ -z "${PYTHONHOME+_}" ] ; then
- _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
- unset PYTHONHOME
-fi
-
-if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
- _OLD_VIRTUAL_PS1="$PS1"
- if [ "x__VIRTUAL_PROMPT__" != x ] ; then
- PS1="__VIRTUAL_PROMPT__$PS1"
- else
- PS1="(`basename \"$VIRTUAL_ENV\"`) $PS1"
- fi
- export PS1
-fi
-
-# Make sure to unalias pydoc if it's already there
-alias pydoc 2>/dev/null >/dev/null && unalias pydoc
-
-pydoc () {
- python -m pydoc "$@"
-}
-
-# This should detect bash and zsh, which have a hash command that must
-# be called to get it to forget past commands. Without forgetting
-# past commands the $PATH changes we made may not be respected
-if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
- hash -r 2>/dev/null
-fi
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate_this.py b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate_this.py
deleted file mode 100644
index f18193bf8..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/activate_this.py
+++ /dev/null
@@ -1,34 +0,0 @@
-"""By using execfile(this_file, dict(__file__=this_file)) you will
-activate this virtualenv environment.
-
-This can be used when you must use an existing Python interpreter, not
-the virtualenv bin/python
-"""
-
-try:
- __file__
-except NameError:
- raise AssertionError(
- "You must run this like execfile('path/to/activate_this.py', dict(__file__='path/to/activate_this.py'))")
-import sys
-import os
-
-old_os_path = os.environ.get('PATH', '')
-os.environ['PATH'] = os.path.dirname(os.path.abspath(__file__)) + os.pathsep + old_os_path
-base = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-if sys.platform == 'win32':
- site_packages = os.path.join(base, 'Lib', 'site-packages')
-else:
- site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages')
-prev_sys_path = list(sys.path)
-import site
-site.addsitedir(site_packages)
-sys.real_prefix = sys.prefix
-sys.prefix = base
-# Move the added items to the front of the path:
-new_sys_path = []
-for item in list(sys.path):
- if item not in prev_sys_path:
- new_sys_path.append(item)
- sys.path.remove(item)
-sys.path[:0] = new_sys_path
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/deactivate.bat b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/deactivate.bat
deleted file mode 100644
index 9228d3171..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/deactivate.bat
+++ /dev/null
@@ -1,19 +0,0 @@
-@echo off
-
-set VIRTUAL_ENV=
-
-REM Don't use () to avoid problems with them in %PATH%
-if not defined _OLD_VIRTUAL_PROMPT goto ENDIFVPROMPT
- set "PROMPT=%_OLD_VIRTUAL_PROMPT%"
- set _OLD_VIRTUAL_PROMPT=
-:ENDIFVPROMPT
-
-if not defined _OLD_VIRTUAL_PYTHONHOME goto ENDIFVHOME
- set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%"
- set _OLD_VIRTUAL_PYTHONHOME=
-:ENDIFVHOME
-
-if not defined _OLD_VIRTUAL_PATH goto ENDIFVPATH
- set "PATH=%_OLD_VIRTUAL_PATH%"
- set _OLD_VIRTUAL_PATH=
-:ENDIFVPATH \ No newline at end of file
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils-init.py b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils-init.py
deleted file mode 100644
index 29fc1da45..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils-init.py
+++ /dev/null
@@ -1,101 +0,0 @@
-import os
-import sys
-import warnings
-import imp
-import opcode # opcode is not a virtualenv module, so we can use it to find the stdlib
- # Important! To work on pypy, this must be a module that resides in the
- # lib-python/modified-x.y.z directory
-
-dirname = os.path.dirname
-
-distutils_path = os.path.join(os.path.dirname(opcode.__file__), 'distutils')
-if os.path.normpath(distutils_path) == os.path.dirname(os.path.normpath(__file__)):
- warnings.warn(
- "The virtualenv distutils package at %s appears to be in the same location as the system distutils?")
-else:
- __path__.insert(0, distutils_path)
- real_distutils = imp.load_module("_virtualenv_distutils", None, distutils_path, ('', '', imp.PKG_DIRECTORY))
- # Copy the relevant attributes
- try:
- __revision__ = real_distutils.__revision__
- except AttributeError:
- pass
- __version__ = real_distutils.__version__
-
-from distutils import dist, sysconfig
-
-try:
- basestring
-except NameError:
- basestring = str
-
-## patch build_ext (distutils doesn't know how to get the libs directory
-## path on windows - it hardcodes the paths around the patched sys.prefix)
-
-if sys.platform == 'win32':
- from distutils.command.build_ext import build_ext as old_build_ext
- class build_ext(old_build_ext):
- def finalize_options (self):
- if self.library_dirs is None:
- self.library_dirs = []
- elif isinstance(self.library_dirs, basestring):
- self.library_dirs = self.library_dirs.split(os.pathsep)
-
- self.library_dirs.insert(0, os.path.join(sys.real_prefix, "Libs"))
- old_build_ext.finalize_options(self)
-
- from distutils.command import build_ext as build_ext_module
- build_ext_module.build_ext = build_ext
-
-## distutils.dist patches:
-
-old_find_config_files = dist.Distribution.find_config_files
-def find_config_files(self):
- found = old_find_config_files(self)
- system_distutils = os.path.join(distutils_path, 'distutils.cfg')
- #if os.path.exists(system_distutils):
- # found.insert(0, system_distutils)
- # What to call the per-user config file
- if os.name == 'posix':
- user_filename = ".pydistutils.cfg"
- else:
- user_filename = "pydistutils.cfg"
- user_filename = os.path.join(sys.prefix, user_filename)
- if os.path.isfile(user_filename):
- for item in list(found):
- if item.endswith('pydistutils.cfg'):
- found.remove(item)
- found.append(user_filename)
- return found
-dist.Distribution.find_config_files = find_config_files
-
-## distutils.sysconfig patches:
-
-old_get_python_inc = sysconfig.get_python_inc
-def sysconfig_get_python_inc(plat_specific=0, prefix=None):
- if prefix is None:
- prefix = sys.real_prefix
- return old_get_python_inc(plat_specific, prefix)
-sysconfig_get_python_inc.__doc__ = old_get_python_inc.__doc__
-sysconfig.get_python_inc = sysconfig_get_python_inc
-
-old_get_python_lib = sysconfig.get_python_lib
-def sysconfig_get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
- if standard_lib and prefix is None:
- prefix = sys.real_prefix
- return old_get_python_lib(plat_specific, standard_lib, prefix)
-sysconfig_get_python_lib.__doc__ = old_get_python_lib.__doc__
-sysconfig.get_python_lib = sysconfig_get_python_lib
-
-old_get_config_vars = sysconfig.get_config_vars
-def sysconfig_get_config_vars(*args):
- real_vars = old_get_config_vars(*args)
- if sys.platform == 'win32':
- lib_dir = os.path.join(sys.real_prefix, "libs")
- if isinstance(real_vars, dict) and 'LIBDIR' not in real_vars:
- real_vars['LIBDIR'] = lib_dir # asked for all
- elif isinstance(real_vars, list) and 'LIBDIR' in args:
- real_vars = real_vars + [lib_dir] # asked for list
- return real_vars
-sysconfig_get_config_vars.__doc__ = old_get_config_vars.__doc__
-sysconfig.get_config_vars = sysconfig_get_config_vars
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils.cfg b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils.cfg
deleted file mode 100644
index 1af230ec9..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/distutils.cfg
+++ /dev/null
@@ -1,6 +0,0 @@
-# This is a config file local to this virtualenv installation
-# You may include options that will be used by all distutils commands,
-# and by easy_install. For instance:
-#
-# [easy_install]
-# find_links = http://mylocalsite
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/python-config b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/python-config
deleted file mode 100644
index 5e7a7c901..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/python-config
+++ /dev/null
@@ -1,78 +0,0 @@
-#!__VIRTUAL_ENV__/__BIN_NAME__/python
-
-import sys
-import getopt
-import sysconfig
-
-valid_opts = ['prefix', 'exec-prefix', 'includes', 'libs', 'cflags',
- 'ldflags', 'help']
-
-if sys.version_info >= (3, 2):
- valid_opts.insert(-1, 'extension-suffix')
- valid_opts.append('abiflags')
-if sys.version_info >= (3, 3):
- valid_opts.append('configdir')
-
-
-def exit_with_usage(code=1):
- sys.stderr.write("Usage: {0} [{1}]\n".format(
- sys.argv[0], '|'.join('--'+opt for opt in valid_opts)))
- sys.exit(code)
-
-try:
- opts, args = getopt.getopt(sys.argv[1:], '', valid_opts)
-except getopt.error:
- exit_with_usage()
-
-if not opts:
- exit_with_usage()
-
-pyver = sysconfig.get_config_var('VERSION')
-getvar = sysconfig.get_config_var
-
-opt_flags = [flag for (flag, val) in opts]
-
-if '--help' in opt_flags:
- exit_with_usage(code=0)
-
-for opt in opt_flags:
- if opt == '--prefix':
- print(sysconfig.get_config_var('prefix'))
-
- elif opt == '--exec-prefix':
- print(sysconfig.get_config_var('exec_prefix'))
-
- elif opt in ('--includes', '--cflags'):
- flags = ['-I' + sysconfig.get_path('include'),
- '-I' + sysconfig.get_path('platinclude')]
- if opt == '--cflags':
- flags.extend(getvar('CFLAGS').split())
- print(' '.join(flags))
-
- elif opt in ('--libs', '--ldflags'):
- abiflags = getattr(sys, 'abiflags', '')
- libs = ['-lpython' + pyver + abiflags]
- libs += getvar('LIBS').split()
- libs += getvar('SYSLIBS').split()
- # add the prefix/lib/pythonX.Y/config dir, but only if there is no
- # shared library in prefix/lib/.
- if opt == '--ldflags':
- if not getvar('Py_ENABLE_SHARED'):
- libs.insert(0, '-L' + getvar('LIBPL'))
- if not getvar('PYTHONFRAMEWORK'):
- libs.extend(getvar('LINKFORSHARED').split())
- print(' '.join(libs))
-
- elif opt == '--extension-suffix':
- ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
- if ext_suffix is None:
- ext_suffix = sysconfig.get_config_var('SO')
- print(ext_suffix)
-
- elif opt == '--abiflags':
- if not getattr(sys, 'abiflags', None):
- exit_with_usage()
- print(sys.abiflags)
-
- elif opt == '--configdir':
- print(sysconfig.get_config_var('LIBPL'))
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/site.py b/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/site.py
deleted file mode 100644
index 7969769c3..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_embedded/site.py
+++ /dev/null
@@ -1,758 +0,0 @@
-"""Append module search paths for third-party packages to sys.path.
-
-****************************************************************
-* This module is automatically imported during initialization. *
-****************************************************************
-
-In earlier versions of Python (up to 1.5a3), scripts or modules that
-needed to use site-specific modules would place ``import site''
-somewhere near the top of their code. Because of the automatic
-import, this is no longer necessary (but code that does it still
-works).
-
-This will append site-specific paths to the module search path. On
-Unix, it starts with sys.prefix and sys.exec_prefix (if different) and
-appends lib/python<version>/site-packages as well as lib/site-python.
-It also supports the Debian convention of
-lib/python<version>/dist-packages. On other platforms (mainly Mac and
-Windows), it uses just sys.prefix (and sys.exec_prefix, if different,
-but this is unlikely). The resulting directories, if they exist, are
-appended to sys.path, and also inspected for path configuration files.
-
-FOR DEBIAN, this sys.path is augmented with directories in /usr/local.
-Local addons go into /usr/local/lib/python<version>/site-packages
-(resp. /usr/local/lib/site-python), Debian addons install into
-/usr/{lib,share}/python<version>/dist-packages.
-
-A path configuration file is a file whose name has the form
-<package>.pth; its contents are additional directories (one per line)
-to be added to sys.path. Non-existing directories (or
-non-directories) are never added to sys.path; no directory is added to
-sys.path more than once. Blank lines and lines beginning with
-'#' are skipped. Lines starting with 'import' are executed.
-
-For example, suppose sys.prefix and sys.exec_prefix are set to
-/usr/local and there is a directory /usr/local/lib/python2.X/site-packages
-with three subdirectories, foo, bar and spam, and two path
-configuration files, foo.pth and bar.pth. Assume foo.pth contains the
-following:
-
- # foo package configuration
- foo
- bar
- bletch
-
-and bar.pth contains:
-
- # bar package configuration
- bar
-
-Then the following directories are added to sys.path, in this order:
-
- /usr/local/lib/python2.X/site-packages/bar
- /usr/local/lib/python2.X/site-packages/foo
-
-Note that bletch is omitted because it doesn't exist; bar precedes foo
-because bar.pth comes alphabetically before foo.pth; and spam is
-omitted because it is not mentioned in either path configuration file.
-
-After these path manipulations, an attempt is made to import a module
-named sitecustomize, which can perform arbitrary additional
-site-specific customizations. If this import fails with an
-ImportError exception, it is silently ignored.
-
-"""
-
-import sys
-import os
-try:
- import __builtin__ as builtins
-except ImportError:
- import builtins
-try:
- set
-except NameError:
- from sets import Set as set
-
-# Prefixes for site-packages; add additional prefixes like /usr/local here
-PREFIXES = [sys.prefix, sys.exec_prefix]
-# Enable per user site-packages directory
-# set it to False to disable the feature or True to force the feature
-ENABLE_USER_SITE = None
-# for distutils.commands.install
-USER_SITE = None
-USER_BASE = None
-
-_is_64bit = (getattr(sys, 'maxsize', None) or getattr(sys, 'maxint')) > 2**32
-_is_pypy = hasattr(sys, 'pypy_version_info')
-_is_jython = sys.platform[:4] == 'java'
-if _is_jython:
- ModuleType = type(os)
-
-def makepath(*paths):
- dir = os.path.join(*paths)
- if _is_jython and (dir == '__classpath__' or
- dir.startswith('__pyclasspath__')):
- return dir, dir
- dir = os.path.abspath(dir)
- return dir, os.path.normcase(dir)
-
-def abs__file__():
- """Set all module' __file__ attribute to an absolute path"""
- for m in sys.modules.values():
- if ((_is_jython and not isinstance(m, ModuleType)) or
- hasattr(m, '__loader__')):
- # only modules need the abspath in Jython. and don't mess
- # with a PEP 302-supplied __file__
- continue
- f = getattr(m, '__file__', None)
- if f is None:
- continue
- m.__file__ = os.path.abspath(f)
-
-def removeduppaths():
- """ Remove duplicate entries from sys.path along with making them
- absolute"""
- # This ensures that the initial path provided by the interpreter contains
- # only absolute pathnames, even if we're running from the build directory.
- L = []
- known_paths = set()
- for dir in sys.path:
- # Filter out duplicate paths (on case-insensitive file systems also
- # if they only differ in case); turn relative paths into absolute
- # paths.
- dir, dircase = makepath(dir)
- if not dircase in known_paths:
- L.append(dir)
- known_paths.add(dircase)
- sys.path[:] = L
- return known_paths
-
-# XXX This should not be part of site.py, since it is needed even when
-# using the -S option for Python. See http://www.python.org/sf/586680
-def addbuilddir():
- """Append ./build/lib.<platform> in case we're running in the build dir
- (especially for Guido :-)"""
- from distutils.util import get_platform
- s = "build/lib.%s-%.3s" % (get_platform(), sys.version)
- if hasattr(sys, 'gettotalrefcount'):
- s += '-pydebug'
- s = os.path.join(os.path.dirname(sys.path[-1]), s)
- sys.path.append(s)
-
-def _init_pathinfo():
- """Return a set containing all existing directory entries from sys.path"""
- d = set()
- for dir in sys.path:
- try:
- if os.path.isdir(dir):
- dir, dircase = makepath(dir)
- d.add(dircase)
- except TypeError:
- continue
- return d
-
-def addpackage(sitedir, name, known_paths):
- """Add a new path to known_paths by combining sitedir and 'name' or execute
- sitedir if it starts with 'import'"""
- if known_paths is None:
- _init_pathinfo()
- reset = 1
- else:
- reset = 0
- fullname = os.path.join(sitedir, name)
- try:
- f = open(fullname, "rU")
- except IOError:
- return
- try:
- for line in f:
- if line.startswith("#"):
- continue
- if line.startswith("import"):
- exec(line)
- continue
- line = line.rstrip()
- dir, dircase = makepath(sitedir, line)
- if not dircase in known_paths and os.path.exists(dir):
- sys.path.append(dir)
- known_paths.add(dircase)
- finally:
- f.close()
- if reset:
- known_paths = None
- return known_paths
-
-def addsitedir(sitedir, known_paths=None):
- """Add 'sitedir' argument to sys.path if missing and handle .pth files in
- 'sitedir'"""
- if known_paths is None:
- known_paths = _init_pathinfo()
- reset = 1
- else:
- reset = 0
- sitedir, sitedircase = makepath(sitedir)
- if not sitedircase in known_paths:
- sys.path.append(sitedir) # Add path component
- try:
- names = os.listdir(sitedir)
- except os.error:
- return
- names.sort()
- for name in names:
- if name.endswith(os.extsep + "pth"):
- addpackage(sitedir, name, known_paths)
- if reset:
- known_paths = None
- return known_paths
-
-def addsitepackages(known_paths, sys_prefix=sys.prefix, exec_prefix=sys.exec_prefix):
- """Add site-packages (and possibly site-python) to sys.path"""
- prefixes = [os.path.join(sys_prefix, "local"), sys_prefix]
- if exec_prefix != sys_prefix:
- prefixes.append(os.path.join(exec_prefix, "local"))
-
- for prefix in prefixes:
- if prefix:
- if sys.platform in ('os2emx', 'riscos') or _is_jython:
- sitedirs = [os.path.join(prefix, "Lib", "site-packages")]
- elif _is_pypy:
- sitedirs = [os.path.join(prefix, 'site-packages')]
- elif sys.platform == 'darwin' and prefix == sys_prefix:
-
- if prefix.startswith("/System/Library/Frameworks/"): # Apple's Python
-
- sitedirs = [os.path.join("/Library/Python", sys.version[:3], "site-packages"),
- os.path.join(prefix, "Extras", "lib", "python")]
-
- else: # any other Python distros on OSX work this way
- sitedirs = [os.path.join(prefix, "lib",
- "python" + sys.version[:3], "site-packages")]
-
- elif os.sep == '/':
- sitedirs = [os.path.join(prefix,
- "lib",
- "python" + sys.version[:3],
- "site-packages"),
- os.path.join(prefix, "lib", "site-python"),
- os.path.join(prefix, "python" + sys.version[:3], "lib-dynload")]
- lib64_dir = os.path.join(prefix, "lib64", "python" + sys.version[:3], "site-packages")
- if (os.path.exists(lib64_dir) and
- os.path.realpath(lib64_dir) not in [os.path.realpath(p) for p in sitedirs]):
- if _is_64bit:
- sitedirs.insert(0, lib64_dir)
- else:
- sitedirs.append(lib64_dir)
- try:
- # sys.getobjects only available in --with-pydebug build
- sys.getobjects
- sitedirs.insert(0, os.path.join(sitedirs[0], 'debug'))
- except AttributeError:
- pass
- # Debian-specific dist-packages directories:
- sitedirs.append(os.path.join(prefix, "local/lib",
- "python" + sys.version[:3],
- "dist-packages"))
- if sys.version[0] == '2':
- sitedirs.append(os.path.join(prefix, "lib",
- "python" + sys.version[:3],
- "dist-packages"))
- else:
- sitedirs.append(os.path.join(prefix, "lib",
- "python" + sys.version[0],
- "dist-packages"))
- sitedirs.append(os.path.join(prefix, "lib", "dist-python"))
- else:
- sitedirs = [prefix, os.path.join(prefix, "lib", "site-packages")]
- if sys.platform == 'darwin':
- # for framework builds *only* we add the standard Apple
- # locations. Currently only per-user, but /Library and
- # /Network/Library could be added too
- if 'Python.framework' in prefix:
- home = os.environ.get('HOME')
- if home:
- sitedirs.append(
- os.path.join(home,
- 'Library',
- 'Python',
- sys.version[:3],
- 'site-packages'))
- for sitedir in sitedirs:
- if os.path.isdir(sitedir):
- addsitedir(sitedir, known_paths)
- return None
-
-def check_enableusersite():
- """Check if user site directory is safe for inclusion
-
- The function tests for the command line flag (including environment var),
- process uid/gid equal to effective uid/gid.
-
- None: Disabled for security reasons
- False: Disabled by user (command line option)
- True: Safe and enabled
- """
- if hasattr(sys, 'flags') and getattr(sys.flags, 'no_user_site', False):
- return False
-
- if hasattr(os, "getuid") and hasattr(os, "geteuid"):
- # check process uid == effective uid
- if os.geteuid() != os.getuid():
- return None
- if hasattr(os, "getgid") and hasattr(os, "getegid"):
- # check process gid == effective gid
- if os.getegid() != os.getgid():
- return None
-
- return True
-
-def addusersitepackages(known_paths):
- """Add a per user site-package to sys.path
-
- Each user has its own python directory with site-packages in the
- home directory.
-
- USER_BASE is the root directory for all Python versions
-
- USER_SITE is the user specific site-packages directory
-
- USER_SITE/.. can be used for data.
- """
- global USER_BASE, USER_SITE, ENABLE_USER_SITE
- env_base = os.environ.get("PYTHONUSERBASE", None)
-
- def joinuser(*args):
- return os.path.expanduser(os.path.join(*args))
-
- #if sys.platform in ('os2emx', 'riscos'):
- # # Don't know what to put here
- # USER_BASE = ''
- # USER_SITE = ''
- if os.name == "nt":
- base = os.environ.get("APPDATA") or "~"
- if env_base:
- USER_BASE = env_base
- else:
- USER_BASE = joinuser(base, "Python")
- USER_SITE = os.path.join(USER_BASE,
- "Python" + sys.version[0] + sys.version[2],
- "site-packages")
- else:
- if env_base:
- USER_BASE = env_base
- else:
- USER_BASE = joinuser("~", ".local")
- USER_SITE = os.path.join(USER_BASE, "lib",
- "python" + sys.version[:3],
- "site-packages")
-
- if ENABLE_USER_SITE and os.path.isdir(USER_SITE):
- addsitedir(USER_SITE, known_paths)
- if ENABLE_USER_SITE:
- for dist_libdir in ("lib", "local/lib"):
- user_site = os.path.join(USER_BASE, dist_libdir,
- "python" + sys.version[:3],
- "dist-packages")
- if os.path.isdir(user_site):
- addsitedir(user_site, known_paths)
- return known_paths
-
-
-
-def setBEGINLIBPATH():
- """The OS/2 EMX port has optional extension modules that do double duty
- as DLLs (and must use the .DLL file extension) for other extensions.
- The library search path needs to be amended so these will be found
- during module import. Use BEGINLIBPATH so that these are at the start
- of the library search path.
-
- """
- dllpath = os.path.join(sys.prefix, "Lib", "lib-dynload")
- libpath = os.environ['BEGINLIBPATH'].split(';')
- if libpath[-1]:
- libpath.append(dllpath)
- else:
- libpath[-1] = dllpath
- os.environ['BEGINLIBPATH'] = ';'.join(libpath)
-
-
-def setquit():
- """Define new built-ins 'quit' and 'exit'.
- These are simply strings that display a hint on how to exit.
-
- """
- if os.sep == ':':
- eof = 'Cmd-Q'
- elif os.sep == '\\':
- eof = 'Ctrl-Z plus Return'
- else:
- eof = 'Ctrl-D (i.e. EOF)'
-
- class Quitter(object):
- def __init__(self, name):
- self.name = name
- def __repr__(self):
- return 'Use %s() or %s to exit' % (self.name, eof)
- def __call__(self, code=None):
- # Shells like IDLE catch the SystemExit, but listen when their
- # stdin wrapper is closed.
- try:
- sys.stdin.close()
- except:
- pass
- raise SystemExit(code)
- builtins.quit = Quitter('quit')
- builtins.exit = Quitter('exit')
-
-
-class _Printer(object):
- """interactive prompt objects for printing the license text, a list of
- contributors and the copyright notice."""
-
- MAXLINES = 23
-
- def __init__(self, name, data, files=(), dirs=()):
- self.__name = name
- self.__data = data
- self.__files = files
- self.__dirs = dirs
- self.__lines = None
-
- def __setup(self):
- if self.__lines:
- return
- data = None
- for dir in self.__dirs:
- for filename in self.__files:
- filename = os.path.join(dir, filename)
- try:
- fp = open(filename, "rU")
- data = fp.read()
- fp.close()
- break
- except IOError:
- pass
- if data:
- break
- if not data:
- data = self.__data
- self.__lines = data.split('\n')
- self.__linecnt = len(self.__lines)
-
- def __repr__(self):
- self.__setup()
- if len(self.__lines) <= self.MAXLINES:
- return "\n".join(self.__lines)
- else:
- return "Type %s() to see the full %s text" % ((self.__name,)*2)
-
- def __call__(self):
- self.__setup()
- prompt = 'Hit Return for more, or q (and Return) to quit: '
- lineno = 0
- while 1:
- try:
- for i in range(lineno, lineno + self.MAXLINES):
- print(self.__lines[i])
- except IndexError:
- break
- else:
- lineno += self.MAXLINES
- key = None
- while key is None:
- try:
- key = raw_input(prompt)
- except NameError:
- key = input(prompt)
- if key not in ('', 'q'):
- key = None
- if key == 'q':
- break
-
-def setcopyright():
- """Set 'copyright' and 'credits' in __builtin__"""
- builtins.copyright = _Printer("copyright", sys.copyright)
- if _is_jython:
- builtins.credits = _Printer(
- "credits",
- "Jython is maintained by the Jython developers (www.jython.org).")
- elif _is_pypy:
- builtins.credits = _Printer(
- "credits",
- "PyPy is maintained by the PyPy developers: http://pypy.org/")
- else:
- builtins.credits = _Printer("credits", """\
- Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands
- for supporting Python development. See www.python.org for more information.""")
- here = os.path.dirname(os.__file__)
- builtins.license = _Printer(
- "license", "See http://www.python.org/%.3s/license.html" % sys.version,
- ["LICENSE.txt", "LICENSE"],
- [os.path.join(here, os.pardir), here, os.curdir])
-
-
-class _Helper(object):
- """Define the built-in 'help'.
- This is a wrapper around pydoc.help (with a twist).
-
- """
-
- def __repr__(self):
- return "Type help() for interactive help, " \
- "or help(object) for help about object."
- def __call__(self, *args, **kwds):
- import pydoc
- return pydoc.help(*args, **kwds)
-
-def sethelper():
- builtins.help = _Helper()
-
-def aliasmbcs():
- """On Windows, some default encodings are not provided by Python,
- while they are always available as "mbcs" in each locale. Make
- them usable by aliasing to "mbcs" in such a case."""
- if sys.platform == 'win32':
- import locale, codecs
- enc = locale.getdefaultlocale()[1]
- if enc.startswith('cp'): # "cp***" ?
- try:
- codecs.lookup(enc)
- except LookupError:
- import encodings
- encodings._cache[enc] = encodings._unknown
- encodings.aliases.aliases[enc] = 'mbcs'
-
-def setencoding():
- """Set the string encoding used by the Unicode implementation. The
- default is 'ascii', but if you're willing to experiment, you can
- change this."""
- encoding = "ascii" # Default value set by _PyUnicode_Init()
- if 0:
- # Enable to support locale aware default string encodings.
- import locale
- loc = locale.getdefaultlocale()
- if loc[1]:
- encoding = loc[1]
- if 0:
- # Enable to switch off string to Unicode coercion and implicit
- # Unicode to string conversion.
- encoding = "undefined"
- if encoding != "ascii":
- # On Non-Unicode builds this will raise an AttributeError...
- sys.setdefaultencoding(encoding) # Needs Python Unicode build !
-
-
-def execsitecustomize():
- """Run custom site specific code, if available."""
- try:
- import sitecustomize
- except ImportError:
- pass
-
-def virtual_install_main_packages():
- f = open(os.path.join(os.path.dirname(__file__), 'orig-prefix.txt'))
- sys.real_prefix = f.read().strip()
- f.close()
- pos = 2
- hardcoded_relative_dirs = []
- if sys.path[0] == '':
- pos += 1
- if _is_jython:
- paths = [os.path.join(sys.real_prefix, 'Lib')]
- elif _is_pypy:
- if sys.version_info > (3, 2):
- cpyver = '%d' % sys.version_info[0]
- elif sys.pypy_version_info >= (1, 5):
- cpyver = '%d.%d' % sys.version_info[:2]
- else:
- cpyver = '%d.%d.%d' % sys.version_info[:3]
- paths = [os.path.join(sys.real_prefix, 'lib_pypy'),
- os.path.join(sys.real_prefix, 'lib-python', cpyver)]
- if sys.pypy_version_info < (1, 9):
- paths.insert(1, os.path.join(sys.real_prefix,
- 'lib-python', 'modified-%s' % cpyver))
- hardcoded_relative_dirs = paths[:] # for the special 'darwin' case below
- #
- # This is hardcoded in the Python executable, but relative to sys.prefix:
- for path in paths[:]:
- plat_path = os.path.join(path, 'plat-%s' % sys.platform)
- if os.path.exists(plat_path):
- paths.append(plat_path)
- elif sys.platform == 'win32':
- paths = [os.path.join(sys.real_prefix, 'Lib'), os.path.join(sys.real_prefix, 'DLLs')]
- else:
- paths = [os.path.join(sys.real_prefix, 'lib', 'python'+sys.version[:3])]
- hardcoded_relative_dirs = paths[:] # for the special 'darwin' case below
- lib64_path = os.path.join(sys.real_prefix, 'lib64', 'python'+sys.version[:3])
- if os.path.exists(lib64_path):
- if _is_64bit:
- paths.insert(0, lib64_path)
- else:
- paths.append(lib64_path)
- # This is hardcoded in the Python executable, but relative to
- # sys.prefix. Debian change: we need to add the multiarch triplet
- # here, which is where the real stuff lives. As per PEP 421, in
- # Python 3.3+, this lives in sys.implementation, while in Python 2.7
- # it lives in sys.
- try:
- arch = getattr(sys, 'implementation', sys)._multiarch
- except AttributeError:
- # This is a non-multiarch aware Python. Fallback to the old way.
- arch = sys.platform
- plat_path = os.path.join(sys.real_prefix, 'lib',
- 'python'+sys.version[:3],
- 'plat-%s' % arch)
- if os.path.exists(plat_path):
- paths.append(plat_path)
- # This is hardcoded in the Python executable, but
- # relative to sys.prefix, so we have to fix up:
- for path in list(paths):
- tk_dir = os.path.join(path, 'lib-tk')
- if os.path.exists(tk_dir):
- paths.append(tk_dir)
-
- # These are hardcoded in the Apple's Python executable,
- # but relative to sys.prefix, so we have to fix them up:
- if sys.platform == 'darwin':
- hardcoded_paths = [os.path.join(relative_dir, module)
- for relative_dir in hardcoded_relative_dirs
- for module in ('plat-darwin', 'plat-mac', 'plat-mac/lib-scriptpackages')]
-
- for path in hardcoded_paths:
- if os.path.exists(path):
- paths.append(path)
-
- sys.path.extend(paths)
-
-def force_global_eggs_after_local_site_packages():
- """
- Force easy_installed eggs in the global environment to get placed
- in sys.path after all packages inside the virtualenv. This
- maintains the "least surprise" result that packages in the
- virtualenv always mask global packages, never the other way
- around.
-
- """
- egginsert = getattr(sys, '__egginsert', 0)
- for i, path in enumerate(sys.path):
- if i > egginsert and path.startswith(sys.prefix):
- egginsert = i
- sys.__egginsert = egginsert + 1
-
-def virtual_addsitepackages(known_paths):
- force_global_eggs_after_local_site_packages()
- return addsitepackages(known_paths, sys_prefix=sys.real_prefix)
-
-def fixclasspath():
- """Adjust the special classpath sys.path entries for Jython. These
- entries should follow the base virtualenv lib directories.
- """
- paths = []
- classpaths = []
- for path in sys.path:
- if path == '__classpath__' or path.startswith('__pyclasspath__'):
- classpaths.append(path)
- else:
- paths.append(path)
- sys.path = paths
- sys.path.extend(classpaths)
-
-def execusercustomize():
- """Run custom user specific code, if available."""
- try:
- import usercustomize
- except ImportError:
- pass
-
-
-def main():
- global ENABLE_USER_SITE
- virtual_install_main_packages()
- abs__file__()
- paths_in_sys = removeduppaths()
- if (os.name == "posix" and sys.path and
- os.path.basename(sys.path[-1]) == "Modules"):
- addbuilddir()
- if _is_jython:
- fixclasspath()
- GLOBAL_SITE_PACKAGES = not os.path.exists(os.path.join(os.path.dirname(__file__), 'no-global-site-packages.txt'))
- if not GLOBAL_SITE_PACKAGES:
- ENABLE_USER_SITE = False
- if ENABLE_USER_SITE is None:
- ENABLE_USER_SITE = check_enableusersite()
- paths_in_sys = addsitepackages(paths_in_sys)
- paths_in_sys = addusersitepackages(paths_in_sys)
- if GLOBAL_SITE_PACKAGES:
- paths_in_sys = virtual_addsitepackages(paths_in_sys)
- if sys.platform == 'os2emx':
- setBEGINLIBPATH()
- setquit()
- setcopyright()
- sethelper()
- aliasmbcs()
- setencoding()
- execsitecustomize()
- if ENABLE_USER_SITE:
- execusercustomize()
- # Remove sys.setdefaultencoding() so that users cannot change the
- # encoding after initialization. The test for presence is needed when
- # this module is run as a script, because this code is executed twice.
- if hasattr(sys, "setdefaultencoding"):
- del sys.setdefaultencoding
-
-main()
-
-def _script():
- help = """\
- %s [--user-base] [--user-site]
-
- Without arguments print some useful information
- With arguments print the value of USER_BASE and/or USER_SITE separated
- by '%s'.
-
- Exit codes with --user-base or --user-site:
- 0 - user site directory is enabled
- 1 - user site directory is disabled by user
- 2 - uses site directory is disabled by super user
- or for security reasons
- >2 - unknown error
- """
- args = sys.argv[1:]
- if not args:
- print("sys.path = [")
- for dir in sys.path:
- print(" %r," % (dir,))
- print("]")
- def exists(path):
- if os.path.isdir(path):
- return "exists"
- else:
- return "doesn't exist"
- print("USER_BASE: %r (%s)" % (USER_BASE, exists(USER_BASE)))
- print("USER_SITE: %r (%s)" % (USER_SITE, exists(USER_BASE)))
- print("ENABLE_USER_SITE: %r" % ENABLE_USER_SITE)
- sys.exit(0)
-
- buffer = []
- if '--user-base' in args:
- buffer.append(USER_BASE)
- if '--user-site' in args:
- buffer.append(USER_SITE)
-
- if buffer:
- print(os.pathsep.join(buffer))
- if ENABLE_USER_SITE:
- sys.exit(0)
- elif ENABLE_USER_SITE is False:
- sys.exit(1)
- elif ENABLE_USER_SITE is None:
- sys.exit(2)
- else:
- sys.exit(3)
- else:
- import textwrap
- print(textwrap.dedent(help % (sys.argv[0], os.pathsep)))
- sys.exit(10)
-
-if __name__ == '__main__':
- _script()
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_support/__init__.py b/testing/mozharness/external_tools/virtualenv/virtualenv_support/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_support/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_support/argparse-1.4.0-py2.py3-none-any.whl b/testing/mozharness/external_tools/virtualenv/virtualenv_support/argparse-1.4.0-py2.py3-none-any.whl
deleted file mode 100644
index dfef51d44..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_support/argparse-1.4.0-py2.py3-none-any.whl
+++ /dev/null
Binary files differ
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_support/pip-8.1.2-py2.py3-none-any.whl b/testing/mozharness/external_tools/virtualenv/virtualenv_support/pip-8.1.2-py2.py3-none-any.whl
deleted file mode 100644
index cc49227a0..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_support/pip-8.1.2-py2.py3-none-any.whl
+++ /dev/null
Binary files differ
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_support/setuptools-25.2.0-py2.py3-none-any.whl b/testing/mozharness/external_tools/virtualenv/virtualenv_support/setuptools-25.2.0-py2.py3-none-any.whl
deleted file mode 100644
index 02c8ce873..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_support/setuptools-25.2.0-py2.py3-none-any.whl
+++ /dev/null
Binary files differ
diff --git a/testing/mozharness/external_tools/virtualenv/virtualenv_support/wheel-0.29.0-py2.py3-none-any.whl b/testing/mozharness/external_tools/virtualenv/virtualenv_support/wheel-0.29.0-py2.py3-none-any.whl
deleted file mode 100644
index 506d5e520..000000000
--- a/testing/mozharness/external_tools/virtualenv/virtualenv_support/wheel-0.29.0-py2.py3-none-any.whl
+++ /dev/null
Binary files differ
diff --git a/testing/mozharness/mach_commands.py b/testing/mozharness/mach_commands.py
deleted file mode 100644
index f453397db..000000000
--- a/testing/mozharness/mach_commands.py
+++ /dev/null
@@ -1,196 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import argparse
-import os
-import re
-import subprocess
-import sys
-import urllib
-import urlparse
-
-import mozinfo
-
-from mach.decorators import (
- CommandArgument,
- CommandProvider,
- Command,
-)
-
-from mozbuild.base import MachCommandBase, MozbuildObject
-from mozbuild.base import MachCommandConditions as conditions
-from argparse import ArgumentParser
-
-def get_parser():
- parser = argparse.ArgumentParser()
- parser.add_argument("suite_name", nargs=1, type=str, action="store",
- help="Suite to run in mozharness")
- parser.add_argument("mozharness_args", nargs=argparse.REMAINDER,
- help="Extra arguments to pass to mozharness")
- return parser
-
-class MozharnessRunner(MozbuildObject):
- def __init__(self, *args, **kwargs):
- MozbuildObject.__init__(self, *args, **kwargs)
-
-
- self.test_packages_url = self._test_packages_url()
- self.installer_url = self._installer_url()
-
- desktop_unittest_config = [
- "--config-file", lambda: self.config_path("unittests",
- "%s_unittest.py" % mozinfo.info['os']),
- "--config-file", lambda: self.config_path("developer_config.py")]
-
- self.config = {
- "__defaults__": {
- "config": ["--no-read-buildbot-config",
- "--download-symbols", "ondemand",
- "--installer-url", self.installer_url,
- "--test-packages-url", self.test_packages_url]
- },
-
- "mochitest-valgrind": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--mochitest-suite", "valgrind-plain"]
- },
- "mochitest": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--mochitest-suite", "plain"]
- },
- "mochitest-chrome": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--mochitest-suite", "chrome"]
- },
- "mochitest-browser-chrome": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--mochitest-suite", "browser-chrome"]
- },
- "mochitest-devtools-chrome": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--mochitest-suite", "mochitest-devtools-chrome"]
- },
- "crashtest": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--reftest-suite", "crashtest"]
- },
- "jsreftest": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--reftest-suite", "jsreftest"]
- },
- "reftest": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--reftest-suite", "reftest"]
- },
- "reftest-no-accel": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--reftest-suite", "reftest-no-accel"]
- },
- "cppunittest": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--cppunittest-suite", "cppunittest"]
- },
- "xpcshell": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--xpcshell-suite", "xpcshell"]
- },
- "xpcshell-addons": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--xpcshell-suite", "xpcshell-addons"]
- },
- "jittest": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--jittest-suite", "jittest"]
- },
- "mozbase": {
- "script": "desktop_unittest.py",
- "config": desktop_unittest_config + [
- "--mozbase-suite", "mozbase"]
- },
- "marionette": {
- "script": "marionette.py",
- "config": ["--config-file", self.config_path("marionette",
- "test_config.py")]
- },
- "web-platform-tests": {
- "script": "web_platform_tests.py",
- "config": ["--config-file", self.config_path("web_platform_tests",
- self.wpt_config)]
- },
- }
-
-
- def path_to_url(self, path):
- return urlparse.urljoin('file:', urllib.pathname2url(path))
-
- def _installer_url(self):
- package_re = {
- "linux": re.compile("^firefox-\d+\..+\.tar\.bz2$"),
- "win": re.compile("^firefox-\d+\..+\.installer\.exe$"),
- "mac": re.compile("^firefox-\d+\..+\.mac(?:64)?\.dmg$"),
- }[mozinfo.info['os']]
- dist_path = os.path.join(self.topobjdir, "dist")
- filenames = [item for item in os.listdir(dist_path) if
- package_re.match(item)]
- assert len(filenames) == 1
- return self.path_to_url(os.path.join(dist_path, filenames[0]))
-
- def _test_packages_url(self):
- dist_path = os.path.join(self.topobjdir, "dist")
- filenames = [item for item in os.listdir(dist_path) if
- item.endswith('test_packages.json')]
- assert len(filenames) == 1
- return self.path_to_url(os.path.join(dist_path, filenames[0]))
-
- def config_path(self, *parts):
- return self.path_to_url(os.path.join(self.topsrcdir, "testing", "mozharness",
- "configs", *parts))
-
- @property
- def wpt_config(self):
- return "test_config.py" if mozinfo.info['os'] != "win" else "test_config_windows.py"
-
- def run_suite(self, suite, **kwargs):
- default_config = self.config.get("__defaults__")
- suite_config = self.config.get(suite)
-
- if suite_config is None:
- print("Unknown suite %s" % suite)
- return 1
-
- script = os.path.join(self.topsrcdir, "testing", "mozharness",
- "scripts", suite_config["script"])
- options = [item() if callable(item) else item
- for item in default_config["config"] + suite_config["config"]]
-
- cmd = [script] + options
-
- rv = subprocess.call(cmd, cwd=os.path.dirname(script))
- return rv
-
-
-@CommandProvider
-class MozharnessCommands(MachCommandBase):
- @Command('mozharness', category='testing',
- description='Run tests using mozharness.',
- conditions=[conditions.is_firefox],
- parser=get_parser)
- def mozharness(self, **kwargs):
- runner = self._spawn(MozharnessRunner)
- return runner.run_suite(kwargs.pop("suite_name")[0], **kwargs)
diff --git a/testing/mozharness/mozfile/__init__.py b/testing/mozharness/mozfile/__init__.py
deleted file mode 100644
index 37b8babb8..000000000
--- a/testing/mozharness/mozfile/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from mozfile import *
diff --git a/testing/mozharness/mozfile/mozfile.py b/testing/mozharness/mozfile/mozfile.py
deleted file mode 100644
index ac0edcab4..000000000
--- a/testing/mozharness/mozfile/mozfile.py
+++ /dev/null
@@ -1,372 +0,0 @@
-# -*- coding: utf-8 -*-
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from contextlib import contextmanager
-import os
-import shutil
-import stat
-import tarfile
-import tempfile
-import urlparse
-import urllib2
-import zipfile
-import time
-
-__all__ = ['extract_tarball',
- 'extract_zip',
- 'extract',
- 'is_url',
- 'load',
- 'remove',
- 'rmtree',
- 'tree',
- 'NamedTemporaryFile',
- 'TemporaryDirectory']
-
-try:
- WindowsError
-except NameError:
- WindowsError = None # so we can unconditionally catch it later...
-
-
-### utilities for extracting archives
-
-def extract_tarball(src, dest):
- """extract a .tar file"""
-
- bundle = tarfile.open(src)
- namelist = bundle.getnames()
-
- for name in namelist:
- bundle.extract(name, path=dest)
- bundle.close()
- return namelist
-
-
-def extract_zip(src, dest):
- """extract a zip file"""
-
- if isinstance(src, zipfile.ZipFile):
- bundle = src
- else:
- try:
- bundle = zipfile.ZipFile(src)
- except Exception, e:
- print "src: %s" % src
- raise
-
- namelist = bundle.namelist()
-
- for name in namelist:
- filename = os.path.realpath(os.path.join(dest, name))
- if name.endswith('/'):
- if not os.path.isdir(filename):
- os.makedirs(filename)
- else:
- path = os.path.dirname(filename)
- if not os.path.isdir(path):
- os.makedirs(path)
- _dest = open(filename, 'wb')
- _dest.write(bundle.read(name))
- _dest.close()
- mode = bundle.getinfo(name).external_attr >> 16 & 0x1FF
- os.chmod(filename, mode)
- bundle.close()
- return namelist
-
-
-def extract(src, dest=None):
- """
- Takes in a tar or zip file and extracts it to dest
-
- If dest is not specified, extracts to os.path.dirname(src)
-
- Returns the list of top level files that were extracted
- """
-
- assert os.path.exists(src), "'%s' does not exist" % src
-
- if dest is None:
- dest = os.path.dirname(src)
- elif not os.path.isdir(dest):
- os.makedirs(dest)
- assert not os.path.isfile(dest), "dest cannot be a file"
-
- if zipfile.is_zipfile(src):
- namelist = extract_zip(src, dest)
- elif tarfile.is_tarfile(src):
- namelist = extract_tarball(src, dest)
- else:
- raise Exception("mozfile.extract: no archive format found for '%s'" %
- src)
-
- # namelist returns paths with forward slashes even in windows
- top_level_files = [os.path.join(dest, name.rstrip('/')) for name in namelist
- if len(name.rstrip('/').split('/')) == 1]
-
- # namelist doesn't include folders, append these to the list
- for name in namelist:
- index = name.find('/')
- if index != -1:
- root = os.path.join(dest, name[:index])
- if root not in top_level_files:
- top_level_files.append(root)
-
- return top_level_files
-
-
-### utilities for removal of files and directories
-
-def rmtree(dir):
- """Deprecated wrapper method to remove a directory tree.
-
- Ensure to update your code to use mozfile.remove() directly
-
- :param dir: directory to be removed
- """
-
- return remove(dir)
-
-
-def remove(path):
- """Removes the specified file, link, or directory tree
-
- This is a replacement for shutil.rmtree that works better under
- windows.
-
- :param path: path to be removed
- """
-
- def _call_with_windows_retry(func, path, retry_max=5, retry_delay=0.5):
- """
- It's possible to see spurious errors on Windows due to various things
- keeping a handle to the directory open (explorer, virus scanners, etc)
- So we try a few times if it fails with a known error.
- """
- retry_count = 0
- while True:
- try:
- func(path)
- break
- except WindowsError as e:
- # Error 5 == Access is denied
- # Error 32 == The process cannot access the file because it is
- # being used by another process
- # Error 145 == The directory is not empty
-
- if retry_count == retry_max or e.winerror not in [5, 32, 145]:
- raise
- retry_count += 1
-
- print 'Retrying to remove "%s" because it is in use.' % path
- time.sleep(retry_delay)
-
- if not os.path.exists(path):
- return
-
- path_stats = os.stat(path)
-
- if os.path.isfile(path) or os.path.islink(path):
- # Verify the file or link is read/write for the current user
- os.chmod(path, path_stats.st_mode | stat.S_IRUSR | stat.S_IWUSR)
- _call_with_windows_retry(os.remove, path)
-
- elif os.path.isdir(path):
- # Verify the directory is read/write/execute for the current user
- os.chmod(path, path_stats.st_mode | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
- _call_with_windows_retry(shutil.rmtree, path)
-
-def depth(directory):
- """returns the integer depth of a directory or path relative to '/' """
-
- directory = os.path.abspath(directory)
- level = 0
- while True:
- directory, remainder = os.path.split(directory)
- level += 1
- if not remainder:
- break
- return level
-
-# ASCII delimeters
-ascii_delimeters = {
- 'vertical_line' : '|',
- 'item_marker' : '+',
- 'last_child' : '\\'
- }
-
-# unicode delimiters
-unicode_delimeters = {
- 'vertical_line' : '│',
- 'item_marker' : '├',
- 'last_child' : 'â””'
- }
-
-def tree(directory,
- item_marker=unicode_delimeters['item_marker'],
- vertical_line=unicode_delimeters['vertical_line'],
- last_child=unicode_delimeters['last_child'],
- sort_key=lambda x: x.lower()):
- """
- display tree directory structure for `directory`
- """
-
- retval = []
- indent = []
- last = {}
- top = depth(directory)
-
- for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
-
- abspath = os.path.abspath(dirpath)
- basename = os.path.basename(abspath)
- parent = os.path.dirname(abspath)
- level = depth(abspath) - top
-
- # sort articles of interest
- for resource in (dirnames, filenames):
- resource[:] = sorted(resource, key=sort_key)
-
- files_end = item_marker
- dirpath_marker = item_marker
-
- if level > len(indent):
- indent.append(vertical_line)
- indent = indent[:level]
-
- if dirnames:
- files_end = item_marker
- last[abspath] = dirnames[-1]
- else:
- files_end = last_child
-
- if last.get(parent) == os.path.basename(abspath):
- # last directory of parent
- dirpath_mark = last_child
- indent[-1] = ' '
- elif not indent:
- dirpath_mark = ''
- else:
- dirpath_mark = item_marker
-
- # append the directory and piece of tree structure
- # if the top-level entry directory, print as passed
- retval.append('%s%s%s'% (''.join(indent[:-1]),
- dirpath_mark,
- basename if retval else directory))
- # add the files
- if filenames:
- last_file = filenames[-1]
- retval.extend([('%s%s%s' % (''.join(indent),
- files_end if filename == last_file else item_marker,
- filename))
- for index, filename in enumerate(filenames)])
-
- return '\n'.join(retval)
-
-
-### utilities for temporary resources
-
-class NamedTemporaryFile(object):
- """
- Like tempfile.NamedTemporaryFile except it works on Windows
- in the case where you open the created file a second time.
-
- This behaves very similarly to tempfile.NamedTemporaryFile but may
- not behave exactly the same. For example, this function does not
- prevent fd inheritance by children.
-
- Example usage:
-
- with NamedTemporaryFile() as fh:
- fh.write(b'foobar')
-
- print('Filename: %s' % fh.name)
-
- see https://bugzilla.mozilla.org/show_bug.cgi?id=821362
- """
- def __init__(self, mode='w+b', bufsize=-1, suffix='', prefix='tmp',
- dir=None, delete=True):
-
- fd, path = tempfile.mkstemp(suffix, prefix, dir, 't' in mode)
- os.close(fd)
-
- self.file = open(path, mode)
- self._path = path
- self._delete = delete
- self._unlinked = False
-
- def __getattr__(self, k):
- return getattr(self.__dict__['file'], k)
-
- def __iter__(self):
- return self.__dict__['file']
-
- def __enter__(self):
- self.file.__enter__()
- return self
-
- def __exit__(self, exc, value, tb):
- self.file.__exit__(exc, value, tb)
- if self.__dict__['_delete']:
- os.unlink(self.__dict__['_path'])
- self._unlinked = True
-
- def __del__(self):
- if self.__dict__['_unlinked']:
- return
- self.file.__exit__(None, None, None)
- if self.__dict__['_delete']:
- os.unlink(self.__dict__['_path'])
-
-
-@contextmanager
-def TemporaryDirectory():
- """
- create a temporary directory using tempfile.mkdtemp, and then clean it up.
-
- Example usage:
- with TemporaryDirectory() as tmp:
- open(os.path.join(tmp, "a_temp_file"), "w").write("data")
-
- """
- tempdir = tempfile.mkdtemp()
- try:
- yield tempdir
- finally:
- shutil.rmtree(tempdir)
-
-
-### utilities dealing with URLs
-
-def is_url(thing):
- """
- Return True if thing looks like a URL.
- """
-
- parsed = urlparse.urlparse(thing)
- if 'scheme' in parsed:
- return len(parsed.scheme) >= 2
- else:
- return len(parsed[0]) >= 2
-
-def load(resource):
- """
- open a file or URL for reading. If the passed resource string is not a URL,
- or begins with 'file://', return a ``file``. Otherwise, return the
- result of urllib2.urlopen()
- """
-
- # handle file URLs separately due to python stdlib limitations
- if resource.startswith('file://'):
- resource = resource[len('file://'):]
-
- if not is_url(resource):
- # if no scheme is given, it is a file path
- return file(resource)
-
- return urllib2.urlopen(resource)
-
diff --git a/testing/mozharness/mozharness/__init__.py b/testing/mozharness/mozharness/__init__.py
deleted file mode 100644
index 609f98f33..000000000
--- a/testing/mozharness/mozharness/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-version = (0, 7)
-version_string = '.'.join(['%d' % i for i in version])
diff --git a/testing/mozharness/mozharness/base/__init__.py b/testing/mozharness/mozharness/base/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/mozharness/base/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/mozharness/base/config.py b/testing/mozharness/mozharness/base/config.py
deleted file mode 100644
index 9c17b3381..000000000
--- a/testing/mozharness/mozharness/base/config.py
+++ /dev/null
@@ -1,569 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Generic config parsing and dumping, the way I remember it from scripts
-gone by.
-
-The config should be built from script-level defaults, overlaid by
-config-file defaults, overlaid by command line options.
-
- (For buildbot-analogues that would be factory-level defaults,
- builder-level defaults, and build request/scheduler settings.)
-
-The config should then be locked (set to read-only, to prevent runtime
-alterations). Afterwards we should dump the config to a file that is
-uploaded with the build, and can be used to debug or replicate the build
-at a later time.
-
-TODO:
-
-* check_required_settings or something -- run at init, assert that
- these settings are set.
-"""
-
-from copy import deepcopy
-from optparse import OptionParser, Option, OptionGroup
-import os
-import sys
-import urllib2
-import socket
-import time
-try:
- import simplejson as json
-except ImportError:
- import json
-
-from mozharness.base.log import DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL
-
-
-# optparse {{{1
-class ExtendedOptionParser(OptionParser):
- """OptionParser, but with ExtendOption as the option_class.
- """
- def __init__(self, **kwargs):
- kwargs['option_class'] = ExtendOption
- OptionParser.__init__(self, **kwargs)
-
-
-class ExtendOption(Option):
- """from http://docs.python.org/library/optparse.html?highlight=optparse#adding-new-actions"""
- ACTIONS = Option.ACTIONS + ("extend",)
- STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",)
- TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",)
- ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + ("extend",)
-
- def take_action(self, action, dest, opt, value, values, parser):
- if action == "extend":
- lvalue = value.split(",")
- values.ensure_value(dest, []).extend(lvalue)
- else:
- Option.take_action(
- self, action, dest, opt, value, values, parser)
-
-
-def make_immutable(item):
- if isinstance(item, list) or isinstance(item, tuple):
- result = LockedTuple(item)
- elif isinstance(item, dict):
- result = ReadOnlyDict(item)
- result.lock()
- else:
- result = item
- return result
-
-
-class LockedTuple(tuple):
- def __new__(cls, items):
- return tuple.__new__(cls, (make_immutable(x) for x in items))
- def __deepcopy__(self, memo):
- return [deepcopy(elem, memo) for elem in self]
-
-
-# ReadOnlyDict {{{1
-class ReadOnlyDict(dict):
- def __init__(self, dictionary):
- self._lock = False
- self.update(dictionary.copy())
-
- def _check_lock(self):
- assert not self._lock, "ReadOnlyDict is locked!"
-
- def lock(self):
- for (k, v) in self.items():
- self[k] = make_immutable(v)
- self._lock = True
-
- def __setitem__(self, *args):
- self._check_lock()
- return dict.__setitem__(self, *args)
-
- def __delitem__(self, *args):
- self._check_lock()
- return dict.__delitem__(self, *args)
-
- def clear(self, *args):
- self._check_lock()
- return dict.clear(self, *args)
-
- def pop(self, *args):
- self._check_lock()
- return dict.pop(self, *args)
-
- def popitem(self, *args):
- self._check_lock()
- return dict.popitem(self, *args)
-
- def setdefault(self, *args):
- self._check_lock()
- return dict.setdefault(self, *args)
-
- def update(self, *args):
- self._check_lock()
- dict.update(self, *args)
-
- def __deepcopy__(self, memo):
- cls = self.__class__
- result = cls.__new__(cls)
- memo[id(self)] = result
- for k, v in self.__dict__.items():
- setattr(result, k, deepcopy(v, memo))
- result._lock = False
- for k, v in self.items():
- result[k] = deepcopy(v, memo)
- return result
-
-# parse_config_file {{{1
-def parse_config_file(file_name, quiet=False, search_path=None,
- config_dict_name="config"):
- """Read a config file and return a dictionary.
- """
- file_path = None
- if os.path.exists(file_name):
- file_path = file_name
- else:
- if not search_path:
- search_path = ['.', os.path.join(sys.path[0], '..', 'configs'),
- os.path.join(sys.path[0], '..', '..', 'configs')]
- for path in search_path:
- if os.path.exists(os.path.join(path, file_name)):
- file_path = os.path.join(path, file_name)
- break
- else:
- raise IOError("Can't find %s in %s!" % (file_name, search_path))
- if file_name.endswith('.py'):
- global_dict = {}
- local_dict = {}
- execfile(file_path, global_dict, local_dict)
- config = local_dict[config_dict_name]
- elif file_name.endswith('.json'):
- fh = open(file_path)
- config = {}
- json_config = json.load(fh)
- config = dict(json_config)
- fh.close()
- else:
- raise RuntimeError("Unknown config file type %s!" % file_name)
- # TODO return file_path
- return config
-
-
-def download_config_file(url, file_name):
- n = 0
- attempts = 5
- sleeptime = 60
- max_sleeptime = 5 * 60
- while True:
- if n >= attempts:
- print "Failed to download from url %s after %d attempts, quiting..." % (url, attempts)
- raise SystemError(-1)
- try:
- contents = urllib2.urlopen(url, timeout=30).read()
- break
- except urllib2.URLError, e:
- print "Error downloading from url %s: %s" % (url, str(e))
- except socket.timeout, e:
- print "Time out accessing %s: %s" % (url, str(e))
- except socket.error, e:
- print "Socket error when accessing %s: %s" % (url, str(e))
- print "Sleeping %d seconds before retrying" % sleeptime
- time.sleep(sleeptime)
- sleeptime = sleeptime * 2
- if sleeptime > max_sleeptime:
- sleeptime = max_sleeptime
- n += 1
-
- try:
- f = open(file_name, 'w')
- f.write(contents)
- f.close()
- except IOError, e:
- print "Error writing downloaded contents to file %s: %s" % (file_name, str(e))
- raise SystemError(-1)
-
-
-# BaseConfig {{{1
-class BaseConfig(object):
- """Basic config setting/getting.
- """
- def __init__(self, config=None, initial_config_file=None, config_options=None,
- all_actions=None, default_actions=None,
- volatile_config=None, option_args=None,
- require_config_file=False,
- append_env_variables_from_configs=False,
- usage="usage: %prog [options]"):
- self._config = {}
- self.all_cfg_files_and_dicts = []
- self.actions = []
- self.config_lock = False
- self.require_config_file = require_config_file
- # It allows to append env variables from multiple config files
- self.append_env_variables_from_configs = append_env_variables_from_configs
-
- if all_actions:
- self.all_actions = all_actions[:]
- else:
- self.all_actions = ['clobber', 'build']
- if default_actions:
- self.default_actions = default_actions[:]
- else:
- self.default_actions = self.all_actions[:]
- if volatile_config is None:
- self.volatile_config = {
- 'actions': None,
- 'add_actions': None,
- 'no_actions': None,
- }
- else:
- self.volatile_config = deepcopy(volatile_config)
-
- if config:
- self.set_config(config)
- if initial_config_file:
- initial_config = parse_config_file(initial_config_file)
- self.all_cfg_files_and_dicts.append(
- (initial_config_file, initial_config)
- )
- self.set_config(initial_config)
- # Since initial_config_file is only set when running unit tests,
- # if no option_args have been specified, then the parser will
- # parse sys.argv which in this case would be the command line
- # options specified to run the tests, e.g. nosetests -v. Clearly,
- # the options passed to nosetests (such as -v) should not be
- # interpreted by mozharness as mozharness options, so we specify
- # a dummy command line with no options, so that the parser does
- # not add anything from the test invocation command line
- # arguments to the mozharness options.
- if option_args is None:
- option_args=['dummy_mozharness_script_with_no_command_line_options.py']
- if config_options is None:
- config_options = []
- self._create_config_parser(config_options, usage)
- # we allow manually passing of option args for things like nosetests
- self.parse_args(args=option_args)
-
- def get_read_only_config(self):
- return ReadOnlyDict(self._config)
-
- def _create_config_parser(self, config_options, usage):
- self.config_parser = ExtendedOptionParser(usage=usage)
- self.config_parser.add_option(
- "--work-dir", action="store", dest="work_dir",
- type="string", default="build",
- help="Specify the work_dir (subdir of base_work_dir)"
- )
- self.config_parser.add_option(
- "--base-work-dir", action="store", dest="base_work_dir",
- type="string", default=os.getcwd(),
- help="Specify the absolute path of the parent of the working directory"
- )
- self.config_parser.add_option(
- "-c", "--config-file", "--cfg", action="extend", dest="config_files",
- type="string", help="Specify a config file; can be repeated"
- )
- self.config_parser.add_option(
- "-C", "--opt-config-file", "--opt-cfg", action="extend",
- dest="opt_config_files", type="string", default=[],
- help="Specify an optional config file, like --config-file but with no "
- "error if the file is missing; can be repeated"
- )
- self.config_parser.add_option(
- "--dump-config", action="store_true",
- dest="dump_config",
- help="List and dump the config generated from this run to "
- "a JSON file."
- )
- self.config_parser.add_option(
- "--dump-config-hierarchy", action="store_true",
- dest="dump_config_hierarchy",
- help="Like --dump-config but will list and dump which config "
- "files were used making up the config and specify their own "
- "keys/values that were not overwritten by another cfg -- "
- "held the highest hierarchy."
- )
-
- # Logging
- log_option_group = OptionGroup(self.config_parser, "Logging")
- log_option_group.add_option(
- "--log-level", action="store",
- type="choice", dest="log_level", default=INFO,
- choices=[DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL],
- help="Set log level (debug|info|warning|error|critical|fatal)"
- )
- log_option_group.add_option(
- "-q", "--quiet", action="store_false", dest="log_to_console",
- default=True, help="Don't log to the console"
- )
- log_option_group.add_option(
- "--append-to-log", action="store_true",
- dest="append_to_log", default=False,
- help="Append to the log"
- )
- log_option_group.add_option(
- "--multi-log", action="store_const", const="multi",
- dest="log_type", help="Log using MultiFileLogger"
- )
- log_option_group.add_option(
- "--simple-log", action="store_const", const="simple",
- dest="log_type", help="Log using SimpleFileLogger"
- )
- self.config_parser.add_option_group(log_option_group)
-
- # Actions
- action_option_group = OptionGroup(
- self.config_parser, "Actions",
- "Use these options to list or enable/disable actions."
- )
- action_option_group.add_option(
- "--list-actions", action="store_true",
- dest="list_actions",
- help="List all available actions, then exit"
- )
- action_option_group.add_option(
- "--add-action", action="extend",
- dest="add_actions", metavar="ACTIONS",
- help="Add action %s to the list of actions" % self.all_actions
- )
- action_option_group.add_option(
- "--no-action", action="extend",
- dest="no_actions", metavar="ACTIONS",
- help="Don't perform action"
- )
- for action in self.all_actions:
- action_option_group.add_option(
- "--%s" % action, action="append_const",
- dest="actions", const=action,
- help="Add %s to the limited list of actions" % action
- )
- action_option_group.add_option(
- "--no-%s" % action, action="append_const",
- dest="no_actions", const=action,
- help="Remove %s from the list of actions to perform" % action
- )
- self.config_parser.add_option_group(action_option_group)
- # Child-specified options
- # TODO error checking for overlapping options
- if config_options:
- for option in config_options:
- self.config_parser.add_option(*option[0], **option[1])
-
- # Initial-config-specified options
- config_options = self._config.get('config_options', None)
- if config_options:
- for option in config_options:
- self.config_parser.add_option(*option[0], **option[1])
-
- def set_config(self, config, overwrite=False):
- """This is probably doable some other way."""
- if self._config and not overwrite:
- self._config.update(config)
- else:
- self._config = config
- return self._config
-
- def get_actions(self):
- return self.actions
-
- def verify_actions(self, action_list, quiet=False):
- for action in action_list:
- if action not in self.all_actions:
- if not quiet:
- print("Invalid action %s not in %s!" % (action,
- self.all_actions))
- raise SystemExit(-1)
- return action_list
-
- def verify_actions_order(self, action_list):
- try:
- indexes = [ self.all_actions.index(elt) for elt in action_list ]
- sorted_indexes = sorted(indexes)
- for i in range(len(indexes)):
- if indexes[i] != sorted_indexes[i]:
- print(("Action %s comes in different order in %s\n" +
- "than in %s") % (action_list[i], action_list, self.all_actions))
- raise SystemExit(-1)
- except ValueError as e:
- print("Invalid action found: " + str(e))
- raise SystemExit(-1)
-
- def list_actions(self):
- print "Actions available:"
- for a in self.all_actions:
- print " " + ("*" if a in self.default_actions else " "), a
- raise SystemExit(0)
-
- def get_cfgs_from_files(self, all_config_files, options):
- """Returns the configuration derived from the list of configuration
- files. The result is represented as a list of `(filename,
- config_dict)` tuples; they will be combined with keys in later
- dictionaries taking precedence over earlier.
-
- `all_config_files` is all files specified with `--config-file` and
- `--opt-config-file`; `options` is the argparse options object giving
- access to any other command-line options.
-
- This function is also responsible for downloading any configuration
- files specified by URL. It uses ``parse_config_file`` in this module
- to parse individual files.
-
- This method can be overridden in a subclass to add extra logic to the
- way that self.config is made up. See
- `mozharness.mozilla.building.buildbase.BuildingConfig` for an example.
- """
- all_cfg_files_and_dicts = []
- for cf in all_config_files:
- try:
- if '://' in cf: # config file is an url
- file_name = os.path.basename(cf)
- file_path = os.path.join(os.getcwd(), file_name)
- download_config_file(cf, file_path)
- all_cfg_files_and_dicts.append(
- (file_path, parse_config_file(file_path))
- )
- else:
- all_cfg_files_and_dicts.append((cf, parse_config_file(cf)))
- except Exception:
- if cf in options.opt_config_files:
- print(
- "WARNING: optional config file not found %s" % cf
- )
- else:
- raise
- return all_cfg_files_and_dicts
-
- def parse_args(self, args=None):
- """Parse command line arguments in a generic way.
- Return the parser object after adding the basic options, so
- child objects can manipulate it.
- """
- self.command_line = ' '.join(sys.argv)
- if args is None:
- args = sys.argv[1:]
- (options, args) = self.config_parser.parse_args(args)
-
- defaults = self.config_parser.defaults.copy()
-
- if not options.config_files:
- if self.require_config_file:
- if options.list_actions:
- self.list_actions()
- print("Required config file not set! (use --config-file option)")
- raise SystemExit(-1)
- else:
- # this is what get_cfgs_from_files returns. It will represent each
- # config file name and its assoctiated dict
- # eg ('builds/branch_specifics.py', {'foo': 'bar'})
- # let's store this to self for things like --interpret-config-files
- self.all_cfg_files_and_dicts.extend(self.get_cfgs_from_files(
- # append opt_config to allow them to overwrite previous configs
- options.config_files + options.opt_config_files, options=options
- ))
- config = {}
- if self.append_env_variables_from_configs:
- # We only append values from various configs for the 'env' entry
- # For everything else we follow the standard behaviour
- for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
- for v in c_dict.keys():
- if v == 'env' and v in config:
- config[v].update(c_dict[v])
- else:
- config[v] = c_dict[v]
- else:
- for i, (c_file, c_dict) in enumerate(self.all_cfg_files_and_dicts):
- config.update(c_dict)
- # assign or update self._config depending on if it exists or not
- # NOTE self._config will be passed to ReadOnlyConfig's init -- a
- # dict subclass with immutable locking capabilities -- and serve
- # as the keys/values that make up that instance. Ultimately,
- # this becomes self.config during BaseScript's init
- self.set_config(config)
- for key in defaults.keys():
- value = getattr(options, key)
- if value is None:
- continue
- # Don't override config_file defaults with config_parser defaults
- if key in defaults and value == defaults[key] and key in self._config:
- continue
- self._config[key] = value
-
- # The idea behind the volatile_config is we don't want to save this
- # info over multiple runs. This defaults to the action-specific
- # config options, but can be anything.
- for key in self.volatile_config.keys():
- if self._config.get(key) is not None:
- self.volatile_config[key] = self._config[key]
- del(self._config[key])
-
- self.update_actions()
- if options.list_actions:
- self.list_actions()
-
- # Keep? This is for saving the volatile config in the dump_config
- self._config['volatile_config'] = self.volatile_config
-
- self.options = options
- self.args = args
- return (self.options, self.args)
-
- def update_actions(self):
- """ Update actions after reading in config.
-
- Seems a little complex, but the logic goes:
-
- First, if default_actions is specified in the config, set our
- default actions even if the script specifies other default actions.
-
- Without any other action-specific options, run with default actions.
-
- If we specify --ACTION or --only-ACTION once or multiple times,
- we want to override the default_actions list with the one(s) we list.
-
- Otherwise, if we specify --add-action ACTION, we want to add an
- action to the list.
-
- Finally, if we specify --no-ACTION, remove that from the list of
- actions to perform.
- """
- if self._config.get('default_actions'):
- default_actions = self.verify_actions(self._config['default_actions'])
- self.default_actions = default_actions
- self.verify_actions_order(self.default_actions)
- self.actions = self.default_actions[:]
- if self.volatile_config['actions']:
- actions = self.verify_actions(self.volatile_config['actions'])
- self.actions = actions
- elif self.volatile_config['add_actions']:
- actions = self.verify_actions(self.volatile_config['add_actions'])
- self.actions.extend(actions)
- if self.volatile_config['no_actions']:
- actions = self.verify_actions(self.volatile_config['no_actions'])
- for action in actions:
- if action in self.actions:
- self.actions.remove(action)
-
-
-# __main__ {{{1
-if __name__ == '__main__':
- pass
diff --git a/testing/mozharness/mozharness/base/diskutils.py b/testing/mozharness/mozharness/base/diskutils.py
deleted file mode 100644
index 745384ff9..000000000
--- a/testing/mozharness/mozharness/base/diskutils.py
+++ /dev/null
@@ -1,156 +0,0 @@
-"""Disk utility module, no mixins here!
-
- examples:
- 1) get disk size
- from mozharness.base.diskutils import DiskInfo, DiskutilsError
- ...
- try:
- DiskSize().get_size(path='/', unit='Mb')
- except DiskutilsError as e:
- # manage the exception e.g: log.error(e)
- pass
- log.info("%s" % di)
-
-
- 2) convert disk size:
- from mozharness.base.diskutils import DiskutilsError, convert_to
- ...
- file_size = <function that gets file size in bytes>
- # convert file_size to GB
- try:
- file_size = convert_to(file_size, from_unit='bytes', to_unit='GB')
- except DiskutilsError as e:
- # manage the exception e.g: log.error(e)
- pass
-
-"""
-import ctypes
-import os
-import sys
-import logging
-from mozharness.base.log import INFO, numeric_log_level
-
-# use mozharness log
-log = logging.getLogger(__name__)
-
-
-class DiskutilsError(Exception):
- """Exception thrown by Diskutils module"""
- pass
-
-
-def convert_to(size, from_unit, to_unit):
- """Helper method to convert filesystem sizes to kB/ MB/ GB/ TB/
- valid values for source_format and destination format are:
- * bytes
- * kB
- * MB
- * GB
- * TB
- returns: size converted from source_format to destination_format.
- """
- sizes = {'bytes': 1,
- 'kB': 1024,
- 'MB': 1024 * 1024,
- 'GB': 1024 * 1024 * 1024,
- 'TB': 1024 * 1024 * 1024 * 1024}
- try:
- df = sizes[to_unit]
- sf = sizes[from_unit]
- return size * sf / df
- except KeyError:
- raise DiskutilsError('conversion error: Invalid source or destination format')
- except TypeError:
- raise DiskutilsError('conversion error: size (%s) is not a number' % size)
-
-
-class DiskInfo(object):
- """Stores basic information about the disk"""
- def __init__(self):
- self.unit = 'bytes'
- self.free = 0
- self.used = 0
- self.total = 0
-
- def __str__(self):
- string = ['Disk space info (in %s)' % self.unit]
- string += ['total: %s' % self.total]
- string += ['used: %s' % self.used]
- string += ['free: %s' % self.free]
- return " ".join(string)
-
- def _to(self, unit):
- from_unit = self.unit
- to_unit = unit
- self.free = convert_to(self.free, from_unit=from_unit, to_unit=to_unit)
- self.used = convert_to(self.used, from_unit=from_unit, to_unit=to_unit)
- self.total = convert_to(self.total, from_unit=from_unit, to_unit=to_unit)
- self.unit = unit
-
-
-class DiskSize(object):
- """DiskSize object
- """
- @staticmethod
- def _posix_size(path):
- """returns the disk size in bytes
- disk size is relative to path
- """
- # we are on a POSIX system
- st = os.statvfs(path)
- disk_info = DiskInfo()
- disk_info.free = st.f_bavail * st.f_frsize
- disk_info.used = (st.f_blocks - st.f_bfree) * st.f_frsize
- disk_info.total = st.f_blocks * st.f_frsize
- return disk_info
-
- @staticmethod
- def _windows_size(path):
- """returns size in bytes, works only on windows platforms"""
- # we're on a non POSIX system (windows)
- # DLL call
- disk_info = DiskInfo()
- dummy = ctypes.c_ulonglong() # needed by the dll call but not used
- total = ctypes.c_ulonglong() # stores the total space value
- free = ctypes.c_ulonglong() # stores the free space value
- # depending on path format (unicode or not) and python version (2 or 3)
- # we need to call GetDiskFreeSpaceExW or GetDiskFreeSpaceExA
- called_function = ctypes.windll.kernel32.GetDiskFreeSpaceExA
- if isinstance(path, unicode) or sys.version_info >= (3,):
- called_function = ctypes.windll.kernel32.GetDiskFreeSpaceExW
- # we're ready for the dll call. On error it returns 0
- if called_function(path,
- ctypes.byref(dummy),
- ctypes.byref(total),
- ctypes.byref(free)) != 0:
- # success, we can use the values returned by the dll call
- disk_info.free = free.value
- disk_info.total = total.value
- disk_info.used = total.value - free.value
- return disk_info
-
- @staticmethod
- def get_size(path, unit, log_level=INFO):
- """Disk info stats:
- total => size of the disk
- used => space used
- free => free space
- In case of error raises a DiskutilError Exception
- """
- try:
- # let's try to get the disk size using os module
- disk_info = DiskSize()._posix_size(path)
- except AttributeError:
- try:
- # os module failed. let's try to get the size using
- # ctypes.windll...
- disk_info = DiskSize()._windows_size(path)
- except AttributeError:
- # No luck! This is not a posix nor window platform
- # raise an exception
- raise DiskutilsError('Unsupported platform')
-
- disk_info._to(unit)
- lvl = numeric_log_level(log_level)
- log.log(lvl, msg="%s" % disk_info)
- return disk_info
diff --git a/testing/mozharness/mozharness/base/errors.py b/testing/mozharness/mozharness/base/errors.py
deleted file mode 100755
index 9d2f3ebe1..000000000
--- a/testing/mozharness/mozharness/base/errors.py
+++ /dev/null
@@ -1,213 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Generic error lists.
-
-Error lists are used to parse output in mozharness.base.log.OutputParser.
-
-Each line of output is matched against each substring or regular expression
-in the error list. On a match, we determine the 'level' of that line,
-whether IGNORE, DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL.
-
-TODO: Context lines (requires work on the OutputParser side)
-
-TODO: We could also create classes that generate these, but with the
-appropriate level (please don't die on any errors; please die on any
-warning; etc.) or platform or language or whatever.
-"""
-
-import re
-
-from mozharness.base.log import DEBUG, WARNING, ERROR, CRITICAL, FATAL
-
-
-# Exceptions
-class VCSException(Exception):
- pass
-
-# ErrorLists {{{1
-BaseErrorList = [{
- 'substr': r'''command not found''',
- 'level': ERROR
-}]
-
-# For ssh, scp, rsync over ssh
-SSHErrorList = BaseErrorList + [{
- 'substr': r'''Name or service not known''',
- 'level': ERROR
-}, {
- 'substr': r'''Could not resolve hostname''',
- 'level': ERROR
-}, {
- 'substr': r'''POSSIBLE BREAK-IN ATTEMPT''',
- 'level': WARNING
-}, {
- 'substr': r'''Network error:''',
- 'level': ERROR
-}, {
- 'substr': r'''Access denied''',
- 'level': ERROR
-}, {
- 'substr': r'''Authentication refused''',
- 'level': ERROR
-}, {
- 'substr': r'''Out of memory''',
- 'level': ERROR
-}, {
- 'substr': r'''Connection reset by peer''',
- 'level': WARNING
-}, {
- 'substr': r'''Host key verification failed''',
- 'level': ERROR
-}, {
- 'substr': r'''WARNING:''',
- 'level': WARNING
-}, {
- 'substr': r'''rsync error:''',
- 'level': ERROR
-}, {
- 'substr': r'''Broken pipe:''',
- 'level': ERROR
-}, {
- 'substr': r'''Permission denied:''',
- 'level': ERROR
-}, {
- 'substr': r'''connection unexpectedly closed''',
- 'level': ERROR
-}, {
- 'substr': r'''Warning: Identity file''',
- 'level': ERROR
-}, {
- 'substr': r'''command-line line 0: Missing argument''',
- 'level': ERROR
-}]
-
-HgErrorList = BaseErrorList + [{
- 'regex': re.compile(r'''^abort:'''),
- 'level': ERROR,
- 'explanation': 'Automation Error: hg not responding'
-}, {
- 'substr': r'''unknown exception encountered''',
- 'level': ERROR,
- 'explanation': 'Automation Error: python exception in hg'
-}, {
- 'substr': r'''failed to import extension''',
- 'level': WARNING,
- 'explanation': 'Automation Error: hg extension missing'
-}]
-
-GitErrorList = BaseErrorList + [
- {'substr': r'''Permission denied (publickey).''', 'level': ERROR},
- {'substr': r'''fatal: The remote end hung up unexpectedly''', 'level': ERROR},
- {'substr': r'''does not appear to be a git repository''', 'level': ERROR},
- {'substr': r'''error: src refspec''', 'level': ERROR},
- {'substr': r'''invalid author/committer line -''', 'level': ERROR},
- {'substr': r'''remote: fatal: Error in object''', 'level': ERROR},
- {'substr': r'''fatal: sha1 file '<stdout>' write error: Broken pipe''', 'level': ERROR},
- {'substr': r'''error: failed to push some refs to ''', 'level': ERROR},
- {'substr': r'''remote: error: denying non-fast-forward ''', 'level': ERROR},
- {'substr': r'''! [remote rejected] ''', 'level': ERROR},
- {'regex': re.compile(r'''remote:.*No such file or directory'''), 'level': ERROR},
-]
-
-PythonErrorList = BaseErrorList + [
- {'regex': re.compile(r'''Warning:.*Error: '''), 'level': WARNING},
- {'substr': r'''Traceback (most recent call last)''', 'level': ERROR},
- {'substr': r'''SyntaxError: ''', 'level': ERROR},
- {'substr': r'''TypeError: ''', 'level': ERROR},
- {'substr': r'''NameError: ''', 'level': ERROR},
- {'substr': r'''ZeroDivisionError: ''', 'level': ERROR},
- {'regex': re.compile(r'''raise \w*Exception: '''), 'level': CRITICAL},
- {'regex': re.compile(r'''raise \w*Error: '''), 'level': CRITICAL},
-]
-
-VirtualenvErrorList = [
- {'substr': r'''not found or a compiler error:''', 'level': WARNING},
- {'regex': re.compile('''\d+: error: '''), 'level': ERROR},
- {'regex': re.compile('''\d+: warning: '''), 'level': WARNING},
- {'regex': re.compile(r'''Downloading .* \(.*\): *([0-9]+%)? *[0-9\.]+[kmKM]b'''), 'level': DEBUG},
-] + PythonErrorList
-
-
-# We may need to have various MakefileErrorLists for differing amounts of
-# warning-ignoring-ness.
-MakefileErrorList = BaseErrorList + PythonErrorList + [
- {'substr': r'''No rule to make target ''', 'level': ERROR},
- {'regex': re.compile(r'''akefile.*was not found\.'''), 'level': ERROR},
- {'regex': re.compile(r'''Stop\.$'''), 'level': ERROR},
- {'regex': re.compile(r''':\d+: error:'''), 'level': ERROR},
- {'regex': re.compile(r'''make\[\d+\]: \*\*\* \[.*\] Error \d+'''), 'level': ERROR},
- {'regex': re.compile(r''':\d+: warning:'''), 'level': WARNING},
- {'regex': re.compile(r'''make(?:\[\d+\])?: \*\*\*/'''), 'level': ERROR},
- {'substr': r'''Warning: ''', 'level': WARNING},
-]
-
-TarErrorList = BaseErrorList + [
- {'substr': r'''(stdin) is not a bzip2 file.''', 'level': ERROR},
- {'regex': re.compile(r'''Child returned status [1-9]'''), 'level': ERROR},
- {'substr': r'''Error exit delayed from previous errors''', 'level': ERROR},
- {'substr': r'''stdin: unexpected end of file''', 'level': ERROR},
- {'substr': r'''stdin: not in gzip format''', 'level': ERROR},
- {'substr': r'''Cannot exec: No such file or directory''', 'level': ERROR},
- {'substr': r''': Error is not recoverable: exiting now''', 'level': ERROR},
-]
-
-ADBErrorList = BaseErrorList + [
- {'substr': r'''INSTALL_FAILED_''', 'level': ERROR},
- {'substr': r'''Android Debug Bridge version''', 'level': ERROR},
- {'substr': r'''error: protocol fault''', 'level': ERROR},
- {'substr': r'''unable to connect to ''', 'level': ERROR},
-]
-
-JarsignerErrorList = [{
- 'substr': r'''command not found''',
- 'level': FATAL
-}, {
- 'substr': r'''jarsigner error: java.lang.RuntimeException: keystore load: Keystore was tampered with, or password was incorrect''',
- 'level': FATAL,
- 'explanation': r'''The store passphrase is probably incorrect!''',
-}, {
- 'regex': re.compile(r'''jarsigner: key associated with .* not a private key'''),
- 'level': FATAL,
- 'explanation': r'''The key passphrase is probably incorrect!''',
-}, {
- 'regex': re.compile(r'''jarsigner error: java.lang.RuntimeException: keystore load: .* .No such file or directory'''),
- 'level': FATAL,
- 'explanation': r'''The keystore doesn't exist!''',
-}, {
- 'substr': r'''jarsigner: unable to open jar file:''',
- 'level': FATAL,
- 'explanation': r'''The apk is missing!''',
-}]
-
-ZipErrorList = BaseErrorList + [{
- 'substr': r'''zip warning:''',
- 'level': WARNING,
-}, {
- 'substr': r'''zip error:''',
- 'level': ERROR,
-}, {
- 'substr': r'''Cannot open file: it does not appear to be a valid archive''',
- 'level': ERROR,
-}]
-
-ZipalignErrorList = BaseErrorList + [{
- 'regex': re.compile(r'''Unable to open .* as a zip archive'''),
- 'level': ERROR,
-}, {
- 'regex': re.compile(r'''Output file .* exists'''),
- 'level': ERROR,
-}, {
- 'substr': r'''Input and output can't be the same file''',
- 'level': ERROR,
-}]
-
-
-# __main__ {{{1
-if __name__ == '__main__':
- '''TODO: unit tests.
- '''
- pass
diff --git a/testing/mozharness/mozharness/base/log.py b/testing/mozharness/mozharness/base/log.py
deleted file mode 100755
index 2c18b50c3..000000000
--- a/testing/mozharness/mozharness/base/log.py
+++ /dev/null
@@ -1,694 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Generic logging classes and functionalities for single and multi file logging.
-Capturing console output and providing general logging functionalities.
-
-Attributes:
- FATAL_LEVEL (int): constant logging level value set based on the logging.CRITICAL
- value
- DEBUG (str): mozharness `debug` log name
- INFO (str): mozharness `info` log name
- WARNING (str): mozharness `warning` log name
- CRITICAL (str): mozharness `critical` log name
- FATAL (str): mozharness `fatal` log name
- IGNORE (str): mozharness `ignore` log name
- LOG_LEVELS (dict): mapping of the mozharness log level names to logging values
- ROOT_LOGGER (logging.Logger): instance of a logging.Logger class
-
-TODO:
-- network logging support.
-- log rotation config
-"""
-
-from datetime import datetime
-import logging
-import os
-import sys
-import traceback
-
-# Define our own FATAL_LEVEL
-FATAL_LEVEL = logging.CRITICAL + 10
-logging.addLevelName(FATAL_LEVEL, 'FATAL')
-
-# mozharness log levels.
-DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL, IGNORE = (
- 'debug', 'info', 'warning', 'error', 'critical', 'fatal', 'ignore')
-
-
-LOG_LEVELS = {
- DEBUG: logging.DEBUG,
- INFO: logging.INFO,
- WARNING: logging.WARNING,
- ERROR: logging.ERROR,
- CRITICAL: logging.CRITICAL,
- FATAL: FATAL_LEVEL
-}
-
-# mozharness root logger
-ROOT_LOGGER = logging.getLogger()
-
-
-# LogMixin {{{1
-class LogMixin(object):
- """This is a mixin for any object to access similar logging functionality
-
- The logging functionality described here is specially useful for those
- objects with self.config and self.log_obj member variables
- """
-
- def _log_level_at_least(self, level):
- """ Check if the current logging level is greater or equal than level
-
- Args:
- level (str): log level name to compare against mozharness log levels
- names
-
- Returns:
- bool: True if the current logging level is great or equal than level,
- False otherwise
- """
- log_level = INFO
- levels = [DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL]
- if hasattr(self, 'config'):
- log_level = self.config.get('log_level', INFO)
- return levels.index(level) >= levels.index(log_level)
-
- def _print(self, message, stderr=False):
- """ prints a message to the sys.stdout or sys.stderr according to the
- value of the stderr argument.
-
- Args:
- message (str): The message to be printed
- stderr (bool, optional): if True, message will be printed to
- sys.stderr. Defaults to False.
-
- Returns:
- None
- """
- if not hasattr(self, 'config') or self.config.get('log_to_console', True):
- if stderr:
- print >> sys.stderr, message
- else:
- print message
-
- def log(self, message, level=INFO, exit_code=-1):
- """ log the message passed to it according to level, exit if level == FATAL
-
- Args:
- message (str): message to be logged
- level (str, optional): logging level of the message. Defaults to INFO
- exit_code (int, optional): exit code to log before the scripts calls
- SystemExit.
-
- Returns:
- None
- """
- if self.log_obj:
- return self.log_obj.log_message(
- message, level=level,
- exit_code=exit_code,
- post_fatal_callback=self._post_fatal,
- )
- if level == INFO:
- if self._log_level_at_least(level):
- self._print(message)
- elif level == DEBUG:
- if self._log_level_at_least(level):
- self._print('DEBUG: %s' % message)
- elif level in (WARNING, ERROR, CRITICAL):
- if self._log_level_at_least(level):
- self._print("%s: %s" % (level.upper(), message), stderr=True)
- elif level == FATAL:
- if self._log_level_at_least(level):
- self._print("FATAL: %s" % message, stderr=True)
- raise SystemExit(exit_code)
-
- def worst_level(self, target_level, existing_level, levels=None):
- """Compare target_level with existing_level according to levels values
- and return the worst among them.
-
- Args:
- target_level (str): minimum logging level to which the current object
- should be set
- existing_level (str): current logging level
- levels (list(str), optional): list of logging levels names to compare
- target_level and existing_level against.
- Defaults to mozharness log level
- list sorted from most to less critical.
-
- Returns:
- str: the logging lavel that is closest to the first levels value,
- i.e. levels[0]
- """
- if not levels:
- levels = [FATAL, CRITICAL, ERROR, WARNING, INFO, DEBUG, IGNORE]
- if target_level not in levels:
- self.fatal("'%s' not in %s'." % (target_level, levels))
- for l in levels:
- if l in (target_level, existing_level):
- return l
-
- # Copying Bear's dumpException():
- # https://hg.mozilla.org/build/tools/annotate/1485f23c38e0/sut_tools/sut_lib.py#l23
- def exception(self, message=None, level=ERROR):
- """ log an exception message base on the log level passed to it.
-
- This function fetches the information of the current exception being handled and
- adds it to the message argument.
-
- Args:
- message (str, optional): message to be printed at the beginning of the log.
- Default to an empty string.
- level (str, optional): log level to use for the logging. Defaults to ERROR
-
- Returns:
- None
- """
- tb_type, tb_value, tb_traceback = sys.exc_info()
- if message is None:
- message = ""
- else:
- message = "%s\n" % message
- for s in traceback.format_exception(tb_type, tb_value, tb_traceback):
- message += "%s\n" % s
- # Log at the end, as a fatal will attempt to exit after the 1st line.
- self.log(message, level=level)
-
- def debug(self, message):
- """ calls the log method with DEBUG as logging level
-
- Args:
- message (str): message to log
- """
- self.log(message, level=DEBUG)
-
- def info(self, message):
- """ calls the log method with INFO as logging level
-
- Args:
- message (str): message to log
- """
- self.log(message, level=INFO)
-
- def warning(self, message):
- """ calls the log method with WARNING as logging level
-
- Args:
- message (str): message to log
- """
- self.log(message, level=WARNING)
-
- def error(self, message):
- """ calls the log method with ERROR as logging level
-
- Args:
- message (str): message to log
- """
- self.log(message, level=ERROR)
-
- def critical(self, message):
- """ calls the log method with CRITICAL as logging level
-
- Args:
- message (str): message to log
- """
- self.log(message, level=CRITICAL)
-
- def fatal(self, message, exit_code=-1):
- """ calls the log method with FATAL as logging level
-
- Args:
- message (str): message to log
- exit_code (int, optional): exit code to use for the SystemExit
- exception to be raised. Default to -1.
- """
- self.log(message, level=FATAL, exit_code=exit_code)
-
- def _post_fatal(self, message=None, exit_code=None):
- """ Sometimes you want to create a report or cleanup
- or notify on fatal(); override this method to do so.
-
- Please don't use this for anything significantly long-running.
-
- Args:
- message (str, optional): message to report. Default to None
- exit_code (int, optional): exit code to use for the SystemExit
- exception to be raised. Default to None
- """
- pass
-
-
-# OutputParser {{{1
-class OutputParser(LogMixin):
- """ Helper object to parse command output.
-
- This will buffer output if needed, so we can go back and mark
- [(linenum - 10) : linenum+10] as errors if need be, without having to
- get all the output first.
-
- linenum+10 will be easy; we can set self.num_post_context_lines to 10,
- and self.num_post_context_lines-- as we mark each line to at least error
- level X.
-
- linenum-10 will be trickier. We'll not only need to save the line
- itself, but also the level that we've set for that line previously,
- whether by matching on that line, or by a previous line's context.
- We should only log that line if all output has ended (self.finish() ?);
- otherwise store a list of dictionaries in self.context_buffer that is
- buffered up to self.num_pre_context_lines (set to the largest
- pre-context-line setting in error_list.)
- """
-
- def __init__(self, config=None, log_obj=None, error_list=None, log_output=True, **kwargs):
- """Initialization method for the OutputParser class
-
- Args:
- config (dict, optional): dictionary containing values such as `log_level`
- or `log_to_console`. Defaults to `None`.
- log_obj (BaseLogger, optional): instance of the BaseLogger class. Defaults
- to `None`.
- error_list (list, optional): list of the error to look for. Defaults to
- `None`.
- log_output (boolean, optional): flag for deciding if the commands
- output should be logged or not.
- Defaults to `True`.
- """
- self.config = config
- self.log_obj = log_obj
- self.error_list = error_list or []
- self.log_output = log_output
- self.num_errors = 0
- self.num_warnings = 0
- # TODO context_lines.
- # Not in use yet, but will be based off error_list.
- self.context_buffer = []
- self.num_pre_context_lines = 0
- self.num_post_context_lines = 0
- self.worst_log_level = INFO
-
- def parse_single_line(self, line):
- """ parse a console output line and check if it matches one in `error_list`,
- if so then log it according to `log_output`.
-
- Args:
- line (str): command line output to parse.
- """
- for error_check in self.error_list:
- # TODO buffer for context_lines.
- match = False
- if 'substr' in error_check:
- if error_check['substr'] in line:
- match = True
- elif 'regex' in error_check:
- if error_check['regex'].search(line):
- match = True
- else:
- self.warning("error_list: 'substr' and 'regex' not in %s" %
- error_check)
- if match:
- log_level = error_check.get('level', INFO)
- if self.log_output:
- message = ' %s' % line
- if error_check.get('explanation'):
- message += '\n %s' % error_check['explanation']
- if error_check.get('summary'):
- self.add_summary(message, level=log_level)
- else:
- self.log(message, level=log_level)
- if log_level in (ERROR, CRITICAL, FATAL):
- self.num_errors += 1
- if log_level == WARNING:
- self.num_warnings += 1
- self.worst_log_level = self.worst_level(log_level,
- self.worst_log_level)
- break
- else:
- if self.log_output:
- self.info(' %s' % line)
-
- def add_lines(self, output):
- """ process a string or list of strings, decode them to utf-8,strip
- them of any trailing whitespaces and parse them using `parse_single_line`
-
- strings consisting only of whitespaces are ignored.
-
- Args:
- output (str | list): string or list of string to parse
- """
-
- if isinstance(output, basestring):
- output = [output]
- for line in output:
- if not line or line.isspace():
- continue
- line = line.decode("utf-8", 'replace').rstrip()
- self.parse_single_line(line)
-
-
-# BaseLogger {{{1
-class BaseLogger(object):
- """ Base class in charge of logging handling logic such as creating logging
- files, dirs, attaching to the console output and managing its output.
-
- Attributes:
- LEVELS (dict): flat copy of the `LOG_LEVELS` attribute of the `log` module.
-
- TODO: status? There may be a status object or status capability in
- either logging or config that allows you to count the number of
- error,critical,fatal messages for us to count up at the end (aiming
- for 0).
- """
- LEVELS = LOG_LEVELS
-
- def __init__(
- self, log_level=INFO,
- log_format='%(message)s',
- log_date_format='%H:%M:%S',
- log_name='test',
- log_to_console=True,
- log_dir='.',
- log_to_raw=False,
- logger_name='',
- append_to_log=False,
- ):
- """ BaseLogger constructor
-
- Args:
- log_level (str, optional): mozharness log level name. Defaults to INFO.
- log_format (str, optional): message format string to instantiate a
- `logging.Formatter`. Defaults to '%(message)s'
- log_date_format (str, optional): date format string to instantiate a
- `logging.Formatter`. Defaults to '%H:%M:%S'
- log_name (str, optional): name to use for the log files to be created.
- Defaults to 'test'
- log_to_console (bool, optional): set to True in order to create a Handler
- instance base on the `Logger`
- current instance. Defaults to True.
- log_dir (str, optional): directory location to store the log files.
- Defaults to '.', i.e. current working directory.
- log_to_raw (bool, optional): set to True in order to create a *raw.log
- file. Defaults to False.
- logger_name (str, optional): currently useless parameter. According
- to the code comments, it could be useful
- if we were to have multiple logging
- objects that don't trample each other.
- append_to_log (bool, optional): set to True if the logging content should
- be appended to old logging files. Defaults to False
- """
-
- self.log_format = log_format
- self.log_date_format = log_date_format
- self.log_to_console = log_to_console
- self.log_to_raw = log_to_raw
- self.log_level = log_level
- self.log_name = log_name
- self.log_dir = log_dir
- self.append_to_log = append_to_log
-
- # Not sure what I'm going to use this for; useless unless we
- # can have multiple logging objects that don't trample each other
- self.logger_name = logger_name
-
- self.all_handlers = []
- self.log_files = {}
-
- self.create_log_dir()
-
- def create_log_dir(self):
- """ create a logging directory if it doesn't exits. If there is a file with
- same name as the future logging directory it will be deleted.
- """
-
- if os.path.exists(self.log_dir):
- if not os.path.isdir(self.log_dir):
- os.remove(self.log_dir)
- if not os.path.exists(self.log_dir):
- os.makedirs(self.log_dir)
- self.abs_log_dir = os.path.abspath(self.log_dir)
-
- def init_message(self, name=None):
- """ log an init message stating the name passed to it, the current date
- and time and, the current working directory.
-
- Args:
- name (str, optional): name to use for the init log message. Defaults to
- the current instance class name.
- """
-
- if not name:
- name = self.__class__.__name__
- self.log_message("%s online at %s in %s" %
- (name, datetime.now().strftime("%Y%m%d %H:%M:%S"),
- os.getcwd()))
-
- def get_logger_level(self, level=None):
- """ translate the level name passed to it and return its numeric value
- according to `LEVELS` values.
-
- Args:
- level (str, optional): level name to be translated. Defaults to the current
- instance `log_level`.
-
- Returns:
- int: numeric value of the log level name passed to it or 0 (NOTSET) if the
- name doesn't exists
- """
-
- if not level:
- level = self.log_level
- return self.LEVELS.get(level, logging.NOTSET)
-
- def get_log_formatter(self, log_format=None, date_format=None):
- """ create a `logging.Formatter` base on the log and date format.
-
- Args:
- log_format (str, optional): log format to use for the Formatter constructor.
- Defaults to the current instance log format.
- date_format (str, optional): date format to use for the Formatter constructor.
- Defaults to the current instance date format.
-
- Returns:
- logging.Formatter: instance created base on the passed arguments
- """
-
- if not log_format:
- log_format = self.log_format
- if not date_format:
- date_format = self.log_date_format
- return logging.Formatter(log_format, date_format)
-
- def new_logger(self):
- """ Create a new logger based on the ROOT_LOGGER instance. By default there are no handlers.
- The new logger becomes a member variable of the current instance as `self.logger`.
- """
-
- self.logger = ROOT_LOGGER
- self.logger.setLevel(self.get_logger_level())
- self._clear_handlers()
- if self.log_to_console:
- self.add_console_handler()
- if self.log_to_raw:
- self.log_files['raw'] = '%s_raw.log' % self.log_name
- self.add_file_handler(os.path.join(self.abs_log_dir,
- self.log_files['raw']),
- log_format='%(message)s')
-
- def _clear_handlers(self):
- """ remove all handlers stored in `self.all_handlers`.
-
- To prevent dups -- logging will preserve Handlers across
- objects :(
- """
- attrs = dir(self)
- if 'all_handlers' in attrs and 'logger' in attrs:
- for handler in self.all_handlers:
- self.logger.removeHandler(handler)
- self.all_handlers = []
-
- def __del__(self):
- """ BaseLogger class destructor; shutdown, flush and remove all handlers"""
- logging.shutdown()
- self._clear_handlers()
-
- def add_console_handler(self, log_level=None, log_format=None,
- date_format=None):
- """ create a `logging.StreamHandler` using `sys.stderr` for logging the console
- output and add it to the `all_handlers` member variable
-
- Args:
- log_level (str, optional): useless argument. Not used here.
- Defaults to None.
- log_format (str, optional): format used for the Formatter attached to the
- StreamHandler. Defaults to None.
- date_format (str, optional): format used for the Formatter attached to the
- StreamHandler. Defaults to None.
- """
-
- console_handler = logging.StreamHandler()
- console_handler.setFormatter(self.get_log_formatter(log_format=log_format,
- date_format=date_format))
- self.logger.addHandler(console_handler)
- self.all_handlers.append(console_handler)
-
- def add_file_handler(self, log_path, log_level=None, log_format=None,
- date_format=None):
- """ create a `logging.FileHandler` base on the path, log and date format
- and add it to the `all_handlers` member variable.
-
- Args:
- log_path (str): filepath to use for the `FileHandler`.
- log_level (str, optional): useless argument. Not used here.
- Defaults to None.
- log_format (str, optional): log format to use for the Formatter constructor.
- Defaults to the current instance log format.
- date_format (str, optional): date format to use for the Formatter constructor.
- Defaults to the current instance date format.
- """
-
- if not self.append_to_log and os.path.exists(log_path):
- os.remove(log_path)
- file_handler = logging.FileHandler(log_path)
- file_handler.setLevel(self.get_logger_level(log_level))
- file_handler.setFormatter(self.get_log_formatter(log_format=log_format,
- date_format=date_format))
- self.logger.addHandler(file_handler)
- self.all_handlers.append(file_handler)
-
- def log_message(self, message, level=INFO, exit_code=-1, post_fatal_callback=None):
- """ Generic log method.
- There should be more options here -- do or don't split by line,
- use os.linesep instead of assuming \n, be able to pass in log level
- by name or number.
-
- Adding the IGNORE special level for runCommand.
-
- Args:
- message (str): message to log using the current `logger`
- level (str, optional): log level of the message. Defaults to INFO.
- exit_code (int, optional): exit code to use in case of a FATAL level is used.
- Defaults to -1.
- post_fatal_callback (function, optional): function to callback in case of
- of a fatal log level. Defaults None.
- """
-
- if level == IGNORE:
- return
- for line in message.splitlines():
- self.logger.log(self.get_logger_level(level), line)
- if level == FATAL:
- if callable(post_fatal_callback):
- self.logger.log(FATAL_LEVEL, "Running post_fatal callback...")
- post_fatal_callback(message=message, exit_code=exit_code)
- self.logger.log(FATAL_LEVEL, 'Exiting %d' % exit_code)
- raise SystemExit(exit_code)
-
-
-# SimpleFileLogger {{{1
-class SimpleFileLogger(BaseLogger):
- """ Subclass of the BaseLogger.
-
- Create one logFile. Possibly also output to the terminal and a raw log
- (no prepending of level or date)
- """
-
- def __init__(self,
- log_format='%(asctime)s %(levelname)8s - %(message)s',
- logger_name='Simple', log_dir='logs', **kwargs):
- """ SimpleFileLogger constructor. Calls its superclass constructor,
- creates a new logger instance and log an init message.
-
- Args:
- log_format (str, optional): message format string to instantiate a
- `logging.Formatter`. Defaults to
- '%(asctime)s %(levelname)8s - %(message)s'
- log_name (str, optional): name to use for the log files to be created.
- Defaults to 'Simple'
- log_dir (str, optional): directory location to store the log files.
- Defaults to 'logs'
- **kwargs: Arbitrary keyword arguments passed to the BaseLogger constructor
- """
-
- BaseLogger.__init__(self, logger_name=logger_name, log_format=log_format,
- log_dir=log_dir, **kwargs)
- self.new_logger()
- self.init_message()
-
- def new_logger(self):
- """ calls the BaseLogger.new_logger method and adds a file handler to it."""
-
- BaseLogger.new_logger(self)
- self.log_path = os.path.join(self.abs_log_dir, '%s.log' % self.log_name)
- self.log_files['default'] = self.log_path
- self.add_file_handler(self.log_path)
-
-
-# MultiFileLogger {{{1
-class MultiFileLogger(BaseLogger):
- """Subclass of the BaseLogger class. Create a log per log level in log_dir.
- Possibly also output to the terminal and a raw log (no prepending of level or date)
- """
-
- def __init__(self, logger_name='Multi',
- log_format='%(asctime)s %(levelname)8s - %(message)s',
- log_dir='logs', log_to_raw=True, **kwargs):
- """ MultiFileLogger constructor. Calls its superclass constructor,
- creates a new logger instance and log an init message.
-
- Args:
- log_format (str, optional): message format string to instantiate a
- `logging.Formatter`. Defaults to
- '%(asctime)s %(levelname)8s - %(message)s'
- log_name (str, optional): name to use for the log files to be created.
- Defaults to 'Multi'
- log_dir (str, optional): directory location to store the log files.
- Defaults to 'logs'
- log_to_raw (bool, optional): set to True in order to create a *raw.log
- file. Defaults to False.
- **kwargs: Arbitrary keyword arguments passed to the BaseLogger constructor
- """
-
- BaseLogger.__init__(self, logger_name=logger_name,
- log_format=log_format,
- log_to_raw=log_to_raw, log_dir=log_dir,
- **kwargs)
-
- self.new_logger()
- self.init_message()
-
- def new_logger(self):
- """ calls the BaseLogger.new_logger method and adds a file handler per
- logging level in the `LEVELS` class attribute.
- """
-
- BaseLogger.new_logger(self)
- min_logger_level = self.get_logger_level(self.log_level)
- for level in self.LEVELS.keys():
- if self.get_logger_level(level) >= min_logger_level:
- self.log_files[level] = '%s_%s.log' % (self.log_name,
- level)
- self.add_file_handler(os.path.join(self.abs_log_dir,
- self.log_files[level]),
- log_level=level)
-
-
-def numeric_log_level(level):
- """Converts a mozharness log level (string) to the corresponding logger
- level (number). This function makes possible to set the log level
- in functions that do not inherit from LogMixin
-
- Args:
- level (str): log level name to convert.
-
- Returns:
- int: numeric value of the log level name.
- """
- return LOG_LEVELS[level]
-
-# __main__ {{{1
-if __name__ == '__main__':
- """ Useless comparison, due to the `pass` keyword on its body"""
- pass
diff --git a/testing/mozharness/mozharness/base/parallel.py b/testing/mozharness/mozharness/base/parallel.py
deleted file mode 100755
index b20b9c97c..000000000
--- a/testing/mozharness/mozharness/base/parallel.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Generic ways to parallelize jobs.
-"""
-
-
-# ChunkingMixin {{{1
-class ChunkingMixin(object):
- """Generic signing helper methods.
- """
- def query_chunked_list(self, possible_list, this_chunk, total_chunks,
- sort=False):
- """Split a list of items into a certain number of chunks and
- return the subset of that will occur in this chunk.
-
- Ported from build.l10n.getLocalesForChunk in build/tools.
- """
- if sort:
- possible_list = sorted(possible_list)
- else:
- # Copy to prevent altering
- possible_list = possible_list[:]
- length = len(possible_list)
- for c in range(1, total_chunks + 1):
- n = length / total_chunks
- # If the total number of items isn't evenly divisible by the
- # number of chunks, we need to append one more onto some chunks
- if c <= (length % total_chunks):
- n += 1
- if c == this_chunk:
- return possible_list[0:n]
- del possible_list[0:n]
diff --git a/testing/mozharness/mozharness/base/python.py b/testing/mozharness/mozharness/base/python.py
deleted file mode 100644
index cb5bfbc46..000000000
--- a/testing/mozharness/mozharness/base/python.py
+++ /dev/null
@@ -1,743 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-'''Python usage, esp. virtualenv.
-'''
-
-import errno
-import os
-import subprocess
-import sys
-import json
-import socket
-import traceback
-import urlparse
-
-import mozharness
-from mozharness.base.script import (
- PostScriptAction,
- PostScriptRun,
- PreScriptAction,
- ScriptMixin,
-)
-from mozharness.base.errors import VirtualenvErrorList
-from mozharness.base.log import WARNING, FATAL
-from mozharness.mozilla.proxxy import Proxxy
-
-external_tools_path = os.path.join(
- os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
- 'external_tools',
-)
-
-def get_tlsv1_post():
- # Monkeypatch to work around SSL errors in non-bleeding-edge Python.
- # Taken from https://lukasa.co.uk/2013/01/Choosing_SSL_Version_In_Requests/
- import requests
- from requests.packages.urllib3.poolmanager import PoolManager
- import ssl
-
- class TLSV1Adapter(requests.adapters.HTTPAdapter):
- def init_poolmanager(self, connections, maxsize, block=False):
- self.poolmanager = PoolManager(num_pools=connections,
- maxsize=maxsize,
- block=block,
- ssl_version=ssl.PROTOCOL_TLSv1)
- s = requests.Session()
- s.mount('https://', TLSV1Adapter())
- return s.post
-
-# Virtualenv {{{1
-virtualenv_config_options = [
- [["--virtualenv-path"], {
- "action": "store",
- "dest": "virtualenv_path",
- "default": "venv",
- "help": "Specify the path to the virtualenv top level directory"
- }],
- [["--find-links"], {
- "action": "extend",
- "dest": "find_links",
- "help": "URL to look for packages at"
- }],
- [["--pip-index"], {
- "action": "store_true",
- "default": True,
- "dest": "pip_index",
- "help": "Use pip indexes (default)"
- }],
- [["--no-pip-index"], {
- "action": "store_false",
- "dest": "pip_index",
- "help": "Don't use pip indexes"
- }],
-]
-
-
-class VirtualenvMixin(object):
- '''BaseScript mixin, designed to create and use virtualenvs.
-
- Config items:
- * virtualenv_path points to the virtualenv location on disk.
- * virtualenv_modules lists the module names.
- * MODULE_url list points to the module URLs (optional)
- Requires virtualenv to be in PATH.
- Depends on ScriptMixin
- '''
- python_paths = {}
- site_packages_path = None
-
- def __init__(self, *args, **kwargs):
- self._virtualenv_modules = []
- super(VirtualenvMixin, self).__init__(*args, **kwargs)
-
- def register_virtualenv_module(self, name=None, url=None, method=None,
- requirements=None, optional=False,
- two_pass=False, editable=False):
- """Register a module to be installed with the virtualenv.
-
- This method can be called up until create_virtualenv() to register
- modules that should be installed in the virtualenv.
-
- See the documentation for install_module for how the arguments are
- applied.
- """
- self._virtualenv_modules.append((name, url, method, requirements,
- optional, two_pass, editable))
-
- def query_virtualenv_path(self):
- """Determine the absolute path to the virtualenv."""
- dirs = self.query_abs_dirs()
-
- if 'abs_virtualenv_dir' in dirs:
- return dirs['abs_virtualenv_dir']
-
- p = self.config['virtualenv_path']
- if not p:
- self.fatal('virtualenv_path config option not set; '
- 'this should never happen')
-
- if os.path.isabs(p):
- return p
- else:
- return os.path.join(dirs['abs_work_dir'], p)
-
- def query_python_path(self, binary="python"):
- """Return the path of a binary inside the virtualenv, if
- c['virtualenv_path'] is set; otherwise return the binary name.
- Otherwise return None
- """
- if binary not in self.python_paths:
- bin_dir = 'bin'
- if self._is_windows():
- bin_dir = 'Scripts'
- virtualenv_path = self.query_virtualenv_path()
- self.python_paths[binary] = os.path.abspath(os.path.join(virtualenv_path, bin_dir, binary))
-
- return self.python_paths[binary]
-
- def query_python_site_packages_path(self):
- if self.site_packages_path:
- return self.site_packages_path
- python = self.query_python_path()
- self.site_packages_path = self.get_output_from_command(
- [python, '-c',
- 'from distutils.sysconfig import get_python_lib; ' +
- 'print(get_python_lib())'])
- return self.site_packages_path
-
- def package_versions(self, pip_freeze_output=None, error_level=WARNING, log_output=False):
- """
- reads packages from `pip freeze` output and returns a dict of
- {package_name: 'version'}
- """
- packages = {}
-
- if pip_freeze_output is None:
- # get the output from `pip freeze`
- pip = self.query_python_path("pip")
- if not pip:
- self.log("package_versions: Program pip not in path", level=error_level)
- return {}
- pip_freeze_output = self.get_output_from_command([pip, "freeze"], silent=True, ignore_errors=True)
- if not isinstance(pip_freeze_output, basestring):
- self.fatal("package_versions: Error encountered running `pip freeze`: %s" % pip_freeze_output)
-
- for line in pip_freeze_output.splitlines():
- # parse the output into package, version
- line = line.strip()
- if not line:
- # whitespace
- continue
- if line.startswith('-'):
- # not a package, probably like '-e http://example.com/path#egg=package-dev'
- continue
- if '==' not in line:
- self.fatal("pip_freeze_packages: Unrecognized output line: %s" % line)
- package, version = line.split('==', 1)
- packages[package] = version
-
- if log_output:
- self.info("Current package versions:")
- for package in sorted(packages):
- self.info(" %s == %s" % (package, packages[package]))
-
- return packages
-
- def is_python_package_installed(self, package_name, error_level=WARNING):
- """
- Return whether the package is installed
- """
- packages = self.package_versions(error_level=error_level).keys()
- return package_name.lower() in [package.lower() for package in packages]
-
- def install_module(self, module=None, module_url=None, install_method=None,
- requirements=(), optional=False, global_options=[],
- no_deps=False, editable=False):
- """
- Install module via pip.
-
- module_url can be a url to a python package tarball, a path to
- a directory containing a setup.py (absolute or relative to work_dir)
- or None, in which case it will default to the module name.
-
- requirements is a list of pip requirements files. If specified, these
- will be combined with the module_url (if any), like so:
-
- pip install -r requirements1.txt -r requirements2.txt module_url
- """
- c = self.config
- dirs = self.query_abs_dirs()
- env = self.query_env()
- venv_path = self.query_virtualenv_path()
- self.info("Installing %s into virtualenv %s" % (module, venv_path))
- if not module_url:
- module_url = module
- if install_method in (None, 'pip'):
- if not module_url and not requirements:
- self.fatal("Must specify module and/or requirements")
- pip = self.query_python_path("pip")
- if c.get("verbose_pip"):
- command = [pip, "-v", "install"]
- else:
- command = [pip, "install"]
- if no_deps:
- command += ["--no-deps"]
- # To avoid timeouts with our pypi server, increase default timeout:
- # https://bugzilla.mozilla.org/show_bug.cgi?id=1007230#c802
- command += ['--timeout', str(c.get('pip_timeout', 120))]
- for requirement in requirements:
- command += ["-r", requirement]
- if c.get('find_links') and not c["pip_index"]:
- command += ['--no-index']
- for opt in global_options:
- command += ["--global-option", opt]
- elif install_method == 'easy_install':
- if not module:
- self.fatal("module parameter required with install_method='easy_install'")
- if requirements:
- # Install pip requirements files separately, since they're
- # not understood by easy_install.
- self.install_module(requirements=requirements,
- install_method='pip')
- # Allow easy_install to be overridden by
- # self.config['exes']['easy_install']
- default = 'easy_install'
- command = self.query_exe('easy_install', default=default, return_type="list")
- else:
- self.fatal("install_module() doesn't understand an install_method of %s!" % install_method)
-
- # Add --find-links pages to look at. Add --trusted-host automatically if
- # the host isn't secure. This allows modern versions of pip to connect
- # without requiring an override.
- proxxy = Proxxy(self.config, self.log_obj)
- trusted_hosts = set()
- for link in proxxy.get_proxies_and_urls(c.get('find_links', [])):
- parsed = urlparse.urlparse(link)
-
- try:
- socket.gethostbyname(parsed.hostname)
- except socket.gaierror as e:
- self.info('error resolving %s (ignoring): %s' %
- (parsed.hostname, e.message))
- continue
-
- command.extend(["--find-links", link])
- if parsed.scheme != 'https':
- trusted_hosts.add(parsed.hostname)
-
- if install_method != 'easy_install':
- for host in sorted(trusted_hosts):
- command.extend(['--trusted-host', host])
-
- # module_url can be None if only specifying requirements files
- if module_url:
- if editable:
- if install_method in (None, 'pip'):
- command += ['-e']
- else:
- self.fatal("editable installs not supported for install_method %s" % install_method)
- command += [module_url]
-
- # If we're only installing a single requirements file, use
- # the file's directory as cwd, so relative paths work correctly.
- cwd = dirs['abs_work_dir']
- if not module and len(requirements) == 1:
- cwd = os.path.dirname(requirements[0])
-
- quoted_command = subprocess.list2cmdline(command)
- # Allow for errors while building modules, but require a
- # return status of 0.
- self.retry(
- self.run_command,
- # None will cause default value to be used
- attempts=1 if optional else None,
- good_statuses=(0,),
- error_level=WARNING if optional else FATAL,
- error_message='Could not install python package: ' + quoted_command + ' failed after %(attempts)d tries!',
- args=[command, ],
- kwargs={
- 'error_list': VirtualenvErrorList,
- 'cwd': cwd,
- 'env': env,
- # WARNING only since retry will raise final FATAL if all
- # retry attempts are unsuccessful - and we only want
- # an ERROR of FATAL if *no* retry attempt works
- 'error_level': WARNING,
- }
- )
-
- def create_virtualenv(self, modules=(), requirements=()):
- """
- Create a python virtualenv.
-
- The virtualenv exe can be defined in c['virtualenv'] or
- c['exes']['virtualenv'], as a string (path) or list (path +
- arguments).
-
- c['virtualenv_python_dll'] is an optional config item that works
- around an old windows virtualenv bug.
-
- virtualenv_modules can be a list of module names to install, e.g.
-
- virtualenv_modules = ['module1', 'module2']
-
- or it can be a heterogeneous list of modules names and dicts that
- define a module by its name, url-or-path, and a list of its global
- options.
-
- virtualenv_modules = [
- {
- 'name': 'module1',
- 'url': None,
- 'global_options': ['--opt', '--without-gcc']
- },
- {
- 'name': 'module2',
- 'url': 'http://url/to/package',
- 'global_options': ['--use-clang']
- },
- {
- 'name': 'module3',
- 'url': os.path.join('path', 'to', 'setup_py', 'dir')
- 'global_options': []
- },
- 'module4'
- ]
-
- virtualenv_requirements is an optional list of pip requirements files to
- use when invoking pip, e.g.,
-
- virtualenv_requirements = [
- '/path/to/requirements1.txt',
- '/path/to/requirements2.txt'
- ]
- """
- c = self.config
- dirs = self.query_abs_dirs()
- venv_path = self.query_virtualenv_path()
- self.info("Creating virtualenv %s" % venv_path)
-
- # Always use the virtualenv that is vendored since that is deterministic.
- # TODO Bug 1408051 - Use the copy of virtualenv under
- # third_party/python/virtualenv once everything is off buildbot
- virtualenv = [
- sys.executable,
- os.path.join(external_tools_path, 'virtualenv', 'virtualenv.py'),
- ]
- virtualenv_options = c.get('virtualenv_options', [])
- # Don't create symlinks. If we don't do this, permissions issues may
- # hinder virtualenv creation or operation.
- virtualenv_options.append('--always-copy')
-
- if os.path.exists(self.query_python_path()):
- self.info("Virtualenv %s appears to already exist; skipping virtualenv creation." % self.query_python_path())
- else:
- self.mkdir_p(dirs['abs_work_dir'])
- self.run_command(virtualenv + virtualenv_options + [venv_path],
- cwd=dirs['abs_work_dir'],
- error_list=VirtualenvErrorList,
- partial_env={'VIRTUALENV_NO_DOWNLOAD': "1"},
- halt_on_failure=True)
-
- if not modules:
- modules = c.get('virtualenv_modules', [])
- if not requirements:
- requirements = c.get('virtualenv_requirements', [])
- if not modules and requirements:
- self.install_module(requirements=requirements,
- install_method='pip')
- for module in modules:
- module_url = module
- global_options = []
- if isinstance(module, dict):
- if module.get('name', None):
- module_name = module['name']
- else:
- self.fatal("Can't install module without module name: %s" %
- str(module))
- module_url = module.get('url', None)
- global_options = module.get('global_options', [])
- else:
- module_url = self.config.get('%s_url' % module, module_url)
- module_name = module
- install_method = 'pip'
- if module_name in ('pywin32',):
- install_method = 'easy_install'
- self.install_module(module=module_name,
- module_url=module_url,
- install_method=install_method,
- requirements=requirements,
- global_options=global_options)
-
- for module, url, method, requirements, optional, two_pass, editable in \
- self._virtualenv_modules:
- if two_pass:
- self.install_module(
- module=module, module_url=url,
- install_method=method, requirements=requirements or (),
- optional=optional, no_deps=True, editable=editable
- )
- self.install_module(
- module=module, module_url=url,
- install_method=method, requirements=requirements or (),
- optional=optional, editable=editable
- )
-
- self.info("Done creating virtualenv %s." % venv_path)
-
- self.package_versions(log_output=True)
-
- def activate_virtualenv(self):
- """Import the virtualenv's packages into this Python interpreter."""
- bin_dir = os.path.dirname(self.query_python_path())
- activate = os.path.join(bin_dir, 'activate_this.py')
- execfile(activate, dict(__file__=activate))
-
-
-# This is (sadly) a mixin for logging methods.
-class PerfherderResourceOptionsMixin(ScriptMixin):
- def perfherder_resource_options(self):
- """Obtain a list of extraOptions values to identify the env."""
- opts = []
-
- if 'TASKCLUSTER_INSTANCE_TYPE' in os.environ:
- # Include the instance type so results can be grouped.
- opts.append('taskcluster-%s' % os.environ['TASKCLUSTER_INSTANCE_TYPE'])
- else:
- # We assume !taskcluster => buildbot.
- instance = 'unknown'
-
- # Try to load EC2 instance type from metadata file. This file
- # may not exist in many scenarios (including when inside a chroot).
- # So treat it as optional.
- # TODO support Windows.
- try:
- # This file should exist on Linux in EC2.
- with open('/etc/instance_metadata.json', 'rb') as fh:
- im = json.load(fh)
- instance = im['aws_instance_type'].encode('ascii')
- except IOError as e:
- if e.errno != errno.ENOENT:
- raise
- self.info('instance_metadata.json not found; unable to '
- 'determine instance type')
- except Exception:
- self.warning('error reading instance_metadata: %s' %
- traceback.format_exc())
-
- opts.append('buildbot-%s' % instance)
-
- return opts
-
-
-class ResourceMonitoringMixin(PerfherderResourceOptionsMixin):
- """Provides resource monitoring capabilities to scripts.
-
- When this class is in the inheritance chain, resource usage stats of the
- executing script will be recorded.
-
- This class requires the VirtualenvMixin in order to install a package used
- for recording resource usage.
-
- While we would like to record resource usage for the entirety of a script,
- since we require an external package, we can only record resource usage
- after that package is installed (as part of creating the virtualenv).
- That's just the way things have to be.
- """
- def __init__(self, *args, **kwargs):
- super(ResourceMonitoringMixin, self).__init__(*args, **kwargs)
-
- self.register_virtualenv_module('psutil>=3.1.1', method='pip',
- optional=True)
- self.register_virtualenv_module('mozsystemmonitor==0.3',
- method='pip', optional=True)
- self.register_virtualenv_module('jsonschema==2.5.1',
- method='pip')
- # explicitly install functools32, because some slaves aren't using
- # a version of pip recent enough to install it automatically with
- # jsonschema (which depends on it)
- # https://github.com/Julian/jsonschema/issues/233
- self.register_virtualenv_module('functools32==3.2.3-2',
- method='pip')
- self._resource_monitor = None
-
- # 2-tuple of (name, options) to assign Perfherder resource monitor
- # metrics to. This needs to be assigned by a script in order for
- # Perfherder metrics to be reported.
- self.resource_monitor_perfherder_id = None
-
- @PostScriptAction('create-virtualenv')
- def _start_resource_monitoring(self, action, success=None):
- self.activate_virtualenv()
-
- # Resource Monitor requires Python 2.7, however it's currently optional.
- # Remove when all machines have had their Python version updated (bug 711299).
- if sys.version_info[:2] < (2, 7):
- self.warning('Resource monitoring will not be enabled! Python 2.7+ required.')
- return
-
- try:
- from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
-
- self.info("Starting resource monitoring.")
- self._resource_monitor = SystemResourceMonitor(poll_interval=1.0)
- self._resource_monitor.start()
- except Exception:
- self.warning("Unable to start resource monitor: %s" %
- traceback.format_exc())
-
- @PreScriptAction
- def _resource_record_pre_action(self, action):
- # Resource monitor isn't available until after create-virtualenv.
- if not self._resource_monitor:
- return
-
- self._resource_monitor.begin_phase(action)
-
- @PostScriptAction
- def _resource_record_post_action(self, action, success=None):
- # Resource monitor isn't available until after create-virtualenv.
- if not self._resource_monitor:
- return
-
- self._resource_monitor.finish_phase(action)
-
- @PostScriptRun
- def _resource_record_post_run(self):
- if not self._resource_monitor:
- return
-
- # This should never raise an exception. This is a workaround until
- # mozsystemmonitor is fixed. See bug 895388.
- try:
- self._resource_monitor.stop()
- self._log_resource_usage()
-
- # Upload a JSON file containing the raw resource data.
- try:
- upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
- if not os.path.exists(upload_dir):
- os.makedirs(upload_dir)
- with open(os.path.join(upload_dir, 'resource-usage.json'), 'wb') as fh:
- json.dump(self._resource_monitor.as_dict(), fh,
- sort_keys=True, indent=4)
- except (AttributeError, KeyError):
- self.exception('could not upload resource usage JSON',
- level=WARNING)
-
- except Exception:
- self.warning("Exception when reporting resource usage: %s" %
- traceback.format_exc())
-
- def _log_resource_usage(self):
- # Delay import because not available until virtualenv is populated.
- import jsonschema
-
- rm = self._resource_monitor
-
- if rm.start_time is None:
- return
-
- def resources(phase):
- cpu_percent = rm.aggregate_cpu_percent(phase=phase, per_cpu=False)
- cpu_times = rm.aggregate_cpu_times(phase=phase, per_cpu=False)
- io = rm.aggregate_io(phase=phase)
-
- swap_in = sum(m.swap.sin for m in rm.measurements)
- swap_out = sum(m.swap.sout for m in rm.measurements)
-
- return cpu_percent, cpu_times, io, (swap_in, swap_out)
-
- def log_usage(prefix, duration, cpu_percent, cpu_times, io):
- message = '{prefix} - Wall time: {duration:.0f}s; ' \
- 'CPU: {cpu_percent}; ' \
- 'Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; ' \
- 'Read time: {io_read_time}; Write time: {io_write_time}'
-
- # XXX Some test harnesses are complaining about a string being
- # being fed into a 'f' formatter. This will help diagnose the
- # issue.
- cpu_percent_str = str(round(cpu_percent)) + '%' if cpu_percent else "Can't collect data"
-
- try:
- self.info(
- message.format(
- prefix=prefix, duration=duration,
- cpu_percent=cpu_percent_str, io_read_bytes=io.read_bytes,
- io_write_bytes=io.write_bytes, io_read_time=io.read_time,
- io_write_time=io.write_time
- )
- )
-
- except ValueError:
- self.warning("Exception when formatting: %s" %
- traceback.format_exc())
-
- cpu_percent, cpu_times, io, (swap_in, swap_out) = resources(None)
- duration = rm.end_time - rm.start_time
-
- # Write out Perfherder data if configured.
- if self.resource_monitor_perfherder_id:
- perfherder_name, perfherder_options = self.resource_monitor_perfherder_id
-
- suites = []
- overall = []
-
- if cpu_percent:
- overall.append({
- 'name': 'cpu_percent',
- 'value': cpu_percent,
- })
-
- overall.extend([
- {'name': 'io_write_bytes', 'value': io.write_bytes},
- {'name': 'io.read_bytes', 'value': io.read_bytes},
- {'name': 'io_write_time', 'value': io.write_time},
- {'name': 'io_read_time', 'value': io.read_time},
- ])
-
- suites.append({
- 'name': '%s.overall' % perfherder_name,
- 'extraOptions': perfherder_options + self.perfherder_resource_options(),
- 'subtests': overall,
-
- })
-
- for phase in rm.phases.keys():
- phase_duration = rm.phases[phase][1] - rm.phases[phase][0]
- subtests = [
- {
- 'name': 'time',
- 'value': phase_duration,
- }
- ]
- cpu_percent = rm.aggregate_cpu_percent(phase=phase,
- per_cpu=False)
- if cpu_percent is not None:
- subtests.append({
- 'name': 'cpu_percent',
- 'value': rm.aggregate_cpu_percent(phase=phase,
- per_cpu=False),
- })
-
- # We don't report I/O during each step because measured I/O
- # is system I/O and that I/O can be delayed (e.g. writes will
- # buffer before being flushed and recorded in our metrics).
- suites.append({
- 'name': '%s.%s' % (perfherder_name, phase),
- 'subtests': subtests,
- })
-
- data = {
- 'framework': {'name': 'job_resource_usage'},
- 'suites': suites,
- }
-
- schema_path = os.path.join(external_tools_path,
- 'performance-artifact-schema.json')
- with open(schema_path, 'rb') as fh:
- schema = json.load(fh)
-
- # this will throw an exception that causes the job to fail if the
- # perfherder data is not valid -- please don't change this
- # behaviour, otherwise people will inadvertently break this
- # functionality
- self.info('Validating Perfherder data against %s' % schema_path)
- jsonschema.validate(data, schema)
- self.info('PERFHERDER_DATA: %s' % json.dumps(data))
-
- log_usage('Total resource usage', duration, cpu_percent, cpu_times, io)
-
- # Print special messages so usage shows up in Treeherder.
- if cpu_percent:
- self._tinderbox_print('CPU usage<br/>{:,.1f}%'.format(
- cpu_percent))
-
- self._tinderbox_print('I/O read bytes / time<br/>{:,} / {:,}'.format(
- io.read_bytes, io.read_time))
- self._tinderbox_print('I/O write bytes / time<br/>{:,} / {:,}'.format(
- io.write_bytes, io.write_time))
-
- # Print CPU components having >1%. "cpu_times" is a data structure
- # whose attributes are measurements. Ideally we'd have an API that
- # returned just the measurements as a dict or something.
- cpu_attrs = []
- for attr in sorted(dir(cpu_times)):
- if attr.startswith('_'):
- continue
- if attr in ('count', 'index'):
- continue
- cpu_attrs.append(attr)
-
- cpu_total = sum(getattr(cpu_times, attr) for attr in cpu_attrs)
-
- for attr in cpu_attrs:
- value = getattr(cpu_times, attr)
- percent = value / cpu_total * 100.0
- if percent > 1.00:
- self._tinderbox_print('CPU {}<br/>{:,.1f} ({:,.1f}%)'.format(
- attr, value, percent))
-
- # Swap on Windows isn't reported by psutil.
- if not self._is_windows():
- self._tinderbox_print('Swap in / out<br/>{:,} / {:,}'.format(
- swap_in, swap_out))
-
- for phase in rm.phases.keys():
- start_time, end_time = rm.phases[phase]
- cpu_percent, cpu_times, io, swap = resources(phase)
- log_usage(phase, end_time - start_time, cpu_percent, cpu_times, io)
-
- def _tinderbox_print(self, message):
- self.info('TinderboxPrint: %s' % message)
-
-
-# __main__ {{{1
-
-if __name__ == '__main__':
- '''TODO: unit tests.
- '''
- pass
diff --git a/testing/mozharness/mozharness/base/script.py b/testing/mozharness/mozharness/base/script.py
deleted file mode 100755
index 828f4e39e..000000000
--- a/testing/mozharness/mozharness/base/script.py
+++ /dev/null
@@ -1,2273 +0,0 @@
-
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Generic script objects.
-
-script.py, along with config.py and log.py, represents the core of
-mozharness.
-"""
-
-import codecs
-from contextlib import contextmanager
-import datetime
-import errno
-import fnmatch
-import functools
-import gzip
-import inspect
-import itertools
-import os
-import platform
-import pprint
-import re
-import shutil
-import socket
-import subprocess
-import sys
-import tarfile
-import time
-import traceback
-import urllib2
-import zipfile
-import httplib
-import urlparse
-import hashlib
-if os.name == 'nt':
- try:
- import win32file
- import win32api
- PYWIN32 = True
- except ImportError:
- PYWIN32 = False
-
-try:
- import simplejson as json
- assert json
-except ImportError:
- import json
-
-from io import BytesIO
-
-from mozprocess import ProcessHandler
-from mozharness.base.config import BaseConfig
-from mozharness.base.log import SimpleFileLogger, MultiFileLogger, \
- LogMixin, OutputParser, DEBUG, INFO, ERROR, FATAL
-
-
-class FetchedIncorrectFilesize(Exception):
- pass
-
-
-def platform_name():
- pm = PlatformMixin()
-
- if pm._is_linux() and pm._is_64_bit():
- return 'linux64'
- elif pm._is_linux() and not pm._is_64_bit():
- return 'linux'
- elif pm._is_darwin():
- return 'macosx'
- elif pm._is_windows() and pm._is_64_bit():
- return 'win64'
- elif pm._is_windows() and not pm._is_64_bit():
- return 'win32'
- else:
- return None
-
-
-class PlatformMixin(object):
- def _is_windows(self):
- """ check if the current operating system is Windows.
-
- Returns:
- bool: True if the current platform is Windows, False otherwise
- """
- system = platform.system()
- if system in ("Windows", "Microsoft"):
- return True
- if system.startswith("CYGWIN"):
- return True
- if os.name == 'nt':
- return True
-
- def _is_darwin(self):
- """ check if the current operating system is Darwin.
-
- Returns:
- bool: True if the current platform is Darwin, False otherwise
- """
- if platform.system() in ("Darwin"):
- return True
- if sys.platform.startswith("darwin"):
- return True
-
- def _is_linux(self):
- """ check if the current operating system is a Linux distribution.
-
- Returns:
- bool: True if the current platform is a Linux distro, False otherwise
- """
- if platform.system() in ("Linux"):
- return True
- if sys.platform.startswith("linux"):
- return True
-
- def _is_64_bit(self):
- if self._is_darwin():
- # osx is a special snowflake and to ensure the arch, it is better to use the following
- return sys.maxsize > 2**32 # context: https://docs.python.org/2/library/platform.html
- else:
- return '64' in platform.architecture()[0] # architecture() returns (bits, linkage)
-
-
-# ScriptMixin {{{1
-class ScriptMixin(PlatformMixin):
- """This mixin contains simple filesystem commands and the like.
-
- It also contains some very special but very complex methods that,
- together with logging and config, provide the base for all scripts
- in this harness.
-
- WARNING !!!
- This class depends entirely on `LogMixin` methods in such a way that it will
- only works if a class inherits from both `ScriptMixin` and `LogMixin`
- simultaneously.
-
- Depends on self.config of some sort.
-
- Attributes:
- env (dict): a mapping object representing the string environment.
- script_obj (ScriptMixin): reference to a ScriptMixin instance.
- """
-
- env = None
- script_obj = None
-
- def platform_name(self):
- """ Return the platform name on which the script is running on.
- Returns:
- None: for failure to determine the platform.
- str: The name of the platform (e.g. linux64)
- """
- return platform_name()
-
- # Simple filesystem commands {{{2
- def mkdir_p(self, path, error_level=ERROR):
- """ Create a directory if it doesn't exists.
- This method also logs the creation, error or current existence of the
- directory to be created.
-
- Args:
- path (str): path of the directory to be created.
- error_level (str): log level name to be used in case of error.
-
- Returns:
- None: for sucess.
- int: -1 on error
- """
-
- if not os.path.exists(path):
- self.info("mkdir: %s" % path)
- try:
- os.makedirs(path)
- except OSError:
- self.log("Can't create directory %s!" % path,
- level=error_level)
- return -1
- else:
- self.debug("mkdir_p: %s Already exists." % path)
-
- def rmtree(self, path, log_level=INFO, error_level=ERROR,
- exit_code=-1):
- """ Delete an entire directory tree and log its result.
- This method also logs the platform rmtree function, its retries, errors,
- and current existence of the directory.
-
- Args:
- path (str): path to the directory tree root to remove.
- log_level (str, optional): log level name to for this operation. Defaults
- to `INFO`.
- error_level (str, optional): log level name to use in case of error.
- Defaults to `ERROR`.
- exit_code (int, optional): useless parameter, not use here.
- Defaults to -1
-
- Returns:
- None: for success
- """
-
- self.log("rmtree: %s" % path, level=log_level)
- error_message = "Unable to remove %s!" % path
- if self._is_windows():
- # Call _rmtree_windows() directly, since even checking
- # os.path.exists(path) will hang if path is longer than MAX_PATH.
- self.info("Using _rmtree_windows ...")
- return self.retry(
- self._rmtree_windows,
- error_level=error_level,
- error_message=error_message,
- args=(path, ),
- log_level=log_level,
- )
- if os.path.exists(path):
- if os.path.isdir(path):
- return self.retry(
- shutil.rmtree,
- error_level=error_level,
- error_message=error_message,
- retry_exceptions=(OSError, ),
- args=(path, ),
- log_level=log_level,
- )
- else:
- return self.retry(
- os.remove,
- error_level=error_level,
- error_message=error_message,
- retry_exceptions=(OSError, ),
- args=(path, ),
- log_level=log_level,
- )
- else:
- self.debug("%s doesn't exist." % path)
-
- def query_msys_path(self, path):
- """ replaces the Windows harddrive letter path style with a linux
- path style, e.g. C:// --> /C/
- Note: method, not used in any script.
-
- Args:
- path (str?): path to convert to the linux path style.
- Returns:
- str: in case `path` is a string. The result is the path with the new notation.
- type(path): `path` itself is returned in case `path` is not str type.
- """
- if not isinstance(path, basestring):
- return path
- path = path.replace("\\", "/")
-
- def repl(m):
- return '/%s/' % m.group(1)
- path = re.sub(r'''^([a-zA-Z]):/''', repl, path)
- return path
-
- def _rmtree_windows(self, path):
- """ Windows-specific rmtree that handles path lengths longer than MAX_PATH.
- Ported from clobberer.py.
-
- Args:
- path (str): directory path to remove.
-
- Returns:
- None: if the path doesn't exists.
- int: the return number of calling `self.run_command`
- int: in case the path specified is not a directory but a file.
- 0 on success, non-zero on error. Note: The returned value
- is the result of calling `win32file.DeleteFile`
- """
-
- assert self._is_windows()
- path = os.path.realpath(path)
- full_path = '\\\\?\\' + path
- if not os.path.exists(full_path):
- return
- if not PYWIN32:
- if not os.path.isdir(path):
- return self.run_command('del /F /Q "%s"' % path)
- else:
- return self.run_command('rmdir /S /Q "%s"' % path)
- # Make sure directory is writable
- win32file.SetFileAttributesW('\\\\?\\' + path, win32file.FILE_ATTRIBUTE_NORMAL)
- # Since we call rmtree() with a file, sometimes
- if not os.path.isdir('\\\\?\\' + path):
- return win32file.DeleteFile('\\\\?\\' + path)
-
- for ffrec in win32api.FindFiles('\\\\?\\' + path + '\\*.*'):
- file_attr = ffrec[0]
- name = ffrec[8]
- if name == '.' or name == '..':
- continue
- full_name = os.path.join(path, name)
-
- if file_attr & win32file.FILE_ATTRIBUTE_DIRECTORY:
- self._rmtree_windows(full_name)
- else:
- try:
- win32file.SetFileAttributesW('\\\\?\\' + full_name, win32file.FILE_ATTRIBUTE_NORMAL)
- win32file.DeleteFile('\\\\?\\' + full_name)
- except:
- # DeleteFile fails on long paths, del /f /q works just fine
- self.run_command('del /F /Q "%s"' % full_name)
-
- win32file.RemoveDirectory('\\\\?\\' + path)
-
- def get_filename_from_url(self, url):
- """ parse a filename base on an url.
-
- Args:
- url (str): url to parse for the filename
-
- Returns:
- str: filename parsed from the url, or `netloc` network location part
- of the url.
- """
-
- parsed = urlparse.urlsplit(url.rstrip('/'))
- if parsed.path != '':
- return parsed.path.rsplit('/', 1)[-1]
- else:
- return parsed.netloc
-
- def _urlopen(self, url, **kwargs):
- """ open the url `url` using `urllib2`.
- This method can be overwritten to extend its complexity
-
- Args:
- url (str | urllib2.Request): url to open
- kwargs: Arbitrary keyword arguments passed to the `urllib2.urlopen` function.
-
- Returns:
- file-like: file-like object with additional methods as defined in
- `urllib2.urlopen`_.
- None: None may be returned if no handler handles the request.
-
- Raises:
- urllib2.URLError: on errors
-
- .. _urllib2.urlopen:
- https://docs.python.org/2/library/urllib2.html#urllib2.urlopen
- """
- # http://bugs.python.org/issue13359 - urllib2 does not automatically quote the URL
- url_quoted = urllib2.quote(url, safe='%/:=&?~#+!$,;\'@()*[]|')
- return urllib2.urlopen(url_quoted, **kwargs)
-
-
-
- def fetch_url_into_memory(self, url):
- ''' Downloads a file from a url into memory instead of disk.
-
- Args:
- url (str): URL path where the file to be downloaded is located.
-
- Raises:
- IOError: When the url points to a file on disk and cannot be found
- FetchedIncorrectFilesize: When the size of the fetched file does not match the
- expected file size.
- ValueError: When the scheme of a url is not what is expected.
-
- Returns:
- BytesIO: contents of url
- '''
- self.info('Fetch {} into memory'.format(url))
- parsed_url = urlparse.urlparse(url)
-
- if parsed_url.scheme in ('', 'file'):
- if not os.path.isfile(url):
- raise IOError('Could not find file to extract: {}'.format(url))
-
- expected_file_size = os.stat(url.replace('file://', '')).st_size
-
- # In case we're referrencing a file without file://
- if parsed_url.scheme == '':
- url = 'file://%s' % os.path.abspath(url)
- parsed_url = urlparse.urlparse(url)
-
- request = urllib2.Request(url)
- # When calling fetch_url_into_memory() you should retry when we raise one of these exceptions:
- # * Bug 1300663 - HTTPError: HTTP Error 404: Not Found
- # * Bug 1300413 - HTTPError: HTTP Error 500: Internal Server Error
- # * Bug 1300943 - HTTPError: HTTP Error 503: Service Unavailable
- # * Bug 1300953 - URLError: <urlopen error [Errno -2] Name or service not known>
- # * Bug 1301594 - URLError: <urlopen error [Errno 10054] An existing connection was ...
- # * Bug 1301597 - URLError: <urlopen error [Errno 8] _ssl.c:504: EOF occurred in ...
- # * Bug 1301855 - URLError: <urlopen error [Errno 60] Operation timed out>
- # * Bug 1302237 - URLError: <urlopen error [Errno 104] Connection reset by peer>
- # * Bug 1301807 - BadStatusLine: ''
- #
- # Bug 1309912 - Adding timeout in hopes to solve blocking on response.read() (bug 1300413)
- response = urllib2.urlopen(request, timeout=30)
-
- if parsed_url.scheme in ('http', 'https'):
- expected_file_size = int(response.headers.get('Content-Length'))
-
- self.info('Http code: {}'.format(response.getcode()))
- for k in sorted(response.headers.keys()):
- if k.lower().startswith('x-amz-') or k in ('Content-Encoding', 'Content-Type', 'via'):
- self.info('{}: {}'.format(k, response.headers.get(k)))
-
- file_contents = response.read()
- obtained_file_size = len(file_contents)
- self.info('Expected file size: {}'.format(expected_file_size))
- self.info('Obtained file size: {}'.format(obtained_file_size))
-
- if obtained_file_size != expected_file_size:
- raise FetchedIncorrectFilesize(
- 'The expected file size is {} while we got instead {}'.format(
- expected_file_size, obtained_file_size)
- )
-
- # Use BytesIO instead of StringIO
- # http://stackoverflow.com/questions/34162017/unzip-buffer-with-python/34162395#34162395
- return BytesIO(file_contents)
-
-
- def _download_file(self, url, file_name):
- """ Helper script for download_file()
- Additionaly this function logs all exceptions as warnings before
- re-raising them
-
- Args:
- url (str): string containing the URL with the file location
- file_name (str): name of the file where the downloaded file
- is written.
-
- Returns:
- str: filename of the written file on disk
-
- Raises:
- urllib2.URLError: on incomplete download.
- urllib2.HTTPError: on Http error code
- socket.timeout: on connection timeout
- socket.error: on socket error
- """
- # If our URLs look like files, prefix them with file:// so they can
- # be loaded like URLs.
- if not (url.startswith("http") or url.startswith("file://")):
- if not os.path.isfile(url):
- self.fatal("The file %s does not exist" % url)
- url = 'file://%s' % os.path.abspath(url)
-
- try:
- f_length = None
- f = self._urlopen(url, timeout=30)
-
- if f.info().get('content-length') is not None:
- f_length = int(f.info()['content-length'])
- got_length = 0
- local_file = open(file_name, 'wb')
- while True:
- block = f.read(1024 ** 2)
- if not block:
- if f_length is not None and got_length != f_length:
- raise urllib2.URLError("Download incomplete; content-length was %d, but only received %d" % (f_length, got_length))
- break
- local_file.write(block)
- if f_length is not None:
- got_length += len(block)
- local_file.close()
- return file_name
- except urllib2.HTTPError, e:
- self.warning("Server returned status %s %s for %s" % (str(e.code), str(e), url))
- raise
- except urllib2.URLError, e:
- self.warning("URL Error: %s" % url)
-
- # Failures due to missing local files won't benefit from retry.
- # Raise the original OSError.
- if isinstance(e.args[0], OSError) and e.args[0].errno == errno.ENOENT:
- raise e.args[0]
-
- remote_host = urlparse.urlsplit(url)[1]
- if remote_host:
- nslookup = self.query_exe('nslookup')
- error_list = [{
- 'substr': "server can't find %s" % remote_host,
- 'level': ERROR,
- 'explanation': "Either %s is an invalid hostname, or DNS is busted." % remote_host,
- }]
- self.run_command([nslookup, remote_host],
- error_list=error_list)
- raise
- except socket.timeout, e:
- self.warning("Timed out accessing %s: %s" % (url, str(e)))
- raise
- except socket.error, e:
- self.warning("Socket error when accessing %s: %s" % (url, str(e)))
- raise
-
- def _retry_download(self, url, error_level, file_name=None, retry_config=None):
- """ Helper method to retry download methods.
-
- This method calls `self.retry` on `self._download_file` using the passed
- parameters if a file_name is specified. If no file is specified, we will
- instead call `self._urlopen`, which grabs the contents of a url but does
- not create a file on disk.
-
- Args:
- url (str): URL path where the file is located.
- file_name (str): file_name where the file will be written to.
- error_level (str): log level to use in case an error occurs.
- retry_config (dict, optional): key-value pairs to be passed to
- `self.retry`. Defaults to `None`
-
- Returns:
- str: `self._download_file` return value is returned
- unknown: `self.retry` `failure_status` is returned on failure, which
- defaults to -1
- """
- retry_args = dict(
- failure_status=None,
- retry_exceptions=(urllib2.HTTPError, urllib2.URLError,
- httplib.BadStatusLine,
- socket.timeout, socket.error),
- error_message="Can't download from %s to %s!" % (url, file_name),
- error_level=error_level,
- )
-
- if retry_config:
- retry_args.update(retry_config)
-
- download_func = self._urlopen
- kwargs = {"url": url}
- if file_name:
- download_func = self._download_file
- kwargs = {"url": url, "file_name": file_name}
-
- return self.retry(
- download_func,
- kwargs=kwargs,
- **retry_args
- )
-
-
- def _filter_entries(self, namelist, extract_dirs):
- """Filter entries of the archive based on the specified list of to extract dirs."""
- filter_partial = functools.partial(fnmatch.filter, namelist)
- entries = itertools.chain(*map(filter_partial, extract_dirs or ['*']))
-
- for entry in entries:
- yield entry
-
-
- def unzip(self, compressed_file, extract_to, extract_dirs='*', verbose=False):
- """This method allows to extract a zip file without writing to disk first.
-
- Args:
- compressed_file (object): File-like object with the contents of a compressed zip file.
- extract_to (str): where to extract the compressed file.
- extract_dirs (list, optional): directories inside the archive file to extract.
- Defaults to '*'.
- verbose (bool, optional): whether or not extracted content should be displayed.
- Defaults to False.
-
- Raises:
- zipfile.BadZipfile: on contents of zipfile being invalid
- """
- with zipfile.ZipFile(compressed_file) as bundle:
- entries = self._filter_entries(bundle.namelist(), extract_dirs)
-
- for entry in entries:
- if verbose:
- self.info(' {}'.format(entry))
-
- # Exception to be retried:
- # Bug 1301645 - BadZipfile: Bad CRC-32 for file ...
- # http://stackoverflow.com/questions/5624669/strange-badzipfile-bad-crc-32-problem/5626098#5626098
- # Bug 1301802 - error: Error -3 while decompressing: invalid stored block lengths
- bundle.extract(entry, path=extract_to)
-
- # ZipFile doesn't preserve permissions during extraction:
- # http://bugs.python.org/issue15795
- fname = os.path.realpath(os.path.join(extract_to, entry))
- try:
- # getinfo() can raise KeyError
- mode = bundle.getinfo(entry).external_attr >> 16 & 0x1FF
- # Only set permissions if attributes are available. Otherwise all
- # permissions will be removed eg. on Windows.
- if mode:
- os.chmod(fname, mode)
-
- except KeyError:
- self.warning('{} was not found in the zip file'.format(entry))
-
-
- def deflate(self, compressed_file, mode, extract_to='.', *args, **kwargs):
- """This method allows to extract a compressed file from a tar, tar.bz2 and tar.gz files.
-
- Args:
- compressed_file (object): File-like object with the contents of a compressed file.
- mode (str): string of the form 'filemode[:compression]' (e.g. 'r:gz' or 'r:bz2')
- extract_to (str, optional): where to extract the compressed file.
- """
- t = tarfile.open(fileobj=compressed_file, mode=mode)
- t.extractall(path=extract_to)
-
-
- def download_unpack(self, url, extract_to='.', extract_dirs='*', verbose=False):
- """Generic method to download and extract a compressed file without writing it to disk first.
-
- Args:
- url (str): URL where the file to be downloaded is located.
- extract_to (str, optional): directory where the downloaded file will
- be extracted to.
- extract_dirs (list, optional): directories inside the archive to extract.
- Defaults to `*`. It currently only applies to zip files.
- verbose (bool, optional): whether or not extracted content should be displayed.
- Defaults to False.
-
- """
- def _determine_extraction_method_and_kwargs(url):
- EXTENSION_TO_MIMETYPE = {
- 'bz2': 'application/x-bzip2',
- 'gz': 'application/x-gzip',
- 'tar': 'application/x-tar',
- 'zip': 'application/zip',
- }
- MIMETYPES = {
- 'application/x-bzip2': {
- 'function': self.deflate,
- 'kwargs': {'mode': 'r:bz2'},
- },
- 'application/x-gzip': {
- 'function': self.deflate,
- 'kwargs': {'mode': 'r:gz'},
- },
- 'application/x-tar': {
- 'function': self.deflate,
- 'kwargs': {'mode': 'r'},
- },
- 'application/zip': {
- 'function': self.unzip,
- },
- 'application/x-zip-compressed': {
- 'function': self.unzip,
- },
- }
-
- filename = url.split('/')[-1]
- # XXX: bz2/gz instead of tar.{bz2/gz}
- extension = filename[filename.rfind('.')+1:]
- mimetype = EXTENSION_TO_MIMETYPE[extension]
- self.debug('Mimetype: {}'.format(mimetype))
-
- function = MIMETYPES[mimetype]['function']
- kwargs = {
- 'compressed_file': compressed_file,
- 'extract_to': extract_to,
- 'extract_dirs': extract_dirs,
- 'verbose': verbose,
- }
- kwargs.update(MIMETYPES[mimetype].get('kwargs', {}))
-
- return function, kwargs
-
- # Many scripts overwrite this method and set extract_dirs to None
- extract_dirs = '*' if extract_dirs is None else extract_dirs
- self.info('Downloading and extracting to {} these dirs {} from {}'.format(
- extract_to,
- ', '.join(extract_dirs),
- url,
- ))
-
- # 1) Let's fetch the file
- retry_args = dict(
- retry_exceptions=(
- urllib2.HTTPError,
- urllib2.URLError,
- httplib.BadStatusLine,
- socket.timeout,
- socket.error,
- FetchedIncorrectFilesize,
- ),
- error_message="Can't download from {}".format(url),
- error_level=FATAL,
- )
- compressed_file = self.retry(
- self.fetch_url_into_memory,
- kwargs={'url': url},
- **retry_args
- )
-
- # 2) We're guaranteed to have download the file with error_level=FATAL
- # Let's unpack the file
- function, kwargs = _determine_extraction_method_and_kwargs(url)
- try:
- function(**kwargs)
- except zipfile.BadZipfile:
- # Bug 1306189 - Sometimes a good download turns out to be a
- # corrupted zipfile. Let's create a signature that is easy to match
- self.fatal('Check bug 1306189 for details on downloading a truncated zip file.')
-
-
- def load_json_url(self, url, error_level=None, *args, **kwargs):
- """ Returns a json object from a url (it retries). """
- contents = self._retry_download(
- url=url, error_level=error_level, *args, **kwargs
- )
- return json.loads(contents.read())
-
- # http://www.techniqal.com/blog/2008/07/31/python-file-read-write-with-urllib2/
- # TODO thinking about creating a transfer object.
- def download_file(self, url, file_name=None, parent_dir=None,
- create_parent_dir=True, error_level=ERROR,
- exit_code=3, retry_config=None):
- """ Python wget.
- Download the filename at `url` into `file_name` and put it on `parent_dir`.
- On error log with the specified `error_level`, on fatal exit with `exit_code`.
- Execute all the above based on `retry_config` parameter.
-
- Args:
- url (str): URL path where the file to be downloaded is located.
- file_name (str, optional): file_name where the file will be written to.
- Defaults to urls' filename.
- parent_dir (str, optional): directory where the downloaded file will
- be written to. Defaults to current working
- directory
- create_parent_dir (bool, optional): create the parent directory if it
- doesn't exist. Defaults to `True`
- error_level (str, optional): log level to use in case an error occurs.
- Defaults to `ERROR`
- retry_config (dict, optional): key-value pairs to be passed to
- `self.retry`. Defaults to `None`
-
- Returns:
- str: filename where the downloaded file was written to.
- unknown: on failure, `failure_status` is returned.
- """
- if not file_name:
- try:
- file_name = self.get_filename_from_url(url)
- except AttributeError:
- self.log("Unable to get filename from %s; bad url?" % url,
- level=error_level, exit_code=exit_code)
- return
- if parent_dir:
- file_name = os.path.join(parent_dir, file_name)
- if create_parent_dir:
- self.mkdir_p(parent_dir, error_level=error_level)
- self.info("Downloading %s to %s" % (url, file_name))
- status = self._retry_download(
- url=url,
- error_level=error_level,
- file_name=file_name,
- retry_config=retry_config
- )
- if status == file_name:
- self.info("Downloaded %d bytes." % os.path.getsize(file_name))
- return status
-
- def move(self, src, dest, log_level=INFO, error_level=ERROR,
- exit_code=-1):
- """ recursively move a file or directory (src) to another location (dest).
-
- Args:
- src (str): file or directory path to move.
- dest (str): file or directory path where to move the content to.
- log_level (str): log level to use for normal operation. Defaults to
- `INFO`
- error_level (str): log level to use on error. Defaults to `ERROR`
-
- Returns:
- int: 0 on success. -1 on error.
- """
- self.log("Moving %s to %s" % (src, dest), level=log_level)
- try:
- shutil.move(src, dest)
- # http://docs.python.org/tutorial/errors.html
- except IOError, e:
- self.log("IO error: %s" % str(e),
- level=error_level, exit_code=exit_code)
- return -1
- except shutil.Error, e:
- self.log("shutil error: %s" % str(e),
- level=error_level, exit_code=exit_code)
- return -1
- return 0
-
- def chmod(self, path, mode):
- """ change `path` mode to `mode`.
-
- Args:
- path (str): path whose mode will be modified.
- mode (hex): one of the values defined at `stat`_
-
- .. _stat:
- https://docs.python.org/2/library/os.html#os.chmod
- """
-
- self.info("Chmoding %s to %s" % (path, str(oct(mode))))
- os.chmod(path, mode)
-
- def copyfile(self, src, dest, log_level=INFO, error_level=ERROR, copystat=False, compress=False):
- """ copy or compress `src` into `dest`.
-
- Args:
- src (str): filepath to copy.
- dest (str): filepath where to move the content to.
- log_level (str, optional): log level to use for normal operation. Defaults to
- `INFO`
- error_level (str, optional): log level to use on error. Defaults to `ERROR`
- copystat (bool, optional): whether or not to copy the files metadata.
- Defaults to `False`.
- compress (bool, optional): whether or not to compress the destination file.
- Defaults to `False`.
-
- Returns:
- int: -1 on error
- None: on success
- """
-
- if compress:
- self.log("Compressing %s to %s" % (src, dest), level=log_level)
- try:
- infile = open(src, "rb")
- outfile = gzip.open(dest, "wb")
- outfile.writelines(infile)
- outfile.close()
- infile.close()
- except IOError, e:
- self.log("Can't compress %s to %s: %s!" % (src, dest, str(e)),
- level=error_level)
- return -1
- else:
- self.log("Copying %s to %s" % (src, dest), level=log_level)
- try:
- shutil.copyfile(src, dest)
- except (IOError, shutil.Error), e:
- self.log("Can't copy %s to %s: %s!" % (src, dest, str(e)),
- level=error_level)
- return -1
-
- if copystat:
- try:
- shutil.copystat(src, dest)
- except (IOError, shutil.Error), e:
- self.log("Can't copy attributes of %s to %s: %s!" % (src, dest, str(e)),
- level=error_level)
- return -1
-
- def copytree(self, src, dest, overwrite='no_overwrite', log_level=INFO,
- error_level=ERROR):
- """ An implementation of `shutil.copytree` that allows for `dest` to exist
- and implements different overwrite levels:
- - 'no_overwrite' will keep all(any) existing files in destination tree
- - 'overwrite_if_exists' will only overwrite destination paths that have
- the same path names relative to the root of the
- src and destination tree
- - 'clobber' will replace the whole destination tree(clobber) if it exists
-
- Args:
- src (str): directory path to move.
- dest (str): directory path where to move the content to.
- overwrite (str): string specifying the overwrite level.
- log_level (str, optional): log level to use for normal operation. Defaults to
- `INFO`
- error_level (str, optional): log level to use on error. Defaults to `ERROR`
-
- Returns:
- int: -1 on error
- None: on success
- """
-
- self.info('copying tree: %s to %s' % (src, dest))
- try:
- if overwrite == 'clobber' or not os.path.exists(dest):
- self.rmtree(dest)
- shutil.copytree(src, dest)
- elif overwrite == 'no_overwrite' or overwrite == 'overwrite_if_exists':
- files = os.listdir(src)
- for f in files:
- abs_src_f = os.path.join(src, f)
- abs_dest_f = os.path.join(dest, f)
- if not os.path.exists(abs_dest_f):
- if os.path.isdir(abs_src_f):
- self.mkdir_p(abs_dest_f)
- self.copytree(abs_src_f, abs_dest_f,
- overwrite='clobber')
- else:
- shutil.copy2(abs_src_f, abs_dest_f)
- elif overwrite == 'no_overwrite': # destination path exists
- if os.path.isdir(abs_src_f) and os.path.isdir(abs_dest_f):
- self.copytree(abs_src_f, abs_dest_f,
- overwrite='no_overwrite')
- else:
- self.debug('ignoring path: %s as destination: \
- %s exists' % (abs_src_f, abs_dest_f))
- else: # overwrite == 'overwrite_if_exists' and destination exists
- self.debug('overwriting: %s with: %s' %
- (abs_dest_f, abs_src_f))
- self.rmtree(abs_dest_f)
-
- if os.path.isdir(abs_src_f):
- self.mkdir_p(abs_dest_f)
- self.copytree(abs_src_f, abs_dest_f,
- overwrite='overwrite_if_exists')
- else:
- shutil.copy2(abs_src_f, abs_dest_f)
- else:
- self.fatal("%s is not a valid argument for param overwrite" % (overwrite))
- except (IOError, shutil.Error):
- self.exception("There was an error while copying %s to %s!" % (src, dest),
- level=error_level)
- return -1
-
- def write_to_file(self, file_path, contents, verbose=True,
- open_mode='w', create_parent_dir=False,
- error_level=ERROR):
- """ Write `contents` to `file_path`, according to `open_mode`.
-
- Args:
- file_path (str): filepath where the content will be written to.
- contents (str): content to write to the filepath.
- verbose (bool, optional): whether or not to log `contents` value.
- Defaults to `True`
- open_mode (str, optional): open mode to use for openning the file.
- Defaults to `w`
- create_parent_dir (bool, optional): whether or not to create the
- parent directory of `file_path`
- error_level (str, optional): log level to use on error. Defaults to `ERROR`
-
- Returns:
- str: `file_path` on success
- None: on error.
- """
- self.info("Writing to file %s" % file_path)
- if verbose:
- self.info("Contents:")
- for line in contents.splitlines():
- self.info(" %s" % line)
- if create_parent_dir:
- parent_dir = os.path.dirname(file_path)
- self.mkdir_p(parent_dir, error_level=error_level)
- try:
- fh = open(file_path, open_mode)
- try:
- fh.write(contents)
- except UnicodeEncodeError:
- fh.write(contents.encode('utf-8', 'replace'))
- fh.close()
- return file_path
- except IOError:
- self.log("%s can't be opened for writing!" % file_path,
- level=error_level)
-
- @contextmanager
- def opened(self, file_path, verbose=True, open_mode='r',
- error_level=ERROR):
- """ Create a context manager to use on a with statement.
-
- Args:
- file_path (str): filepath of the file to open.
- verbose (bool, optional): useless parameter, not used here.
- Defaults to True.
- open_mode (str, optional): open mode to use for openning the file.
- Defaults to `r`
- error_level (str, optional): log level name to use on error.
- Defaults to `ERROR`
-
- Yields:
- tuple: (file object, error) pair. In case of error `None` is yielded
- as file object, together with the corresponding error.
- If there is no error, `None` is returned as the error.
- """
- # See opened_w_error in http://www.python.org/dev/peps/pep-0343/
- self.info("Reading from file %s" % file_path)
- try:
- fh = open(file_path, open_mode)
- except IOError, err:
- self.log("unable to open %s: %s" % (file_path, err.strerror),
- level=error_level)
- yield None, err
- else:
- try:
- yield fh, None
- finally:
- fh.close()
-
- def read_from_file(self, file_path, verbose=True, open_mode='r',
- error_level=ERROR):
- """ Use `self.opened` context manager to open a file and read its
- content.
-
- Args:
- file_path (str): filepath of the file to read.
- verbose (bool, optional): whether or not to log the file content.
- Defaults to True.
- open_mode (str, optional): open mode to use for openning the file.
- Defaults to `r`
- error_level (str, optional): log level name to use on error.
- Defaults to `ERROR`
-
- Returns:
- None: on error.
- str: file content on success.
- """
- with self.opened(file_path, verbose, open_mode, error_level) as (fh, err):
- if err:
- return None
- contents = fh.read()
- if verbose:
- self.info("Contents:")
- for line in contents.splitlines():
- self.info(" %s" % line)
- return contents
-
- def chdir(self, dir_name):
- self.log("Changing directory to %s." % dir_name)
- os.chdir(dir_name)
-
- def is_exe(self, fpath):
- """
- Determine if fpath is a file and if it is executable.
- """
- return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
-
- def which(self, program):
- """ OS independent implementation of Unix's which command
-
- Args:
- program (str): name or path to the program whose executable is
- being searched.
-
- Returns:
- None: if the executable was not found.
- str: filepath of the executable file.
- """
- if self._is_windows() and not program.endswith(".exe"):
- program += ".exe"
- fpath, fname = os.path.split(program)
- if fpath:
- if self.is_exe(program):
- return program
- else:
- # If the exe file is defined in the configs let's use that
- exe = self.query_exe(program)
- if self.is_exe(exe):
- return exe
-
- # If not defined, let's look for it in the $PATH
- env = self.query_env()
- for path in env["PATH"].split(os.pathsep):
- exe_file = os.path.join(path, program)
- if self.is_exe(exe_file):
- return exe_file
- return None
-
- # More complex commands {{{2
- def retry(self, action, attempts=None, sleeptime=60, max_sleeptime=5 * 60,
- retry_exceptions=(Exception, ), good_statuses=None, cleanup=None,
- error_level=ERROR, error_message="%(action)s failed after %(attempts)d tries!",
- failure_status=-1, log_level=INFO, args=(), kwargs={}):
- """ generic retry command. Ported from `util.retry`_
-
- Args:
- action (func): callable object to retry.
- attempts (int, optinal): maximum number of times to call actions.
- Defaults to `self.config.get('global_retries', 5)`
- sleeptime (int, optional): number of seconds to wait between
- attempts. Defaults to 60 and doubles each retry attempt, to
- a maximum of `max_sleeptime'
- max_sleeptime (int, optional): maximum value of sleeptime. Defaults
- to 5 minutes
- retry_exceptions (tuple, optional): Exceptions that should be caught.
- If exceptions other than those listed in `retry_exceptions' are
- raised from `action', they will be raised immediately. Defaults
- to (Exception)
- good_statuses (object, optional): return values which, if specified,
- will result in retrying if the return value isn't listed.
- Defaults to `None`.
- cleanup (func, optional): If `cleanup' is provided and callable
- it will be called immediately after an Exception is caught.
- No arguments will be passed to it. If your cleanup function
- requires arguments it is recommended that you wrap it in an
- argumentless function.
- Defaults to `None`.
- error_level (str, optional): log level name in case of error.
- Defaults to `ERROR`.
- error_message (str, optional): string format to use in case
- none of the attempts success. Defaults to
- '%(action)s failed after %(attempts)d tries!'
- failure_status (int, optional): flag to return in case the retries
- were not successfull. Defaults to -1.
- log_level (str, optional): log level name to use for normal activity.
- Defaults to `INFO`.
- args (tuple, optional): positional arguments to pass onto `action`.
- kwargs (dict, optional): key-value arguments to pass onto `action`.
-
- Returns:
- object: return value of `action`.
- int: failure status in case of failure retries.
- """
- if not callable(action):
- self.fatal("retry() called with an uncallable method %s!" % action)
- if cleanup and not callable(cleanup):
- self.fatal("retry() called with an uncallable cleanup method %s!" % cleanup)
- if not attempts:
- attempts = self.config.get("global_retries", 5)
- if max_sleeptime < sleeptime:
- self.debug("max_sleeptime %d less than sleeptime %d" % (
- max_sleeptime, sleeptime))
- n = 0
- while n <= attempts:
- retry = False
- n += 1
- try:
- self.log("retry: Calling %s with args: %s, kwargs: %s, attempt #%d" %
- (action.__name__, str(args), str(kwargs), n), level=log_level)
- status = action(*args, **kwargs)
- if good_statuses and status not in good_statuses:
- retry = True
- except retry_exceptions, e:
- retry = True
- error_message = "%s\nCaught exception: %s" % (error_message, str(e))
- self.log('retry: attempt #%d caught exception: %s' % (n, str(e)), level=INFO)
-
- if not retry:
- return status
- else:
- if cleanup:
- cleanup()
- if n == attempts:
- self.log(error_message % {'action': action, 'attempts': n}, level=error_level)
- return failure_status
- if sleeptime > 0:
- self.log("retry: Failed, sleeping %d seconds before retrying" %
- sleeptime, level=log_level)
- time.sleep(sleeptime)
- sleeptime = sleeptime * 2
- if sleeptime > max_sleeptime:
- sleeptime = max_sleeptime
-
- def query_env(self, partial_env=None, replace_dict=None,
- purge_env=(),
- set_self_env=None, log_level=DEBUG,
- avoid_host_env=False):
- """ Environment query/generation method.
- The default, self.query_env(), will look for self.config['env']
- and replace any special strings in there ( %(PATH)s ).
- It will then store it as self.env for speeding things up later.
-
- If you specify partial_env, partial_env will be used instead of
- self.config['env'], and we don't save self.env as it's a one-off.
-
-
- Args:
- partial_env (dict, optional): key-value pairs of the name and value
- of different environment variables. Defaults to an empty dictionary.
- replace_dict (dict, optional): key-value pairs to replace the old
- environment variables.
- purge_env (list): environment names to delete from the final
- environment dictionary.
- set_self_env (boolean, optional): whether or not the environment
- variables dictionary should be copied to `self`.
- Defaults to True.
- log_level (str, optional): log level name to use on normal operation.
- Defaults to `DEBUG`.
- avoid_host_env (boolean, optional): if set to True, we will not use
- any environment variables set on the host except PATH.
- Defaults to False.
-
- Returns:
- dict: environment variables names with their values.
- """
- if partial_env is None:
- if self.env is not None:
- return self.env
- partial_env = self.config.get('env', None)
- if partial_env is None:
- partial_env = {}
- if set_self_env is None:
- set_self_env = True
-
- env = {'PATH': os.environ['PATH']} if avoid_host_env else os.environ.copy()
-
- default_replace_dict = self.query_abs_dirs()
- default_replace_dict['PATH'] = os.environ['PATH']
- if not replace_dict:
- replace_dict = default_replace_dict
- else:
- for key in default_replace_dict:
- if key not in replace_dict:
- replace_dict[key] = default_replace_dict[key]
- for key in partial_env.keys():
- env[key] = partial_env[key] % replace_dict
- self.log("ENV: %s is now %s" % (key, env[key]), level=log_level)
- for k in purge_env:
- if k in env:
- del env[k]
- if set_self_env:
- self.env = env
- return env
-
- def query_exe(self, exe_name, exe_dict='exes', default=None,
- return_type=None, error_level=FATAL):
- """One way to work around PATH rewrites.
-
- By default, return exe_name, and we'll fall through to searching
- os.environ["PATH"].
- However, if self.config[exe_dict][exe_name] exists, return that.
- This lets us override exe paths via config file.
-
- If we need runtime setting, we can build in self.exes support later.
-
- Args:
- exe_name (str): name of the executable to search for.
- exe_dict(str, optional): name of the dictionary of executables
- present in `self.config`. Defaults to `exes`.
- default (str, optional): default name of the executable to search
- for. Defaults to `exe_name`.
- return_type (str, optional): type to which the original return
- value will be turn into. Only 'list', 'string' and `None` are
- supported. Defaults to `None`.
- error_level (str, optional): log level name to use on error.
-
- Returns:
- list: in case return_type is 'list'
- str: in case return_type is 'string'
- None: in case return_type is `None`
- Any: if the found executable is not of type list, tuple nor str.
- """
- if default is None:
- default = exe_name
- exe = self.config.get(exe_dict, {}).get(exe_name, default)
- repl_dict = {}
- if hasattr(self.script_obj, 'query_abs_dirs'):
- # allow for 'make': '%(abs_work_dir)s/...' etc.
- dirs = self.script_obj.query_abs_dirs()
- repl_dict.update(dirs)
- if isinstance(exe, dict):
- found = False
- # allow for searchable paths of the buildbot exe
- for name, path in exe.iteritems():
- if isinstance(path, list) or isinstance(path, tuple):
- path = [x % repl_dict for x in path]
- if all([os.path.exists(section) for section in path]):
- found = True
- elif isinstance(path, str):
- path = path % repl_dict
- if os.path.exists(path):
- found = True
- else:
- self.log("a exes %s dict's value is not a string, list, or tuple. Got key "
- "%s and value %s" % (exe_name, name, str(path)), level=error_level)
- if found:
- exe = path
- break
- else:
- self.log("query_exe was a searchable dict but an existing path could not be "
- "determined. Tried searching in paths: %s" % (str(exe)), level=error_level)
- return None
- elif isinstance(exe, list) or isinstance(exe, tuple):
- exe = [x % repl_dict for x in exe]
- elif isinstance(exe, str):
- exe = exe % repl_dict
- else:
- self.log("query_exe: %s is not a list, tuple, dict, or string: "
- "%s!" % (exe_name, str(exe)), level=error_level)
- return exe
- if return_type == "list":
- if isinstance(exe, str):
- exe = [exe]
- elif return_type == "string":
- if isinstance(exe, list):
- exe = subprocess.list2cmdline(exe)
- elif return_type is not None:
- self.log("Unknown return_type type %s requested in query_exe!" % return_type, level=error_level)
- return exe
-
- def run_command(self, command, cwd=None, error_list=None,
- halt_on_failure=False, success_codes=None,
- env=None, partial_env=None, return_type='status',
- throw_exception=False, output_parser=None,
- output_timeout=None, fatal_exit_code=2,
- error_level=ERROR, **kwargs):
- """Run a command, with logging and error parsing.
- TODO: context_lines
-
- error_list example:
- [{'regex': re.compile('^Error: LOL J/K'), level=IGNORE},
- {'regex': re.compile('^Error:'), level=ERROR, contextLines='5:5'},
- {'substr': 'THE WORLD IS ENDING', level=FATAL, contextLines='20:'}
- ]
- (context_lines isn't written yet)
-
- Args:
- command (str | list | tuple): command or sequence of commands to
- execute and log.
- cwd (str, optional): directory path from where to execute the
- command. Defaults to `None`.
- error_list (list, optional): list of errors to pass to
- `mozharness.base.log.OutputParser`. Defaults to `None`.
- halt_on_failure (bool, optional): whether or not to redefine the
- log level as `FATAL` on errors. Defaults to False.
- success_codes (int, optional): numeric value to compare against
- the command return value.
- env (dict, optional): key-value of environment values to use to
- run the command. Defaults to None.
- partial_env (dict, optional): key-value of environment values to
- replace from the current environment values. Defaults to None.
- return_type (str, optional): if equal to 'num_errors' then the
- amount of errors matched by `error_list` is returned. Defaults
- to 'status'.
- throw_exception (bool, optional): whether or not to raise an
- exception if the return value of the command doesn't match
- any of the `success_codes`. Defaults to False.
- output_parser (OutputParser, optional): lets you provide an
- instance of your own OutputParser subclass. Defaults to `OutputParser`.
- output_timeout (int): amount of seconds to wait for output before
- the process is killed.
- fatal_exit_code (int, optional): call `self.fatal` if the return value
- of the command is not in `success_codes`. Defaults to 2.
- error_level (str, optional): log level name to use on error. Defaults
- to `ERROR`.
- **kwargs: Arbitrary keyword arguments.
-
- Returns:
- int: -1 on error.
- Any: `command` return value is returned otherwise.
- """
- if success_codes is None:
- success_codes = [0]
- if cwd is not None:
- if not os.path.isdir(cwd):
- level = error_level
- if halt_on_failure:
- level = FATAL
- self.log("Can't run command %s in non-existent directory '%s'!" %
- (command, cwd), level=level)
- return -1
- self.info("Running command: %s in %s" % (command, cwd))
- else:
- self.info("Running command: %s" % command)
- if isinstance(command, list) or isinstance(command, tuple):
- self.info("Copy/paste: %s" % subprocess.list2cmdline(command))
- shell = True
- if isinstance(command, list) or isinstance(command, tuple):
- shell = False
- if env is None:
- if partial_env:
- self.info("Using partial env: %s" % pprint.pformat(partial_env))
- env = self.query_env(partial_env=partial_env)
- else:
- self.info("Using env: %s" % pprint.pformat(env))
-
- if output_parser is None:
- parser = OutputParser(config=self.config, log_obj=self.log_obj,
- error_list=error_list)
- else:
- parser = output_parser
-
- try:
- if output_timeout:
- def processOutput(line):
- parser.add_lines(line)
-
- def onTimeout():
- self.info("Automation Error: mozprocess timed out after %s seconds running %s" % (str(output_timeout), str(command)))
-
- p = ProcessHandler(command,
- shell=shell,
- env=env,
- cwd=cwd,
- storeOutput=False,
- onTimeout=(onTimeout,),
- processOutputLine=[processOutput])
- self.info("Calling %s with output_timeout %d" % (command, output_timeout))
- p.run(outputTimeout=output_timeout)
- p.wait()
- if p.timedOut:
- self.log(
- 'timed out after %s seconds of no output' % output_timeout,
- level=error_level
- )
- returncode = int(p.proc.returncode)
- else:
- p = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE,
- cwd=cwd, stderr=subprocess.STDOUT, env=env,
- bufsize=0)
- loop = True
- while loop:
- if p.poll() is not None:
- """Avoid losing the final lines of the log?"""
- loop = False
- while True:
- line = p.stdout.readline()
- if not line:
- break
- parser.add_lines(line)
- returncode = p.returncode
- except OSError, e:
- level = error_level
- if halt_on_failure:
- level = FATAL
- self.log('caught OS error %s: %s while running %s' % (e.errno,
- e.strerror, command), level=level)
- return -1
-
- return_level = INFO
- if returncode not in success_codes:
- return_level = error_level
- if throw_exception:
- raise subprocess.CalledProcessError(returncode, command)
- self.log("Return code: %d" % returncode, level=return_level)
-
- if halt_on_failure:
- _fail = False
- if returncode not in success_codes:
- self.log(
- "%s not in success codes: %s" % (returncode, success_codes),
- level=error_level
- )
- _fail = True
- if parser.num_errors:
- self.log("failures found while parsing output", level=error_level)
- _fail = True
- if _fail:
- self.return_code = fatal_exit_code
- self.fatal("Halting on failure while running %s" % command,
- exit_code=fatal_exit_code)
- if return_type == 'num_errors':
- return parser.num_errors
- return returncode
-
- def get_output_from_command(self, command, cwd=None,
- halt_on_failure=False, env=None,
- silent=False, log_level=INFO,
- tmpfile_base_path='tmpfile',
- return_type='output', save_tmpfiles=False,
- throw_exception=False, fatal_exit_code=2,
- ignore_errors=False, success_codes=None):
- """Similar to run_command, but where run_command is an
- os.system(command) analog, get_output_from_command is a `command`
- analog.
-
- Less error checking by design, though if we figure out how to
- do it without borking the output, great.
-
- TODO: binary mode? silent is kinda like that.
- TODO: since p.wait() can take a long time, optionally log something
- every N seconds?
- TODO: optionally only keep the first or last (N) line(s) of output?
- TODO: optionally only return the tmp_stdout_filename?
-
- ignore_errors=True is for the case where a command might produce standard
- error output, but you don't particularly care; setting to True will
- cause standard error to be logged at DEBUG rather than ERROR
-
- Args:
- command (str | list): command or list of commands to
- execute and log.
- cwd (str, optional): directory path from where to execute the
- command. Defaults to `None`.
- halt_on_failure (bool, optional): whether or not to redefine the
- log level as `FATAL` on error. Defaults to False.
- env (dict, optional): key-value of environment values to use to
- run the command. Defaults to None.
- silent (bool, optional): whether or not to output the stdout of
- executing the command. Defaults to False.
- log_level (str, optional): log level name to use on normal execution.
- Defaults to `INFO`.
- tmpfile_base_path (str, optional): base path of the file to which
- the output will be writen to. Defaults to 'tmpfile'.
- return_type (str, optional): if equal to 'output' then the complete
- output of the executed command is returned, otherwise the written
- filenames are returned. Defaults to 'output'.
- save_tmpfiles (bool, optional): whether or not to save the temporary
- files created from the command output. Defaults to False.
- throw_exception (bool, optional): whether or not to raise an
- exception if the return value of the command is not zero.
- Defaults to False.
- fatal_exit_code (int, optional): call self.fatal if the return value
- of the command match this value.
- ignore_errors (bool, optional): whether or not to change the log
- level to `ERROR` for the output of stderr. Defaults to False.
- success_codes (int, optional): numeric value to compare against
- the command return value.
-
- Returns:
- None: if the cwd is not a directory.
- None: on IOError.
- tuple: stdout and stderr filenames.
- str: stdout output.
- """
- if cwd:
- if not os.path.isdir(cwd):
- level = ERROR
- if halt_on_failure:
- level = FATAL
- self.log("Can't run command %s in non-existent directory %s!" %
- (command, cwd), level=level)
- return None
- self.info("Getting output from command: %s in %s" % (command, cwd))
- else:
- self.info("Getting output from command: %s" % command)
- if isinstance(command, list):
- self.info("Copy/paste: %s" % subprocess.list2cmdline(command))
- # This could potentially return something?
- tmp_stdout = None
- tmp_stderr = None
- tmp_stdout_filename = '%s_stdout' % tmpfile_base_path
- tmp_stderr_filename = '%s_stderr' % tmpfile_base_path
- if success_codes is None:
- success_codes = [0]
-
- # TODO probably some more elegant solution than 2 similar passes
- try:
- tmp_stdout = open(tmp_stdout_filename, 'w')
- except IOError:
- level = ERROR
- if halt_on_failure:
- level = FATAL
- self.log("Can't open %s for writing!" % tmp_stdout_filename +
- self.exception(), level=level)
- return None
- try:
- tmp_stderr = open(tmp_stderr_filename, 'w')
- except IOError:
- level = ERROR
- if halt_on_failure:
- level = FATAL
- self.log("Can't open %s for writing!" % tmp_stderr_filename +
- self.exception(), level=level)
- return None
- shell = True
- if isinstance(command, list):
- shell = False
- p = subprocess.Popen(command, shell=shell, stdout=tmp_stdout,
- cwd=cwd, stderr=tmp_stderr, env=env)
- # XXX: changed from self.debug to self.log due to this error:
- # TypeError: debug() takes exactly 1 argument (2 given)
- self.log("Temporary files: %s and %s" % (tmp_stdout_filename, tmp_stderr_filename), level=DEBUG)
- p.wait()
- tmp_stdout.close()
- tmp_stderr.close()
- return_level = DEBUG
- output = None
- if os.path.exists(tmp_stdout_filename) and os.path.getsize(tmp_stdout_filename):
- output = self.read_from_file(tmp_stdout_filename,
- verbose=False)
- if not silent:
- self.log("Output received:", level=log_level)
- output_lines = output.rstrip().splitlines()
- for line in output_lines:
- if not line or line.isspace():
- continue
- line = line.decode("utf-8")
- self.log(' %s' % line, level=log_level)
- output = '\n'.join(output_lines)
- if os.path.exists(tmp_stderr_filename) and os.path.getsize(tmp_stderr_filename):
- if not ignore_errors:
- return_level = ERROR
- self.log("Errors received:", level=return_level)
- errors = self.read_from_file(tmp_stderr_filename,
- verbose=False)
- for line in errors.rstrip().splitlines():
- if not line or line.isspace():
- continue
- line = line.decode("utf-8")
- self.log(' %s' % line, level=return_level)
- elif p.returncode not in success_codes and not ignore_errors:
- return_level = ERROR
- # Clean up.
- if not save_tmpfiles:
- self.rmtree(tmp_stderr_filename, log_level=DEBUG)
- self.rmtree(tmp_stdout_filename, log_level=DEBUG)
- if p.returncode and throw_exception:
- raise subprocess.CalledProcessError(p.returncode, command)
- self.log("Return code: %d" % p.returncode, level=return_level)
- if halt_on_failure and return_level == ERROR:
- self.return_code = fatal_exit_code
- self.fatal("Halting on failure while running %s" % command,
- exit_code=fatal_exit_code)
- # Hm, options on how to return this? I bet often we'll want
- # output_lines[0] with no newline.
- if return_type != 'output':
- return (tmp_stdout_filename, tmp_stderr_filename)
- else:
- return output
-
- def _touch_file(self, file_name, times=None, error_level=FATAL):
- """touch a file.
-
- Args:
- file_name (str): name of the file to touch.
- times (tuple, optional): 2-tuple as specified by `os.utime`_
- Defaults to None.
- error_level (str, optional): log level name in case of error.
- Defaults to `FATAL`.
-
- .. _`os.utime`:
- https://docs.python.org/3.4/library/os.html?highlight=os.utime#os.utime
- """
- self.info("Touching: %s" % file_name)
- try:
- os.utime(file_name, times)
- except OSError:
- try:
- open(file_name, 'w').close()
- except IOError as e:
- msg = "I/O error(%s): %s" % (e.errno, e.strerror)
- self.log(msg, error_level=error_level)
- os.utime(file_name, times)
-
- def unpack(self, filename, extract_to, extract_dirs=None,
- error_level=ERROR, fatal_exit_code=2, verbose=False):
- """The method allows to extract a file regardless of its extension.
-
- Args:
- filename (str): filename of the compressed file.
- extract_to (str): where to extract the compressed file.
- extract_dirs (list, optional): directories inside the archive file to extract.
- Defaults to `None`.
- fatal_exit_code (int, optional): call `self.fatal` if the return value
- of the command is not in `success_codes`. Defaults to 2.
- verbose (bool, optional): whether or not extracted content should be displayed.
- Defaults to False.
-
- Raises:
- IOError: on `filename` file not found.
-
- """
- if not os.path.isfile(filename):
- raise IOError('Could not find file to extract: %s' % filename)
-
- if zipfile.is_zipfile(filename):
- try:
- self.info('Using ZipFile to extract {} to {}'.format(filename, extract_to))
- with zipfile.ZipFile(filename) as bundle:
- for entry in self._filter_entries(bundle.namelist(), extract_dirs):
- if verbose:
- self.info(' %s' % entry)
- bundle.extract(entry, path=extract_to)
-
- # ZipFile doesn't preserve permissions during extraction:
- # http://bugs.python.org/issue15795
- fname = os.path.realpath(os.path.join(extract_to, entry))
- mode = bundle.getinfo(entry).external_attr >> 16 & 0x1FF
- # Only set permissions if attributes are available. Otherwise all
- # permissions will be removed eg. on Windows.
- if mode:
- os.chmod(fname, mode)
- except zipfile.BadZipfile as e:
- self.log('%s (%s)' % (e.message, filename),
- level=error_level, exit_code=fatal_exit_code)
-
- # Bug 1211882 - is_tarfile cannot be trusted for dmg files
- elif tarfile.is_tarfile(filename) and not filename.lower().endswith('.dmg'):
- try:
- self.info('Using TarFile to extract {} to {}'.format(filename, extract_to))
- with tarfile.open(filename) as bundle:
- for entry in self._filter_entries(bundle.getnames(), extract_dirs):
- if verbose:
- self.info(' %s' % entry)
- bundle.extract(entry, path=extract_to)
- except tarfile.TarError as e:
- self.log('%s (%s)' % (e.message, filename),
- level=error_level, exit_code=fatal_exit_code)
- else:
- self.log('No extraction method found for: %s' % filename,
- level=error_level, exit_code=fatal_exit_code)
-
- def is_taskcluster(self):
- """Returns boolean indicating if we're running in TaskCluster."""
- # This may need expanding in the future to work on
- return 'TASKCLUSTER_WORKER_TYPE' in os.environ
-
-
-def PreScriptRun(func):
- """Decorator for methods that will be called before script execution.
-
- Each method on a BaseScript having this decorator will be called at the
- beginning of BaseScript.run().
-
- The return value is ignored. Exceptions will abort execution.
- """
- func._pre_run_listener = True
- return func
-
-
-def PostScriptRun(func):
- """Decorator for methods that will be called after script execution.
-
- This is similar to PreScriptRun except it is called at the end of
- execution. The method will always be fired, even if execution fails.
- """
- func._post_run_listener = True
- return func
-
-
-def PreScriptAction(action=None):
- """Decorator for methods that will be called at the beginning of each action.
-
- Each method on a BaseScript having this decorator will be called during
- BaseScript.run() before an individual action is executed. The method will
- receive the action's name as an argument.
-
- If no values are passed to the decorator, it will be applied to every
- action. If a string is passed, the decorated function will only be called
- for the action of that name.
-
- The return value of the method is ignored. Exceptions will abort execution.
- """
- def _wrapped(func):
- func._pre_action_listener = action
- return func
-
- def _wrapped_none(func):
- func._pre_action_listener = None
- return func
-
- if type(action) == type(_wrapped):
- return _wrapped_none(action)
-
- return _wrapped
-
-
-def PostScriptAction(action=None):
- """Decorator for methods that will be called at the end of each action.
-
- This behaves similarly to PreScriptAction. It varies in that it is called
- after execution of the action.
-
- The decorated method will receive the action name as a positional argument.
- It will then receive the following named arguments:
-
- success - Bool indicating whether the action finished successfully.
-
- The decorated method will always be called, even if the action threw an
- exception.
-
- The return value is ignored.
- """
- def _wrapped(func):
- func._post_action_listener = action
- return func
-
- def _wrapped_none(func):
- func._post_action_listener = None
- return func
-
- if type(action) == type(_wrapped):
- return _wrapped_none(action)
-
- return _wrapped
-
-
-# BaseScript {{{1
-class BaseScript(ScriptMixin, LogMixin, object):
- def __init__(self, config_options=None, ConfigClass=BaseConfig,
- default_log_level="info", **kwargs):
- self._return_code = 0
- super(BaseScript, self).__init__()
-
- # Collect decorated methods. We simply iterate over the attributes of
- # the current class instance and look for signatures deposited by
- # the decorators.
- self._listeners = dict(
- pre_run=[],
- pre_action=[],
- post_action=[],
- post_run=[],
- )
- for k in dir(self):
- item = getattr(self, k)
-
- # We only decorate methods, so ignore other types.
- if not inspect.ismethod(item):
- continue
-
- if hasattr(item, '_pre_run_listener'):
- self._listeners['pre_run'].append(k)
-
- if hasattr(item, '_pre_action_listener'):
- self._listeners['pre_action'].append((
- k,
- item._pre_action_listener))
-
- if hasattr(item, '_post_action_listener'):
- self._listeners['post_action'].append((
- k,
- item._post_action_listener))
-
- if hasattr(item, '_post_run_listener'):
- self._listeners['post_run'].append(k)
-
- self.log_obj = None
- self.abs_dirs = None
- if config_options is None:
- config_options = []
- self.summary_list = []
- self.failures = []
- rw_config = ConfigClass(config_options=config_options, **kwargs)
- self.config = rw_config.get_read_only_config()
- self.actions = tuple(rw_config.actions)
- self.all_actions = tuple(rw_config.all_actions)
- self.env = None
- self.new_log_obj(default_log_level=default_log_level)
- self.script_obj = self
-
- # Indicate we're a source checkout if VCS directory is present at the
- # appropriate place. This code will break if this file is ever moved
- # to another directory.
- self.topsrcdir = None
-
- srcreldir = 'testing/mozharness/mozharness/base'
- here = os.path.normpath(os.path.dirname(__file__))
- if here.replace('\\', '/').endswith(srcreldir):
- topsrcdir = os.path.normpath(os.path.join(here, '..', '..',
- '..', '..'))
- hg_dir = os.path.join(topsrcdir, '.hg')
- git_dir = os.path.join(topsrcdir, '.git')
- if os.path.isdir(hg_dir) or os.path.isdir(git_dir):
- self.topsrcdir = topsrcdir
-
- # Set self.config to read-only.
- #
- # We can create intermediate config info programmatically from
- # this in a repeatable way, with logs; this is how we straddle the
- # ideal-but-not-user-friendly static config and the
- # easy-to-write-hard-to-debug writable config.
- #
- # To allow for other, script-specific configurations
- # (e.g., buildbot props json parsing), before locking,
- # call self._pre_config_lock(). If needed, this method can
- # alter self.config.
- self._pre_config_lock(rw_config)
- self._config_lock()
-
- self.info("Run as %s" % rw_config.command_line)
- if self.config.get("dump_config_hierarchy"):
- # we only wish to dump and display what self.config is made up of,
- # against the current script + args, without actually running any
- # actions
- self._dump_config_hierarchy(rw_config.all_cfg_files_and_dicts)
- if self.config.get("dump_config"):
- self.dump_config(exit_on_finish=True)
-
- def _dump_config_hierarchy(self, cfg_files):
- """ interpret each config file used.
-
- This will show which keys/values are being added or overwritten by
- other config files depending on their hierarchy (when they were added).
- """
- # go through each config_file. We will start with the lowest and
- # print its keys/values that are being used in self.config. If any
- # keys/values are present in a config file with a higher precedence,
- # ignore those.
- dirs = self.query_abs_dirs()
- cfg_files_dump_config = {} # we will dump this to file
- # keep track of keys that did not come from a config file
- keys_not_from_file = set(self.config.keys())
- if not cfg_files:
- cfg_files = []
- self.info("Total config files: %d" % (len(cfg_files)))
- if len(cfg_files):
- self.info("cfg files used from lowest precedence to highest:")
- for i, (target_file, target_dict) in enumerate(cfg_files):
- unique_keys = set(target_dict.keys())
- unique_dict = {}
- # iterate through the target_dicts remaining 'higher' cfg_files
- remaining_cfgs = cfg_files[slice(i + 1, len(cfg_files))]
- # where higher == more precedent
- for ii, (higher_file, higher_dict) in enumerate(remaining_cfgs):
- # now only keep keys/values that are not overwritten by a
- # higher config
- unique_keys = unique_keys.difference(set(higher_dict.keys()))
- # unique_dict we know now has only keys/values that are unique to
- # this config file.
- unique_dict = dict(
- (key, target_dict.get(key)) for key in unique_keys
- )
- cfg_files_dump_config[target_file] = unique_dict
- self.action_message("Config File %d: %s" % (i + 1, target_file))
- self.info(pprint.pformat(unique_dict))
- # let's also find out which keys/values from self.config are not
- # from each target config file dict
- keys_not_from_file = keys_not_from_file.difference(
- set(target_dict.keys())
- )
- not_from_file_dict = dict(
- (key, self.config.get(key)) for key in keys_not_from_file
- )
- cfg_files_dump_config["not_from_cfg_file"] = not_from_file_dict
- self.action_message("Not from any config file (default_config, "
- "cmd line options, etc)")
- self.info(pprint.pformat(not_from_file_dict))
-
- # finally, let's dump this output as JSON and exit early
- self.dump_config(
- os.path.join(dirs['abs_log_dir'], "localconfigfiles.json"),
- cfg_files_dump_config, console_output=False, exit_on_finish=True
- )
-
- def _pre_config_lock(self, rw_config):
- """This empty method can allow for config checking and manipulation
- before the config lock, when overridden in scripts.
- """
- pass
-
- def _config_lock(self):
- """After this point, the config is locked and should not be
- manipulated (based on mozharness.base.config.ReadOnlyDict)
- """
- self.config.lock()
-
- def _possibly_run_method(self, method_name, error_if_missing=False):
- """This is here for run().
- """
- if hasattr(self, method_name) and callable(getattr(self, method_name)):
- return getattr(self, method_name)()
- elif error_if_missing:
- self.error("No such method %s!" % method_name)
-
- @PostScriptRun
- def copy_logs_to_upload_dir(self):
- """Copies logs to the upload directory"""
- self.info("Copying logs to upload dir...")
- log_files = ['localconfig.json']
- for log_name in self.log_obj.log_files.keys():
- log_files.append(self.log_obj.log_files[log_name])
- dirs = self.query_abs_dirs()
- for log_file in log_files:
- self.copy_to_upload_dir(os.path.join(dirs['abs_log_dir'], log_file),
- dest=os.path.join('logs', log_file),
- short_desc='%s log' % log_name,
- long_desc='%s log' % log_name,
- max_backups=self.config.get("log_max_rotate", 0))
-
- def run_action(self, action):
- if action not in self.actions:
- self.action_message("Skipping %s step." % action)
- return
-
- method_name = action.replace("-", "_")
- self.action_message("Running %s step." % action)
-
- # An exception during a pre action listener should abort execution.
- for fn, target in self._listeners['pre_action']:
- if target is not None and target != action:
- continue
-
- try:
- self.info("Running pre-action listener: %s" % fn)
- method = getattr(self, fn)
- method(action)
- except Exception:
- self.error("Exception during pre-action for %s: %s" % (
- action, traceback.format_exc()))
-
- for fn, target in self._listeners['post_action']:
- if target is not None and target != action:
- continue
-
- try:
- self.info("Running post-action listener: %s" % fn)
- method = getattr(self, fn)
- method(action, success=False)
- except Exception:
- self.error("An additional exception occurred during "
- "post-action for %s: %s" % (action,
- traceback.format_exc()))
-
- self.fatal("Aborting due to exception in pre-action listener.")
-
- # We always run post action listeners, even if the main routine failed.
- success = False
- try:
- self.info("Running main action method: %s" % method_name)
- self._possibly_run_method("preflight_%s" % method_name)
- self._possibly_run_method(method_name, error_if_missing=True)
- self._possibly_run_method("postflight_%s" % method_name)
- success = True
- finally:
- post_success = True
- for fn, target in self._listeners['post_action']:
- if target is not None and target != action:
- continue
-
- try:
- self.info("Running post-action listener: %s" % fn)
- method = getattr(self, fn)
- method(action, success=success and self.return_code == 0)
- except Exception:
- post_success = False
- self.error("Exception during post-action for %s: %s" % (
- action, traceback.format_exc()))
-
- step_result = 'success' if success else 'failed'
- self.action_message("Finished %s step (%s)" % (action, step_result))
-
- if not post_success:
- self.fatal("Aborting due to failure in post-action listener.")
-
- def run(self):
- """Default run method.
- This is the "do everything" method, based on actions and all_actions.
-
- First run self.dump_config() if it exists.
- Second, go through the list of all_actions.
- If they're in the list of self.actions, try to run
- self.preflight_ACTION(), self.ACTION(), and self.postflight_ACTION().
-
- Preflight is sanity checking before doing anything time consuming or
- destructive.
-
- Postflight is quick testing for success after an action.
-
- """
- for fn in self._listeners['pre_run']:
- try:
- self.info("Running pre-run listener: %s" % fn)
- method = getattr(self, fn)
- method()
- except Exception:
- self.error("Exception during pre-run listener: %s" %
- traceback.format_exc())
-
- for fn in self._listeners['post_run']:
- try:
- method = getattr(self, fn)
- method()
- except Exception:
- self.error("An additional exception occurred during a "
- "post-run listener: %s" % traceback.format_exc())
-
- self.fatal("Aborting due to failure in pre-run listener.")
-
- self.dump_config()
- try:
- for action in self.all_actions:
- self.run_action(action)
- except Exception:
- self.fatal("Uncaught exception: %s" % traceback.format_exc())
- finally:
- post_success = True
- for fn in self._listeners['post_run']:
- try:
- self.info("Running post-run listener: %s" % fn)
- method = getattr(self, fn)
- method()
- except Exception:
- post_success = False
- self.error("Exception during post-run listener: %s" %
- traceback.format_exc())
-
- if not post_success:
- self.fatal("Aborting due to failure in post-run listener.")
- if self.config.get("copy_logs_post_run", True):
- self.copy_logs_to_upload_dir()
-
- return self.return_code
-
- def run_and_exit(self):
- """Runs the script and exits the current interpreter."""
- rc = self.run()
- if rc != 0:
- self.warning("returning nonzero exit status %d" % rc)
- sys.exit(rc)
-
- def clobber(self):
- """
- Delete the working directory
- """
- dirs = self.query_abs_dirs()
- self.rmtree(dirs['abs_work_dir'], error_level=FATAL)
-
- def query_abs_dirs(self):
- """We want to be able to determine where all the important things
- are. Absolute paths lend themselves well to this, though I wouldn't
- be surprised if this causes some issues somewhere.
-
- This should be overridden in any script that has additional dirs
- to query.
-
- The query_* methods tend to set self.VAR variables as their
- runtime cache.
- """
- if self.abs_dirs:
- return self.abs_dirs
- c = self.config
- dirs = {}
- dirs['base_work_dir'] = c['base_work_dir']
- dirs['abs_work_dir'] = os.path.join(c['base_work_dir'], c['work_dir'])
- dirs['abs_upload_dir'] = os.path.join(dirs['abs_work_dir'], 'upload')
- dirs['abs_log_dir'] = os.path.join(c['base_work_dir'], c.get('log_dir', 'logs'))
- self.abs_dirs = dirs
- return self.abs_dirs
-
- def dump_config(self, file_path=None, config=None,
- console_output=True, exit_on_finish=False):
- """Dump self.config to localconfig.json
- """
- config = config or self.config
- dirs = self.query_abs_dirs()
- if not file_path:
- file_path = os.path.join(dirs['abs_log_dir'], "localconfig.json")
- self.info("Dumping config to %s." % file_path)
- self.mkdir_p(os.path.dirname(file_path))
- json_config = json.dumps(config, sort_keys=True, indent=4)
- fh = codecs.open(file_path, encoding='utf-8', mode='w+')
- fh.write(json_config)
- fh.close()
- if console_output:
- self.info(pprint.pformat(config))
- if exit_on_finish:
- sys.exit()
-
- # logging {{{2
- def new_log_obj(self, default_log_level="info"):
- c = self.config
- log_dir = os.path.join(c['base_work_dir'], c.get('log_dir', 'logs'))
- log_config = {
- "logger_name": 'Simple',
- "log_name": 'log',
- "log_dir": log_dir,
- "log_level": default_log_level,
- "log_format": '%(asctime)s %(levelname)8s - %(message)s',
- "log_to_console": True,
- "append_to_log": False,
- }
- log_type = self.config.get("log_type", "multi")
- for key in log_config.keys():
- value = self.config.get(key, None)
- if value is not None:
- log_config[key] = value
- if log_type == "multi":
- self.log_obj = MultiFileLogger(**log_config)
- else:
- self.log_obj = SimpleFileLogger(**log_config)
-
- def action_message(self, message):
- self.info("[mozharness: %sZ] %s" % (
- datetime.datetime.utcnow().isoformat(' '), message))
-
- def summary(self):
- """Print out all the summary lines added via add_summary()
- throughout the script.
-
- I'd like to revisit how to do this in a prettier fashion.
- """
- self.action_message("%s summary:" % self.__class__.__name__)
- if self.summary_list:
- for item in self.summary_list:
- try:
- self.log(item['message'], level=item['level'])
- except ValueError:
- """log is closed; print as a default. Ran into this
- when calling from __del__()"""
- print "### Log is closed! (%s)" % item['message']
-
- def add_summary(self, message, level=INFO):
- self.summary_list.append({'message': message, 'level': level})
- # TODO write to a summary-only log?
- # Summaries need a lot more love.
- self.log(message, level=level)
-
- def add_failure(self, key, message="%(key)s failed.", level=ERROR,
- increment_return_code=True):
- if key not in self.failures:
- self.failures.append(key)
- self.add_summary(message % {'key': key}, level=level)
- if increment_return_code:
- self.return_code += 1
-
- def query_failure(self, key):
- return key in self.failures
-
- def summarize_success_count(self, success_count, total_count,
- message="%d of %d successful.",
- level=None):
- if level is None:
- level = INFO
- if success_count < total_count:
- level = ERROR
- self.add_summary(message % (success_count, total_count),
- level=level)
-
- def copy_to_upload_dir(self, target, dest=None, short_desc="unknown",
- long_desc="unknown", log_level=DEBUG,
- error_level=ERROR, max_backups=None,
- compress=False, upload_dir=None):
- """Copy target file to upload_dir/dest.
-
- Potentially update a manifest in the future if we go that route.
-
- Currently only copies a single file; would be nice to allow for
- recursive copying; that would probably done by creating a helper
- _copy_file_to_upload_dir().
-
- short_desc and long_desc are placeholders for if/when we add
- upload_dir manifests.
- """
- dest_filename_given = dest is not None
- if upload_dir is None:
- upload_dir = self.query_abs_dirs()['abs_upload_dir']
- if dest is None:
- dest = os.path.basename(target)
- if dest.endswith('/'):
- dest_file = os.path.basename(target)
- dest_dir = os.path.join(upload_dir, dest)
- dest_filename_given = False
- else:
- dest_file = os.path.basename(dest)
- dest_dir = os.path.join(upload_dir, os.path.dirname(dest))
- if compress and not dest_filename_given:
- dest_file += ".gz"
- dest = os.path.join(dest_dir, dest_file)
- if not os.path.exists(target):
- self.log("%s doesn't exist!" % target, level=error_level)
- return None
- self.mkdir_p(dest_dir)
- if os.path.exists(dest):
- if os.path.isdir(dest):
- self.log("%s exists and is a directory!" % dest, level=error_level)
- return -1
- if max_backups:
- # Probably a better way to do this
- oldest_backup = 0
- backup_regex = re.compile("^%s\.(\d+)$" % dest_file)
- for filename in os.listdir(dest_dir):
- r = backup_regex.match(filename)
- if r and int(r.groups()[0]) > oldest_backup:
- oldest_backup = int(r.groups()[0])
- for backup_num in range(oldest_backup, 0, -1):
- # TODO more error checking?
- if backup_num >= max_backups:
- self.rmtree(os.path.join(dest_dir, "%s.%d" % (dest_file, backup_num)),
- log_level=log_level)
- else:
- self.move(os.path.join(dest_dir, "%s.%d" % (dest_file, backup_num)),
- os.path.join(dest_dir, "%s.%d" % (dest_file, backup_num + 1)),
- log_level=log_level)
- if self.move(dest, "%s.1" % dest, log_level=log_level):
- self.log("Unable to move %s!" % dest, level=error_level)
- return -1
- else:
- if self.rmtree(dest, log_level=log_level):
- self.log("Unable to remove %s!" % dest, level=error_level)
- return -1
- self.copyfile(target, dest, log_level=log_level, compress=compress)
- if os.path.exists(dest):
- return dest
- else:
- self.log("%s doesn't exist after copy!" % dest, level=error_level)
- return None
-
- def get_hash_for_file(self, file_path, hash_type="sha512"):
- bs = 65536
- hasher = hashlib.new(hash_type)
- with open(file_path, 'rb') as fh:
- buf = fh.read(bs)
- while len(buf) > 0:
- hasher.update(buf)
- buf = fh.read(bs)
- return hasher.hexdigest()
-
- @property
- def return_code(self):
- return self._return_code
-
- @return_code.setter
- def return_code(self, code):
- old_return_code, self._return_code = self._return_code, code
- if old_return_code != code:
- self.warning("setting return code to %d" % code)
-
-# __main__ {{{1
-if __name__ == '__main__':
- """ Useless comparison, due to the `pass` keyword on its body"""
- pass
diff --git a/testing/mozharness/mozharness/base/signing.py b/testing/mozharness/mozharness/base/signing.py
deleted file mode 100755
index d0fe05da2..000000000
--- a/testing/mozharness/mozharness/base/signing.py
+++ /dev/null
@@ -1,164 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Generic signing methods.
-"""
-
-import getpass
-import hashlib
-import os
-import re
-import subprocess
-
-from mozharness.base.errors import JarsignerErrorList, ZipErrorList, ZipalignErrorList
-from mozharness.base.log import OutputParser, IGNORE, DEBUG, INFO, ERROR, FATAL
-
-UnsignApkErrorList = [{
- 'regex': re.compile(r'''zip warning: name not matched: '?META-INF/'''),
- 'level': INFO,
- 'explanation': r'''This apk is already unsigned.''',
-}, {
- 'substr': r'''zip error: Nothing to do!''',
- 'level': IGNORE,
-}] + ZipErrorList
-
-TestJarsignerErrorList = [{
- "substr": "jarsigner: unable to open jar file:",
- "level": IGNORE,
-}] + JarsignerErrorList
-
-
-# BaseSigningMixin {{{1
-class BaseSigningMixin(object):
- """Generic signing helper methods.
- """
- def query_filesize(self, file_path):
- self.info("Determining filesize for %s" % file_path)
- length = os.path.getsize(file_path)
- self.info(" %s" % str(length))
- return length
-
- # TODO this should be parallelized with the to-be-written BaseHelper!
- def query_sha512sum(self, file_path):
- self.info("Determining sha512sum for %s" % file_path)
- m = hashlib.sha512()
- contents = self.read_from_file(file_path, verbose=False,
- open_mode='rb')
- m.update(contents)
- sha512 = m.hexdigest()
- self.info(" %s" % sha512)
- return sha512
-
-
-# AndroidSigningMixin {{{1
-class AndroidSigningMixin(object):
- """
- Generic Android apk signing methods.
-
- Dependent on BaseScript.
- """
- # TODO port build/tools/release/signing/verify-android-signature.sh here
-
- key_passphrase = os.environ.get('android_keypass')
- store_passphrase = os.environ.get('android_storepass')
-
- def passphrase(self):
- if not self.store_passphrase:
- self.store_passphrase = getpass.getpass("Store passphrase: ")
- if not self.key_passphrase:
- self.key_passphrase = getpass.getpass("Key passphrase: ")
-
- def _verify_passphrases(self, keystore, key_alias, error_level=FATAL):
- self.info("Verifying passphrases...")
- status = self.sign_apk("NOTAREALAPK", keystore,
- self.store_passphrase, self.key_passphrase,
- key_alias, remove_signature=False,
- log_level=DEBUG, error_level=DEBUG,
- error_list=TestJarsignerErrorList)
- if status == 0:
- self.info("Passphrases are good.")
- elif status < 0:
- self.log("Encountered errors while trying to sign!",
- level=error_level)
- else:
- self.log("Unable to verify passphrases!",
- level=error_level)
- return status
-
- def verify_passphrases(self):
- c = self.config
- self._verify_passphrases(c['keystore'], c['key_alias'])
-
- def postflight_passphrase(self):
- self.verify_passphrases()
-
- def sign_apk(self, apk, keystore, storepass, keypass, key_alias,
- remove_signature=True, error_list=None,
- log_level=INFO, error_level=ERROR):
- """
- Signs an apk with jarsigner.
- """
- jarsigner = self.query_exe('jarsigner')
- if remove_signature:
- status = self.unsign_apk(apk)
- if status:
- self.error("Can't remove signature in %s!" % apk)
- return -1
- if error_list is None:
- error_list = JarsignerErrorList[:]
- # This needs to run silently, so no run_command() or
- # get_output_from_command() (though I could add a
- # suppress_command_echo=True or something?)
- self.log("(signing %s)" % apk, level=log_level)
- try:
- p = subprocess.Popen([jarsigner, "-keystore", keystore,
- "-storepass", storepass,
- "-keypass", keypass,
- apk, key_alias],
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
- except OSError:
- self.exception("Error while signing %s (missing %s?):" % (apk, jarsigner))
- return -2
- except ValueError:
- self.exception("Popen called with invalid arguments during signing?")
- return -3
- parser = OutputParser(config=self.config, log_obj=self.log_obj,
- error_list=error_list)
- loop = True
- while loop:
- if p.poll() is not None:
- """Avoid losing the final lines of the log?"""
- loop = False
- for line in p.stdout:
- parser.add_lines(line)
- if parser.num_errors:
- self.log("(failure)", level=error_level)
- else:
- self.log("(success)", level=log_level)
- return parser.num_errors
-
- def unsign_apk(self, apk, **kwargs):
- zip_bin = self.query_exe("zip")
- return self.run_command([zip_bin, apk, '-d', 'META-INF/*'],
- error_list=UnsignApkErrorList,
- success_codes=[0, 12],
- return_type='num_errors', **kwargs)
-
- def align_apk(self, unaligned_apk, aligned_apk, error_level=ERROR):
- """
- Zipalign apk.
- Returns None on success, not None on failure.
- """
- dirs = self.query_abs_dirs()
- zipalign = self.query_exe("zipalign")
- if self.run_command([zipalign, '-f', '4',
- unaligned_apk, aligned_apk],
- return_type='num_errors',
- cwd=dirs['abs_work_dir'],
- error_list=ZipalignErrorList):
- self.log("Unable to zipalign %s to %s!" % (unaligned_apk, aligned_apk), level=error_level)
- return -1
diff --git a/testing/mozharness/mozharness/base/transfer.py b/testing/mozharness/mozharness/base/transfer.py
deleted file mode 100755
index 014c665a1..000000000
--- a/testing/mozharness/mozharness/base/transfer.py
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Generic ways to upload + download files.
-"""
-
-import os
-import pprint
-import urllib2
-try:
- import simplejson as json
- assert json
-except ImportError:
- import json
-
-from mozharness.base.errors import SSHErrorList
-from mozharness.base.log import DEBUG, ERROR
-
-
-# TransferMixin {{{1
-class TransferMixin(object):
- """
- Generic transfer methods.
-
- Dependent on BaseScript.
- """
- def rsync_upload_directory(self, local_path, ssh_key, ssh_user,
- remote_host, remote_path,
- rsync_options=None,
- error_level=ERROR,
- create_remote_directory=True,
- ):
- """
- Create a remote directory and upload the contents of
- a local directory to it via rsync+ssh.
-
- Returns:
- None: on success
- -1: if local_path is not a directory
- -2: if the remote_directory cannot be created
- (it only makes sense if create_remote_directory is True)
- -3: rsync fails to copy to the remote directory
- """
- dirs = self.query_abs_dirs()
- self.info("Uploading the contents of %s to %s:%s" % (local_path, remote_host, remote_path))
- rsync = self.query_exe("rsync")
- ssh = self.query_exe("ssh")
- if rsync_options is None:
- rsync_options = ['-azv']
- if not os.path.isdir(local_path):
- self.log("%s isn't a directory!" % local_path,
- level=ERROR)
- return -1
- if create_remote_directory:
- mkdir_error_list = [{
- 'substr': r'''exists but is not a directory''',
- 'level': ERROR
- }] + SSHErrorList
- if self.run_command([ssh, '-oIdentityFile=%s' % ssh_key,
- '%s@%s' % (ssh_user, remote_host),
- 'mkdir', '-p', remote_path],
- cwd=dirs['abs_work_dir'],
- return_type='num_errors',
- error_list=mkdir_error_list):
- self.log("Unable to create remote directory %s:%s!" % (remote_host, remote_path), level=error_level)
- return -2
- if self.run_command([rsync, '-e',
- '%s -oIdentityFile=%s' % (ssh, ssh_key)
- ] + rsync_options + ['.',
- '%s@%s:%s/' % (ssh_user, remote_host, remote_path)],
- cwd=local_path,
- return_type='num_errors',
- error_list=SSHErrorList):
- self.log("Unable to rsync %s to %s:%s!" % (local_path, remote_host, remote_path), level=error_level)
- return -3
-
- def rsync_download_directory(self, ssh_key, ssh_user, remote_host,
- remote_path, local_path,
- rsync_options=None,
- error_level=ERROR,
- ):
- """
- rsync+ssh the content of a remote directory to local_path
-
- Returns:
- None: on success
- -1: if local_path is not a directory
- -3: rsync fails to download from the remote directory
- """
- self.info("Downloading the contents of %s:%s to %s" % (remote_host, remote_path, local_path))
- rsync = self.query_exe("rsync")
- ssh = self.query_exe("ssh")
- if rsync_options is None:
- rsync_options = ['-azv']
- if not os.path.isdir(local_path):
- self.log("%s isn't a directory!" % local_path,
- level=error_level)
- return -1
- if self.run_command([rsync, '-e',
- '%s -oIdentityFile=%s' % (ssh, ssh_key)
- ] + rsync_options + [
- '%s@%s:%s/' % (ssh_user, remote_host, remote_path),
- '.'],
- cwd=local_path,
- return_type='num_errors',
- error_list=SSHErrorList):
- self.log("Unable to rsync %s:%s to %s!" % (remote_host, remote_path, local_path), level=error_level)
- return -3
-
- def load_json_from_url(self, url, timeout=30, log_level=DEBUG):
- self.log("Attempting to download %s; timeout=%i" % (url, timeout),
- level=log_level)
- try:
- r = urllib2.urlopen(url, timeout=timeout)
- j = json.load(r)
- self.log(pprint.pformat(j), level=log_level)
- except:
- self.exception(message="Unable to download %s!" % url)
- raise
- return j
diff --git a/testing/mozharness/mozharness/base/vcs/__init__.py b/testing/mozharness/mozharness/base/vcs/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/mozharness/base/vcs/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/mozharness/base/vcs/gittool.py b/testing/mozharness/mozharness/base/vcs/gittool.py
deleted file mode 100644
index d6c609ea0..000000000
--- a/testing/mozharness/mozharness/base/vcs/gittool.py
+++ /dev/null
@@ -1,95 +0,0 @@
-import os
-import re
-import urlparse
-
-from mozharness.base.script import ScriptMixin
-from mozharness.base.log import LogMixin, OutputParser
-from mozharness.base.errors import GitErrorList, VCSException
-
-
-class GittoolParser(OutputParser):
- """
- A class that extends OutputParser such that it can find the "Got revision"
- string from gittool.py output
- """
-
- got_revision_exp = re.compile(r'Got revision (\w+)')
- got_revision = None
-
- def parse_single_line(self, line):
- m = self.got_revision_exp.match(line)
- if m:
- self.got_revision = m.group(1)
- super(GittoolParser, self).parse_single_line(line)
-
-
-class GittoolVCS(ScriptMixin, LogMixin):
- def __init__(self, log_obj=None, config=None, vcs_config=None,
- script_obj=None):
- super(GittoolVCS, self).__init__()
-
- self.log_obj = log_obj
- self.script_obj = script_obj
- if config:
- self.config = config
- else:
- self.config = {}
- # vcs_config = {
- # repo: repository,
- # branch: branch,
- # revision: revision,
- # ssh_username: ssh_username,
- # ssh_key: ssh_key,
- # }
- self.vcs_config = vcs_config
- self.gittool = self.query_exe('gittool.py', return_type='list')
-
- def ensure_repo_and_revision(self):
- """Makes sure that `dest` is has `revision` or `branch` checked out
- from `repo`.
-
- Do what it takes to make that happen, including possibly clobbering
- dest.
- """
- c = self.vcs_config
- for conf_item in ('dest', 'repo'):
- assert self.vcs_config[conf_item]
- dest = os.path.abspath(c['dest'])
- repo = c['repo']
- revision = c.get('revision')
- branch = c.get('branch')
- clean = c.get('clean')
- share_base = c.get('vcs_share_base', os.environ.get("GIT_SHARE_BASE_DIR", None))
- env = {'PATH': os.environ.get('PATH')}
- env.update(c.get('env', {}))
- if self._is_windows():
- # git.exe is not in the PATH by default
- env['PATH'] = '%s;C:/mozilla-build/Git/bin' % env['PATH']
- # SYSTEMROOT is needed for 'import random'
- if 'SYSTEMROOT' not in env:
- env['SYSTEMROOT'] = os.environ.get('SYSTEMROOT')
- if share_base is not None:
- env['GIT_SHARE_BASE_DIR'] = share_base
-
- cmd = self.gittool[:]
- if branch:
- cmd.extend(['-b', branch])
- if revision:
- cmd.extend(['-r', revision])
- if clean:
- cmd.append('--clean')
-
- for base_mirror_url in self.config.get('gittool_base_mirror_urls', self.config.get('vcs_base_mirror_urls', [])):
- bits = urlparse.urlparse(repo)
- mirror_url = urlparse.urljoin(base_mirror_url, bits.path)
- cmd.extend(['--mirror', mirror_url])
-
- cmd.extend([repo, dest])
- parser = GittoolParser(config=self.config, log_obj=self.log_obj,
- error_list=GitErrorList)
- retval = self.run_command(cmd, error_list=GitErrorList, env=env, output_parser=parser)
-
- if retval != 0:
- raise VCSException("Unable to checkout")
-
- return parser.got_revision
diff --git a/testing/mozharness/mozharness/base/vcs/mercurial.py b/testing/mozharness/mozharness/base/vcs/mercurial.py
deleted file mode 100755
index 71e5e3ea0..000000000
--- a/testing/mozharness/mozharness/base/vcs/mercurial.py
+++ /dev/null
@@ -1,497 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Mercurial VCS support.
-"""
-
-import os
-import re
-import subprocess
-from collections import namedtuple
-from urlparse import urlsplit
-import hashlib
-
-import sys
-sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(sys.path[0]))))
-
-import mozharness
-from mozharness.base.errors import HgErrorList, VCSException
-from mozharness.base.log import LogMixin, OutputParser
-from mozharness.base.script import ScriptMixin
-from mozharness.base.transfer import TransferMixin
-
-external_tools_path = os.path.join(
- os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
- 'external_tools',
-)
-
-
-HG_OPTIONS = ['--config', 'ui.merge=internal:merge']
-
-# MercurialVCS {{{1
-# TODO Make the remaining functions more mozharness-friendly.
-# TODO Add the various tag functionality that are currently in
-# build/tools/scripts to MercurialVCS -- generic tagging logic belongs here.
-REVISION, BRANCH = 0, 1
-
-
-class RepositoryUpdateRevisionParser(OutputParser):
- """Parse `hg pull` output for "repository unrelated" errors."""
- revision = None
- RE_UPDATED = re.compile('^updated to ([a-f0-9]{40})$')
-
- def parse_single_line(self, line):
- m = self.RE_UPDATED.match(line)
- if m:
- self.revision = m.group(1)
-
- return super(RepositoryUpdateRevisionParser, self).parse_single_line(line)
-
-
-def make_hg_url(hg_host, repo_path, protocol='http', revision=None,
- filename=None):
- """Helper function.
-
- Construct a valid hg url from a base hg url (hg.mozilla.org),
- repo_path, revision and possible filename
- """
- base = '%s://%s' % (protocol, hg_host)
- repo = '/'.join(p.strip('/') for p in [base, repo_path])
- if not filename:
- if not revision:
- return repo
- else:
- return '/'.join([p.strip('/') for p in [repo, 'rev', revision]])
- else:
- assert revision
- return '/'.join([p.strip('/') for p in [repo, 'raw-file', revision, filename]])
-
-
-class MercurialVCS(ScriptMixin, LogMixin, TransferMixin):
- # For the most part, scripts import mercurial, update
- # tag-release.py imports
- # apply_and_push, update, get_revision, out, BRANCH, REVISION,
- # get_branches, cleanOutgoingRevs
-
- def __init__(self, log_obj=None, config=None, vcs_config=None,
- script_obj=None):
- super(MercurialVCS, self).__init__()
- self.can_share = None
- self.log_obj = log_obj
- self.script_obj = script_obj
- if config:
- self.config = config
- else:
- self.config = {}
- # vcs_config = {
- # hg_host: hg_host,
- # repo: repository,
- # branch: branch,
- # revision: revision,
- # ssh_username: ssh_username,
- # ssh_key: ssh_key,
- # }
- self.vcs_config = vcs_config or {}
- self.hg = self.query_exe("hg", return_type="list") + HG_OPTIONS
-
- def _make_absolute(self, repo):
- if repo.startswith("file://"):
- path = repo[len("file://"):]
- repo = "file://%s" % os.path.abspath(path)
- elif "://" not in repo:
- repo = os.path.abspath(repo)
- return repo
-
- def get_repo_name(self, repo):
- return repo.rstrip('/').split('/')[-1]
-
- def get_repo_path(self, repo):
- repo = self._make_absolute(repo)
- if repo.startswith("/"):
- return repo.lstrip("/")
- else:
- return urlsplit(repo).path.lstrip("/")
-
- def get_revision_from_path(self, path):
- """Returns which revision directory `path` currently has checked out."""
- return self.get_output_from_command(
- self.hg + ['parent', '--template', '{node}'], cwd=path
- )
-
- def get_branch_from_path(self, path):
- branch = self.get_output_from_command(self.hg + ['branch'], cwd=path)
- return str(branch).strip()
-
- def get_branches_from_path(self, path):
- branches = []
- for line in self.get_output_from_command(self.hg + ['branches', '-c'],
- cwd=path).splitlines():
- branches.append(line.split()[0])
- return branches
-
- def hg_ver(self):
- """Returns the current version of hg, as a tuple of
- (major, minor, build)"""
- ver_string = self.get_output_from_command(self.hg + ['-q', 'version'])
- match = re.search("\(version ([0-9.]+)\)", ver_string)
- if match:
- bits = match.group(1).split(".")
- if len(bits) < 3:
- bits += (0,)
- ver = tuple(int(b) for b in bits)
- else:
- ver = (0, 0, 0)
- self.debug("Running hg version %s" % str(ver))
- return ver
-
- def update(self, dest, branch=None, revision=None):
- """Updates working copy `dest` to `branch` or `revision`.
- If revision is set, branch will be ignored.
- If neither is set then the working copy will be updated to the
- latest revision on the current branch. Local changes will be
- discarded.
- """
- # If we have a revision, switch to that
- msg = "Updating %s" % dest
- if branch:
- msg += " to branch %s" % branch
- if revision:
- msg += " revision %s" % revision
- self.info("%s." % msg)
- if revision is not None:
- cmd = self.hg + ['update', '-C', '-r', revision]
- if self.run_command(cmd, cwd=dest, error_list=HgErrorList):
- raise VCSException("Unable to update %s to %s!" % (dest, revision))
- else:
- # Check & switch branch
- local_branch = self.get_branch_from_path(dest)
-
- cmd = self.hg + ['update', '-C']
-
- # If this is different, checkout the other branch
- if branch and branch != local_branch:
- cmd.append(branch)
-
- if self.run_command(cmd, cwd=dest, error_list=HgErrorList):
- raise VCSException("Unable to update %s!" % dest)
- return self.get_revision_from_path(dest)
-
- def clone(self, repo, dest, branch=None, revision=None, update_dest=True):
- """Clones hg repo and places it at `dest`, replacing whatever else
- is there. The working copy will be empty.
-
- If `revision` is set, only the specified revision and its ancestors
- will be cloned. If revision is set, branch is ignored.
-
- If `update_dest` is set, then `dest` will be updated to `revision`
- if set, otherwise to `branch`, otherwise to the head of default.
- """
- msg = "Cloning %s to %s" % (repo, dest)
- if branch:
- msg += " on branch %s" % branch
- if revision:
- msg += " to revision %s" % revision
- self.info("%s." % msg)
- parent_dest = os.path.dirname(dest)
- if parent_dest and not os.path.exists(parent_dest):
- self.mkdir_p(parent_dest)
- if os.path.exists(dest):
- self.info("Removing %s before clone." % dest)
- self.rmtree(dest)
-
- cmd = self.hg + ['clone']
- if not update_dest:
- cmd.append('-U')
-
- if revision:
- cmd.extend(['-r', revision])
- elif branch:
- # hg >= 1.6 supports -b branch for cloning
- ver = self.hg_ver()
- if ver >= (1, 6, 0):
- cmd.extend(['-b', branch])
-
- cmd.extend([repo, dest])
- output_timeout = self.config.get("vcs_output_timeout",
- self.vcs_config.get("output_timeout"))
- if self.run_command(cmd, error_list=HgErrorList,
- output_timeout=output_timeout) != 0:
- raise VCSException("Unable to clone %s to %s!" % (repo, dest))
-
- if update_dest:
- return self.update(dest, branch, revision)
-
- def common_args(self, revision=None, branch=None, ssh_username=None,
- ssh_key=None):
- """Fill in common hg arguments, encapsulating logic checks that
- depend on mercurial versions and provided arguments
- """
- args = []
- if ssh_username or ssh_key:
- opt = ['-e', 'ssh']
- if ssh_username:
- opt[1] += ' -l %s' % ssh_username
- if ssh_key:
- opt[1] += ' -i %s' % ssh_key
- args.extend(opt)
- if revision:
- args.extend(['-r', revision])
- elif branch:
- if self.hg_ver() >= (1, 6, 0):
- args.extend(['-b', branch])
- return args
-
- def pull(self, repo, dest, update_dest=True, **kwargs):
- """Pulls changes from hg repo and places it in `dest`.
-
- If `revision` is set, only the specified revision and its ancestors
- will be pulled.
-
- If `update_dest` is set, then `dest` will be updated to `revision`
- if set, otherwise to `branch`, otherwise to the head of default.
- """
- msg = "Pulling %s to %s" % (repo, dest)
- if update_dest:
- msg += " and updating"
- self.info("%s." % msg)
- if not os.path.exists(dest):
- # Error or clone?
- # If error, should we have a halt_on_error=False above?
- self.error("Can't hg pull in nonexistent directory %s." % dest)
- return -1
- # Convert repo to an absolute path if it's a local repository
- repo = self._make_absolute(repo)
- cmd = self.hg + ['pull']
- cmd.extend(self.common_args(**kwargs))
- cmd.append(repo)
- output_timeout = self.config.get("vcs_output_timeout",
- self.vcs_config.get("output_timeout"))
- if self.run_command(cmd, cwd=dest, error_list=HgErrorList,
- output_timeout=output_timeout) != 0:
- raise VCSException("Can't pull in %s!" % dest)
-
- if update_dest:
- branch = self.vcs_config.get('branch')
- revision = self.vcs_config.get('revision')
- return self.update(dest, branch=branch, revision=revision)
-
- # Defines the places of attributes in the tuples returned by `out'
-
- def out(self, src, remote, **kwargs):
- """Check for outgoing changesets present in a repo"""
- self.info("Checking for outgoing changesets from %s to %s." % (src, remote))
- cmd = self.hg + ['-q', 'out', '--template', '{node} {branches}\n']
- cmd.extend(self.common_args(**kwargs))
- cmd.append(remote)
- if os.path.exists(src):
- try:
- revs = []
- for line in self.get_output_from_command(cmd, cwd=src, throw_exception=True).rstrip().split("\n"):
- try:
- rev, branch = line.split()
- # Mercurial displays no branch at all if the revision
- # is on "default"
- except ValueError:
- rev = line.rstrip()
- branch = "default"
- revs.append((rev, branch))
- return revs
- except subprocess.CalledProcessError, inst:
- # In some situations, some versions of Mercurial return "1"
- # if no changes are found, so we need to ignore this return
- # code
- if inst.returncode == 1:
- return []
- raise
-
- def push(self, src, remote, push_new_branches=True, **kwargs):
- # This doesn't appear to work with hg_ver < (1, 6, 0).
- # Error out, or let you try?
- self.info("Pushing new changes from %s to %s." % (src, remote))
- cmd = self.hg + ['push']
- cmd.extend(self.common_args(**kwargs))
- if push_new_branches and self.hg_ver() >= (1, 6, 0):
- cmd.append('--new-branch')
- cmd.append(remote)
- status = self.run_command(cmd, cwd=src, error_list=HgErrorList, success_codes=(0, 1),
- return_type="num_errors")
- if status:
- raise VCSException("Can't push %s to %s!" % (src, remote))
- return status
-
- @property
- def robustcheckout_path(self):
- """Path to the robustcheckout extension."""
- ext = os.path.join(external_tools_path, 'robustcheckout.py')
- if os.path.exists(ext):
- return ext
-
- def ensure_repo_and_revision(self):
- """Makes sure that `dest` is has `revision` or `branch` checked out
- from `repo`.
-
- Do what it takes to make that happen, including possibly clobbering
- dest.
- """
- c = self.vcs_config
- dest = c['dest']
- repo_url = c['repo']
- rev = c.get('revision')
- branch = c.get('branch')
- purge = c.get('clone_with_purge', False)
- upstream = c.get('clone_upstream_url')
-
- # The API here is kind of bad because we're relying on state in
- # self.vcs_config instead of passing arguments. This confuses
- # scripts that have multiple repos. This includes the clone_tools()
- # step :(
-
- if not rev and not branch:
- self.warning('did not specify revision or branch; assuming "default"')
- branch = 'default'
-
- share_base = c.get('vcs_share_base') or os.environ.get('HG_SHARE_BASE_DIR')
- if share_base and c.get('use_vcs_unique_share'):
- # Bug 1277041 - update migration scripts to support robustcheckout
- # fake a share but don't really share
- share_base = os.path.join(share_base, hashlib.md5(dest).hexdigest())
-
- # We require shared storage is configured because it guarantees we
- # only have 1 local copy of logical repo stores.
- if not share_base:
- raise VCSException('vcs share base not defined; '
- 'refusing to operate sub-optimally')
-
- if not self.robustcheckout_path:
- raise VCSException('could not find the robustcheckout Mercurial extension')
-
- # Log HG version and install info to aid debugging.
- self.run_command(self.hg + ['--version'])
- self.run_command(self.hg + ['debuginstall'])
-
- args = self.hg + [
- '--config', 'extensions.robustcheckout=%s' % self.robustcheckout_path,
- 'robustcheckout', repo_url, dest, '--sharebase', share_base,
- ]
- if purge:
- args.append('--purge')
- if upstream:
- args.extend(['--upstream', upstream])
-
- if rev:
- args.extend(['--revision', rev])
- if branch:
- args.extend(['--branch', branch])
-
- parser = RepositoryUpdateRevisionParser(config=self.config,
- log_obj=self.log_obj)
- if self.run_command(args, output_parser=parser):
- raise VCSException('repo checkout failed!')
-
- if not parser.revision:
- raise VCSException('could not identify revision updated to')
-
- return parser.revision
-
- def apply_and_push(self, localrepo, remote, changer, max_attempts=10,
- ssh_username=None, ssh_key=None):
- """This function calls `changer' to make changes to the repo, and
- tries its hardest to get them to the origin repo. `changer' must be
- a callable object that receives two arguments: the directory of the
- local repository, and the attempt number. This function will push
- ALL changesets missing from remote.
- """
- self.info("Applying and pushing local changes from %s to %s." % (localrepo, remote))
- assert callable(changer)
- branch = self.get_branch_from_path(localrepo)
- changer(localrepo, 1)
- for n in range(1, max_attempts + 1):
- try:
- new_revs = self.out(src=localrepo, remote=remote,
- ssh_username=ssh_username,
- ssh_key=ssh_key)
- if len(new_revs) < 1:
- raise VCSException("No revs to push")
- self.push(src=localrepo, remote=remote,
- ssh_username=ssh_username,
- ssh_key=ssh_key)
- return
- except VCSException, e:
- self.debug("Hit error when trying to push: %s" % str(e))
- if n == max_attempts:
- self.debug("Tried %d times, giving up" % max_attempts)
- for r in reversed(new_revs):
- self.run_command(self.hg + ['strip', '-n', r[REVISION]],
- cwd=localrepo, error_list=HgErrorList)
- raise VCSException("Failed to push")
- self.pull(remote, localrepo, update_dest=False,
- ssh_username=ssh_username, ssh_key=ssh_key)
- # After we successfully rebase or strip away heads the push
- # is is attempted again at the start of the loop
- try:
- self.run_command(self.hg + ['rebase'], cwd=localrepo,
- error_list=HgErrorList,
- throw_exception=True)
- except subprocess.CalledProcessError, e:
- self.debug("Failed to rebase: %s" % str(e))
- # clean up any hanging rebase. ignore errors if we aren't
- # in the middle of a rebase.
- self.run_command(self.hg + ['rebase', '--abort'],
- cwd=localrepo, success_codes=[0, 255])
- self.update(localrepo, branch=branch)
- for r in reversed(new_revs):
- self.run_command(self.hg + ['strip', '-n', r[REVISION]],
- cwd=localrepo, error_list=HgErrorList)
- changer(localrepo, n + 1)
-
- def cleanOutgoingRevs(self, reponame, remote, username, sshKey):
- # TODO retry
- self.info("Wiping outgoing local changes from %s to %s." % (reponame, remote))
- outgoingRevs = self.out(src=reponame, remote=remote,
- ssh_username=username, ssh_key=sshKey)
- for r in reversed(outgoingRevs):
- self.run_command(self.hg + ['strip', '-n', r[REVISION]],
- cwd=reponame, error_list=HgErrorList)
-
- def query_pushinfo(self, repository, revision):
- """Query the pushdate and pushid of a repository/revision.
- This is intended to be used on hg.mozilla.org/mozilla-central and
- similar. It may or may not work for other hg repositories.
- """
- PushInfo = namedtuple('PushInfo', ['pushid', 'pushdate'])
-
- try:
- url = '%s/json-pushes?changeset=%s' % (repository, revision)
- self.info('Pushdate URL is: %s' % url)
- contents = self.retry(self.load_json_from_url, args=(url,))
-
- # The contents should be something like:
- # {
- # "28537": {
- # "changesets": [
- # "1d0a914ae676cc5ed203cdc05c16d8e0c22af7e5",
- # ],
- # "date": 1428072488,
- # "user": "user@mozilla.com"
- # }
- # }
- #
- # So we grab the first element ("28537" in this case) and then pull
- # out the 'date' field.
- pushid = contents.iterkeys().next()
- self.info('Pushid is: %s' % pushid)
- pushdate = contents[pushid]['date']
- self.info('Pushdate is: %s' % pushdate)
- return PushInfo(pushid, pushdate)
-
- except Exception:
- self.exception("Failed to get push info from hg.mozilla.org")
- raise
-
-
-# __main__ {{{1
-if __name__ == '__main__':
- pass
diff --git a/testing/mozharness/mozharness/base/vcs/tcvcs.py b/testing/mozharness/mozharness/base/vcs/tcvcs.py
deleted file mode 100644
index 55fca4afd..000000000
--- a/testing/mozharness/mozharness/base/vcs/tcvcs.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import os.path
-from mozharness.base.script import ScriptMixin
-from mozharness.base.log import LogMixin
-
-class TcVCS(ScriptMixin, LogMixin):
- def __init__(self, log_obj=None, config=None, vcs_config=None,
- script_obj=None):
- super(TcVCS, self).__init__()
-
- self.log_obj = log_obj
- self.script_obj = script_obj
- if config:
- self.config = config
- else:
- self.config = {}
- # vcs_config = {
- # repo: repository,
- # branch: branch,
- # revision: revision,
- # ssh_username: ssh_username,
- # ssh_key: ssh_key,
- # }
- self.vcs_config = vcs_config
- self.tc_vcs = self.query_exe('tc-vcs', return_type='list')
-
- def ensure_repo_and_revision(self):
- """Makes sure that `dest` is has `revision` or `branch` checked out
- from `repo`.
-
- Do what it takes to make that happen, including possibly clobbering
- dest.
- """
- c = self.vcs_config
- for conf_item in ('dest', 'repo'):
- assert self.vcs_config[conf_item]
-
- dest = os.path.abspath(c['dest'])
- repo = c['repo']
- branch = c.get('branch', '')
- revision = c.get('revision', '')
- if revision is None:
- revision = ''
- base_repo = self.config.get('base_repo', repo)
-
- cmd = [self.tc_vcs[:][0], 'checkout', dest, base_repo, repo, revision, branch]
- self.run_command(cmd)
-
- cmd = [self.tc_vcs[:][0], 'revision', dest]
- return self.get_output_from_command(cmd)
diff --git a/testing/mozharness/mozharness/base/vcs/vcsbase.py b/testing/mozharness/mozharness/base/vcs/vcsbase.py
deleted file mode 100755
index 60ba5b79c..000000000
--- a/testing/mozharness/mozharness/base/vcs/vcsbase.py
+++ /dev/null
@@ -1,149 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Generic VCS support.
-"""
-
-from copy import deepcopy
-import os
-import sys
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(sys.path[0]))))
-
-from mozharness.base.errors import VCSException
-from mozharness.base.log import FATAL
-from mozharness.base.script import BaseScript
-from mozharness.base.vcs.mercurial import MercurialVCS
-from mozharness.base.vcs.gittool import GittoolVCS
-from mozharness.base.vcs.tcvcs import TcVCS
-
-# Update this with supported VCS name : VCS object
-VCS_DICT = {
- 'hg': MercurialVCS,
- 'gittool': GittoolVCS,
- 'tc-vcs': TcVCS,
-}
-
-
-# VCSMixin {{{1
-class VCSMixin(object):
- """Basic VCS methods that are vcs-agnostic.
- The vcs_class handles all the vcs-specific tasks.
- """
- def query_dest(self, kwargs):
- if 'dest' in kwargs:
- return kwargs['dest']
- dest = os.path.basename(kwargs['repo'])
- # Git fun
- if dest.endswith('.git'):
- dest = dest.replace('.git', '')
- return dest
-
- def _get_revision(self, vcs_obj, dest):
- try:
- got_revision = vcs_obj.ensure_repo_and_revision()
- if got_revision:
- return got_revision
- except VCSException:
- self.rmtree(dest)
- raise
-
- def _get_vcs_class(self, vcs):
- vcs = vcs or self.config.get('default_vcs', getattr(self, 'default_vcs', None))
- vcs_class = VCS_DICT.get(vcs)
- return vcs_class
-
- def vcs_checkout(self, vcs=None, error_level=FATAL, **kwargs):
- """ Check out a single repo.
- """
- c = self.config
- vcs_class = self._get_vcs_class(vcs)
- if not vcs_class:
- self.error("Running vcs_checkout with kwargs %s" % str(kwargs))
- raise VCSException("No VCS set!")
- # need a better way to do this.
- if 'dest' not in kwargs:
- kwargs['dest'] = self.query_dest(kwargs)
- if 'vcs_share_base' not in kwargs:
- kwargs['vcs_share_base'] = c.get('%s_share_base' % vcs, c.get('vcs_share_base'))
- vcs_obj = vcs_class(
- log_obj=self.log_obj,
- config=self.config,
- vcs_config=kwargs,
- script_obj=self,
- )
- return self.retry(
- self._get_revision,
- error_level=error_level,
- error_message="Automation Error: Can't checkout %s!" % kwargs['repo'],
- args=(vcs_obj, kwargs['dest']),
- )
-
- def vcs_checkout_repos(self, repo_list, parent_dir=None,
- tag_override=None, **kwargs):
- """Check out a list of repos.
- """
- orig_dir = os.getcwd()
- c = self.config
- if not parent_dir:
- parent_dir = os.path.join(c['base_work_dir'], c['work_dir'])
- self.mkdir_p(parent_dir)
- self.chdir(parent_dir)
- revision_dict = {}
- kwargs_orig = deepcopy(kwargs)
- for repo_dict in repo_list:
- kwargs = deepcopy(kwargs_orig)
- kwargs.update(repo_dict)
- if tag_override:
- kwargs['branch'] = tag_override
- dest = self.query_dest(kwargs)
- revision_dict[dest] = {'repo': kwargs['repo']}
- revision_dict[dest]['revision'] = self.vcs_checkout(**kwargs)
- self.chdir(orig_dir)
- return revision_dict
-
- def vcs_query_pushinfo(self, repository, revision, vcs=None):
- """Query the pushid/pushdate of a repository/revision
- Returns a namedtuple with "pushid" and "pushdate" elements
- """
- vcs_class = self._get_vcs_class(vcs)
- if not vcs_class:
- raise VCSException("No VCS set in vcs_query_pushinfo!")
- vcs_obj = vcs_class(
- log_obj=self.log_obj,
- config=self.config,
- script_obj=self,
- )
- return vcs_obj.query_pushinfo(repository, revision)
-
-
-class VCSScript(VCSMixin, BaseScript):
- def __init__(self, **kwargs):
- super(VCSScript, self).__init__(**kwargs)
-
- def pull(self, repos=None, parent_dir=None):
- repos = repos or self.config.get('repos')
- if not repos:
- self.info("Pull has nothing to do!")
- return
- dirs = self.query_abs_dirs()
- parent_dir = parent_dir or dirs['abs_work_dir']
- return self.vcs_checkout_repos(repos,
- parent_dir=parent_dir)
-
-
-# Specific VCS stubs {{{1
-# For ease of use.
-# This is here instead of mercurial.py because importing MercurialVCS into
-# vcsbase from mercurial, and importing VCSScript into mercurial from
-# vcsbase, was giving me issues.
-class MercurialScript(VCSScript):
- default_vcs = 'hg'
-
-
-# __main__ {{{1
-if __name__ == '__main__':
- pass
diff --git a/testing/mozharness/mozharness/base/vcs/vcssync.py b/testing/mozharness/mozharness/base/vcs/vcssync.py
deleted file mode 100644
index ffecb16b7..000000000
--- a/testing/mozharness/mozharness/base/vcs/vcssync.py
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Generic VCS support.
-"""
-
-import os
-import smtplib
-import sys
-import time
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(os.path.dirname(sys.path[0]))))
-
-from mozharness.base.log import ERROR, INFO
-from mozharness.base.vcs.vcsbase import VCSScript
-
-
-# VCSSyncScript {{{1
-class VCSSyncScript(VCSScript):
- start_time = time.time()
-
- def __init__(self, **kwargs):
- super(VCSSyncScript, self).__init__(**kwargs)
-
- def notify(self, message=None, fatal=False):
- """ Email people in the notify_config (depending on status and failure_only)
- """
- c = self.config
- dirs = self.query_abs_dirs()
- job_name = c.get('job_name', c.get('conversion_dir', os.getcwd()))
- end_time = time.time()
- seconds = int(end_time - self.start_time)
- self.info("Job took %d seconds." % seconds)
- subject = "[vcs2vcs] Successful conversion for %s" % job_name
- text = ''
- error_contents = ''
- max_log_sample_size = c.get('email_max_log_sample_size') # default defined in vcs_sync.py
- error_log = os.path.join(dirs['abs_log_dir'], self.log_obj.log_files[ERROR])
- info_log = os.path.join(dirs['abs_log_dir'], self.log_obj.log_files[INFO])
- if os.path.exists(error_log) and os.path.getsize(error_log) > 0:
- error_contents = self.get_output_from_command(
- ["egrep", "-C5", "^[0-9:]+ +(ERROR|CRITICAL|FATAL) -", info_log],
- silent=True,
- )
- if fatal:
- subject = "[vcs2vcs] Failed conversion for %s" % job_name
- text = ''
- if len(message) > max_log_sample_size:
- text += '*** Message below has been truncated: it was %s characters, and has been reduced to %s characters:\n\n' % (len(message), max_log_sample_size)
- text += message[0:max_log_sample_size] + '\n\n' # limit message to max_log_sample_size in size (large emails fail to send)
- if not self.successful_repos:
- subject = "[vcs2vcs] Successful no-op conversion for %s" % job_name
- if error_contents and not fatal:
- subject += " with warnings"
- if self.successful_repos:
- if len(self.successful_repos) <= 5:
- subject += ' (' + ','.join(self.successful_repos) + ')'
- else:
- text += "Successful repos: %s\n\n" % ', '.join(self.successful_repos)
- subject += ' (%ds)' % seconds
- if self.summary_list:
- text += 'Summary is non-zero:\n\n'
- for item in self.summary_list:
- text += '%s - %s\n' % (item['level'], item['message'])
- if not fatal and error_contents and not self.summary_list:
- text += 'Summary is empty; the below errors have probably been auto-corrected.\n\n'
- if error_contents:
- if len(error_contents) > max_log_sample_size:
- text += '\n*** Message below has been truncated: it was %s characters, and has been reduced to %s characters:\n' % (len(error_contents), max_log_sample_size)
- text += '\n%s\n\n' % error_contents[0:max_log_sample_size] # limit message to 100KB in size (large emails fail to send)
- if not text:
- subject += " <EOM>"
- for notify_config in c.get('notify_config', []):
- if not fatal:
- if notify_config.get('failure_only'):
- self.info("Skipping notification for %s (failure_only)" % notify_config['to'])
- continue
- if not text and notify_config.get('skip_empty_messages'):
- self.info("Skipping notification for %s (skip_empty_messages)" % notify_config['to'])
- continue
- fromaddr = notify_config.get('from', c['default_notify_from'])
- message = '\r\n'.join((
- "From: %s" % fromaddr,
- "To: %s" % notify_config['to'],
- "CC: %s" % ','.join(notify_config.get('cc', [])),
- "Subject: %s" % subject,
- "",
- text
- ))
- toaddrs = [notify_config['to']] + notify_config.get('cc', [])
- # TODO allow for a different smtp server
- # TODO deal with failures
- server = smtplib.SMTP('localhost')
- self.retry(
- server.sendmail,
- args=(fromaddr, toaddrs, message),
- )
- server.quit()
diff --git a/testing/mozharness/mozharness/lib/__init__.py b/testing/mozharness/mozharness/lib/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/mozharness/lib/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/mozharness/lib/python/__init__.py b/testing/mozharness/mozharness/lib/python/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/mozharness/lib/python/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/mozharness/lib/python/authentication.py b/testing/mozharness/mozharness/lib/python/authentication.py
deleted file mode 100644
index 2e5f83f37..000000000
--- a/testing/mozharness/mozharness/lib/python/authentication.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-"""module for http authentication operations"""
-import getpass
-import os
-
-CREDENTIALS_PATH = os.path.expanduser("~/.mozilla/credentials.cfg")
-DIRNAME = os.path.dirname(CREDENTIALS_PATH)
-LDAP_PASSWORD = None
-
-def get_credentials():
- """ Returns http credentials.
-
- The user's email address is stored on disk (for convenience in the future)
- while the password is requested from the user on first invocation.
- """
- global LDAP_PASSWORD
- if not os.path.exists(DIRNAME):
- os.makedirs(DIRNAME)
-
- if os.path.isfile(CREDENTIALS_PATH):
- with open(CREDENTIALS_PATH, 'r') as file_handler:
- content = file_handler.read().splitlines()
-
- https_username = content[0].strip()
-
- if len(content) > 1:
- # We want to remove files which contain the password
- os.remove(CREDENTIALS_PATH)
- else:
- https_username = \
- raw_input("Please enter your full LDAP email address: ")
-
- with open(CREDENTIALS_PATH, "w+") as file_handler:
- file_handler.write("%s\n" % https_username)
-
- os.chmod(CREDENTIALS_PATH, 0600)
-
- if not LDAP_PASSWORD:
- print "Please enter your LDAP password (we won't store it):"
- LDAP_PASSWORD = getpass.getpass()
-
- return https_username, LDAP_PASSWORD
-
-def get_credentials_path():
- if os.path.isfile(CREDENTIALS_PATH):
- get_credentials()
-
- return CREDENTIALS_PATH
diff --git a/testing/mozharness/mozharness/mozilla/__init__.py b/testing/mozharness/mozharness/mozilla/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/mozharness/mozilla/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/mozharness/mozilla/aws.py b/testing/mozharness/mozharness/mozilla/aws.py
deleted file mode 100644
index 264c39037..000000000
--- a/testing/mozharness/mozharness/mozilla/aws.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import os
-
-
-def pop_aws_auth_from_env():
- """
- retrieves aws creds and deletes them from os.environ if present.
- """
- aws_key_id = os.environ.pop("AWS_ACCESS_KEY_ID", None)
- aws_secret_key = os.environ.pop("AWS_SECRET_ACCESS_KEY", None)
-
- return aws_key_id, aws_secret_key
diff --git a/testing/mozharness/mozharness/mozilla/blob_upload.py b/testing/mozharness/mozharness/mozilla/blob_upload.py
deleted file mode 100644
index 1607ddf99..000000000
--- a/testing/mozharness/mozharness/mozilla/blob_upload.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-import os
-
-from mozharness.base.python import VirtualenvMixin
-from mozharness.base.script import PostScriptRun
-
-blobupload_config_options = [
- [["--blob-upload-branch"],
- {"dest": "blob_upload_branch",
- "help": "Branch for blob server's metadata",
- }],
- [["--blob-upload-server"],
- {"dest": "blob_upload_servers",
- "action": "extend",
- "help": "Blob servers's location",
- }]
- ]
-
-
-class BlobUploadMixin(VirtualenvMixin):
- """Provides mechanism to automatically upload files written in
- MOZ_UPLOAD_DIR to the blobber upload server at the end of the
- running script.
-
- This is dependent on ScriptMixin and BuildbotMixin.
- The testing script inheriting this class is to specify as cmdline
- options the <blob-upload-branch> and <blob-upload-server>
-
- """
- def __init__(self, *args, **kwargs):
- requirements = [
- 'blobuploader==1.2.4',
- ]
- super(BlobUploadMixin, self).__init__(*args, **kwargs)
- for req in requirements:
- self.register_virtualenv_module(req, method='pip')
-
- def upload_blobber_files(self):
- self.debug("Check branch and server cmdline options.")
- if self.config.get('blob_upload_branch') and \
- (self.config.get('blob_upload_servers') or
- self.config.get('default_blob_upload_servers')) and \
- self.config.get('blob_uploader_auth_file'):
-
- self.info("Blob upload gear active.")
- upload = [self.query_python_path(), self.query_python_path("blobberc.py")]
-
- dirs = self.query_abs_dirs()
- self.debug("Get the directory from which to upload the files.")
- if dirs.get('abs_blob_upload_dir'):
- blob_dir = dirs['abs_blob_upload_dir']
- else:
- self.warning("Couldn't find the blob upload folder's path!")
- return
-
- if not os.path.isdir(blob_dir):
- self.warning("Blob upload directory does not exist!")
- return
-
- if not os.listdir(blob_dir):
- self.info("There are no files to upload in the directory. "
- "Skipping the blob upload mechanism ...")
- return
-
- self.info("Preparing to upload files from %s." % blob_dir)
- auth_file = self.config.get('blob_uploader_auth_file')
- if not os.path.isfile(auth_file):
- self.warning("Could not find the credentials files!")
- return
- blob_branch = self.config.get('blob_upload_branch')
- blob_servers_list = self.config.get('blob_upload_servers',
- self.config.get('default_blob_upload_servers'))
-
- servers = []
- for server in blob_servers_list:
- servers.extend(['-u', server])
- auth = ['-a', auth_file]
- branch = ['-b', blob_branch]
- dir_to_upload = ['-d', blob_dir]
- # We want blobberc to tell us if a summary file was uploaded through this manifest file
- manifest_path = os.path.join(dirs['abs_work_dir'], 'uploaded_files.json')
- record_uploaded_files = ['--output-manifest', manifest_path]
- self.info("Files from %s are to be uploaded with <%s> branch at "
- "the following location(s): %s" % (blob_dir, blob_branch,
- ", ".join(["%s" % s for s in blob_servers_list])))
-
- # call blob client to upload files to server
- self.run_command(upload + servers + auth + branch + dir_to_upload + record_uploaded_files)
-
- uploaded_files = '{}'
- if os.path.isfile(manifest_path):
- with open(manifest_path, 'r') as f:
- uploaded_files = f.read()
- self.rmtree(manifest_path)
-
- self.set_buildbot_property(prop_name='blobber_files',
- prop_value=uploaded_files, write_to_file=True)
- else:
- self.warning("Blob upload gear skipped. Missing cmdline options.")
-
- @PostScriptRun
- def _upload_blobber_files(self):
- self.upload_blobber_files()
diff --git a/testing/mozharness/mozharness/mozilla/bouncer/__init__.py b/testing/mozharness/mozharness/mozilla/bouncer/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/mozharness/mozilla/bouncer/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/mozharness/mozilla/bouncer/submitter.py b/testing/mozharness/mozharness/mozilla/bouncer/submitter.py
deleted file mode 100644
index 43983dca8..000000000
--- a/testing/mozharness/mozharness/mozilla/bouncer/submitter.py
+++ /dev/null
@@ -1,114 +0,0 @@
-import base64
-import httplib
-import socket
-import sys
-import traceback
-import urllib
-import urllib2
-from xml.dom.minidom import parseString
-
-from mozharness.base.log import FATAL
-
-
-class BouncerSubmitterMixin(object):
- def query_credentials(self):
- if self.credentials:
- return self.credentials
- global_dict = {}
- local_dict = {}
- execfile(self.config["credentials_file"], global_dict, local_dict)
- self.credentials = (local_dict["tuxedoUsername"],
- local_dict["tuxedoPassword"])
- return self.credentials
-
- def api_call(self, route, data, error_level=FATAL, retry_config=None):
- retry_args = dict(
- failure_status=None,
- retry_exceptions=(urllib2.HTTPError, urllib2.URLError,
- httplib.BadStatusLine,
- socket.timeout, socket.error),
- error_message="call to %s failed" % (route),
- error_level=error_level,
- )
-
- if retry_config:
- retry_args.update(retry_config)
-
- return self.retry(
- self._api_call,
- args=(route, data),
- **retry_args
- )
-
- def _api_call(self, route, data):
- api_prefix = self.config["bouncer-api-prefix"]
- api_url = "%s/%s" % (api_prefix, route)
- request = urllib2.Request(api_url)
- if data:
- post_data = urllib.urlencode(data, doseq=True)
- request.add_data(post_data)
- self.info("POST data: %s" % post_data)
- credentials = self.query_credentials()
- if credentials:
- auth = base64.encodestring('%s:%s' % credentials)
- request.add_header("Authorization", "Basic %s" % auth.strip())
- try:
- self.info("Submitting to %s" % api_url)
- res = urllib2.urlopen(request, timeout=60).read()
- self.info("Server response")
- self.info(res)
- return res
- except urllib2.HTTPError as e:
- self.warning("Cannot access %s" % api_url)
- traceback.print_exc(file=sys.stdout)
- self.warning("Returned page source:")
- self.warning(e.read())
- raise
- except urllib2.URLError:
- traceback.print_exc(file=sys.stdout)
- self.warning("Cannot access %s" % api_url)
- raise
- except socket.timeout as e:
- self.warning("Timed out accessing %s: %s" % (api_url, e))
- raise
- except socket.error as e:
- self.warning("Socket error when accessing %s: %s" % (api_url, e))
- raise
- except httplib.BadStatusLine as e:
- self.warning('BadStatusLine accessing %s: %s' % (api_url, e))
- raise
-
- def product_exists(self, product_name):
- self.info("Checking if %s already exists" % product_name)
- res = self.api_call("product_show?product=%s" %
- urllib.quote(product_name), data=None)
- try:
- xml = parseString(res)
- # API returns <products/> if the product doesn't exist
- products_found = len(xml.getElementsByTagName("product"))
- self.info("Products found: %s" % products_found)
- return bool(products_found)
- except Exception as e:
- self.warning("Error parsing XML: %s" % e)
- self.warning("Assuming %s does not exist" % product_name)
- # ignore XML parsing errors
- return False
-
- def api_add_product(self, product_name, add_locales, ssl_only=False):
- data = {
- "product": product_name,
- }
- if self.locales and add_locales:
- data["languages"] = self.locales
- if ssl_only:
- # Send "true" as a string
- data["ssl_only"] = "true"
- self.api_call("product_add/", data)
-
- def api_add_location(self, product_name, bouncer_platform, path):
- data = {
- "product": product_name,
- "os": bouncer_platform,
- "path": path,
- }
- self.api_call("location_add/", data)
diff --git a/testing/mozharness/mozharness/mozilla/buildbot.py b/testing/mozharness/mozharness/mozilla/buildbot.py
deleted file mode 100755
index e17343633..000000000
--- a/testing/mozharness/mozharness/mozilla/buildbot.py
+++ /dev/null
@@ -1,246 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Code to tie into buildbot.
-Ideally this will go away if and when we retire buildbot.
-"""
-
-import copy
-import os
-import re
-import sys
-
-try:
- import simplejson as json
- assert json
-except ImportError:
- import json
-
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.config import parse_config_file
-from mozharness.base.log import INFO, WARNING, ERROR
-
-# BuildbotMixin {{{1
-
-TBPL_SUCCESS = 'SUCCESS'
-TBPL_WARNING = 'WARNING'
-TBPL_FAILURE = 'FAILURE'
-TBPL_EXCEPTION = 'EXCEPTION'
-TBPL_RETRY = 'RETRY'
-TBPL_STATUS_DICT = {
- TBPL_SUCCESS: INFO,
- TBPL_WARNING: WARNING,
- TBPL_FAILURE: ERROR,
- TBPL_EXCEPTION: ERROR,
- TBPL_RETRY: WARNING,
-}
-EXIT_STATUS_DICT = {
- TBPL_SUCCESS: 0,
- TBPL_WARNING: 1,
- TBPL_FAILURE: 2,
- TBPL_EXCEPTION: 3,
- TBPL_RETRY: 4,
-}
-TBPL_WORST_LEVEL_TUPLE = (TBPL_RETRY, TBPL_EXCEPTION, TBPL_FAILURE,
- TBPL_WARNING, TBPL_SUCCESS)
-
-
-class BuildbotMixin(object):
- buildbot_config = None
- buildbot_properties = {}
- worst_buildbot_status = TBPL_SUCCESS
-
- def read_buildbot_config(self):
- c = self.config
- if not c.get("buildbot_json_path"):
- # If we need to fail out, add postflight_read_buildbot_config()
- self.info("buildbot_json_path is not set. Skipping...")
- else:
- # TODO try/except?
- self.buildbot_config = parse_config_file(c['buildbot_json_path'])
- buildbot_properties = copy.deepcopy(self.buildbot_config.get('properties', {}))
- if 'commit_titles' in buildbot_properties:
- # Remove the commit messages since they can cause false positives with
- # Treeherder log parsers. Eg: "Bug X - Fix TEST-UNEPXECTED-FAIL ...".
- del buildbot_properties['commit_titles']
- self.info("Using buildbot properties:")
- self.info(json.dumps(buildbot_properties, indent=4))
-
- def tryserver_email(self):
- pass
-
- def buildbot_status(self, tbpl_status, level=None, set_return_code=True):
- if tbpl_status not in TBPL_STATUS_DICT:
- self.error("buildbot_status() doesn't grok the status %s!" % tbpl_status)
- else:
- # Set failure if our log > buildbot_max_log_size (bug 876159)
- if self.config.get("buildbot_max_log_size") and self.log_obj:
- # Find the path to the default log
- dirs = self.query_abs_dirs()
- log_file = os.path.join(
- dirs['abs_log_dir'],
- self.log_obj.log_files[self.log_obj.log_level]
- )
- if os.path.exists(log_file):
- file_size = os.path.getsize(log_file)
- if file_size > self.config['buildbot_max_log_size']:
- self.error("Log file size %d is greater than max allowed %d! Setting TBPL_FAILURE (was %s)..." % (file_size, self.config['buildbot_max_log_size'], tbpl_status))
- tbpl_status = TBPL_FAILURE
- if not level:
- level = TBPL_STATUS_DICT[tbpl_status]
- self.worst_buildbot_status = self.worst_level(tbpl_status, self.worst_buildbot_status, TBPL_WORST_LEVEL_TUPLE)
- if self.worst_buildbot_status != tbpl_status:
- self.info("Current worst status %s is worse; keeping it." % self.worst_buildbot_status)
- self.add_summary("# TBPL %s #" % self.worst_buildbot_status, level=level)
- if set_return_code:
- self.return_code = EXIT_STATUS_DICT[self.worst_buildbot_status]
-
- def set_buildbot_property(self, prop_name, prop_value, write_to_file=False):
- self.info("Setting buildbot property %s to %s" % (prop_name, prop_value))
- self.buildbot_properties[prop_name] = prop_value
- if write_to_file:
- return self.dump_buildbot_properties(prop_list=[prop_name], file_name=prop_name)
- return self.buildbot_properties[prop_name]
-
- def query_buildbot_property(self, prop_name):
- return self.buildbot_properties.get(prop_name)
-
- def query_is_nightly(self):
- """returns whether or not the script should run as a nightly build.
-
- First will check for 'nightly_build' in self.config and if that is
- not True, we will also allow buildbot_config to determine
- for us. Failing all of that, we default to False.
- Note, dependancy on buildbot_config is being deprecated.
- Putting everything in self.config is the preference.
- """
- if self.config.get('nightly_build'):
- return True
- elif self.buildbot_config and 'properties' in self.buildbot_config:
- return self.buildbot_config['properties'].get('nightly_build', False)
- else:
- return False
-
- def dump_buildbot_properties(self, prop_list=None, file_name="properties", error_level=ERROR):
- c = self.config
- if not os.path.isabs(file_name):
- file_name = os.path.join(c['base_work_dir'], "properties", file_name)
- dir_name = os.path.dirname(file_name)
- if not os.path.isdir(dir_name):
- self.mkdir_p(dir_name)
- if not prop_list:
- prop_list = self.buildbot_properties.keys()
- self.info("Writing buildbot properties to %s" % file_name)
- else:
- if not isinstance(prop_list, (list, tuple)):
- self.log("dump_buildbot_properties: Can't dump non-list prop_list %s!" % str(prop_list), level=error_level)
- return
- self.info("Writing buildbot properties %s to %s" % (str(prop_list), file_name))
- contents = ""
- for prop in prop_list:
- contents += "%s:%s\n" % (prop, self.buildbot_properties.get(prop, "None"))
- return self.write_to_file(file_name, contents)
-
- def invoke_sendchange(self, downloadables=None, branch=None,
- username="sendchange-unittest", sendchange_props=None):
- """ Generic sendchange, currently b2g- and unittest-specific.
- """
- c = self.config
- buildbot = self.query_exe("buildbot", return_type="list")
- if branch is None:
- if c.get("debug_build"):
- platform = re.sub('[_-]debug', '', self.buildbot_config["properties"]["platform"])
- branch = '%s-%s-debug-unittest' % (self.buildbot_config["properties"]["branch"], platform)
- else:
- branch = '%s-%s-opt-unittest' % (self.buildbot_config["properties"]["branch"], self.buildbot_config["properties"]["platform"])
- sendchange = [
- 'sendchange',
- '--master', c.get("sendchange_masters")[0],
- '--username', username,
- '--branch', branch,
- ]
- if self.buildbot_config['sourcestamp'].get("revision"):
- sendchange += ['-r', self.buildbot_config['sourcestamp']["revision"]]
- if len(self.buildbot_config['sourcestamp']['changes']) > 0:
- if self.buildbot_config['sourcestamp']['changes'][0].get('who'):
- sendchange += ['--username', self.buildbot_config['sourcestamp']['changes'][0]['who']]
- if self.buildbot_config['sourcestamp']['changes'][0].get('comments'):
- sendchange += ['--comments', self.buildbot_config['sourcestamp']['changes'][0]['comments'].encode('ascii', 'ignore')]
- if sendchange_props:
- for key, value in sendchange_props.iteritems():
- sendchange.extend(['--property', '%s:%s' % (key, value)])
- else:
- if self.buildbot_config["properties"].get("builduid"):
- sendchange += ['--property', "builduid:%s" % self.buildbot_config["properties"]["builduid"]]
- sendchange += [
- '--property', "buildid:%s" % self.query_buildid(),
- '--property', 'pgo_build:False',
- ]
-
- for d in downloadables:
- sendchange += [d]
-
- retcode = self.run_command(buildbot + sendchange)
- if retcode != 0:
- self.info("The sendchange failed but we don't want to turn the build orange: %s" % retcode)
-
- def query_build_name(self):
- build_name = self.config.get('platform')
- if not build_name:
- self.fatal('Must specify "platform" in the mozharness config for indexing')
-
- return build_name
-
- def query_build_type(self):
- if self.config.get('build_type'):
- build_type = self.config['build_type']
- elif self.config.get('pgo_build'):
- build_type = 'pgo'
- elif self.config.get('debug_build', False):
- build_type = 'debug'
- else:
- build_type = 'opt'
- return build_type
-
- def buildid_to_dict(self, buildid):
- """Returns an dict with the year, month, day, hour, minute, and second
- as keys, as parsed from the buildid"""
- buildidDict = {}
- try:
- # strptime is no good here because it strips leading zeros
- buildidDict['year'] = buildid[0:4]
- buildidDict['month'] = buildid[4:6]
- buildidDict['day'] = buildid[6:8]
- buildidDict['hour'] = buildid[8:10]
- buildidDict['minute'] = buildid[10:12]
- buildidDict['second'] = buildid[12:14]
- except:
- self.fatal('Could not parse buildid into YYYYMMDDHHMMSS: %s' % buildid)
- return buildidDict
-
- def query_who(self):
- """ looks for who triggered the build with a change.
-
- This is used for things like try builds where the upload dir is
- associated with who pushed to try. First it will look in self.config
- and failing that, will poll buildbot_config
- If nothing is found, it will default to returning "nobody@example.com"
- """
- if self.config.get('who'):
- return self.config['who']
- self.read_buildbot_config()
- try:
- return self.buildbot_config['sourcestamp']['changes'][0]['who']
- except (KeyError, IndexError):
- # KeyError: "sourcestamp" or "changes" or "who" not in buildbot_config
- # IndexError: buildbot_config['sourcestamp']['changes'] is empty
- pass
- try:
- return str(self.buildbot_config['properties']['who'])
- except KeyError:
- pass
- return "nobody@example.com"
diff --git a/testing/mozharness/mozharness/mozilla/building/__init__.py b/testing/mozharness/mozharness/mozilla/building/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/mozharness/mozilla/building/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/mozharness/mozilla/building/buildbase.py b/testing/mozharness/mozharness/mozilla/building/buildbase.py
deleted file mode 100755
index 9fd507816..000000000
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ /dev/null
@@ -1,2153 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-""" buildbase.py.
-
-provides a base class for fx desktop builds
-author: Jordan Lund
-
-"""
-import json
-
-import os
-import pprint
-import subprocess
-import time
-import uuid
-import copy
-import glob
-import shlex
-from itertools import chain
-
-# import the power of mozharness ;)
-import sys
-from datetime import datetime
-import re
-from mozharness.base.config import BaseConfig, parse_config_file
-from mozharness.base.log import ERROR, OutputParser, FATAL
-from mozharness.base.script import PostScriptRun
-from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.buildbot import (
- BuildbotMixin,
- EXIT_STATUS_DICT,
- TBPL_STATUS_DICT,
- TBPL_EXCEPTION,
- TBPL_FAILURE,
- TBPL_RETRY,
- TBPL_WARNING,
- TBPL_SUCCESS,
- TBPL_WORST_LEVEL_TUPLE,
-)
-from mozharness.mozilla.purge import PurgeMixin
-from mozharness.mozilla.mock import MockMixin
-from mozharness.mozilla.secrets import SecretsMixin
-from mozharness.mozilla.signing import SigningMixin
-from mozharness.mozilla.mock import ERROR_MSGS as MOCK_ERROR_MSGS
-from mozharness.mozilla.testing.errors import TinderBoxPrintRe
-from mozharness.mozilla.testing.unittest import tbox_print_summary
-from mozharness.mozilla.updates.balrog import BalrogMixin
-from mozharness.mozilla.taskcluster_helper import Taskcluster
-from mozharness.base.python import (
- PerfherderResourceOptionsMixin,
- VirtualenvMixin,
-)
-
-AUTOMATION_EXIT_CODES = EXIT_STATUS_DICT.values()
-AUTOMATION_EXIT_CODES.sort()
-
-MISSING_CFG_KEY_MSG = "The key '%s' could not be determined \
-Please add this to your config."
-
-ERROR_MSGS = {
- 'undetermined_repo_path': 'The repo could not be determined. \
-Please make sure that either "repo" is in your config or, if \
-you are running this in buildbot, "repo_path" is in your buildbot_config.',
- 'comments_undetermined': '"comments" could not be determined. This may be \
-because it was a forced build.',
- 'tooltool_manifest_undetermined': '"tooltool_manifest_src" not set, \
-Skipping run_tooltool...',
-}
-ERROR_MSGS.update(MOCK_ERROR_MSGS)
-
-
-### Output Parsers
-
-TBPL_UPLOAD_ERRORS = [
- {
- 'regex': re.compile("Connection timed out"),
- 'level': TBPL_RETRY,
- },
- {
- 'regex': re.compile("Connection reset by peer"),
- 'level': TBPL_RETRY,
- },
- {
- 'regex': re.compile("Connection refused"),
- 'level': TBPL_RETRY,
- }
-]
-
-
-class MakeUploadOutputParser(OutputParser):
- tbpl_error_list = TBPL_UPLOAD_ERRORS
- # let's create a switch case using name-spaces/dict
- # rather than a long if/else with duplicate code
- property_conditions = [
- # key: property name, value: condition
- ('symbolsUrl', "m.endswith('crashreporter-symbols.zip') or "
- "m.endswith('crashreporter-symbols-full.zip')"),
- ('testsUrl', "m.endswith(('tests.tar.bz2', 'tests.zip'))"),
- ('jsshellUrl', "'jsshell-' in m and m.endswith('.zip')"),
- ('partialMarUrl', "m.endswith('.mar') and '.partial.' in m"),
- ('completeMarUrl', "m.endswith('.mar')"),
- ('codeCoverageUrl', "m.endswith('code-coverage-gcno.zip')"),
- ]
-
- def __init__(self, use_package_as_marfile=False, package_filename=None, **kwargs):
- super(MakeUploadOutputParser, self).__init__(**kwargs)
- self.matches = {}
- self.tbpl_status = TBPL_SUCCESS
- self.use_package_as_marfile = use_package_as_marfile
- self.package_filename = package_filename
-
- def parse_single_line(self, line):
- prop_assigned = False
- pat = r'''^(https?://.*?\.(?:tar\.bz2|dmg|zip|apk|rpm|mar|tar\.gz))$'''
- m = re.compile(pat).match(line)
- if m:
- m = m.group(1)
- for prop, condition in self.property_conditions:
- if eval(condition):
- self.matches[prop] = m
- prop_assigned = True
- break
- if not prop_assigned:
- # if we found a match but haven't identified the prop then this
- # is the packageURL. Alternatively, if we already know the
- # package filename, then use that explicitly so we don't pick up
- # just any random file and assume it's the package.
- if not self.package_filename or m.endswith(self.package_filename):
- self.matches['packageUrl'] = m
-
- # For android builds, the package is also used as the mar file.
- # Grab the first one, since that is the one in the
- # nightly/YYYY/MM directory
- if self.use_package_as_marfile:
- if 'tinderbox-builds' in m or 'nightly/latest-' in m:
- self.info("Skipping wrong packageUrl: %s" % m)
- else:
- if 'completeMarUrl' in self.matches:
- self.fatal("Found multiple package URLs. Please update buildbase.py")
- self.info("Using package as mar file: %s" % m)
- self.matches['completeMarUrl'] = m
- u, self.package_filename = os.path.split(m)
-
- if self.use_package_as_marfile and self.package_filename:
- # The checksum file is also dumped during 'make upload'. Look
- # through here to get the hash and filesize of the android package
- # for balrog submission.
- pat = r'''^([^ ]*) sha512 ([0-9]*) %s$''' % self.package_filename
- m = re.compile(pat).match(line)
- if m:
- self.matches['completeMarHash'] = m.group(1)
- self.matches['completeMarSize'] = m.group(2)
- self.info("Using package as mar file and found package hash=%s size=%s" % (m.group(1), m.group(2)))
-
- # now let's check for retry errors which will give log levels:
- # tbpl status as RETRY and mozharness status as WARNING
- for error_check in self.tbpl_error_list:
- if error_check['regex'].search(line):
- self.num_warnings += 1
- self.warning(line)
- self.tbpl_status = self.worst_level(
- error_check['level'], self.tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE
- )
- break
- else:
- self.info(line)
-
-
-class CheckTestCompleteParser(OutputParser):
- tbpl_error_list = TBPL_UPLOAD_ERRORS
-
- def __init__(self, **kwargs):
- self.matches = {}
- super(CheckTestCompleteParser, self).__init__(**kwargs)
- self.pass_count = 0
- self.fail_count = 0
- self.leaked = False
- self.harness_err_re = TinderBoxPrintRe['harness_error']['full_regex']
- self.tbpl_status = TBPL_SUCCESS
-
- def parse_single_line(self, line):
- # Counts and flags.
- # Regular expression for crash and leak detections.
- if "TEST-PASS" in line:
- self.pass_count += 1
- return self.info(line)
- if "TEST-UNEXPECTED-" in line:
- # Set the error flags.
- # Or set the failure count.
- m = self.harness_err_re.match(line)
- if m:
- r = m.group(1)
- if r == "missing output line for total leaks!":
- self.leaked = None
- else:
- self.leaked = True
- self.fail_count += 1
- return self.warning(line)
- self.info(line) # else
-
- def evaluate_parser(self, return_code, success_codes=None):
- success_codes = success_codes or [0]
-
- if self.num_errors: # ran into a script error
- self.tbpl_status = self.worst_level(TBPL_FAILURE, self.tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE)
-
- if self.fail_count > 0:
- self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE)
-
- # Account for the possibility that no test summary was output.
- if self.pass_count == 0 and self.fail_count == 0:
- self.error('No tests run or test summary not found')
- self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE)
-
- if return_code not in success_codes:
- self.tbpl_status = self.worst_level(TBPL_FAILURE, self.tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE)
-
- # Print the summary.
- summary = tbox_print_summary(self.pass_count,
- self.fail_count,
- self.leaked)
- self.info("TinderboxPrint: check<br/>%s\n" % summary)
-
- return self.tbpl_status
-
-
-class BuildingConfig(BaseConfig):
- # TODO add nosetests for this class
- def get_cfgs_from_files(self, all_config_files, options):
- """
- Determine the configuration from the normal options and from
- `--branch`, `--build-pool`, and `--custom-build-variant-cfg`. If the
- files for any of the latter options are also given with `--config-file`
- or `--opt-config-file`, they are only parsed once.
-
- The build pool has highest precedence, followed by branch, build
- variant, and any normally-specified configuration files.
- """
- # override from BaseConfig
-
- # this is what we will return. It will represent each config
- # file name and its associated dict
- # eg ('builds/branch_specifics.py', {'foo': 'bar'})
- all_config_dicts = []
- # important config files
- variant_cfg_file = branch_cfg_file = pool_cfg_file = ''
-
- # we want to make the order in which the options were given
- # not matter. ie: you can supply --branch before --build-pool
- # or vice versa and the hierarchy will not be different
-
- #### The order from highest precedence to lowest is:
- ## There can only be one of these...
- # 1) build_pool: this can be either staging, pre-prod, and prod cfgs
- # 2) branch: eg: mozilla-central, cedar, cypress, etc
- # 3) build_variant: these could be known like asan and debug
- # or a custom config
- ##
- ## There can be many of these
- # 4) all other configs: these are any configs that are passed with
- # --cfg and --opt-cfg. There order is kept in
- # which they were passed on the cmd line. This
- # behaviour is maintains what happens by default
- # in mozharness
- ##
- ####
-
- # so, let's first assign the configs that hold a known position of
- # importance (1 through 3)
- for i, cf in enumerate(all_config_files):
- if options.build_pool:
- if cf == BuildOptionParser.build_pool_cfg_file:
- pool_cfg_file = all_config_files[i]
-
- if cf == BuildOptionParser.branch_cfg_file:
- branch_cfg_file = all_config_files[i]
-
- if cf == options.build_variant:
- variant_cfg_file = all_config_files[i]
-
- # now remove these from the list if there was any.
- # we couldn't pop() these in the above loop as mutating a list while
- # iterating through it causes spurious results :)
- for cf in [pool_cfg_file, branch_cfg_file, variant_cfg_file]:
- if cf:
- all_config_files.remove(cf)
-
- # now let's update config with the remaining config files.
- # this functionality is the same as the base class
- all_config_dicts.extend(
- super(BuildingConfig, self).get_cfgs_from_files(all_config_files,
- options)
- )
-
- # stack variant, branch, and pool cfg files on top of that,
- # if they are present, in that order
- if variant_cfg_file:
- # take the whole config
- all_config_dicts.append(
- (variant_cfg_file, parse_config_file(variant_cfg_file))
- )
- if branch_cfg_file:
- # take only the specific branch, if present
- branch_configs = parse_config_file(branch_cfg_file)
- if branch_configs.get(options.branch or ""):
- all_config_dicts.append(
- (branch_cfg_file, branch_configs[options.branch])
- )
- if pool_cfg_file:
- # take only the specific pool. If we are here, the pool
- # must be present
- build_pool_configs = parse_config_file(pool_cfg_file)
- all_config_dicts.append(
- (pool_cfg_file, build_pool_configs[options.build_pool])
- )
- return all_config_dicts
-
-
-# noinspection PyUnusedLocal
-class BuildOptionParser(object):
- # TODO add nosetests for this class
- platform = None
- bits = None
- config_file_search_path = [
- '.', os.path.join(sys.path[0], '..', 'configs'),
- os.path.join(sys.path[0], '..', '..', 'configs')
- ]
-
- # add to this list and you can automagically do things like
- # --custom-build-variant-cfg asan
- # and the script will pull up the appropriate path for the config
- # against the current platform and bits.
- # *It will warn and fail if there is not a config for the current
- # platform/bits
- build_variants = {
- 'add-on-devel': 'builds/releng_sub_%s_configs/%s_add-on-devel.py',
- 'asan': 'builds/releng_sub_%s_configs/%s_asan.py',
- 'asan-tc': 'builds/releng_sub_%s_configs/%s_asan_tc.py',
- 'tsan': 'builds/releng_sub_%s_configs/%s_tsan.py',
- 'cross-debug': 'builds/releng_sub_%s_configs/%s_cross_debug.py',
- 'cross-opt': 'builds/releng_sub_%s_configs/%s_cross_opt.py',
- 'cross-universal': 'builds/releng_sub_%s_configs/%s_cross_universal.py',
- 'debug': 'builds/releng_sub_%s_configs/%s_debug.py',
- 'asan-and-debug': 'builds/releng_sub_%s_configs/%s_asan_and_debug.py',
- 'asan-tc-and-debug': 'builds/releng_sub_%s_configs/%s_asan_tc_and_debug.py',
- 'stat-and-debug': 'builds/releng_sub_%s_configs/%s_stat_and_debug.py',
- 'code-coverage': 'builds/releng_sub_%s_configs/%s_code_coverage.py',
- 'source': 'builds/releng_sub_%s_configs/%s_source.py',
- 'api-15-gradle-dependencies': 'builds/releng_sub_%s_configs/%s_api_15_gradle_dependencies.py',
- 'api-15': 'builds/releng_sub_%s_configs/%s_api_15.py',
- 'api-15-debug': 'builds/releng_sub_%s_configs/%s_api_15_debug.py',
- 'api-15-gradle': 'builds/releng_sub_%s_configs/%s_api_15_gradle.py',
- 'x86': 'builds/releng_sub_%s_configs/%s_x86.py',
- 'api-15-partner-sample1': 'builds/releng_sub_%s_configs/%s_api_15_partner_sample1.py',
- 'android-test': 'builds/releng_sub_%s_configs/%s_test.py',
- 'android-checkstyle': 'builds/releng_sub_%s_configs/%s_checkstyle.py',
- 'android-lint': 'builds/releng_sub_%s_configs/%s_lint.py',
- 'valgrind' : 'builds/releng_sub_%s_configs/%s_valgrind.py',
- 'artifact': 'builds/releng_sub_%s_configs/%s_artifact.py',
- 'debug-artifact': 'builds/releng_sub_%s_configs/%s_debug_artifact.py',
- }
- build_pool_cfg_file = 'builds/build_pool_specifics.py'
- branch_cfg_file = 'builds/branch_specifics.py'
-
- @classmethod
- def _query_pltfrm_and_bits(cls, target_option, options):
- """ determine platform and bits
-
- This can be from either from a supplied --platform and --bits
- or parsed from given config file names.
- """
- error_msg = (
- 'Whoops!\nYou are trying to pass a shortname for '
- '%s. \nHowever, I need to know the %s to find the appropriate '
- 'filename. You can tell me by passing:\n\t"%s" or a config '
- 'filename via "--config" with %s in it. \nIn either case, these '
- 'option arguments must come before --custom-build-variant.'
- )
- current_config_files = options.config_files or []
- if not cls.bits:
- # --bits has not been supplied
- # lets parse given config file names for 32 or 64
- for cfg_file_name in current_config_files:
- if '32' in cfg_file_name:
- cls.bits = '32'
- break
- if '64' in cfg_file_name:
- cls.bits = '64'
- break
- else:
- sys.exit(error_msg % (target_option, 'bits', '--bits',
- '"32" or "64"'))
-
- if not cls.platform:
- # --platform has not been supplied
- # lets parse given config file names for platform
- for cfg_file_name in current_config_files:
- if 'windows' in cfg_file_name:
- cls.platform = 'windows'
- break
- if 'mac' in cfg_file_name:
- cls.platform = 'mac'
- break
- if 'linux' in cfg_file_name:
- cls.platform = 'linux'
- break
- if 'android' in cfg_file_name:
- cls.platform = 'android'
- break
- else:
- sys.exit(error_msg % (target_option, 'platform', '--platform',
- '"linux", "windows", "mac", or "android"'))
- return cls.bits, cls.platform
-
- @classmethod
- def find_variant_cfg_path(cls, opt, value, parser):
- valid_variant_cfg_path = None
- # first let's see if we were given a valid short-name
- if cls.build_variants.get(value):
- bits, pltfrm = cls._query_pltfrm_and_bits(opt, parser.values)
- prospective_cfg_path = cls.build_variants[value] % (pltfrm, bits)
- else:
- # this is either an incomplete path or an invalid key in
- # build_variants
- prospective_cfg_path = value
-
- if os.path.exists(prospective_cfg_path):
- # now let's see if we were given a valid pathname
- valid_variant_cfg_path = value
- else:
- # let's take our prospective_cfg_path and see if we can
- # determine an existing file
- for path in cls.config_file_search_path:
- if os.path.exists(os.path.join(path, prospective_cfg_path)):
- # success! we found a config file
- valid_variant_cfg_path = os.path.join(path,
- prospective_cfg_path)
- break
- return valid_variant_cfg_path, prospective_cfg_path
-
- @classmethod
- def set_build_variant(cls, option, opt, value, parser):
- """ sets an extra config file.
-
- This is done by either taking an existing filepath or by taking a valid
- shortname coupled with known platform/bits.
- """
- valid_variant_cfg_path, prospective_cfg_path = cls.find_variant_cfg_path(
- '--custom-build-variant-cfg', value, parser)
-
- if not valid_variant_cfg_path:
- # either the value was an indeterminable path or an invalid short
- # name
- sys.exit("Whoops!\n'--custom-build-variant' was passed but an "
- "appropriate config file could not be determined. Tried "
- "using: '%s' but it was either not:\n\t-- a valid "
- "shortname: %s \n\t-- a valid path in %s \n\t-- a "
- "valid variant for the given platform and bits." % (
- prospective_cfg_path,
- str(cls.build_variants.keys()),
- str(cls.config_file_search_path)))
- parser.values.config_files.append(valid_variant_cfg_path)
- setattr(parser.values, option.dest, value) # the pool
-
- @classmethod
- def set_build_pool(cls, option, opt, value, parser):
- # first let's add the build pool file where there may be pool
- # specific keys/values. Then let's store the pool name
- parser.values.config_files.append(cls.build_pool_cfg_file)
- setattr(parser.values, option.dest, value) # the pool
-
- @classmethod
- def set_build_branch(cls, option, opt, value, parser):
- # first let's add the branch_specific file where there may be branch
- # specific keys/values. Then let's store the branch name we are using
- parser.values.config_files.append(cls.branch_cfg_file)
- setattr(parser.values, option.dest, value) # the branch name
-
- @classmethod
- def set_platform(cls, option, opt, value, parser):
- cls.platform = value
- setattr(parser.values, option.dest, value)
-
- @classmethod
- def set_bits(cls, option, opt, value, parser):
- cls.bits = value
- setattr(parser.values, option.dest, value)
-
-
-# this global depends on BuildOptionParser and therefore can not go at the
-# top of the file
-BUILD_BASE_CONFIG_OPTIONS = [
- [['--developer-run', '--skip-buildbot-actions'], {
- "action": "store_false",
- "dest": "is_automation",
- "default": True,
- "help": "If this is running outside of Mozilla's build"
- "infrastructure, use this option. It ignores actions"
- "that are not needed and adds config checks."}],
- [['--platform'], {
- "action": "callback",
- "callback": BuildOptionParser.set_platform,
- "type": "string",
- "dest": "platform",
- "help": "Sets the platform we are running this against"
- " valid values: 'windows', 'mac', 'linux'"}],
- [['--bits'], {
- "action": "callback",
- "callback": BuildOptionParser.set_bits,
- "type": "string",
- "dest": "bits",
- "help": "Sets which bits we are building this against"
- " valid values: '32', '64'"}],
- [['--custom-build-variant-cfg'], {
- "action": "callback",
- "callback": BuildOptionParser.set_build_variant,
- "type": "string",
- "dest": "build_variant",
- "help": "Sets the build type and will determine appropriate"
- " additional config to use. Either pass a config path"
- " or use a valid shortname from: "
- "%s" % (BuildOptionParser.build_variants.keys(),)}],
- [['--build-pool'], {
- "action": "callback",
- "callback": BuildOptionParser.set_build_pool,
- "type": "string",
- "dest": "build_pool",
- "help": "This will update the config with specific pool"
- " environment keys/values. The dicts for this are"
- " in %s\nValid values: staging or"
- " production" % ('builds/build_pool_specifics.py',)}],
- [['--branch'], {
- "action": "callback",
- "callback": BuildOptionParser.set_build_branch,
- "type": "string",
- "dest": "branch",
- "help": "This sets the branch we will be building this for."
- " If this branch is in branch_specifics.py, update our"
- " config with specific keys/values from that. See"
- " %s for possibilites" % (
- BuildOptionParser.branch_cfg_file,
- )}],
- [['--scm-level'], {
- "action": "store",
- "type": "int",
- "dest": "scm_level",
- "default": 1,
- "help": "This sets the SCM level for the branch being built."
- " See https://www.mozilla.org/en-US/about/"
- "governance/policies/commit/access-policy/"}],
- [['--enable-pgo'], {
- "action": "store_true",
- "dest": "pgo_build",
- "default": False,
- "help": "Sets the build to run in PGO mode"}],
- [['--enable-nightly'], {
- "action": "store_true",
- "dest": "nightly_build",
- "default": False,
- "help": "Sets the build to run in nightly mode"}],
- [['--who'], {
- "dest": "who",
- "default": '',
- "help": "stores who made the created the buildbot change."}],
- [["--disable-mock"], {
- "dest": "disable_mock",
- "action": "store_true",
- "help": "do not run under mock despite what gecko-config says",
- }],
-
-]
-
-
-def generate_build_ID():
- return time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))
-
-
-def generate_build_UID():
- return uuid.uuid4().hex
-
-
-class BuildScript(BuildbotMixin, PurgeMixin, MockMixin, BalrogMixin,
- SigningMixin, VirtualenvMixin, MercurialScript,
- SecretsMixin, PerfherderResourceOptionsMixin):
- def __init__(self, **kwargs):
- # objdir is referenced in _query_abs_dirs() so let's make sure we
- # have that attribute before calling BaseScript.__init__
- self.objdir = None
- super(BuildScript, self).__init__(**kwargs)
- # epoch is only here to represent the start of the buildbot build
- # that this mozharn script came from. until I can grab bbot's
- # status.build.gettime()[0] this will have to do as a rough estimate
- # although it is about 4s off from the time it would be if it was
- # done through MBF.
- # TODO find out if that time diff matters or if we just use it to
- # separate each build
- self.epoch_timestamp = int(time.mktime(datetime.now().timetuple()))
- self.branch = self.config.get('branch')
- self.stage_platform = self.config.get('stage_platform')
- if not self.branch or not self.stage_platform:
- if not self.branch:
- self.error("'branch' not determined and is required")
- if not self.stage_platform:
- self.error("'stage_platform' not determined and is required")
- self.fatal("Please add missing items to your config")
- self.repo_path = None
- self.buildid = None
- self.builduid = None
- self.query_buildid() # sets self.buildid
- self.query_builduid() # sets self.builduid
- self.generated_build_props = False
- self.client_id = None
- self.access_token = None
-
- # Call this before creating the virtualenv so that we can support
- # substituting config values with other config values.
- self.query_build_env()
-
- # We need to create the virtualenv directly (without using an action) in
- # order to use python modules in PreScriptRun/Action listeners
- self.create_virtualenv()
-
- def _pre_config_lock(self, rw_config):
- c = self.config
- cfg_files_and_dicts = rw_config.all_cfg_files_and_dicts
- build_pool = c.get('build_pool', '')
- build_variant = c.get('build_variant', '')
- variant_cfg = ''
- if build_variant:
- variant_cfg = BuildOptionParser.build_variants[build_variant] % (
- BuildOptionParser.platform,
- BuildOptionParser.bits
- )
- build_pool_cfg = BuildOptionParser.build_pool_cfg_file
- branch_cfg = BuildOptionParser.branch_cfg_file
-
- cfg_match_msg = "Script was run with '%(option)s %(type)s' and \
-'%(type)s' matches a key in '%(type_config_file)s'. Updating self.config with \
-items from that key's value."
- pf_override_msg = "The branch '%(branch)s' has custom behavior for the \
-platform '%(platform)s'. Updating self.config with the following from \
-'platform_overrides' found in '%(pf_cfg_file)s':"
-
- for i, (target_file, target_dict) in enumerate(cfg_files_and_dicts):
- if branch_cfg and branch_cfg in target_file:
- self.info(
- cfg_match_msg % {
- 'option': '--branch',
- 'type': c['branch'],
- 'type_config_file': BuildOptionParser.branch_cfg_file
- }
- )
- if build_pool_cfg and build_pool_cfg in target_file:
- self.info(
- cfg_match_msg % {
- 'option': '--build-pool',
- 'type': build_pool,
- 'type_config_file': build_pool_cfg,
- }
- )
- if variant_cfg and variant_cfg in target_file:
- self.info(
- cfg_match_msg % {
- 'option': '--custom-build-variant-cfg',
- 'type': build_variant,
- 'type_config_file': variant_cfg,
- }
- )
- if c.get("platform_overrides"):
- if c['stage_platform'] in c['platform_overrides'].keys():
- self.info(
- pf_override_msg % {
- 'branch': c['branch'],
- 'platform': c['stage_platform'],
- 'pf_cfg_file': BuildOptionParser.branch_cfg_file
- }
- )
- branch_pf_overrides = c['platform_overrides'][
- c['stage_platform']
- ]
- self.info(pprint.pformat(branch_pf_overrides))
- c.update(branch_pf_overrides)
- self.info('To generate a config file based upon options passed and '
- 'config files used, run script as before but extend options '
- 'with "--dump-config"')
- self.info('For a diff of where self.config got its items, '
- 'run the script again as before but extend options with: '
- '"--dump-config-hierarchy"')
- self.info("Both --dump-config and --dump-config-hierarchy don't "
- "actually run any actions.")
-
- def _assert_cfg_valid_for_action(self, dependencies, action):
- """ assert dependency keys are in config for given action.
-
- Takes a list of dependencies and ensures that each have an
- assoctiated key in the config. Displays error messages as
- appropriate.
-
- """
- # TODO add type and value checking, not just keys
- # TODO solution should adhere to: bug 699343
- # TODO add this to BaseScript when the above is done
- # for now, let's just use this as a way to save typing...
- c = self.config
- undetermined_keys = []
- err_template = "The key '%s' could not be determined \
-and is needed for the action '%s'. Please add this to your config \
-or run without that action (ie: --no-{action})"
- for dep in dependencies:
- if dep not in c:
- undetermined_keys.append(dep)
- if undetermined_keys:
- fatal_msgs = [err_template % (key, action)
- for key in undetermined_keys]
- self.fatal("".join(fatal_msgs))
- # otherwise:
- return # all good
-
- def _query_build_prop_from_app_ini(self, prop, app_ini_path=None):
- dirs = self.query_abs_dirs()
- print_conf_setting_path = os.path.join(dirs['abs_src_dir'],
- 'config',
- 'printconfigsetting.py')
- if not app_ini_path:
- # set the default
- app_ini_path = dirs['abs_app_ini_path']
- if (os.path.exists(print_conf_setting_path) and
- os.path.exists(app_ini_path)):
- python = self.query_exe('python2.7')
- cmd = [
- python, os.path.join(dirs['abs_src_dir'], 'mach'), 'python',
- print_conf_setting_path, app_ini_path,
- 'App', prop
- ]
- env = self.query_build_env()
- # dirs['abs_obj_dir'] can be different from env['MOZ_OBJDIR'] on
- # mac, and that confuses mach.
- del env['MOZ_OBJDIR']
- return self.get_output_from_command_m(cmd,
- cwd=dirs['abs_obj_dir'], env=env)
- else:
- return None
-
- def query_builduid(self):
- c = self.config
- if self.builduid:
- return self.builduid
-
- builduid = None
- if c.get("is_automation"):
- if self.buildbot_config['properties'].get('builduid'):
- self.info("Determining builduid from buildbot properties")
- builduid = self.buildbot_config['properties']['builduid'].encode(
- 'ascii', 'replace'
- )
-
- if not builduid:
- self.info("Creating builduid through uuid hex")
- builduid = generate_build_UID()
-
- if c.get('is_automation'):
- self.set_buildbot_property('builduid',
- builduid,
- write_to_file=True)
- self.builduid = builduid
- return self.builduid
-
- def query_buildid(self):
- c = self.config
- if self.buildid:
- return self.buildid
-
- buildid = None
- if c.get("is_automation"):
- if self.buildbot_config['properties'].get('buildid'):
- self.info("Determining buildid from buildbot properties")
- buildid = self.buildbot_config['properties']['buildid'].encode(
- 'ascii', 'replace'
- )
- else:
- # for taskcluster, there are no buildbot properties, and we pass
- # MOZ_BUILD_DATE into mozharness as an environment variable, only
- # to have it pass the same value out with the same name.
- buildid = os.environ.get('MOZ_BUILD_DATE')
-
- if not buildid:
- self.info("Creating buildid through current time")
- buildid = generate_build_ID()
-
- if c.get('is_automation'):
- self.set_buildbot_property('buildid',
- buildid,
- write_to_file=True)
-
- self.buildid = buildid
- return self.buildid
-
- def _query_objdir(self):
- if self.objdir:
- return self.objdir
-
- if not self.config.get('objdir'):
- return self.fatal(MISSING_CFG_KEY_MSG % ('objdir',))
- self.objdir = self.config['objdir']
- return self.objdir
-
- def _query_repo(self):
- if self.repo_path:
- return self.repo_path
- c = self.config
-
- # we actually supply the repo in mozharness so if it's in
- # the config, we use that (automation does not require it in
- # buildbot props)
- if not c.get('repo_path'):
- repo_path = 'projects/%s' % (self.branch,)
- self.info(
- "repo_path not in config. Using '%s' instead" % (repo_path,)
- )
- else:
- repo_path = c['repo_path']
- self.repo_path = '%s/%s' % (c['repo_base'], repo_path,)
- return self.repo_path
-
- def _skip_buildbot_specific_action(self):
- """ ignore actions from buildbot's infra."""
- self.info("This action is specific to buildbot's infrastructure")
- self.info("Skipping......")
- return
-
- def query_is_nightly_promotion(self):
- platform_enabled = self.config.get('enable_nightly_promotion')
- branch_enabled = self.branch in self.config.get('nightly_promotion_branches')
- return platform_enabled and branch_enabled
-
- def query_build_env(self, **kwargs):
- c = self.config
-
- # let's evoke the base query_env and make a copy of it
- # as we don't always want every key below added to the same dict
- env = copy.deepcopy(
- super(BuildScript, self).query_env(**kwargs)
- )
-
- # first grab the buildid
- env['MOZ_BUILD_DATE'] = self.query_buildid()
-
- # Set the source repository to what we're building from since
- # the default is to query `hg paths` which isn't reliable with pooled
- # storage
- repo_path = self._query_repo()
- assert repo_path
- env['MOZ_SOURCE_REPO'] = repo_path
-
- if self.query_is_nightly() or self.query_is_nightly_promotion():
- if self.query_is_nightly():
- # nightly promotion needs to set update_channel but not do all the 'IS_NIGHTLY'
- # automation parts like uploading symbols for now
- env["IS_NIGHTLY"] = "yes"
- # in branch_specifics.py we might set update_channel explicitly
- if c.get('update_channel'):
- env["MOZ_UPDATE_CHANNEL"] = c['update_channel']
- else: # let's just give the generic channel based on branch
- env["MOZ_UPDATE_CHANNEL"] = "nightly-%s" % (self.branch,)
-
- if self.config.get('pgo_build') or self._compile_against_pgo():
- env['MOZ_PGO'] = '1'
-
- if c.get('enable_signing'):
- if os.environ.get('MOZ_SIGNING_SERVERS'):
- moz_sign_cmd = subprocess.list2cmdline(
- self.query_moz_sign_cmd(formats=None)
- )
- # windows fix. This is passed to mach build env and we call that
- # with python, not with bash so we need to fix the slashes here
- env['MOZ_SIGN_CMD'] = moz_sign_cmd.replace('\\', '\\\\\\\\')
- else:
- self.warning("signing disabled because MOZ_SIGNING_SERVERS is not set")
- elif 'MOZ_SIGN_CMD' in env:
- # Ensure that signing is truly disabled
- # MOZ_SIGN_CMD may be defined by default in buildbot (see MozillaBuildFactory)
- self.warning("Clearing MOZ_SIGN_CMD because we don't have config['enable_signing']")
- del env['MOZ_SIGN_CMD']
-
- # to activate the right behaviour in mozonfigs while we transition
- if c.get('enable_release_promotion'):
- env['ENABLE_RELEASE_PROMOTION'] = "1"
- update_channel = c.get('update_channel', self.branch)
- self.info("Release promotion update channel: %s"
- % (update_channel,))
- env["MOZ_UPDATE_CHANNEL"] = update_channel
-
- # we can't make env an attribute of self because env can change on
- # every call for reasons like MOZ_SIGN_CMD
- return env
-
- def query_mach_build_env(self, multiLocale=None):
- c = self.config
- if multiLocale is None and self.query_is_nightly():
- multiLocale = c.get('multi_locale', False)
- mach_env = {}
- if c.get('upload_env'):
- mach_env.update(c['upload_env'])
- if 'UPLOAD_HOST' in mach_env and 'stage_server' in c:
- mach_env['UPLOAD_HOST'] = mach_env['UPLOAD_HOST'] % {
- 'stage_server': c['stage_server']
- }
- if 'UPLOAD_USER' in mach_env and 'stage_username' in c:
- mach_env['UPLOAD_USER'] = mach_env['UPLOAD_USER'] % {
- 'stage_username': c['stage_username']
- }
- if 'UPLOAD_SSH_KEY' in mach_env and 'stage_ssh_key' in c:
- mach_env['UPLOAD_SSH_KEY'] = mach_env['UPLOAD_SSH_KEY'] % {
- 'stage_ssh_key': c['stage_ssh_key']
- }
-
- # this prevents taskcluster from overwriting the target files with
- # the multilocale files. Put everything from the en-US build in a
- # separate folder.
- if multiLocale and self.config.get('taskcluster_nightly'):
- if 'UPLOAD_PATH' in mach_env:
- mach_env['UPLOAD_PATH'] = os.path.join(mach_env['UPLOAD_PATH'],
- 'en-US')
-
- # _query_post_upload_cmd returns a list (a cmd list), for env sake here
- # let's make it a string
- if c.get('is_automation'):
- pst_up_cmd = ' '.join([str(i) for i in self._query_post_upload_cmd(multiLocale)])
- mach_env['POST_UPLOAD_CMD'] = pst_up_cmd
-
- return mach_env
-
- def _compile_against_pgo(self):
- """determines whether a build should be run with pgo even if it is
- not a classified as a 'pgo build'.
-
- requirements:
- 1) must be a platform that can run against pgo
- 2) either:
- a) must be a nightly build
- b) must be on a branch that runs pgo if it can everytime
- """
- c = self.config
- if self.stage_platform in c['pgo_platforms']:
- if c.get('branch_uses_per_checkin_strategy') or self.query_is_nightly():
- return True
- return False
-
- def query_check_test_env(self):
- c = self.config
- dirs = self.query_abs_dirs()
- check_test_env = {}
- if c.get('check_test_env'):
- for env_var, env_value in c['check_test_env'].iteritems():
- check_test_env[env_var] = env_value % dirs
- return check_test_env
-
- def _query_post_upload_cmd(self, multiLocale):
- c = self.config
- post_upload_cmd = ["post_upload.py"]
- buildid = self.query_buildid()
- revision = self.query_revision()
- platform = self.stage_platform
- who = self.query_who()
- if c.get('pgo_build'):
- platform += '-pgo'
-
- if c.get('tinderbox_build_dir'):
- # TODO find out if we should fail here like we are
- if not who and not revision:
- self.fatal("post upload failed. --tinderbox-builds-dir could "
- "not be determined. 'who' and/or 'revision' unknown")
- # branches like try will use 'tinderbox_build_dir
- tinderbox_build_dir = c['tinderbox_build_dir'] % {
- 'who': who,
- 'got_revision': revision
- }
- else:
- # the default
- tinderbox_build_dir = "%s-%s" % (self.branch, platform)
-
- if who and self.branch == 'try':
- post_upload_cmd.extend(["--who", who])
- if c.get('include_post_upload_builddir'):
- post_upload_cmd.extend(
- ["--builddir", "%s-%s" % (self.branch, platform)]
- )
- elif multiLocale:
- # Android builds with multilocale enabled upload the en-US builds
- # to an en-US subdirectory, and the multilocale builds to the
- # top-level directory.
- post_upload_cmd.extend(
- ["--builddir", "en-US"]
- )
-
- post_upload_cmd.extend(["--tinderbox-builds-dir", tinderbox_build_dir])
- post_upload_cmd.extend(["-p", c['stage_product']])
- post_upload_cmd.extend(['-i', buildid])
- if revision:
- post_upload_cmd.extend(['--revision', revision])
- if c.get('to_tinderbox_dated'):
- post_upload_cmd.append('--release-to-tinderbox-dated-builds')
- if c.get('release_to_try_builds'):
- post_upload_cmd.append('--release-to-try-builds')
- if self.query_is_nightly():
- if c.get('post_upload_include_platform'):
- post_upload_cmd.extend(['-b', '%s-%s' % (self.branch, platform)])
- else:
- post_upload_cmd.extend(['-b', self.branch])
- post_upload_cmd.append('--release-to-dated')
- if c['platform_supports_post_upload_to_latest']:
- post_upload_cmd.append('--release-to-latest')
- post_upload_cmd.extend(c.get('post_upload_extra', []))
-
- return post_upload_cmd
-
- def _ccache_z(self):
- """clear ccache stats."""
- dirs = self.query_abs_dirs()
- env = self.query_build_env()
- self.run_command(command=['ccache', '-z'],
- cwd=dirs['base_work_dir'],
- env=env)
-
- def _ccache_s(self):
- """print ccache stats. only done for unix like platforms"""
- dirs = self.query_abs_dirs()
- env = self.query_build_env()
- cmd = ['ccache', '-s']
- self.run_command(cmd, cwd=dirs['abs_src_dir'], env=env)
-
- def _rm_old_package(self):
- """rm the old package."""
- c = self.config
- dirs = self.query_abs_dirs()
- old_package_paths = []
- old_package_patterns = c.get('old_packages')
-
- self.info("removing old packages...")
- if os.path.exists(dirs['abs_obj_dir']):
- for product in old_package_patterns:
- old_package_paths.extend(
- glob.glob(product % {"objdir": dirs['abs_obj_dir']})
- )
- if old_package_paths:
- for package_path in old_package_paths:
- self.rmtree(package_path)
- else:
- self.info("There wasn't any old packages to remove.")
-
- def _get_mozconfig(self):
- """assign mozconfig."""
- c = self.config
- dirs = self.query_abs_dirs()
- abs_mozconfig_path = ''
-
- # first determine the mozconfig path
- if c.get('src_mozconfig') and not c.get('src_mozconfig_manifest'):
- self.info('Using in-tree mozconfig')
- abs_mozconfig_path = os.path.join(dirs['abs_src_dir'], c.get('src_mozconfig'))
- elif c.get('src_mozconfig_manifest') and not c.get('src_mozconfig'):
- self.info('Using mozconfig based on manifest contents')
- manifest = os.path.join(dirs['abs_work_dir'], c['src_mozconfig_manifest'])
- if not os.path.exists(manifest):
- self.fatal('src_mozconfig_manifest: "%s" not found. Does it exist?' % (manifest,))
- with self.opened(manifest, error_level=ERROR) as (fh, err):
- if err:
- self.fatal("%s exists but coud not read properties" % manifest)
- abs_mozconfig_path = os.path.join(dirs['abs_src_dir'], json.load(fh)['gecko_path'])
- else:
- self.fatal("'src_mozconfig' or 'src_mozconfig_manifest' must be "
- "in the config but not both in order to determine the mozconfig.")
-
- # print its contents
- content = self.read_from_file(abs_mozconfig_path, error_level=FATAL)
- self.info("mozconfig content:")
- self.info(content)
-
- # finally, copy the mozconfig to a path that 'mach build' expects it to be
- self.copyfile(abs_mozconfig_path, os.path.join(dirs['abs_src_dir'], '.mozconfig'))
-
- # TODO: replace with ToolToolMixin
- def _get_tooltool_auth_file(self):
- # set the default authentication file based on platform; this
- # corresponds to where puppet puts the token
- if 'tooltool_authentication_file' in self.config:
- fn = self.config['tooltool_authentication_file']
- elif self._is_windows():
- fn = r'c:\builds\relengapi.tok'
- else:
- fn = '/builds/relengapi.tok'
-
- # if the file doesn't exist, don't pass it to tooltool (it will just
- # fail). In taskcluster, this will work OK as the relengapi-proxy will
- # take care of auth. Everywhere else, we'll get auth failures if
- # necessary.
- if os.path.exists(fn):
- return fn
-
- def _run_tooltool(self):
- self._assert_cfg_valid_for_action(
- ['tooltool_script', 'tooltool_bootstrap', 'tooltool_url'],
- 'build'
- )
- c = self.config
- dirs = self.query_abs_dirs()
- if not c.get('tooltool_manifest_src'):
- return self.warning(ERROR_MSGS['tooltool_manifest_undetermined'])
- fetch_script_path = os.path.join(dirs['abs_tools_dir'],
- 'scripts',
- 'tooltool',
- 'tooltool_wrapper.sh')
- tooltool_manifest_path = os.path.join(dirs['abs_src_dir'],
- c['tooltool_manifest_src'])
- cmd = [
- 'sh',
- fetch_script_path,
- tooltool_manifest_path,
- c['tooltool_url'],
- c['tooltool_bootstrap'],
- ]
- cmd.extend(c['tooltool_script'])
- auth_file = self._get_tooltool_auth_file()
- if auth_file:
- cmd.extend(['--authentication-file', auth_file])
- cache = c['env'].get('TOOLTOOL_CACHE')
- if cache:
- cmd.extend(['-c', cache])
- self.info(str(cmd))
- self.run_command_m(cmd, cwd=dirs['abs_src_dir'], halt_on_failure=True)
-
- def query_revision(self, source_path=None):
- """ returns the revision of the build
-
- first will look for it in buildbot_properties and then in
- buildbot_config. Failing that, it will actually poll the source of
- the repo if it exists yet.
-
- This method is used both to figure out what revision to check out and
- to figure out what revision *was* checked out.
- """
- revision = None
- if 'revision' in self.buildbot_properties:
- revision = self.buildbot_properties['revision']
- elif (self.buildbot_config and
- self.buildbot_config.get('sourcestamp', {}).get('revision')):
- revision = self.buildbot_config['sourcestamp']['revision']
- elif self.buildbot_config and self.buildbot_config.get('revision'):
- revision = self.buildbot_config['revision']
- else:
- if not source_path:
- dirs = self.query_abs_dirs()
- source_path = dirs['abs_src_dir'] # let's take the default
-
- # Look at what we have checked out
- if os.path.exists(source_path):
- hg = self.query_exe('hg', return_type='list')
- revision = self.get_output_from_command(
- hg + ['parent', '--template', '{node}'], cwd=source_path
- )
- return revision.encode('ascii', 'replace') if revision else None
-
- def _checkout_source(self):
- """use vcs_checkout to grab source needed for build."""
- # TODO make this method its own action
- c = self.config
- dirs = self.query_abs_dirs()
- repo = self._query_repo()
- vcs_checkout_kwargs = {
- 'repo': repo,
- 'dest': dirs['abs_src_dir'],
- 'revision': self.query_revision(),
- 'env': self.query_build_env()
- }
- if c.get('clone_by_revision'):
- vcs_checkout_kwargs['clone_by_revision'] = True
-
- if c.get('clone_with_purge'):
- vcs_checkout_kwargs['clone_with_purge'] = True
- vcs_checkout_kwargs['clone_upstream_url'] = c.get('clone_upstream_url')
- rev = self.vcs_checkout(**vcs_checkout_kwargs)
- if c.get('is_automation'):
- changes = self.buildbot_config['sourcestamp']['changes']
- if changes:
- comments = changes[0].get('comments', '')
- self.set_buildbot_property('comments',
- comments,
- write_to_file=True)
- else:
- self.warning(ERROR_MSGS['comments_undetermined'])
- self.set_buildbot_property('got_revision',
- rev,
- write_to_file=True)
-
- def _count_ctors(self):
- """count num of ctors and set testresults."""
- dirs = self.query_abs_dirs()
- python_path = os.path.join(dirs['abs_work_dir'], 'venv', 'bin',
- 'python')
- abs_count_ctors_path = os.path.join(dirs['abs_src_dir'],
- 'build',
- 'util',
- 'count_ctors.py')
- abs_libxul_path = os.path.join(dirs['abs_obj_dir'],
- 'dist',
- 'bin',
- 'libxul.so')
-
- cmd = [python_path, abs_count_ctors_path, abs_libxul_path]
- self.get_output_from_command(cmd, cwd=dirs['abs_src_dir'],
- throw_exception=True)
-
- def _generate_properties_file(self, path):
- # TODO it would be better to grab all the properties that were
- # persisted to file rather than use whats in the buildbot_properties
- # live object so we become less action dependant.
- all_current_props = dict(
- chain(self.buildbot_config['properties'].items(),
- self.buildbot_properties.items())
- )
- # graph_server_post.py expects a file with 'properties' key
- graph_props = dict(properties=all_current_props)
- self.dump_config(path, graph_props)
-
- def _query_props_set_by_mach(self, console_output=True, error_level=FATAL):
- mach_properties_path = os.path.join(
- self.query_abs_dirs()['abs_obj_dir'], 'dist', 'mach_build_properties.json'
- )
- self.info("setting properties set by mach build. Looking in path: %s"
- % mach_properties_path)
- if os.path.exists(mach_properties_path):
- with self.opened(mach_properties_path, error_level=error_level) as (fh, err):
- build_props = json.load(fh)
- if err:
- self.log("%s exists but there was an error reading the "
- "properties. props: `%s` - error: "
- "`%s`" % (mach_properties_path,
- build_props or 'None',
- err or 'No error'),
- error_level)
- if console_output:
- self.info("Properties set from 'mach build'")
- self.info(pprint.pformat(build_props))
- for key, prop in build_props.iteritems():
- if prop != 'UNKNOWN':
- self.set_buildbot_property(key, prop, write_to_file=True)
- else:
- self.info("No mach_build_properties.json found - not importing properties.")
-
- def generate_build_props(self, console_output=True, halt_on_failure=False):
- """sets props found from mach build and, in addition, buildid,
- sourcestamp, appVersion, and appName."""
-
- error_level = ERROR
- if halt_on_failure:
- error_level = FATAL
-
- if self.generated_build_props:
- return
-
- # grab props set by mach if any
- self._query_props_set_by_mach(console_output=console_output,
- error_level=error_level)
-
- dirs = self.query_abs_dirs()
- print_conf_setting_path = os.path.join(dirs['abs_src_dir'],
- 'config',
- 'printconfigsetting.py')
- if (not os.path.exists(print_conf_setting_path) or
- not os.path.exists(dirs['abs_app_ini_path'])):
- self.log("Can't set the following properties: "
- "buildid, sourcestamp, appVersion, and appName. "
- "Required paths missing. Verify both %s and %s "
- "exist. These paths require the 'build' action to be "
- "run prior to this" % (print_conf_setting_path,
- dirs['abs_app_ini_path']),
- level=error_level)
- self.info("Setting properties found in: %s" % dirs['abs_app_ini_path'])
- python = self.query_exe('python2.7')
- base_cmd = [
- python, os.path.join(dirs['abs_src_dir'], 'mach'), 'python',
- print_conf_setting_path, dirs['abs_app_ini_path'], 'App'
- ]
- properties_needed = [
- {'ini_name': 'SourceStamp', 'prop_name': 'sourcestamp'},
- {'ini_name': 'Version', 'prop_name': 'appVersion'},
- {'ini_name': 'Name', 'prop_name': 'appName'}
- ]
- env = self.query_build_env()
- # dirs['abs_obj_dir'] can be different from env['MOZ_OBJDIR'] on
- # mac, and that confuses mach.
- del env['MOZ_OBJDIR']
- for prop in properties_needed:
- prop_val = self.get_output_from_command_m(
- base_cmd + [prop['ini_name']], cwd=dirs['abs_obj_dir'],
- halt_on_failure=halt_on_failure, env=env
- )
- self.set_buildbot_property(prop['prop_name'],
- prop_val,
- write_to_file=True)
-
- if self.config.get('is_automation'):
- self.info("Verifying buildid from application.ini matches buildid "
- "from buildbot")
- app_ini_buildid = self._query_build_prop_from_app_ini('BuildID')
- # it would be hard to imagine query_buildid evaluating to a falsey
- # value (e.g. 0), but incase it does, force it to None
- buildbot_buildid = self.query_buildid() or None
- self.info(
- 'buildid from application.ini: "%s". buildid from buildbot '
- 'properties: "%s"' % (app_ini_buildid, buildbot_buildid)
- )
- if app_ini_buildid == buildbot_buildid != None:
- self.info('buildids match.')
- else:
- self.error(
- 'buildids do not match or values could not be determined'
- )
- # set the build to orange if not already worse
- self.return_code = self.worst_level(
- EXIT_STATUS_DICT[TBPL_WARNING], self.return_code,
- AUTOMATION_EXIT_CODES[::-1]
- )
-
- self.generated_build_props = True
-
- def _initialize_taskcluster(self):
- if self.client_id and self.access_token:
- # Already initialized
- return
-
- dirs = self.query_abs_dirs()
- auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file'])
- credentials = {}
- execfile(auth, credentials)
- self.client_id = credentials.get('taskcluster_clientId')
- self.access_token = credentials.get('taskcluster_accessToken')
-
- # We need to create & activate the virtualenv so that we can import
- # taskcluster (and its dependent modules, like requests and hawk).
- # Normally we could create the virtualenv as an action, but due to some
- # odd dependencies with query_build_env() being called from build(),
- # which is necessary before the virtualenv can be created.
- self.create_virtualenv()
- self.activate_virtualenv()
-
- routes_file = os.path.join(dirs['abs_src_dir'],
- 'testing',
- 'mozharness',
- 'configs',
- 'routes.json')
- with open(routes_file) as f:
- self.routes_json = json.load(f)
-
- def _taskcluster_upload(self, files, templates, locale='en-US',
- property_conditions=[]):
- if not self.client_id or not self.access_token:
- self.warning('Skipping S3 file upload: No taskcluster credentials.')
- return
-
- dirs = self.query_abs_dirs()
- repo = self._query_repo()
- revision = self.query_revision()
- pushinfo = self.vcs_query_pushinfo(repo, revision)
- pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate))
-
- index = self.config.get('taskcluster_index', 'index.garbage.staging')
- fmt = {
- 'index': index,
- 'project': self.buildbot_config['properties']['branch'],
- 'head_rev': revision,
- 'pushdate': pushdate,
- 'year': pushdate[0:4],
- 'month': pushdate[4:6],
- 'day': pushdate[6:8],
- 'build_product': self.config['stage_product'],
- 'build_name': self.query_build_name(),
- 'build_type': self.query_build_type(),
- 'locale': locale,
- }
- fmt.update(self.buildid_to_dict(self.query_buildid()))
- routes = []
- for template in templates:
- routes.append(template.format(**fmt))
- self.info("Using routes: %s" % routes)
-
- tc = Taskcluster(
- branch=self.branch,
- rank=pushinfo.pushdate, # Use pushdate as the rank
- client_id=self.client_id,
- access_token=self.access_token,
- log_obj=self.log_obj,
- # `upload_to_task_id` is used by mozci to have access to where the artifacts
- # will be uploaded
- task_id=self.buildbot_config['properties'].get('upload_to_task_id'),
- )
-
- # TODO: Bug 1165980 - these should be in tree
- routes.extend([
- "%s.buildbot.branches.%s.%s" % (index, self.branch, self.stage_platform),
- "%s.buildbot.revisions.%s.%s.%s" % (index, revision, self.branch, self.stage_platform),
- ])
- task = tc.create_task(routes)
- tc.claim_task(task)
-
- # Only those files uploaded with valid extensions are processed.
- # This ensures that we get the correct packageUrl from the list.
- valid_extensions = (
- '.apk',
- '.dmg',
- '.mar',
- '.rpm',
- '.tar.bz2',
- '.tar.gz',
- '.zip',
- '.json',
- )
-
- for upload_file in files:
- # Create an S3 artifact for each file that gets uploaded. We also
- # check the uploaded file against the property conditions so that we
- # can set the buildbot config with the correct URLs for package
- # locations.
- tc.create_artifact(task, upload_file)
- if upload_file.endswith(valid_extensions):
- for prop, condition in property_conditions:
- if condition(upload_file):
- self.set_buildbot_property(prop, tc.get_taskcluster_url(upload_file))
- break
-
- # Upload a file with all Buildbot properties
- # This is necessary for Buildbot Bridge test jobs work properly
- # until we can migrate to TaskCluster
- properties_path = os.path.join(
- dirs['base_work_dir'],
- 'buildbot_properties.json'
- )
- self._generate_properties_file(properties_path)
- tc.create_artifact(task, properties_path)
-
- tc.report_completed(task)
-
- def upload_files(self):
- self._initialize_taskcluster()
- dirs = self.query_abs_dirs()
-
- if self.query_is_nightly():
- templates = self.routes_json['nightly']
-
- # Nightly builds with l10n counterparts also publish to the
- # 'en-US' locale.
- if self.config.get('publish_nightly_en_US_routes'):
- templates.extend(self.routes_json['l10n'])
- else:
- templates = self.routes_json['routes']
-
- # Some trees may not be setting uploadFiles, so default to []. Normally
- # we'd only expect to get here if the build completes successfully,
- # which means we should have uploadFiles.
- files = self.query_buildbot_property('uploadFiles') or []
- if not files:
- self.warning('No files from the build system to upload to S3: uploadFiles property is missing or empty.')
-
- packageName = self.query_buildbot_property('packageFilename')
- self.info('packageFilename is: %s' % packageName)
-
- if self.config.get('use_package_as_marfile'):
- self.info('Using packageUrl for the MAR file')
- self.set_buildbot_property('completeMarUrl',
- self.query_buildbot_property('packageUrl'),
- write_to_file=True)
-
- # Find the full path to the package in uploadFiles so we can
- # get the size/hash of the mar
- for upload_file in files:
- if upload_file.endswith(packageName):
- self.set_buildbot_property('completeMarSize',
- self.query_filesize(upload_file),
- write_to_file=True)
- self.set_buildbot_property('completeMarHash',
- self.query_sha512sum(upload_file),
- write_to_file=True)
- break
-
- property_conditions = [
- # key: property name, value: condition
- ('symbolsUrl', lambda m: m.endswith('crashreporter-symbols.zip') or
- m.endswith('crashreporter-symbols-full.zip')),
- ('testsUrl', lambda m: m.endswith(('tests.tar.bz2', 'tests.zip'))),
- ('jsshellUrl', lambda m: 'jsshell-' in m and m.endswith('.zip')),
- # Temporarily use "TC" in MarUrl parameters. We don't want to
- # override these to point to taskcluster just yet, and still
- # need to use FTP. However, they can't be removed outright since
- # that can affect packageUrl. See bug 1144985.
- ('completeMarUrlTC', lambda m: m.endswith('.complete.mar')),
- ('partialMarUrlTC', lambda m: m.endswith('.mar') and '.partial.' in m),
- ('codeCoverageURL', lambda m: m.endswith('code-coverage-gcno.zip')),
- ('sdkUrl', lambda m: m.endswith(('sdk.tar.bz2', 'sdk.zip'))),
- ('testPackagesUrl', lambda m: m.endswith('test_packages.json')),
- ('packageUrl', lambda m: m.endswith(packageName)),
- ]
-
- # Also upload our mozharness log files
- files.extend([os.path.join(self.log_obj.abs_log_dir, x) for x in self.log_obj.log_files.values()])
-
- # Also upload our buildprops.json file.
- files.extend([os.path.join(dirs['base_work_dir'], 'buildprops.json')])
-
- self._taskcluster_upload(files, templates,
- property_conditions=property_conditions)
-
- def _set_file_properties(self, file_name, find_dir, prop_type,
- error_level=ERROR):
- c = self.config
- dirs = self.query_abs_dirs()
-
- # windows fix. even bash -c loses two single slashes.
- find_dir = find_dir.replace('\\', '\\\\\\\\')
-
- error_msg = "Not setting props: %s{Filename, Size, Hash}" % prop_type
- cmd = ["bash", "-c",
- "find %s -maxdepth 1 -type f -name %s" % (find_dir, file_name)]
- file_path = self.get_output_from_command(cmd, dirs['abs_work_dir'])
- if not file_path:
- self.error(error_msg)
- self.error("Can't determine filepath with cmd: %s" % (str(cmd),))
- return
-
- cmd = [
- self.query_exe('openssl'), 'dgst',
- '-%s' % (c.get("hash_type", "sha512"),), file_path
- ]
- hash_prop = self.get_output_from_command(cmd, dirs['abs_work_dir'])
- if not hash_prop:
- self.log("undetermined hash_prop with cmd: %s" % (str(cmd),),
- level=error_level)
- self.log(error_msg, level=error_level)
- return
- self.set_buildbot_property(prop_type + 'Filename',
- os.path.split(file_path)[1],
- write_to_file=True)
- self.set_buildbot_property(prop_type + 'Size',
- os.path.getsize(file_path),
- write_to_file=True)
- self.set_buildbot_property(prop_type + 'Hash',
- hash_prop.strip().split(' ', 2)[1],
- write_to_file=True)
-
- def clone_tools(self):
- """clones the tools repo."""
- self._assert_cfg_valid_for_action(['tools_repo'], 'clone_tools')
- c = self.config
- dirs = self.query_abs_dirs()
- repo = {
- 'repo': c['tools_repo'],
- 'vcs': 'hg',
- 'dest': dirs['abs_tools_dir'],
- 'output_timeout': 1200,
- }
- self.vcs_checkout(**repo)
-
- def _create_mozbuild_dir(self, mozbuild_path=None):
- if not mozbuild_path:
- env = self.query_build_env()
- mozbuild_path = env.get('MOZBUILD_STATE_PATH')
- if mozbuild_path:
- self.mkdir_p(mozbuild_path)
- else:
- self.warning("mozbuild_path could not be determined. skipping "
- "creating it.")
-
- def checkout_sources(self):
- self._checkout_source()
-
- def preflight_build(self):
- """set up machine state for a complete build."""
- c = self.config
- if c.get('enable_ccache'):
- self._ccache_z()
- if not self.query_is_nightly():
- # the old package should live in source dir so we don't need to do
- # this for nighties since we clobber the whole work_dir in
- # clobber()
- self._rm_old_package()
- self._get_mozconfig()
- self._run_tooltool()
- self._create_mozbuild_dir()
- mach_props = os.path.join(
- self.query_abs_dirs()['abs_obj_dir'], 'dist', 'mach_build_properties.json'
- )
- if os.path.exists(mach_props):
- self.info("Removing previous mach property file: %s" % mach_props)
- self.rmtree(mach_props)
-
- def build(self):
- """builds application."""
- env = self.query_build_env()
- env.update(self.query_mach_build_env())
-
- # XXX Bug 1037883 - mozconfigs can not find buildprops.json when builds
- # are through mozharness. This is not pretty but it is a stopgap
- # until an alternative solution is made or all builds that touch
- # mozconfig.cache are converted to mozharness.
- dirs = self.query_abs_dirs()
- buildprops = os.path.join(dirs['base_work_dir'], 'buildprops.json')
- # not finding buildprops is not an error outside of buildbot
- if os.path.exists(buildprops):
- self.copyfile(
- buildprops,
- os.path.join(dirs['abs_work_dir'], 'buildprops.json'))
-
- # use mh config override for mach build wrapper, if it exists
- python = self.query_exe('python2.7')
- default_mach_build = [python, 'mach', '--log-no-times', 'build', '-v']
- mach_build = self.query_exe('mach-build', default=default_mach_build)
- return_code = self.run_command_m(
- command=mach_build,
- cwd=dirs['abs_src_dir'],
- env=env,
- output_timeout=self.config.get('max_build_output_timeout', 60 * 40)
- )
- if return_code:
- self.return_code = self.worst_level(
- EXIT_STATUS_DICT[TBPL_FAILURE], self.return_code,
- AUTOMATION_EXIT_CODES[::-1]
- )
- self.fatal("'mach build' did not run successfully. Please check "
- "log for errors.")
-
- def multi_l10n(self):
- if not self.query_is_nightly():
- self.info("Not a nightly build, skipping multi l10n.")
- return
- self._initialize_taskcluster()
-
- dirs = self.query_abs_dirs()
- base_work_dir = dirs['base_work_dir']
- objdir = dirs['abs_obj_dir']
- branch = self.branch
-
- # Building a nightly with the try repository fails because a
- # config-file does not exist for try. Default to mozilla-central
- # settings (arbitrarily).
- if branch == 'try':
- branch = 'mozilla-central'
-
- # Some android versions share the same .json config - if
- # multi_locale_config_platform is set, use that the .json name;
- # otherwise, use the buildbot platform.
- default_platform = self.buildbot_config['properties'].get('platform',
- 'android')
-
- multi_config_pf = self.config.get('multi_locale_config_platform',
- default_platform)
-
- # The l10n script location differs on buildbot and taskcluster
- if self.config.get('taskcluster_nightly'):
- multil10n_path = \
- 'build/src/testing/mozharness/scripts/multil10n.py'
- base_work_dir = os.path.join(base_work_dir, 'workspace')
- else:
- multil10n_path = '%s/scripts/scripts/multil10n.py' % base_work_dir,
-
- cmd = [
- self.query_exe('python'),
- multil10n_path,
- '--config-file',
- 'multi_locale/%s_%s.json' % (branch, multi_config_pf),
- '--config-file',
- 'multi_locale/android-mozharness-build.json',
- '--merge-locales',
- '--pull-locale-source',
- '--add-locales',
- '--package-multi',
- '--summary',
- ]
-
- self.run_command_m(cmd, env=self.query_build_env(), cwd=base_work_dir,
- halt_on_failure=True)
-
- package_cmd = [
- 'make',
- 'echo-variable-PACKAGE',
- 'AB_CD=multi',
- ]
- package_filename = self.get_output_from_command_m(
- package_cmd,
- cwd=objdir,
- )
- if not package_filename:
- self.fatal("Unable to determine the package filename for the multi-l10n build. Was trying to run: %s" % package_cmd)
-
- self.info('Multi-l10n package filename is: %s' % package_filename)
-
- parser = MakeUploadOutputParser(config=self.config,
- log_obj=self.log_obj,
- use_package_as_marfile=True,
- package_filename=package_filename,
- )
- upload_cmd = ['make', 'upload', 'AB_CD=multi']
- self.run_command_m(upload_cmd,
- env=self.query_mach_build_env(multiLocale=False),
- cwd=objdir, halt_on_failure=True,
- output_parser=parser)
- for prop in parser.matches:
- self.set_buildbot_property(prop,
- parser.matches[prop],
- write_to_file=True)
- upload_files_cmd = [
- 'make',
- 'echo-variable-UPLOAD_FILES',
- 'AB_CD=multi',
- ]
- output = self.get_output_from_command_m(
- upload_files_cmd,
- cwd=objdir,
- )
- files = shlex.split(output)
- abs_files = [os.path.abspath(os.path.join(objdir, f)) for f in files]
- self._taskcluster_upload(abs_files, self.routes_json['l10n'],
- locale='multi')
-
- def postflight_build(self, console_output=True):
- """grabs properties from post build and calls ccache -s"""
- self.generate_build_props(console_output=console_output,
- halt_on_failure=True)
- if self.config.get('enable_ccache'):
- self._ccache_s()
-
- # A list of argument lists. Better names gratefully accepted!
- mach_commands = self.config.get('postflight_build_mach_commands', [])
- for mach_command in mach_commands:
- self._execute_postflight_build_mach_command(mach_command)
-
- def _execute_postflight_build_mach_command(self, mach_command_args):
- env = self.query_build_env()
- env.update(self.query_mach_build_env())
- python = self.query_exe('python2.7')
-
- command = [python, 'mach', '--log-no-times']
- command.extend(mach_command_args)
-
- self.run_command_m(
- command=command,
- cwd=self.query_abs_dirs()['abs_src_dir'],
- env=env, output_timeout=self.config.get('max_build_output_timeout', 60 * 20),
- halt_on_failure=True,
- )
-
- def preflight_package_source(self):
- self._get_mozconfig()
-
- def package_source(self):
- """generates source archives and uploads them"""
- env = self.query_build_env()
- env.update(self.query_mach_build_env())
- python = self.query_exe('python2.7')
- dirs = self.query_abs_dirs()
-
- self.run_command_m(
- command=[python, 'mach', '--log-no-times', 'configure'],
- cwd=dirs['abs_src_dir'],
- env=env, output_timeout=60*3, halt_on_failure=True,
- )
- self.run_command_m(
- command=[
- 'make', 'source-package', 'hg-bundle', 'source-upload',
- 'HG_BUNDLE_REVISION=%s' % self.query_revision(),
- 'UPLOAD_HG_BUNDLE=1',
- ],
- cwd=dirs['abs_obj_dir'],
- env=env, output_timeout=60*45, halt_on_failure=True,
- )
-
- def generate_source_signing_manifest(self):
- """Sign source checksum file"""
- env = self.query_build_env()
- env.update(self.query_mach_build_env())
- if env.get("UPLOAD_HOST") != "localhost":
- self.warning("Skipping signing manifest generation. Set "
- "UPLOAD_HOST to `localhost' to enable.")
- return
-
- if not env.get("UPLOAD_PATH"):
- self.warning("Skipping signing manifest generation. Set "
- "UPLOAD_PATH to enable.")
- return
-
- dirs = self.query_abs_dirs()
- objdir = dirs['abs_obj_dir']
-
- output = self.get_output_from_command_m(
- command=['make', 'echo-variable-SOURCE_CHECKSUM_FILE'],
- cwd=objdir,
- )
- files = shlex.split(output)
- abs_files = [os.path.abspath(os.path.join(objdir, f)) for f in files]
- manifest_file = os.path.join(env["UPLOAD_PATH"],
- "signing_manifest.json")
- self.write_to_file(manifest_file,
- self.generate_signing_manifest(abs_files))
-
- def check_test(self):
- if self.config.get('forced_artifact_build'):
- self.info('Skipping due to forced artifact build.')
- return
- c = self.config
- dirs = self.query_abs_dirs()
-
- env = self.query_build_env()
- env.update(self.query_check_test_env())
-
- if c.get('enable_pymake'): # e.g. windows
- pymake_path = os.path.join(dirs['abs_src_dir'], 'build',
- 'pymake', 'make.py')
- cmd = ['python', pymake_path]
- else:
- cmd = ['make']
- cmd.extend(['-k', 'check'])
-
- parser = CheckTestCompleteParser(config=c,
- log_obj=self.log_obj)
- return_code = self.run_command_m(command=cmd,
- cwd=dirs['abs_obj_dir'],
- env=env,
- output_parser=parser)
- tbpl_status = parser.evaluate_parser(return_code)
- return_code = EXIT_STATUS_DICT[tbpl_status]
-
- if return_code:
- self.return_code = self.worst_level(
- return_code, self.return_code,
- AUTOMATION_EXIT_CODES[::-1]
- )
- self.error("'make -k check' did not run successfully. Please check "
- "log for errors.")
-
- def _load_build_resources(self):
- p = self.config.get('build_resources_path') % self.query_abs_dirs()
- if not os.path.exists(p):
- self.info('%s does not exist; not loading build resources' % p)
- return None
-
- with open(p, 'rb') as fh:
- resources = json.load(fh)
-
- if 'duration' not in resources:
- self.info('resource usage lacks duration; ignoring')
- return None
-
- data = {
- 'name': 'build times',
- 'value': resources['duration'],
- 'extraOptions': self.perfherder_resource_options(),
- 'subtests': [],
- }
-
- for phase in resources['phases']:
- if 'duration' not in phase:
- continue
- data['subtests'].append({
- 'name': phase['name'],
- 'value': phase['duration'],
- })
-
- return data
-
- def generate_build_stats(self):
- """grab build stats following a compile.
-
- This action handles all statistics from a build: 'count_ctors'
- and then posts to graph server the results.
- We only post to graph server for non nightly build
- """
- if self.config.get('forced_artifact_build'):
- self.info('Skipping due to forced artifact build.')
- return
-
- import tarfile
- import zipfile
- c = self.config
-
- if c.get('enable_count_ctors'):
- self.info("counting ctors...")
- self._count_ctors()
- else:
- self.info("ctors counts are disabled for this build.")
-
- # Report some important file sizes for display in treeherder
-
- dirs = self.query_abs_dirs()
- packageName = self.query_buildbot_property('packageFilename')
-
- # if packageName is not set because we are not running in Buildbot,
- # then assume we are using MOZ_SIMPLE_PACKAGE_NAME, which means the
- # package is named one of target.{tar.bz2,zip,dmg}.
- if not packageName:
- dist_dir = os.path.join(dirs['abs_obj_dir'], 'dist')
- for ext in ['apk', 'dmg', 'tar.bz2', 'zip']:
- name = 'target.' + ext
- if os.path.exists(os.path.join(dist_dir, name)):
- packageName = name
- break
- else:
- self.fatal("could not determine packageName")
-
- interests = ['libxul.so', 'classes.dex', 'omni.ja']
- installer = os.path.join(dirs['abs_obj_dir'], 'dist', packageName)
- installer_size = 0
- size_measurements = []
-
- if os.path.exists(installer):
- installer_size = self.query_filesize(installer)
- self.info('TinderboxPrint: Size of %s<br/>%s bytes\n' % (
- packageName, installer_size))
- try:
- subtests = {}
- if zipfile.is_zipfile(installer):
- with zipfile.ZipFile(installer, 'r') as zf:
- for zi in zf.infolist():
- name = os.path.basename(zi.filename)
- size = zi.file_size
- if name in interests:
- if name in subtests:
- # File seen twice in same archive;
- # ignore to avoid confusion.
- subtests[name] = None
- else:
- subtests[name] = size
- elif tarfile.is_tarfile(installer):
- with tarfile.open(installer, 'r:*') as tf:
- for ti in tf:
- name = os.path.basename(ti.name)
- size = ti.size
- if name in interests:
- if name in subtests:
- # File seen twice in same archive;
- # ignore to avoid confusion.
- subtests[name] = None
- else:
- subtests[name] = size
- for name in subtests:
- if subtests[name] is not None:
- self.info('TinderboxPrint: Size of %s<br/>%s bytes\n' % (
- name, subtests[name]))
- size_measurements.append({'name': name, 'value': subtests[name]})
- except:
- self.info('Unable to search %s for component sizes.' % installer)
- size_measurements = []
-
- perfherder_data = {
- "framework": {
- "name": "build_metrics"
- },
- "suites": [],
- }
- if installer_size or size_measurements:
- perfherder_data["suites"].append({
- "name": "installer size",
- "value": installer_size,
- "alertThreshold": 0.25,
- "subtests": size_measurements
- })
-
- build_metrics = self._load_build_resources()
- if build_metrics:
- perfherder_data['suites'].append(build_metrics)
-
- if perfherder_data["suites"]:
- self.info('PERFHERDER_DATA: %s' % json.dumps(perfherder_data))
-
- def sendchange(self):
- if os.environ.get('TASK_ID'):
- self.info("We are not running this in buildbot; skipping")
- return
-
- if self.config.get('enable_talos_sendchange'):
- self._do_sendchange('talos')
- else:
- self.info("'enable_talos_sendchange' is false; skipping")
-
- if self.config.get('enable_unittest_sendchange'):
- self._do_sendchange('unittest')
- else:
- self.info("'enable_unittest_sendchange' is false; skipping")
-
- def _do_sendchange(self, test_type):
- c = self.config
-
- # grab any props available from this or previous unclobbered runs
- self.generate_build_props(console_output=False,
- halt_on_failure=False)
-
- installer_url = self.query_buildbot_property('packageUrl')
- if not installer_url:
- # don't burn the job but we should turn orange
- self.error("could not determine packageUrl property to use "
- "against sendchange. Was it set after 'mach build'?")
- self.return_code = self.worst_level(
- 1, self.return_code, AUTOMATION_EXIT_CODES[::-1]
- )
- self.return_code = 1
- return
- tests_url = self.query_buildbot_property('testsUrl')
- # Contains the url to a manifest describing the test packages required
- # for each unittest harness.
- # For the moment this property is only set on desktop builds. Android
- # builds find the packages manifest based on the upload
- # directory of the installer.
- test_packages_url = self.query_buildbot_property('testPackagesUrl')
- pgo_build = c.get('pgo_build', False) or self._compile_against_pgo()
-
- # these cmds are sent to mach through env vars. We won't know the
- # packageUrl or testsUrl until mach runs upload target so we let mach
- # fill in the rest of the cmd
- sendchange_props = {
- 'buildid': self.query_buildid(),
- 'builduid': self.query_builduid(),
- 'pgo_build': pgo_build,
- }
- if self.query_is_nightly():
- sendchange_props['nightly_build'] = True
- if test_type == 'talos':
- if pgo_build:
- build_type = 'pgo-'
- else: # we don't do talos sendchange for debug so no need to check
- build_type = '' # leave 'opt' out of branch for talos
- talos_branch = "%s-%s-%s%s" % (self.branch,
- self.stage_platform,
- build_type,
- 'talos')
- self.invoke_sendchange(downloadables=[installer_url],
- branch=talos_branch,
- username='sendchange',
- sendchange_props=sendchange_props)
- elif test_type == 'unittest':
- # do unittest sendchange
- if c.get('debug_build'):
- build_type = '' # for debug builds we append nothing
- elif pgo_build:
- build_type = '-pgo'
- else: # generic opt build
- build_type = '-opt'
-
- if c.get('unittest_platform'):
- platform = c['unittest_platform']
- else:
- platform = self.stage_platform
-
- platform_and_build_type = "%s%s" % (platform, build_type)
- unittest_branch = "%s-%s-%s" % (self.branch,
- platform_and_build_type,
- 'unittest')
-
- downloadables = [installer_url]
- if test_packages_url:
- downloadables.append(test_packages_url)
- else:
- downloadables.append(tests_url)
-
- self.invoke_sendchange(downloadables=downloadables,
- branch=unittest_branch,
- sendchange_props=sendchange_props)
- else:
- self.fatal('type: "%s" is unknown for sendchange type. valid '
- 'strings are "unittest" or "talos"' % test_type)
-
- def update(self):
- """ submit balrog update steps. """
- if self.config.get('forced_artifact_build'):
- self.info('Skipping due to forced artifact build.')
- return
- if not self.query_is_nightly():
- self.info("Not a nightly build, skipping balrog submission.")
- return
-
- # grab any props available from this or previous unclobbered runs
- self.generate_build_props(console_output=False,
- halt_on_failure=False)
-
- # generate balrog props as artifacts
- if self.config.get('taskcluster_nightly'):
- env = self.query_mach_build_env(multiLocale=False)
- props_path = os.path.join(env["UPLOAD_PATH"],
- 'balrog_props.json')
- self.generate_balrog_props(props_path)
- return
-
- if not self.config.get("balrog_servers"):
- self.fatal("balrog_servers not set; skipping balrog submission.")
- return
-
- if self.submit_balrog_updates():
- # set the build to orange so it is at least caught
- self.return_code = self.worst_level(
- EXIT_STATUS_DICT[TBPL_WARNING], self.return_code,
- AUTOMATION_EXIT_CODES[::-1]
- )
-
- def valgrind_test(self):
- '''Execute mach's valgrind-test for memory leaks'''
- env = self.query_build_env()
- env.update(self.query_mach_build_env())
-
- python = self.query_exe('python2.7')
- return_code = self.run_command_m(
- command=[python, 'mach', 'valgrind-test'],
- cwd=self.query_abs_dirs()['abs_src_dir'],
- env=env, output_timeout=self.config.get('max_build_output_timeout', 60 * 40)
- )
- if return_code:
- self.return_code = self.worst_level(
- EXIT_STATUS_DICT[TBPL_FAILURE], self.return_code,
- AUTOMATION_EXIT_CODES[::-1]
- )
- self.fatal("'mach valgrind-test' did not run successfully. Please check "
- "log for errors.")
-
-
-
- def _post_fatal(self, message=None, exit_code=None):
- if not self.return_code: # only overwrite return_code if it's 0
- self.error('setting return code to 2 because fatal was called')
- self.return_code = 2
-
- @PostScriptRun
- def _summarize(self):
- """ If this is run in automation, ensure the return code is valid and
- set it to one if it's not. Finally, log any summaries we collected
- from the script run.
- """
- if self.config.get("is_automation"):
- # let's ignore all mention of buildbot/tbpl status until this
- # point so it will be easier to manage
- if self.return_code not in AUTOMATION_EXIT_CODES:
- self.error("Return code is set to: %s and is outside of "
- "automation's known values. Setting to 2(failure). "
- "Valid return codes %s" % (self.return_code,
- AUTOMATION_EXIT_CODES))
- self.return_code = 2
- for status, return_code in EXIT_STATUS_DICT.iteritems():
- if return_code == self.return_code:
- self.buildbot_status(status, TBPL_STATUS_DICT[status])
- self.summary()
diff --git a/testing/mozharness/mozharness/mozilla/building/hazards.py b/testing/mozharness/mozharness/mozilla/building/hazards.py
deleted file mode 100644
index 6de235f89..000000000
--- a/testing/mozharness/mozharness/mozilla/building/hazards.py
+++ /dev/null
@@ -1,241 +0,0 @@
-import os
-import json
-import re
-
-from mozharness.base.errors import MakefileErrorList
-from mozharness.mozilla.buildbot import TBPL_WARNING
-
-
-class HazardError(Exception):
- def __init__(self, value):
- self.value = value
-
- def __str__(self):
- return repr(self.value)
-
- # Logging ends up calling splitlines directly on what is being logged, which would fail.
- def splitlines(self):
- return str(self).splitlines()
-
-class HazardAnalysis(object):
- def clobber_shell(self, builder):
- """Clobber the specially-built JS shell used to run the analysis"""
- dirs = builder.query_abs_dirs()
- builder.rmtree(dirs['shell_objdir'])
-
- def configure_shell(self, builder):
- """Configure the specially-built JS shell used to run the analysis"""
- dirs = builder.query_abs_dirs()
-
- if not os.path.exists(dirs['shell_objdir']):
- builder.mkdir_p(dirs['shell_objdir'])
-
- js_src_dir = os.path.join(dirs['gecko_src'], 'js', 'src')
- rc = builder.run_command(['autoconf-2.13'],
- cwd=js_src_dir,
- env=builder.env,
- error_list=MakefileErrorList)
- if rc != 0:
- rc = builder.run_command(['autoconf2.13'],
- cwd=js_src_dir,
- env=builder.env,
- error_list=MakefileErrorList)
- if rc != 0:
- raise HazardError("autoconf failed, can't continue.")
-
- rc = builder.run_command([os.path.join(js_src_dir, 'configure'),
- '--enable-optimize',
- '--disable-debug',
- '--enable-ctypes',
- '--with-system-nspr',
- '--without-intl-api'],
- cwd=dirs['shell_objdir'],
- env=builder.env,
- error_list=MakefileErrorList)
- if rc != 0:
- raise HazardError("Configure failed, can't continue.")
-
- def build_shell(self, builder):
- """Build a JS shell specifically for running the analysis"""
- dirs = builder.query_abs_dirs()
-
- rc = builder.run_command(['make', '-j', str(builder.config.get('concurrency', 4)), '-s'],
- cwd=dirs['shell_objdir'],
- env=builder.env,
- error_list=MakefileErrorList)
- if rc != 0:
- raise HazardError("Build failed, can't continue.")
-
- def clobber(self, builder):
- """Clobber all of the old analysis data. Note that theoretically we could do
- incremental analyses, but they seem to still be buggy."""
- dirs = builder.query_abs_dirs()
- builder.rmtree(dirs['abs_analysis_dir'])
- builder.rmtree(dirs['abs_analyzed_objdir'])
-
- def setup(self, builder):
- """Prepare the config files and scripts for running the analysis"""
- dirs = builder.query_abs_dirs()
- analysis_dir = dirs['abs_analysis_dir']
-
- if not os.path.exists(analysis_dir):
- builder.mkdir_p(analysis_dir)
-
- js_src_dir = os.path.join(dirs['gecko_src'], 'js', 'src')
-
- values = {
- 'js': os.path.join(dirs['shell_objdir'], 'dist', 'bin', 'js'),
- 'analysis_scriptdir': os.path.join(js_src_dir, 'devtools', 'rootAnalysis'),
- 'source_objdir': dirs['abs_analyzed_objdir'],
- 'source': os.path.join(dirs['abs_work_dir'], 'source'),
- 'sixgill': os.path.join(dirs['abs_work_dir'], builder.config['sixgill']),
- 'sixgill_bin': os.path.join(dirs['abs_work_dir'], builder.config['sixgill_bin']),
- 'gcc_bin': os.path.join(dirs['abs_work_dir'], 'gcc'),
- }
- defaults = """
-js = '%(js)s'
-analysis_scriptdir = '%(analysis_scriptdir)s'
-objdir = '%(source_objdir)s'
-source = '%(source)s'
-sixgill = '%(sixgill)s'
-sixgill_bin = '%(sixgill_bin)s'
-gcc_bin = '%(gcc_bin)s'
-jobs = 4
-""" % values
-
- defaults_path = os.path.join(analysis_dir, 'defaults.py')
- file(defaults_path, "w").write(defaults)
- builder.log("Wrote analysis config file " + defaults_path)
-
- build_script = builder.config['build_command']
- builder.copyfile(os.path.join(dirs['mozharness_scriptdir'],
- os.path.join('spidermonkey', build_script)),
- os.path.join(analysis_dir, build_script),
- copystat=True)
-
- def run(self, builder, env, error_list):
- """Execute the analysis, which consists of building all analyzed
- source code with a GCC plugin active that siphons off the interesting
- data, then running some JS scripts over the databases created by
- the plugin."""
- dirs = builder.query_abs_dirs()
- analysis_dir = dirs['abs_analysis_dir']
- analysis_scriptdir = os.path.join(dirs['abs_work_dir'], dirs['analysis_scriptdir'])
-
- build_script = builder.config['build_command']
- build_script = os.path.abspath(os.path.join(analysis_dir, build_script))
-
- cmd = [
- builder.config['python'],
- os.path.join(analysis_scriptdir, 'analyze.py'),
- "--source", dirs['gecko_src'],
- "--buildcommand", build_script,
- ]
- retval = builder.run_command(cmd,
- cwd=analysis_dir,
- env=env,
- error_list=error_list)
- if retval != 0:
- raise HazardError("failed to build")
-
- def collect_output(self, builder):
- """Gather up the analysis output and place in the upload dir."""
- dirs = builder.query_abs_dirs()
- analysis_dir = dirs['abs_analysis_dir']
- upload_dir = dirs['abs_blob_upload_dir']
- builder.mkdir_p(upload_dir)
- files = (('rootingHazards.txt',
- 'rooting_hazards',
- 'list of rooting hazards, unsafe references, and extra roots'),
- ('gcFunctions.txt',
- 'gcFunctions',
- 'list of functions that can gc, and why'),
- ('allFunctions.txt',
- 'allFunctions',
- 'list of all functions that were compiled'),
- ('gcTypes.txt',
- 'gcTypes',
- 'list of types containing unrooted gc pointers'),
- ('unnecessary.txt',
- 'extra',
- 'list of extra roots (rooting with no GC function in scope)'),
- ('refs.txt',
- 'refs',
- 'list of unsafe references to unrooted pointers'),
- ('hazards.txt',
- 'hazards',
- 'list of just the hazards, together with gcFunction reason for each'))
- for f, short, long in files:
- builder.copy_to_upload_dir(os.path.join(analysis_dir, f),
- short_desc=short,
- long_desc=long,
- compress=False, # blobber will compress
- upload_dir=upload_dir)
- print("== Hazards (temporarily inline here, beware weirdly interleaved output, see bug 1211402) ==")
- print(file(os.path.join(analysis_dir, "hazards.txt")).read())
-
- def upload_results(self, builder):
- """Upload the results of the analysis."""
- pass
-
- def check_expectations(self, builder):
- """Compare the actual to expected number of problems."""
- if 'expect_file' not in builder.config:
- builder.info('No expect_file given; skipping comparison with expected hazard count')
- return
-
- dirs = builder.query_abs_dirs()
- analysis_dir = dirs['abs_analysis_dir']
- analysis_scriptdir = os.path.join(dirs['gecko_src'], 'js', 'src', 'devtools', 'rootAnalysis')
- expect_file = os.path.join(analysis_scriptdir, builder.config['expect_file'])
- expect = builder.read_from_file(expect_file)
- if expect is None:
- raise HazardError("could not load expectation file")
- data = json.loads(expect)
-
- num_hazards = 0
- num_refs = 0
- with builder.opened(os.path.join(analysis_dir, "rootingHazards.txt")) as (hazards_fh, err):
- if err:
- raise HazardError("hazards file required")
- for line in hazards_fh:
- m = re.match(r"^Function.*has unrooted.*live across GC call", line)
- if m:
- num_hazards += 1
-
- m = re.match(r'^Function.*takes unsafe address of unrooted', line)
- if m:
- num_refs += 1
-
- expect_hazards = data.get('expect-hazards')
- status = []
- if expect_hazards is None:
- status.append("%d hazards" % num_hazards)
- else:
- status.append("%d/%d hazards allowed" % (num_hazards, expect_hazards))
-
- if expect_hazards is not None and expect_hazards != num_hazards:
- if expect_hazards < num_hazards:
- builder.warning("TEST-UNEXPECTED-FAIL %d more hazards than expected (expected %d, saw %d)" %
- (num_hazards - expect_hazards, expect_hazards, num_hazards))
- builder.buildbot_status(TBPL_WARNING)
- else:
- builder.info("%d fewer hazards than expected! (expected %d, saw %d)" %
- (expect_hazards - num_hazards, expect_hazards, num_hazards))
-
- expect_refs = data.get('expect-refs')
- if expect_refs is None:
- status.append("%d unsafe refs" % num_refs)
- else:
- status.append("%d/%d unsafe refs allowed" % (num_refs, expect_refs))
-
- if expect_refs is not None and expect_refs != num_refs:
- if expect_refs < num_refs:
- builder.warning("TEST-UNEXPECTED-FAIL %d more unsafe refs than expected (expected %d, saw %d)" %
- (num_refs - expect_refs, expect_refs, num_refs))
- builder.buildbot_status(TBPL_WARNING)
- else:
- builder.info("%d fewer unsafe refs than expected! (expected %d, saw %d)" %
- (expect_refs - num_refs, expect_refs, num_refs))
-
- builder.info("TinderboxPrint: " + ", ".join(status))
diff --git a/testing/mozharness/mozharness/mozilla/checksums.py b/testing/mozharness/mozharness/mozilla/checksums.py
deleted file mode 100644
index 6b8997375..000000000
--- a/testing/mozharness/mozharness/mozilla/checksums.py
+++ /dev/null
@@ -1,21 +0,0 @@
-def parse_checksums_file(checksums):
- """Parses checksums files that the build system generates and uploads:
- https://hg.mozilla.org/mozilla-central/file/default/build/checksums.py"""
- fileInfo = {}
- for line in checksums.splitlines():
- hash_, type_, size, file_ = line.split(None, 3)
- size = int(size)
- if size < 0:
- raise ValueError("Found negative value (%d) for size." % size)
- if file_ not in fileInfo:
- fileInfo[file_] = {"hashes": {}}
- # If the file already exists, make sure that the size matches the
- # previous entry.
- elif fileInfo[file_]['size'] != size:
- raise ValueError("Found different sizes for same file %s (%s and %s)" % (file_, fileInfo[file_]['size'], size))
- # Same goes for the hash.
- elif type_ in fileInfo[file_]['hashes'] and fileInfo[file_]['hashes'][type_] != hash_:
- raise ValueError("Found different %s hashes for same file %s (%s and %s)" % (type_, file_, fileInfo[file_]['hashes'][type_], hash_))
- fileInfo[file_]['size'] = size
- fileInfo[file_]['hashes'][type_] = hash_
- return fileInfo
diff --git a/testing/mozharness/mozharness/mozilla/l10n/__init__.py b/testing/mozharness/mozharness/mozilla/l10n/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/mozharness/mozilla/l10n/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/mozharness/mozilla/l10n/locales.py b/testing/mozharness/mozharness/mozilla/l10n/locales.py
deleted file mode 100755
index 24920ae44..000000000
--- a/testing/mozharness/mozharness/mozilla/l10n/locales.py
+++ /dev/null
@@ -1,280 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Localization.
-"""
-
-import os
-from urlparse import urljoin
-import sys
-from copy import deepcopy
-
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.config import parse_config_file
-from mozharness.base.errors import PythonErrorList
-from mozharness.base.parallel import ChunkingMixin
-
-
-# LocalesMixin {{{1
-class LocalesMixin(ChunkingMixin):
- def __init__(self, **kwargs):
- """ Mixins generally don't have an __init__.
- This breaks super().__init__() for children.
- However, this is needed to override the query_abs_dirs()
- """
- self.abs_dirs = None
- self.locales = None
- self.gecko_locale_revisions = None
- self.l10n_revisions = {}
-
- def query_locales(self):
- if self.locales is not None:
- return self.locales
- c = self.config
- ignore_locales = c.get("ignore_locales", [])
- additional_locales = c.get("additional_locales", [])
- # List of locales can be set by using different methods in the
- # following order:
- # 1. "locales" buildbot property: a string of locale:revision separated
- # by space
- # 2. "MOZ_LOCALES" env variable: a string of locale:revision separated
- # by space
- # 3. self.config["locales"] which can be either coming from the config
- # or from --locale command line argument
- # 4. using self.config["locales_file"] l10n changesets file
- locales = None
-
- # Buildbot property
- if hasattr(self, 'read_buildbot_config'):
- self.read_buildbot_config()
- if self.buildbot_config:
- locales = self.buildbot_config['properties'].get("locales")
- if locales:
- self.info("Using locales from buildbot: %s" % locales)
- locales = locales.split()
- else:
- self.info("'read_buildbot_config()' is missing, ignoring buildbot"
- " properties")
-
- # Environment variable
- if not locales and "MOZ_LOCALES" in os.environ:
- self.debug("Using locales from environment: %s" %
- os.environ["MOZ_LOCALES"])
- locales = os.environ["MOZ_LOCALES"].split()
-
- # Command line or config
- if not locales and c.get("locales", None):
- locales = c["locales"]
- self.debug("Using locales from config/CLI: %s" % locales)
-
- # parse locale:revision if set
- if locales:
- for l in locales:
- if ":" in l:
- # revision specified in locale string
- locale, revision = l.split(":", 1)
- self.debug("Using %s:%s" % (locale, revision))
- self.l10n_revisions[locale] = revision
- # clean up locale by removing revisions
- locales = [l.split(":")[0] for l in locales]
-
- if not locales and 'locales_file' in c:
- locales_file = os.path.join(c['base_work_dir'], c['work_dir'],
- c['locales_file'])
- locales = self.parse_locales_file(locales_file)
-
- if not locales:
- self.fatal("No locales set!")
-
- for locale in ignore_locales:
- if locale in locales:
- self.debug("Ignoring locale %s." % locale)
- locales.remove(locale)
- if locale in self.l10n_revisions:
- del self.l10n_revisions[locale]
-
- for locale in additional_locales:
- if locale not in locales:
- self.debug("Adding locale %s." % locale)
- locales.append(locale)
-
- if not locales:
- return None
- if 'total_locale_chunks' and 'this_locale_chunk' in c:
- self.debug("Pre-chunking locale list: %s" % str(locales))
- locales = self.query_chunked_list(locales,
- c['this_locale_chunk'],
- c['total_locale_chunks'],
- sort=True)
- self.debug("Post-chunking locale list: %s" % locales)
- self.locales = locales
- return self.locales
-
- def list_locales(self):
- """ Stub action method.
- """
- self.info("Locale list: %s" % str(self.query_locales()))
-
- def parse_locales_file(self, locales_file):
- locales = []
- c = self.config
- platform = c.get("locales_platform", None)
-
- if locales_file.endswith('json'):
- locales_json = parse_config_file(locales_file)
- for locale in locales_json.keys():
- if isinstance(locales_json[locale], dict):
- if platform and platform not in locales_json[locale]['platforms']:
- continue
- self.l10n_revisions[locale] = locales_json[locale]['revision']
- else:
- # some other way of getting this?
- self.l10n_revisions[locale] = 'default'
- locales.append(locale)
- else:
- locales = self.read_from_file(locales_file).split()
- return locales
-
- def run_compare_locales(self, locale, halt_on_failure=False):
- dirs = self.query_abs_dirs()
- env = self.query_l10n_env()
- python = self.query_exe('python2.7')
- compare_locales_error_list = list(PythonErrorList)
- self.rmtree(dirs['abs_merge_dir'])
- self.mkdir_p(dirs['abs_merge_dir'])
- command = [python, 'mach', 'compare-locales',
- '--merge-dir', dirs['abs_merge_dir'],
- '--l10n-ini', os.path.join(dirs['abs_locales_src_dir'], 'l10n.ini'),
- '--l10n-base', dirs['abs_l10n_dir'], locale]
- self.info("*** BEGIN compare-locales %s" % locale)
- status = self.run_command(command,
- halt_on_failure=halt_on_failure,
- env=env,
- cwd=dirs['abs_mozilla_dir'],
- error_list=compare_locales_error_list)
- self.info("*** END compare-locales %s" % locale)
- return status
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(LocalesMixin, self).query_abs_dirs()
- c = self.config
- dirs = {}
- dirs['abs_work_dir'] = os.path.join(c['base_work_dir'],
- c['work_dir'])
- # TODO prettify this up later
- if 'l10n_dir' in c:
- dirs['abs_l10n_dir'] = os.path.join(dirs['abs_work_dir'],
- c['l10n_dir'])
- if 'mozilla_dir' in c:
- dirs['abs_mozilla_dir'] = os.path.join(dirs['abs_work_dir'],
- c['mozilla_dir'])
- dirs['abs_locales_src_dir'] = os.path.join(dirs['abs_mozilla_dir'],
- c['locales_dir'])
- dirs['abs_compare_locales_dir'] = os.path.join(dirs['abs_mozilla_dir'],
- 'python', 'compare-locales',
- 'compare_locales')
- else:
- # Use old-compare-locales if no mozilla_dir set, needed
- # for clobberer, and existing mozharness tests.
- dirs['abs_compare_locales_dir'] = os.path.join(dirs['abs_work_dir'],
- 'compare-locales')
-
- if 'objdir' in c:
- if os.path.isabs(c['objdir']):
- dirs['abs_objdir'] = c['objdir']
- else:
- dirs['abs_objdir'] = os.path.join(dirs['abs_mozilla_dir'],
- c['objdir'])
- dirs['abs_merge_dir'] = os.path.join(dirs['abs_objdir'],
- 'merged')
- dirs['abs_locales_dir'] = os.path.join(dirs['abs_objdir'],
- c['locales_dir'])
-
- for key in dirs.keys():
- if key not in abs_dirs:
- abs_dirs[key] = dirs[key]
- self.abs_dirs = abs_dirs
- return self.abs_dirs
-
- # This requires self to inherit a VCSMixin.
- def pull_locale_source(self, hg_l10n_base=None, parent_dir=None, vcs='hg'):
- c = self.config
- if not hg_l10n_base:
- hg_l10n_base = c['hg_l10n_base']
- if parent_dir is None:
- parent_dir = self.query_abs_dirs()['abs_l10n_dir']
- self.mkdir_p(parent_dir)
- repos = []
- replace_dict = {}
- # This block is to allow for pulling buildbot-configs in Fennec
- # release builds, since we don't pull it in MBF anymore.
- if c.get("l10n_repos"):
- if c.get("user_repo_override"):
- replace_dict['user_repo_override'] = c['user_repo_override']
- for repo_dict in deepcopy(c['l10n_repos']):
- repo_dict['repo'] = repo_dict['repo'] % replace_dict
- repos.append(repo_dict)
- else:
- repos = c.get("l10n_repos")
- self.vcs_checkout_repos(repos, tag_override=c.get('tag_override'))
- # Pull locales
- locales = self.query_locales()
- locale_repos = []
- if c.get("user_repo_override"):
- hg_l10n_base = hg_l10n_base % {"user_repo_override": c["user_repo_override"]}
- for locale in locales:
- tag = c.get('hg_l10n_tag', 'default')
- if self.l10n_revisions.get(locale):
- tag = self.l10n_revisions[locale]
- locale_repos.append({
- 'repo': "%s/%s" % (hg_l10n_base, locale),
- 'branch': tag,
- 'vcs': vcs
- })
- revs = self.vcs_checkout_repos(repo_list=locale_repos,
- parent_dir=parent_dir,
- tag_override=c.get('tag_override'))
- self.gecko_locale_revisions = revs
-
- def query_l10n_repo(self):
- # Find the name of our repository
- mozilla_dir = self.config['mozilla_dir']
- repo = None
- for repository in self.config['repos']:
- if repository.get('dest') == mozilla_dir:
- repo = repository['repo']
- break
- return repo
-
-# GaiaLocalesMixin {{{1
-class GaiaLocalesMixin(object):
- gaia_locale_revisions = None
-
- def pull_gaia_locale_source(self, l10n_config, locales, base_dir):
- root = l10n_config['root']
- # urljoin will strip the last part of root if it doesn't end with "/"
- if not root.endswith('/'):
- root = root + '/'
- vcs = l10n_config['vcs']
- env = l10n_config.get('env', {})
- repos = []
- for locale in locales:
- repos.append({
- 'repo': urljoin(root, locale),
- 'dest': locale,
- 'vcs': vcs,
- 'env': env,
- })
- self.gaia_locale_revisions = self.vcs_checkout_repos(repo_list=repos, parent_dir=base_dir)
-
-
-# __main__ {{{1
-
-if __name__ == '__main__':
- pass
diff --git a/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py b/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py
deleted file mode 100755
index 5bdbc8011..000000000
--- a/testing/mozharness/mozharness/mozilla/l10n/multi_locale_build.py
+++ /dev/null
@@ -1,254 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""multi_locale_build.py
-
-This should be a mostly generic multilocale build script.
-"""
-
-from copy import deepcopy
-import os
-import sys
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.base.errors import MakefileErrorList, SSHErrorList
-from mozharness.base.log import FATAL
-from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.l10n.locales import LocalesMixin
-
-
-# MultiLocaleBuild {{{1
-class MultiLocaleBuild(LocalesMixin, MercurialScript):
- """ This class targets Fennec multilocale builds.
- We were considering this for potential Firefox desktop multilocale.
- Now that we have a different approach for B2G multilocale,
- it's most likely misnamed. """
- config_options = [[
- ["--locale"],
- {"action": "extend",
- "dest": "locales",
- "type": "string",
- "help": "Specify the locale(s) to repack"
- }
- ], [
- ["--merge-locales"],
- {"action": "store_true",
- "dest": "merge_locales",
- "default": False,
- "help": "Use default [en-US] if there are missing strings"
- }
- ], [
- ["--no-merge-locales"],
- {"action": "store_false",
- "dest": "merge_locales",
- "help": "Do not allow missing strings"
- }
- ], [
- ["--objdir"],
- {"action": "store",
- "dest": "objdir",
- "type": "string",
- "default": "objdir",
- "help": "Specify the objdir"
- }
- ], [
- ["--l10n-base"],
- {"action": "store",
- "dest": "hg_l10n_base",
- "type": "string",
- "help": "Specify the L10n repo base directory"
- }
- ], [
- ["--l10n-tag"],
- {"action": "store",
- "dest": "hg_l10n_tag",
- "type": "string",
- "help": "Specify the L10n tag"
- }
- ], [
- ["--tag-override"],
- {"action": "store",
- "dest": "tag_override",
- "type": "string",
- "help": "Override the tags set for all repos"
- }
- ], [
- ["--user-repo-override"],
- {"action": "store",
- "dest": "user_repo_override",
- "type": "string",
- "help": "Override the user repo path for all repos"
- }
- ], [
- ["--l10n-dir"],
- {"action": "store",
- "dest": "l10n_dir",
- "type": "string",
- "default": "l10n",
- "help": "Specify the l10n dir name"
- }
- ]]
-
- def __init__(self, require_config_file=True):
- LocalesMixin.__init__(self)
- MercurialScript.__init__(self, config_options=self.config_options,
- all_actions=['clobber', 'pull-build-source',
- 'pull-locale-source',
- 'build', 'package-en-US',
- 'upload-en-US',
- 'backup-objdir',
- 'restore-objdir',
- 'add-locales', 'package-multi',
- 'upload-multi', 'summary'],
- require_config_file=require_config_file)
-
- def query_l10n_env(self):
- return self.query_env()
-
- def clobber(self):
- c = self.config
- if c['work_dir'] != '.':
- path = os.path.join(c['base_work_dir'], c['work_dir'])
- if os.path.exists(path):
- self.rmtree(path, error_level=FATAL)
- else:
- self.info("work_dir is '.'; skipping for now.")
-
- def pull_build_source(self):
- c = self.config
- repos = []
- replace_dict = {}
- # Replace %(user_repo_override)s with c['user_repo_override']
- if c.get("user_repo_override"):
- replace_dict['user_repo_override'] = c['user_repo_override']
- for repo_dict in deepcopy(c['repos']):
- repo_dict['repo'] = repo_dict['repo'] % replace_dict
- repos.append(repo_dict)
- else:
- repos = c['repos']
- self.vcs_checkout_repos(repos, tag_override=c.get('tag_override'))
-
- # pull_locale_source() defined in LocalesMixin.
-
- def build(self):
- c = self.config
- dirs = self.query_abs_dirs()
- self.copyfile(os.path.join(dirs['abs_work_dir'], c['mozconfig']),
- os.path.join(dirs['abs_mozilla_dir'], 'mozconfig'),
- error_level=FATAL)
- command = "make -f client.mk build"
- env = self.query_env()
- if self._process_command(command=command,
- cwd=dirs['abs_mozilla_dir'],
- env=env, error_list=MakefileErrorList):
- self.fatal("Erroring out after the build failed.")
-
- def add_locales(self):
- c = self.config
- dirs = self.query_abs_dirs()
- locales = self.query_locales()
-
- for locale in locales:
- self.run_compare_locales(locale, halt_on_failure=True)
- command = 'make chrome-%s L10NBASEDIR=%s' % (locale, dirs['abs_l10n_dir'])
- if c['merge_locales']:
- command += " LOCALE_MERGEDIR=%s" % dirs['abs_merge_dir'].replace(os.sep, '/')
- status = self._process_command(command=command,
- cwd=dirs['abs_locales_dir'],
- error_list=MakefileErrorList)
- if status:
- self.return_code += 1
- self.add_summary("Failed to add locale %s!" % locale,
- level="error")
- else:
- self.add_summary("Added locale %s successfully." % locale)
-
- def package_en_US(self):
- self.package(package_type='en-US')
-
- def preflight_package_multi(self):
- dirs = self.query_abs_dirs()
- self.run_command("rm -rfv dist/fennec*", cwd=dirs['abs_objdir'])
- # bug 933290
- self.run_command(["touch", "mobile/android/installer/Makefile"], cwd=dirs['abs_objdir'])
-
- def package_multi(self):
- self.package(package_type='multi')
-
- def additional_packaging(self, package_type='en-US', env=None):
- dirs = self.query_abs_dirs()
- command = "make package-tests"
- if package_type == 'multi':
- command += " AB_CD=multi"
- self.run_command(command, cwd=dirs['abs_objdir'], env=env,
- error_list=MakefileErrorList,
- halt_on_failure=True)
- # TODO deal with buildsymbols
-
- def package(self, package_type='en-US'):
- dirs = self.query_abs_dirs()
-
- command = "make package"
- env = self.query_env()
- if env is None:
- # This is for Maemo, where we don't want an env for builds
- # but we do for packaging. self.query_env() will return None.
- env = os.environ.copy()
- if package_type == 'multi':
- command += " AB_CD=multi"
- env['MOZ_CHROME_MULTILOCALE'] = "en-US " + \
- ' '.join(self.query_locales())
- self.info("MOZ_CHROME_MULTILOCALE is %s" % env['MOZ_CHROME_MULTILOCALE'])
- self._process_command(command=command, cwd=dirs['abs_objdir'],
- env=env, error_list=MakefileErrorList,
- halt_on_failure=True)
- self.additional_packaging(package_type=package_type, env=env)
-
- def upload_en_US(self):
- # TODO
- self.info("Not written yet.")
-
- def backup_objdir(self):
- dirs = self.query_abs_dirs()
- if not os.path.isdir(dirs['abs_objdir']):
- self.warning("%s doesn't exist! Skipping..." % dirs['abs_objdir'])
- return
- rsync = self.query_exe('rsync')
- backup_dir = '%s-bak' % dirs['abs_objdir']
- self.rmtree(backup_dir)
- self.mkdir_p(backup_dir)
- self.run_command([rsync, '-a', '--delete', '--partial',
- '%s/' % dirs['abs_objdir'],
- '%s/' % backup_dir],
- error_list=SSHErrorList)
-
- def restore_objdir(self):
- dirs = self.query_abs_dirs()
- rsync = self.query_exe('rsync')
- backup_dir = '%s-bak' % dirs['abs_objdir']
- if not os.path.isdir(dirs['abs_objdir']) or not os.path.isdir(backup_dir):
- self.warning("Both %s and %s need to exist to restore the objdir! Skipping..." % (dirs['abs_objdir'], backup_dir))
- return
- self.run_command([rsync, '-a', '--delete', '--partial',
- '%s/' % backup_dir,
- '%s/' % dirs['abs_objdir']],
- error_list=SSHErrorList)
-
- def upload_multi(self):
- # TODO
- self.info("Not written yet.")
-
- def _process_command(self, **kwargs):
- """Stub wrapper function that allows us to call scratchbox in
- MaemoMultiLocaleBuild.
-
- """
- return self.run_command(**kwargs)
-
-# __main__ {{{1
-if __name__ == '__main__':
- pass
diff --git a/testing/mozharness/mozharness/mozilla/mapper.py b/testing/mozharness/mozharness/mozilla/mapper.py
deleted file mode 100644
index c5a2d4895..000000000
--- a/testing/mozharness/mozharness/mozilla/mapper.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Support for hg/git mapper
-"""
-import urllib2
-import time
-try:
- import simplejson as json
-except ImportError:
- import json
-
-
-class MapperMixin:
- def query_mapper(self, mapper_url, project, vcs, rev,
- require_answer=True, attempts=30, sleeptime=30,
- project_name=None):
- """
- Returns the mapped revision for the target vcs via a mapper service
-
- Args:
- mapper_url (str): base url to use for the mapper service
- project (str): The name of the mapper project to use for lookups
- vcs (str): Which vcs you want the revision for. e.g. "git" to get
- the git revision given an hg revision
- rev (str): The original revision you want the mapping for.
- require_answer (bool): Whether you require a valid answer or not.
- If None is acceptable (meaning mapper doesn't know about the
- revision you're asking about), then set this to False. If True,
- then will return the revision, or cause a fatal error.
- attempts (int): How many times to try to do the lookup
- sleeptime (int): How long to sleep between attempts
- project_name (str): Used for logging only to give a more
- descriptive name to the project, otherwise just uses the
- project parameter
-
- Returns:
- A revision string, or None
- """
- if project_name is None:
- project_name = project
- url = mapper_url.format(project=project, vcs=vcs, rev=rev)
- self.info('Mapping %s revision to %s using %s' % (project_name, vcs, url))
- n = 1
- while n <= attempts:
- try:
- r = urllib2.urlopen(url, timeout=10)
- j = json.loads(r.readline())
- if j['%s_rev' % vcs] is None:
- if require_answer:
- raise Exception("Mapper returned a revision of None; maybe it needs more time.")
- else:
- self.warning("Mapper returned a revision of None. Accepting because require_answer is False.")
- return j['%s_rev' % vcs]
- except Exception, err:
- self.warning('Error: %s' % str(err))
- if n == attempts:
- self.fatal('Giving up on %s %s revision for %s.' % (project_name, vcs, rev))
- if sleeptime > 0:
- self.info('Sleeping %i seconds before retrying' % sleeptime)
- time.sleep(sleeptime)
- continue
- finally:
- n += 1
-
- def query_mapper_git_revision(self, url, project, rev, **kwargs):
- """
- Returns the git revision for the given hg revision `rev`
- See query_mapper docs for supported parameters and docstrings
- """
- return self.query_mapper(url, project, "git", rev, **kwargs)
-
- def query_mapper_hg_revision(self, url, project, rev, **kwargs):
- """
- Returns the hg revision for the given git revision `rev`
- See query_mapper docs for supported parameters and docstrings
- """
- return self.query_mapper(url, project, "hg", rev, **kwargs)
diff --git a/testing/mozharness/mozharness/mozilla/mar.py b/testing/mozharness/mozharness/mozilla/mar.py
deleted file mode 100644
index dbe3b96a0..000000000
--- a/testing/mozharness/mozharness/mozilla/mar.py
+++ /dev/null
@@ -1,112 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""MarMixin, manages mar files"""
-
-import os
-import sys
-import ConfigParser
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-
-CONFIG = {
- "buildid_section": 'App',
- "buildid_option": "BuildID",
-}
-
-
-def query_ini_file(ini_file, section, option):
- ini = ConfigParser.SafeConfigParser()
- ini.read(ini_file)
- return ini.get(section, option)
-
-
-def buildid_from_ini(ini_file):
- """reads an ini_file and returns the buildid"""
- return query_ini_file(ini_file,
- CONFIG.get('buildid_section'),
- CONFIG.get('buildid_option'))
-
-
-# MarMixin {{{1
-class MarMixin(object):
- def _mar_tool_dir(self):
- """returns the path or the mar tool directory"""
- config = self.config
- dirs = self.query_abs_dirs()
- return os.path.join(dirs['abs_objdir'], config["local_mar_tool_dir"])
-
- def _incremental_update_script(self):
- """returns the path of incremental update script"""
- config = self.config
- dirs = self.query_abs_dirs()
- return os.path.join(dirs['abs_mozilla_dir'],
- config['incremental_update_script'])
-
- def download_mar_tools(self):
- """downloads mar tools executables (mar,mbsdiff)
- and stores them local_dir()"""
- self.info("getting mar tools")
- dst_dir = self._mar_tool_dir()
- self.mkdir_p(dst_dir)
- config = self.config
- replace_dict = {'platform': config['platform'],
- 'branch': config['branch']}
- url = config['mar_tools_url'] % replace_dict
- binaries = (config['mar'], config['mbsdiff'])
- for binary in binaries:
- from_url = "/".join((url, binary))
- full_path = os.path.join(dst_dir, binary)
- if not os.path.exists(full_path):
- self.download_file(from_url, file_name=full_path)
- self.info("downloaded %s" % full_path)
- else:
- self.info("found %s, skipping download" % full_path)
- self.chmod(full_path, 0755)
-
- def _temp_mar_base_dir(self):
- """a base dir for unpacking mars"""
- dirs = self.query_abs_dirs()
- return dirs['abs_objdir']
-
- def _unpack_mar(self, mar_file, dst_dir):
- """unpacks a mar file into dst_dir"""
- cmd = ['perl', self._unpack_script(), mar_file]
- env = self.query_bootstrap_env()
- self.info("unpacking %s" % mar_file)
- self.mkdir_p(dst_dir)
- return self.run_command(cmd,
- cwd=dst_dir,
- env=env,
- halt_on_failure=True)
-
- def do_incremental_update(self, previous_dir, current_dir, partial_filename):
- """create an incremental update from src_mar to dst_src.
- It stores the result in partial_filename"""
- # Usage: make_incremental_update.sh [OPTIONS] ARCHIVE FROMDIR TODIR
- cmd = [self._incremental_update_script(), partial_filename,
- previous_dir, current_dir]
- env = self.query_bootstrap_env()
- cwd = self._mar_dir('update_mar_dir')
- self.mkdir_p(cwd)
- result = self.run_command(cmd, cwd=cwd, env=env)
- return result
-
- def get_buildid_from_mar_dir(self, mar_unpack_dir):
- """returns the buildid of the current mar file"""
- config = self.config
- ini_file = config['application_ini']
- ini_file = os.path.join(mar_unpack_dir, ini_file)
- self.info("application.ini file: %s" % ini_file)
-
- # log the content of application.ini
- with self.opened(ini_file, 'r') as (ini, error):
- if error:
- self.fatal('cannot open %s' % ini_file)
- self.debug(ini.read())
- return buildid_from_ini(ini_file)
diff --git a/testing/mozharness/mozharness/mozilla/mock.py b/testing/mozharness/mozharness/mozilla/mock.py
deleted file mode 100644
index f8587c0d6..000000000
--- a/testing/mozharness/mozharness/mozilla/mock.py
+++ /dev/null
@@ -1,251 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Code to integrate with mock
-"""
-
-import os.path
-import hashlib
-import subprocess
-import os
-
-ERROR_MSGS = {
- 'undetermined_buildroot_lock': 'buildroot_lock_path does not exist.\
-Nothing to remove.'
-}
-
-
-
-# MockMixin {{{1
-class MockMixin(object):
- """Provides methods to setup and interact with mock environments.
- https://wiki.mozilla.org/ReleaseEngineering/Applications/Mock
-
- This is dependent on ScriptMixin
- """
- done_mock_setup = False
- mock_enabled = False
- default_mock_target = None
-
- def init_mock(self, mock_target):
- "Initialize mock environment defined by `mock_target`"
- cmd = ['mock_mozilla', '-r', mock_target, '--init']
- return super(MockMixin, self).run_command(cmd, halt_on_failure=True,
- fatal_exit_code=3)
-
- def install_mock_packages(self, mock_target, packages):
- "Install `packages` into mock environment `mock_target`"
- cmd = ['mock_mozilla', '-r', mock_target, '--install'] + packages
- # TODO: parse output to see if packages actually were installed
- return super(MockMixin, self).run_command(cmd, halt_on_failure=True,
- fatal_exit_code=3)
-
- def delete_mock_files(self, mock_target, files):
- """Delete files from the mock environment `mock_target`. `files` should
- be an iterable of 2-tuples: (src, dst). Only the dst component is
- deleted."""
- cmd_base = ['mock_mozilla', '-r', mock_target, '--shell']
- for src, dest in files:
- cmd = cmd_base + ['rm -rf %s' % dest]
- super(MockMixin, self).run_command(cmd, halt_on_failure=True,
- fatal_exit_code=3)
-
- def copy_mock_files(self, mock_target, files):
- """Copy files into the mock environment `mock_target`. `files` should
- be an iterable of 2-tuples: (src, dst)"""
- cmd_base = ['mock_mozilla', '-r', mock_target, '--copyin', '--unpriv']
- for src, dest in files:
- cmd = cmd_base + [src, dest]
- super(MockMixin, self).run_command(cmd, halt_on_failure=True,
- fatal_exit_code=3)
- super(MockMixin, self).run_command(
- ['mock_mozilla', '-r', mock_target, '--shell',
- 'chown -R mock_mozilla %s' % dest],
- halt_on_failure=True,
- fatal_exit_code=3)
-
- def get_mock_target(self):
- if self.config.get('disable_mock'):
- return None
- return self.default_mock_target or self.config.get('mock_target')
-
- def enable_mock(self):
- """Wrap self.run_command and self.get_output_from_command to run inside
- the mock environment given by self.config['mock_target']"""
- if not self.get_mock_target():
- return
- self.mock_enabled = True
- self.run_command = self.run_command_m
- self.get_output_from_command = self.get_output_from_command_m
-
- def disable_mock(self):
- """Restore self.run_command and self.get_output_from_command to their
- original versions. This is the opposite of self.enable_mock()"""
- if not self.get_mock_target():
- return
- self.mock_enabled = False
- self.run_command = super(MockMixin, self).run_command
- self.get_output_from_command = super(MockMixin, self).get_output_from_command
-
- def _do_mock_command(self, func, mock_target, command, cwd=None, env=None, **kwargs):
- """Internal helper for preparing commands to run under mock. Used by
- run_mock_command and get_mock_output_from_command."""
- cmd = ['mock_mozilla', '-r', mock_target, '-q']
- if cwd:
- cmd += ['--cwd', cwd]
-
- if not kwargs.get('privileged'):
- cmd += ['--unpriv']
- cmd += ['--shell']
-
- if not isinstance(command, basestring):
- command = subprocess.list2cmdline(command)
-
- # XXX - Hack - gets around AB_CD=%(locale)s type arguments
- command = command.replace("(", "\\(")
- command = command.replace(")", "\\)")
-
- if env:
- env_cmd = ['/usr/bin/env']
- for key, value in env.items():
- # $HOME won't work inside the mock chroot
- if key == 'HOME':
- continue
- value = value.replace(";", "\\;")
- env_cmd += ['%s=%s' % (key, value)]
- cmd.append(subprocess.list2cmdline(env_cmd) + " " + command)
- else:
- cmd.append(command)
- return func(cmd, cwd=cwd, **kwargs)
-
- def run_mock_command(self, mock_target, command, cwd=None, env=None, **kwargs):
- """Same as ScriptMixin.run_command, except runs command inside mock
- environment `mock_target`."""
- return self._do_mock_command(
- super(MockMixin, self).run_command,
- mock_target, command, cwd, env, **kwargs)
-
- def get_mock_output_from_command(self, mock_target, command, cwd=None, env=None, **kwargs):
- """Same as ScriptMixin.get_output_from_command, except runs command
- inside mock environment `mock_target`."""
- return self._do_mock_command(
- super(MockMixin, self).get_output_from_command,
- mock_target, command, cwd, env, **kwargs)
-
- def reset_mock(self, mock_target=None):
- """rm mock lock and reset"""
- c = self.config
- if mock_target is None:
- if not c.get('mock_target'):
- self.fatal("Cound not determine: 'mock_target'")
- mock_target = c.get('mock_target')
- buildroot_lock_path = os.path.join(c.get('mock_mozilla_dir', ''),
- mock_target,
- 'buildroot.lock')
- self.info("Removing buildroot lock at path if exists:O")
- self.info(buildroot_lock_path)
- if not os.path.exists(buildroot_lock_path):
- self.info(ERROR_MSGS['undetermined_buildroot_lock'])
- else:
- rm_lock_cmd = ['rm', '-f', buildroot_lock_path]
- super(MockMixin, self).run_command(rm_lock_cmd,
- halt_on_failure=True,
- fatal_exit_code=3)
- cmd = ['mock_mozilla', '-r', mock_target, '--orphanskill']
- return super(MockMixin, self).run_command(cmd, halt_on_failure=True,
- fatal_exit_code=3)
-
- def setup_mock(self, mock_target=None, mock_packages=None, mock_files=None):
- """Initializes and installs packages, copies files into mock
- environment given by configuration in self.config. The mock
- environment is given by self.config['mock_target'], the list of packges
- to install given by self.config['mock_packages'], and the list of files
- to copy in is self.config['mock_files']."""
- if self.done_mock_setup or self.config.get('disable_mock'):
- return
-
- c = self.config
-
- if mock_target is None:
- assert 'mock_target' in c
- t = c['mock_target']
- else:
- t = mock_target
- self.default_mock_target = t
-
- # Don't re-initialize mock if we're using the same packages as before
- # Put the cache inside the mock root so that if somebody else resets
- # the environment, it invalidates the cache
- mock_root = super(MockMixin, self).get_output_from_command(
- ['mock_mozilla', '-r', t, '--print-root-path']
- )
- package_hash_file = os.path.join(mock_root, "builds/package_list.hash")
- if os.path.exists(package_hash_file):
- old_packages_hash = self.read_from_file(package_hash_file)
- self.info("old package hash: %s" % old_packages_hash)
- else:
- self.info("no previous package list found")
- old_packages_hash = None
-
- if mock_packages is None:
- mock_packages = list(c.get('mock_packages'))
-
- package_list_hash = hashlib.new('sha1')
- if mock_packages:
- for p in sorted(mock_packages):
- package_list_hash.update(p)
- package_list_hash = package_list_hash.hexdigest()
-
- did_init = True
- # This simple hash comparison doesn't take into account depedency
- # changes. If you really care about dependencies, then they should be
- # explicitly listed in the package list.
- if old_packages_hash != package_list_hash:
- self.init_mock(t)
- else:
- self.info("Our list of packages hasn't changed; skipping re-initialization")
- did_init = False
-
- # Still try and install packages here since the package version may
- # have been updated on the server
- if mock_packages:
- self.install_mock_packages(t, mock_packages)
-
- # Save our list of packages
- self.write_to_file(package_hash_file,
- package_list_hash)
-
- if mock_files is None:
- mock_files = list(c.get('mock_files'))
- if mock_files:
- if not did_init:
- # mock complains if you try and copy in files that already
- # exist, so we need to delete them here first
- self.info("Deleting old mock files")
- self.delete_mock_files(t, mock_files)
- self.copy_mock_files(t, mock_files)
-
- self.done_mock_setup = True
-
- def run_command_m(self, *args, **kwargs):
- """Executes self.run_mock_command if we have a mock target set,
- otherwise executes self.run_command."""
- mock_target = self.get_mock_target()
- if mock_target:
- self.setup_mock()
- return self.run_mock_command(mock_target, *args, **kwargs)
- else:
- return super(MockMixin, self).run_command(*args, **kwargs)
-
- def get_output_from_command_m(self, *args, **kwargs):
- """Executes self.get_mock_output_from_command if we have a mock target
- set, otherwise executes self.get_output_from_command."""
- mock_target = self.get_mock_target()
- if mock_target:
- self.setup_mock()
- return self.get_mock_output_from_command(mock_target, *args, **kwargs)
- else:
- return super(MockMixin, self).get_output_from_command(*args, **kwargs)
diff --git a/testing/mozharness/mozharness/mozilla/mozbase.py b/testing/mozharness/mozharness/mozilla/mozbase.py
deleted file mode 100644
index 0201687d1..000000000
--- a/testing/mozharness/mozharness/mozilla/mozbase.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import os
-from mozharness.base.script import PreScriptAction
-
-
-class MozbaseMixin(object):
- """Automatically set virtualenv requirements to use mozbase
- from test package.
- """
- def __init__(self, *args, **kwargs):
- super(MozbaseMixin, self).__init__(*args, **kwargs)
-
- @PreScriptAction('create-virtualenv')
- def _install_mozbase(self, action):
- dirs = self.query_abs_dirs()
-
- requirements = os.path.join(dirs['abs_test_install_dir'],
- 'config',
- 'mozbase_requirements.txt')
- if os.path.isfile(requirements):
- self.register_virtualenv_module(requirements=[requirements],
- two_pass=True)
- return
-
- # XXX Bug 879765: Dependent modules need to be listed before parent
- # modules, otherwise they will get installed from the pypi server.
- # XXX Bug 908356: This block can be removed as soon as the
- # in-tree requirements files propagate to all active trees.
- mozbase_dir = os.path.join('tests', 'mozbase')
- self.register_virtualenv_module(
- 'manifestparser',
- url=os.path.join(mozbase_dir, 'manifestdestiny')
- )
-
- for m in ('mozfile', 'mozlog', 'mozinfo', 'moznetwork', 'mozhttpd',
- 'mozcrash', 'mozinstall', 'mozdevice', 'mozprofile',
- 'mozprocess', 'mozrunner'):
- self.register_virtualenv_module(
- m, url=os.path.join(mozbase_dir, m)
- )
diff --git a/testing/mozharness/mozharness/mozilla/proxxy.py b/testing/mozharness/mozharness/mozilla/proxxy.py
deleted file mode 100644
index b9f14d5f2..000000000
--- a/testing/mozharness/mozharness/mozilla/proxxy.py
+++ /dev/null
@@ -1,167 +0,0 @@
-"""Proxxy module. Defines a Proxxy element that fetches files using local
- proxxy instances (if available). The goal of Proxxy is to lower the traffic
- from the cloud to internal servers.
-"""
-import urlparse
-import socket
-from mozharness.base.log import INFO, ERROR, LogMixin
-from mozharness.base.script import ScriptMixin
-
-
-# Proxxy {{{1
-class Proxxy(ScriptMixin, LogMixin):
- """
- Support downloading files from HTTP caching proxies
-
- Current supports 'proxxy' instances, in which the caching proxy at
- proxxy.domain.com will cache requests for ftp.mozilla.org when passed requests to
- http://ftp.mozilla.org.proxxy.domain.com/...
-
- self.config['proxxy']['urls'] defines the list of backend hosts we are currently caching, and
- the hostname prefix to use for proxxy
-
- self.config['proxxy']['instances'] lists current hostnames for proxxy instances. wildcard DNS
- is set up so that *.proxxy.domain.com is a CNAME to the proxxy instance
- """
-
- # Default configuration. Can be overridden via self.config
- PROXXY_CONFIG = {
- "urls": [
- ('http://ftp.mozilla.org', 'ftp.mozilla.org'),
- ('https://ftp.mozilla.org', 'ftp.mozilla.org'),
- ('https://ftp-ssl.mozilla.org', 'ftp.mozilla.org'),
- # pypi
- ('http://pypi.pvt.build.mozilla.org', 'pypi.pvt.build.mozilla.org'),
- ('http://pypi.pub.build.mozilla.org', 'pypi.pub.build.mozilla.org'),
- ],
- "instances": [
- 'proxxy1.srv.releng.use1.mozilla.com',
- 'proxxy1.srv.releng.usw2.mozilla.com',
- ],
- "regions": [".use1.", ".usw2."],
- }
-
- def __init__(self, config, log_obj):
- # proxxy does not need the need the full configuration,
- # just the 'proxxy' element
- # if configuration has no 'proxxy' section use the default
- # configuration instead
- default_config = {} if self.is_taskcluster() else self.PROXXY_CONFIG
- self.config = config.get('proxxy', default_config)
- self.log_obj = log_obj
-
- def get_proxies_for_url(self, url):
- """Maps url to its proxxy urls
-
- Args:
- url (str): url to be proxxied
- Returns:
- list: of proxy URLs to try, in sorted order.
- please note that url is NOT included in this list.
- """
- config = self.config
- urls = []
-
- self.info("proxxy config: %s" % config)
-
- proxxy_urls = config.get('urls', [])
- proxxy_instances = config.get('instances', [])
-
- url_parts = urlparse.urlsplit(url)
- url_path = url_parts.path
- if url_parts.query:
- url_path += "?" + url_parts.query
- if url_parts.fragment:
- url_path += "#" + url_parts.fragment
-
- for prefix, target in proxxy_urls:
- if url.startswith(prefix):
- self.info("%s matches %s" % (url, prefix))
- for instance in proxxy_instances:
- if not self.query_is_proxxy_local(instance):
- continue
- new_url = "http://%s.%s%s" % (target, instance, url_path)
- urls.append(new_url)
-
- for url in urls:
- self.info("URL Candidate: %s" % url)
- return urls
-
- def get_proxies_and_urls(self, urls):
- """Gets a list of urls and returns a list of proxied urls, the list
- of input urls is appended at the end of the return values
-
- Args:
- urls (list, tuple): urls to be mapped to proxxy urls
-
- Returns:
- list: proxxied urls and urls. urls are appended to the proxxied
- urls list and they are the last elements of the list.
- """
- proxxy_list = []
- for url in urls:
- # get_proxies_for_url returns always a list...
- proxxy_list.extend(self.get_proxies_for_url(url))
- proxxy_list.extend(urls)
- return proxxy_list
-
- def query_is_proxxy_local(self, url):
- """Checks is url is 'proxxable' for the local instance
-
- Args:
- url (string): url to check
-
- Returns:
- bool: True if url maps to a usable proxxy,
- False in any other case
- """
- fqdn = socket.getfqdn()
- config = self.config
- regions = config.get('regions', [])
-
- return any(r in fqdn and r in url for r in regions)
-
- def download_proxied_file(self, url, file_name, parent_dir=None,
- create_parent_dir=True, error_level=ERROR,
- exit_code=3):
- """
- Wrapper around BaseScript.download_file that understands proxies
- retry dict is set to 3 attempts, sleeping time 30 seconds.
-
- Args:
- url (string): url to fetch
- file_name (string, optional): output filename, defaults to None
- if file_name is not defined, the output name is taken from
- the url.
- parent_dir (string, optional): name of the parent directory
- create_parent_dir (bool, optional): if True, creates the parent
- directory. Defaults to True
- error_level (mozharness log level, optional): log error level
- defaults to ERROR
- exit_code (int, optional): return code to log if file_name
- is not defined and it cannot be determined from the url
- Returns:
- string: file_name if the download has succeded, None in case of
- error. In case of error, if error_level is set to FATAL,
- this method interrupts the execution of the script
-
- """
- urls = self.get_proxies_and_urls([url])
-
- for url in urls:
- self.info("trying %s" % url)
- retval = self.download_file(
- url, file_name=file_name, parent_dir=parent_dir,
- create_parent_dir=create_parent_dir, error_level=ERROR,
- exit_code=exit_code,
- retry_config=dict(
- attempts=3,
- sleeptime=30,
- error_level=INFO,
- ))
- if retval:
- return retval
-
- self.log("Failed to download from all available URLs, aborting",
- level=error_level, exit_code=exit_code)
- return retval
diff --git a/testing/mozharness/mozharness/mozilla/purge.py b/testing/mozharness/mozharness/mozilla/purge.py
deleted file mode 100644
index 23ffd9081..000000000
--- a/testing/mozharness/mozharness/mozilla/purge.py
+++ /dev/null
@@ -1,103 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Purge/clobber support
-"""
-
-# Figure out where our external_tools are
-# These are in a sibling directory to the 'mozharness' module
-import os
-import mozharness
-external_tools_path = os.path.join(
- os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
- 'external_tools',
-)
-
-from mozharness.base.log import ERROR
-
-
-# PurgeMixin {{{1
-# Depends on ScriptMixin for self.run_command,
-# and BuildbotMixin for self.buildbot_config and self.query_is_nightly()
-class PurgeMixin(object):
- clobber_tool = os.path.join(external_tools_path, 'clobberer.py')
-
- default_skips = ['info', 'rel-*', 'tb-rel-*']
- default_maxage = 14
- default_periodic_clobber = 7 * 24
-
- def clobberer(self):
- c = self.config
- dirs = self.query_abs_dirs()
- if not self.buildbot_config:
- self.fatal("clobberer requires self.buildbot_config (usually from $PROPERTIES_FILE)")
-
- periodic_clobber = c.get('periodic_clobber') or self.default_periodic_clobber
- clobberer_url = c['clobberer_url']
-
- builddir = os.path.basename(dirs['base_work_dir'])
- branch = self.buildbot_config['properties']['branch']
- buildername = self.buildbot_config['properties']['buildername']
- slave = self.buildbot_config['properties']['slavename']
- master = self.buildbot_config['properties']['master']
-
- cmd = []
- if self._is_windows():
- # The virtualenv isn't setup yet, so just use python directly.
- cmd.append(self.query_exe('python'))
- # Add --dry-run if you don't want to do this for realz
- cmd.extend([self.clobber_tool])
- # TODO configurable list
- cmd.extend(['-s', 'scripts'])
- cmd.extend(['-s', 'logs'])
- cmd.extend(['-s', 'buildprops.json'])
- cmd.extend(['-s', 'token'])
- cmd.extend(['-s', 'oauth.txt'])
-
- if periodic_clobber:
- cmd.extend(['-t', str(periodic_clobber)])
-
- cmd.extend([clobberer_url, branch, buildername, builddir, slave, master])
- error_list = [{
- 'substr': 'Error contacting server', 'level': ERROR,
- 'explanation': 'Error contacting server for clobberer information.'
- }]
-
- retval = self.retry(self.run_command, attempts=3, good_statuses=(0,), args=[cmd],
- kwargs={'cwd':os.path.dirname(dirs['base_work_dir']),
- 'error_list':error_list})
- if retval != 0:
- self.fatal("failed to clobber build", exit_code=2)
-
- def clobber(self, always_clobber_dirs=None):
- """ Mozilla clobberer-type clobber.
- """
- c = self.config
- if c.get('developer_mode'):
- self.info("Suppressing clobber in developer mode for safety.")
- return
- if c.get('is_automation'):
- # Nightly builds always clobber
- do_clobber = False
- if self.query_is_nightly():
- self.info("Clobbering because we're a nightly build")
- do_clobber = True
- if c.get('force_clobber'):
- self.info("Clobbering because our config forced us to")
- do_clobber = True
- if do_clobber:
- super(PurgeMixin, self).clobber()
- else:
- # Delete the upload dir so we don't upload previous stuff by
- # accident
- if always_clobber_dirs is None:
- always_clobber_dirs = []
- for path in always_clobber_dirs:
- self.rmtree(path)
- if 'clobberer_url' in c and c.get('use_clobberer', True):
- self.clobberer()
- else:
- super(PurgeMixin, self).clobber()
diff --git a/testing/mozharness/mozharness/mozilla/release.py b/testing/mozharness/mozharness/mozilla/release.py
deleted file mode 100755
index 52a84cdba..000000000
--- a/testing/mozharness/mozharness/mozilla/release.py
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""release.py
-
-"""
-
-import os
-from distutils.version import LooseVersion, StrictVersion
-
-from mozharness.base.config import parse_config_file
-
-
-# SignAndroid {{{1
-class ReleaseMixin():
- release_config = {}
-
- def query_release_config(self):
- if self.release_config:
- return self.release_config
- c = self.config
- dirs = self.query_abs_dirs()
- if c.get("release_config_file"):
- self.info("Getting release config from %s..." % c["release_config_file"])
- rc = None
- try:
- rc = parse_config_file(
- os.path.join(dirs['abs_work_dir'],
- c["release_config_file"]),
- config_dict_name="releaseConfig"
- )
- except IOError:
- self.fatal("Release config file %s not found!" % c["release_config_file"])
- except RuntimeError:
- self.fatal("Invalid release config file %s!" % c["release_config_file"])
- self.release_config['version'] = rc['version']
- self.release_config['buildnum'] = rc['buildNumber']
- self.release_config['ftp_server'] = rc['stagingServer']
- self.release_config['ftp_user'] = c.get('ftp_user', rc['hgUsername'])
- self.release_config['ftp_ssh_key'] = c.get('ftp_ssh_key', rc['hgSshKey'])
- self.release_config['release_channel'] = rc['releaseChannel']
- else:
- self.info("No release config file; using default config.")
- for key in ('version', 'buildnum',
- 'ftp_server', 'ftp_user', 'ftp_ssh_key'):
- self.release_config[key] = c[key]
- self.info("Release config:\n%s" % self.release_config)
- return self.release_config
-
-
-def get_previous_version(version, partial_versions):
- """ The patcher config bumper needs to know the exact previous version
- We use LooseVersion for ESR because StrictVersion can't parse the trailing
- 'esr', but StrictVersion otherwise because it can sort X.0bN lower than X.0.
- The current version is excluded to avoid an error if build1 is aborted
- before running the updates builder and now we're doing build2
- """
- if version.endswith('esr'):
- return str(max(LooseVersion(v) for v in partial_versions if
- v != version))
- else:
- # StrictVersion truncates trailing zero in versions with more than 1
- # dot. Compose a structure that will be sorted by StrictVersion and
- # return untouched version
- composed = sorted([(v, StrictVersion(v)) for v in partial_versions if
- v != version], key=lambda x: x[1], reverse=True)
- return composed[0][0]
-
-
diff --git a/testing/mozharness/mozharness/mozilla/repo_manifest.py b/testing/mozharness/mozharness/mozilla/repo_manifest.py
deleted file mode 100644
index 2ffb34fe9..000000000
--- a/testing/mozharness/mozharness/mozilla/repo_manifest.py
+++ /dev/null
@@ -1,226 +0,0 @@
-"""
-Module for handling repo style XML manifests
-"""
-import xml.dom.minidom
-import os
-import re
-
-
-def load_manifest(filename):
- """
- Loads manifest from `filename` and returns a single flattened manifest
- Processes any <include name="..." /> nodes recursively
- Removes projects referenced by <remove-project name="..." /> nodes
- Abort on unsupported manifest tags
- Returns the root node of the resulting DOM
- """
- doc = xml.dom.minidom.parse(filename)
-
- # Check that we don't have any unsupported tags
- to_visit = list(doc.childNodes)
- while to_visit:
- node = to_visit.pop()
- # Skip text nodes
- if node.nodeType in (node.TEXT_NODE, node.COMMENT_NODE):
- continue
-
- if node.tagName not in ('include', 'project', 'remote', 'default', 'manifest', 'copyfile', 'remove-project'):
- raise ValueError("Unsupported tag: %s" % node.tagName)
- to_visit.extend(node.childNodes)
-
- # Find all <include> nodes
- for i in doc.getElementsByTagName('include'):
- p = i.parentNode
-
- # The name attribute is relative to where the original manifest lives
- inc_filename = i.getAttribute('name')
- inc_filename = os.path.join(os.path.dirname(filename), inc_filename)
-
- # Parse the included file
- inc_doc = load_manifest(inc_filename).documentElement
- # For all the child nodes in the included manifest, insert into our
- # manifest just before the include node
- # We operate on a copy of childNodes because when we reparent `c`, the
- # list of childNodes is modified.
- for c in inc_doc.childNodes[:]:
- p.insertBefore(c, i)
- # Now we can remove the include node
- p.removeChild(i)
-
- # Remove all projects referenced by <remove-project>
- projects = {}
- manifest = doc.documentElement
- to_remove = []
- for node in manifest.childNodes:
- # Skip text nodes
- if node.nodeType in (node.TEXT_NODE, node.COMMENT_NODE):
- continue
-
- if node.tagName == 'project':
- projects[node.getAttribute('name')] = node
-
- elif node.tagName == 'remove-project':
- project_node = projects[node.getAttribute('name')]
- to_remove.append(project_node)
- to_remove.append(node)
-
- for r in to_remove:
- r.parentNode.removeChild(r)
-
- return doc
-
-
-def rewrite_remotes(manifest, mapping_func, force_all=True):
- """
- Rewrite manifest remotes in place
- Returns the same manifest, with the remotes transformed by mapping_func
- mapping_func should return a modified remote node, or None if no changes
- are required
- If force_all is True, then it is an error for mapping_func to return None;
- a ValueError is raised in this case
- """
- for r in manifest.getElementsByTagName('remote'):
- m = mapping_func(r)
- if not m:
- if force_all:
- raise ValueError("Wasn't able to map %s" % r.toxml())
- continue
-
- r.parentNode.replaceChild(m, r)
-
-
-def add_project(manifest, name, path, remote=None, revision=None):
- """
- Adds a project to the manifest in place
- """
-
- project = manifest.createElement("project")
- project.setAttribute('name', name)
- project.setAttribute('path', path)
- if remote:
- project.setAttribute('remote', remote)
- if revision:
- project.setAttribute('revision', revision)
-
- manifest.documentElement.appendChild(project)
-
-
-def remove_project(manifest, name=None, path=None):
- """
- Removes a project from manifest.
- One of name or path must be set. If path is specified, then the project
- with the given path is removed, otherwise the project with the given name
- is removed.
- """
- assert name or path
- node = get_project(manifest, name, path)
- if node:
- node.parentNode.removeChild(node)
- return node
-
-
-def get_project(manifest, name=None, path=None):
- """
- Gets a project node from the manifest.
- One of name or path must be set. If path is specified, then the project
- with the given path is returned, otherwise the project with the given name
- is returned.
- """
- assert name or path
- for node in manifest.getElementsByTagName('project'):
- if path is not None and node.getAttribute('path') == path:
- return node
- if node.getAttribute('name') == name:
- return node
-
-
-def get_remote(manifest, name):
- for node in manifest.getElementsByTagName('remote'):
- if node.getAttribute('name') == name:
- return node
-
-
-def get_default(manifest):
- default = manifest.getElementsByTagName('default')[0]
- return default
-
-
-def get_project_remote_url(manifest, project):
- """
- Gets the remote URL for the given project node. Will return the default
- remote if the project doesn't explicitly specify one.
- """
- if project.hasAttribute('remote'):
- remote = get_remote(manifest, project.getAttribute('remote'))
- else:
- default = get_default(manifest)
- remote = get_remote(manifest, default.getAttribute('remote'))
- fetch = remote.getAttribute('fetch')
- if not fetch.endswith('/'):
- fetch += '/'
- return "%s%s" % (fetch, project.getAttribute('name'))
-
-
-def get_project_revision(manifest, project):
- """
- Gets the revision for the given project node. Will return the default
- revision if the project doesn't explicitly specify one.
- """
- if project.hasAttribute('revision'):
- return project.getAttribute('revision')
- else:
- default = get_default(manifest)
- return default.getAttribute('revision')
-
-
-def remove_group(manifest, group):
- """
- Removes all projects with groups=`group`
- """
- retval = []
- for node in manifest.getElementsByTagName('project'):
- if group in node.getAttribute('groups').split(","):
- node.parentNode.removeChild(node)
- retval.append(node)
- return retval
-
-
-def map_remote(r, mappings):
- """
- Helper function for mapping git remotes
- """
- remote = r.getAttribute('fetch')
- if remote in mappings:
- r.setAttribute('fetch', mappings[remote])
- # Add a comment about where our original remote was
- comment = r.ownerDocument.createComment("original fetch url was %s" % remote)
- line = r.ownerDocument.createTextNode("\n")
- r.parentNode.insertBefore(comment, r)
- r.parentNode.insertBefore(line, r)
- return r
- return None
-
-
-COMMIT_PATTERN = re.compile("[0-9a-f]{40}")
-
-
-def is_commitid(revision):
- """
- Returns True if revision looks like a commit id
- i.e. 40 character string made up of 0-9a-f
- """
- return bool(re.match(COMMIT_PATTERN, revision))
-
-
-def cleanup(manifest, depth=0):
- """
- Remove any empty text nodes
- """
- for n in manifest.childNodes[:]:
- if n.childNodes:
- n.normalize()
- if n.nodeType == n.TEXT_NODE and not n.data.strip():
- if not n.nextSibling:
- depth -= 2
- n.data = "\n" + (" " * depth)
- cleanup(n, depth + 2)
diff --git a/testing/mozharness/mozharness/mozilla/repo_manupulation.py b/testing/mozharness/mozharness/mozilla/repo_manupulation.py
deleted file mode 100644
index a2dfc46a2..000000000
--- a/testing/mozharness/mozharness/mozilla/repo_manupulation.py
+++ /dev/null
@@ -1,164 +0,0 @@
-import os
-
-from mozharness.base.errors import HgErrorList
-from mozharness.base.log import FATAL, INFO
-from mozharness.base.vcs.mercurial import MercurialVCS
-
-
-class MercurialRepoManipulationMixin(object):
-
- def get_version(self, repo_root,
- version_file="browser/config/version.txt"):
- version_path = os.path.join(repo_root, version_file)
- contents = self.read_from_file(version_path, error_level=FATAL)
- lines = [l for l in contents.splitlines() if l and
- not l.startswith("#")]
- return lines[-1].split(".")
-
- def replace(self, file_name, from_, to_):
- """ Replace text in a file.
- """
- text = self.read_from_file(file_name, error_level=FATAL)
- new_text = text.replace(from_, to_)
- if text == new_text:
- self.fatal("Cannot replace '%s' to '%s' in '%s'" %
- (from_, to_, file_name))
- self.write_to_file(file_name, new_text, error_level=FATAL)
-
- def query_hg_revision(self, path):
- """ Avoid making 'pull' a required action every run, by being able
- to fall back to figuring out the revision from the cloned repo
- """
- m = MercurialVCS(log_obj=self.log_obj, config=self.config)
- revision = m.get_revision_from_path(path)
- return revision
-
- def hg_commit(self, cwd, message, user=None, ignore_no_changes=False):
- """ Commit changes to hg.
- """
- cmd = self.query_exe('hg', return_type='list') + [
- 'commit', '-m', message]
- if user:
- cmd.extend(['-u', user])
- success_codes = [0]
- if ignore_no_changes:
- success_codes.append(1)
- self.run_command(
- cmd, cwd=cwd, error_list=HgErrorList,
- halt_on_failure=True,
- success_codes=success_codes
- )
- return self.query_hg_revision(cwd)
-
- def clean_repos(self):
- """ We may end up with contaminated local repos at some point, but
- we don't want to have to clobber and reclone from scratch every
- time.
-
- This is an attempt to clean up the local repos without needing a
- clobber.
- """
- dirs = self.query_abs_dirs()
- hg = self.query_exe("hg", return_type="list")
- hg_repos = self.query_repos()
- hg_strip_error_list = [{
- 'substr': r'''abort: empty revision set''', 'level': INFO,
- 'explanation': "Nothing to clean up; we're good!",
- }] + HgErrorList
- for repo_config in hg_repos:
- repo_name = repo_config["dest"]
- repo_path = os.path.join(dirs['abs_work_dir'], repo_name)
- if os.path.exists(repo_path):
- # hg up -C to discard uncommitted changes
- self.run_command(
- hg + ["up", "-C", "-r", repo_config['branch']],
- cwd=repo_path,
- error_list=HgErrorList,
- halt_on_failure=True,
- )
- # discard unpushed commits
- status = self.retry(
- self.run_command,
- args=(hg + ["--config", "extensions.mq=", "strip",
- "--no-backup", "outgoing()"], ),
- kwargs={
- 'cwd': repo_path,
- 'error_list': hg_strip_error_list,
- 'return_type': 'num_errors',
- 'success_codes': (0, 255),
- },
- )
- if status not in [0, 255]:
- self.fatal("Issues stripping outgoing revisions!")
- # 2nd hg up -C to make sure we're not on a stranded head
- # which can happen when reverting debugsetparents
- self.run_command(
- hg + ["up", "-C", "-r", repo_config['branch']],
- cwd=repo_path,
- error_list=HgErrorList,
- halt_on_failure=True,
- )
-
- def commit_changes(self):
- """ Do the commit.
- """
- hg = self.query_exe("hg", return_type="list")
- for cwd in self.query_commit_dirs():
- self.run_command(hg + ["diff"], cwd=cwd)
- self.hg_commit(
- cwd, user=self.config['hg_user'],
- message=self.query_commit_message(),
- ignore_no_changes=self.config.get("ignore_no_changes", False)
- )
- self.info("Now verify |hg out| and |hg out --patch| if you're paranoid, and --push")
-
- def hg_tag(self, cwd, tags, user=None, message=None, revision=None,
- force=None, halt_on_failure=True):
- if isinstance(tags, basestring):
- tags = [tags]
- cmd = self.query_exe('hg', return_type='list') + ['tag']
- if not message:
- message = "No bug - Tagging %s" % os.path.basename(cwd)
- if revision:
- message = "%s %s" % (message, revision)
- message = "%s with %s" % (message, ', '.join(tags))
- message += " a=release DONTBUILD CLOSED TREE"
- self.info(message)
- cmd.extend(['-m', message])
- if user:
- cmd.extend(['-u', user])
- if revision:
- cmd.extend(['-r', revision])
- if force:
- cmd.append('-f')
- cmd.extend(tags)
- return self.run_command(
- cmd, cwd=cwd, halt_on_failure=halt_on_failure,
- error_list=HgErrorList
- )
-
- def push(self):
- """
- """
- error_message = """Push failed! If there was a push race, try rerunning
-the script (--clean-repos --pull --migrate). The second run will be faster."""
- hg = self.query_exe("hg", return_type="list")
- for cwd in self.query_push_dirs():
- if not cwd:
- self.warning("Skipping %s" % cwd)
- continue
- push_cmd = hg + ['push'] + self.query_push_args(cwd)
- if self.config.get("push_dest"):
- push_cmd.append(self.config["push_dest"])
- status = self.run_command(
- push_cmd,
- cwd=cwd,
- error_list=HgErrorList,
- success_codes=[0, 1],
- )
- if status == 1:
- self.warning("No changes for %s!" % cwd)
- elif status:
- self.fatal(error_message)
-
-
diff --git a/testing/mozharness/mozharness/mozilla/secrets.py b/testing/mozharness/mozharness/mozilla/secrets.py
deleted file mode 100644
index d40964bd6..000000000
--- a/testing/mozharness/mozharness/mozilla/secrets.py
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Support for fetching secrets from the secrets API
-"""
-
-import os
-import mozharness
-import urllib2
-import json
-from mozharness.base.log import ERROR
-
-
-class SecretsMixin(object):
-
- def _fetch_secret(self, secret_name):
- self.info("fetching secret {} from API".format(secret_name))
- # fetch from http://taskcluster, which points to the taskcluster proxy
- # within a taskcluster task. Outside of that environment, do not
- # use this action.
- url = "http://taskcluster/secrets/v1/secret/" + secret_name
- res = urllib2.urlopen(url)
- if res.getcode() != 200:
- self.fatal("Error fetching from secrets API:" + res.read())
-
- return json.load(res)['secret']['content']
-
- def get_secrets(self):
- """
- Get the secrets specified by the `secret_files` configuration. This is
- a list of dictionaries, one for each secret. The `secret_name` key
- names the key in the TaskCluster secrets API to fetch (see
- http://docs.taskcluster.net/services/secrets/). It can contain
- %-substitutions based on the `subst` dictionary below.
-
- Since secrets must be JSON objects, the `content` property of the
- secret is used as the value to be written to disk.
-
- The `filename` key in the dictionary gives the filename to which the
- secret should be written.
-
- The optional `min_scm_level` key gives a minimum SCM level at which this
- secret is required. For lower levels, the value of the 'default` key
- is used, or no secret is written.
- """
- if self.config.get('forced_artifact_build'):
- self.info('Skipping due to forced artifact build.')
- return
-
- secret_files = self.config.get('secret_files', [])
-
- scm_level = self.config.get('scm-level', 1)
- subst = {
- 'scm-level': scm_level,
- }
-
- for sf in secret_files:
- filename = sf['filename']
- secret_name = sf['secret_name'] % subst
- min_scm_level = sf.get('min_scm_level', 0)
- if scm_level <= min_scm_level:
- if 'default' in sf:
- self.info("Using default value for " + filename)
- secret = sf['default']
- else:
- self.info("No default for secret; not writing " + filename)
- continue
- else:
- secret = self._fetch_secret(secret_name)
-
- open(filename, "w").write(secret)
diff --git a/testing/mozharness/mozharness/mozilla/selfserve.py b/testing/mozharness/mozharness/mozilla/selfserve.py
deleted file mode 100644
index 69e243059..000000000
--- a/testing/mozharness/mozharness/mozilla/selfserve.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import json
-import site
-
-# SelfServeMixin {{{1
-class SelfServeMixin(object):
- def _get_session(self):
- site_packages_path = self.query_python_site_packages_path()
- site.addsitedir(site_packages_path)
- import requests
- session = requests.Session()
- adapter = requests.adapters.HTTPAdapter(max_retries=5)
- session.mount("http://", adapter)
- session.mount("https://", adapter)
- return session
-
- def _get_base_url(self):
- return self.config["selfserve_url"].rstrip("/")
-
- def trigger_nightly_builds(self, branch, revision, auth):
- session = self._get_session()
-
- selfserve_base = self._get_base_url()
- url = "%s/%s/rev/%s/nightly" % (selfserve_base, branch, revision)
-
- data = {
- "revision": revision,
- }
- self.info("Triggering nightly builds via %s" % url)
- return session.post(url, data=data, auth=auth).raise_for_status()
-
- def trigger_arbitrary_job(self, builder, branch, revision, auth, files=None):
- session = self._get_session()
-
- selfserve_base = self._get_base_url()
- url = "%s/%s/builders/%s/%s" % (selfserve_base, branch, builder, revision)
-
- data = {
- "properties": json.dumps({
- "branch": branch,
- "revision": revision
- }),
- }
- if files:
- data["files"] = json.dumps(files)
-
- self.info("Triggering arbritrary job at %s" % url)
- return session.post(url, data=data, auth=auth).raise_for_status()
diff --git a/testing/mozharness/mozharness/mozilla/signing.py b/testing/mozharness/mozharness/mozilla/signing.py
deleted file mode 100755
index 3b16ce595..000000000
--- a/testing/mozharness/mozharness/mozilla/signing.py
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Mozilla-specific signing methods.
-"""
-
-import os
-import re
-import json
-
-from mozharness.base.errors import BaseErrorList
-from mozharness.base.log import ERROR, FATAL
-from mozharness.base.signing import AndroidSigningMixin, BaseSigningMixin
-
-AndroidSignatureVerificationErrorList = BaseErrorList + [{
- "regex": re.compile(r'''^Invalid$'''),
- "level": FATAL,
- "explanation": "Signature is invalid!"
-}, {
- "substr": "filename not matched",
- "level": ERROR,
-}, {
- "substr": "ERROR: Could not unzip",
- "level": ERROR,
-}, {
- "regex": re.compile(r'''Are you sure this is a (nightly|release) package'''),
- "level": FATAL,
- "explanation": "Not signed!"
-}]
-
-
-# SigningMixin {{{1
-
-class SigningMixin(BaseSigningMixin):
- """Generic signing helper methods."""
- def query_moz_sign_cmd(self, formats=['gpg']):
- if 'MOZ_SIGNING_SERVERS' not in os.environ:
- self.fatal("MOZ_SIGNING_SERVERS not in env; no MOZ_SIGN_CMD for you!")
- dirs = self.query_abs_dirs()
- signing_dir = os.path.join(dirs['abs_work_dir'], 'tools', 'release', 'signing')
- cache_dir = os.path.join(dirs['abs_work_dir'], 'signing_cache')
- token = os.path.join(dirs['base_work_dir'], 'token')
- nonce = os.path.join(dirs['base_work_dir'], 'nonce')
- host_cert = os.path.join(signing_dir, 'host.cert')
- python = self.query_exe('python')
- cmd = [
- python,
- os.path.join(signing_dir, 'signtool.py'),
- '--cachedir', cache_dir,
- '-t', token,
- '-n', nonce,
- '-c', host_cert,
- ]
- if formats:
- for f in formats:
- cmd += ['-f', f]
- for h in os.environ['MOZ_SIGNING_SERVERS'].split(","):
- cmd += ['-H', h]
- return cmd
-
- def generate_signing_manifest(self, files):
- """Generate signing manifest for signingworkers
-
- Every entry in the manifest requires a dictionary of
- "file_to_sign" (basename) and "hash" (SHA512) of every file to be
- signed. Signing format is defined in the signing task.
- """
- manifest_content = [
- {
- "file_to_sign": os.path.basename(f),
- "hash": self.query_sha512sum(f)
- }
- for f in files
- ]
- return json.dumps(manifest_content)
-
-
-# MobileSigningMixin {{{1
-class MobileSigningMixin(AndroidSigningMixin, SigningMixin):
- def verify_android_signature(self, apk, script=None, key_alias="nightly",
- tools_dir="tools/", env=None):
- """Runs mjessome's android signature verification script.
- This currently doesn't check to see if the apk exists; you may want
- to do that before calling the method.
- """
- c = self.config
- dirs = self.query_abs_dirs()
- if script is None:
- script = c.get('signature_verification_script')
- if env is None:
- env = self.query_env()
- return self.run_command(
- [script, "--tools-dir=%s" % tools_dir, "--%s" % key_alias,
- "--apk=%s" % apk],
- cwd=dirs['abs_work_dir'],
- env=env,
- error_list=AndroidSignatureVerificationErrorList
- )
diff --git a/testing/mozharness/mozharness/mozilla/structuredlog.py b/testing/mozharness/mozharness/mozilla/structuredlog.py
deleted file mode 100644
index d87c5ebdc..000000000
--- a/testing/mozharness/mozharness/mozilla/structuredlog.py
+++ /dev/null
@@ -1,173 +0,0 @@
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-import json
-
-from mozharness.base import log
-from mozharness.base.log import OutputParser, WARNING, INFO, ERROR
-from mozharness.mozilla.buildbot import TBPL_WARNING, TBPL_FAILURE
-from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE
-from mozharness.mozilla.testing.unittest import tbox_print_summary
-
-
-class StructuredOutputParser(OutputParser):
- # The script class using this must inherit the MozbaseMixin to ensure
- # that mozlog is available.
- def __init__(self, **kwargs):
- """Object that tracks the overall status of the test run"""
- # The 'strict' argument dictates whether the presence of output
- # from the harness process other than line-delimited json indicates
- # failure. If it does not, the errors_list parameter may be used
- # to detect additional failure output from the harness process.
- if 'strict' in kwargs:
- self.strict = kwargs.pop('strict')
- else:
- self.strict = True
-
- self.suite_category = kwargs.pop('suite_category', None)
-
- tbpl_compact = kwargs.pop("log_compact", False)
- super(StructuredOutputParser, self).__init__(**kwargs)
-
- mozlog = self._get_mozlog_module()
- self.formatter = mozlog.formatters.TbplFormatter(compact=tbpl_compact)
- self.handler = mozlog.handlers.StatusHandler()
- self.log_actions = mozlog.structuredlog.log_actions()
-
- self.worst_log_level = INFO
- self.tbpl_status = TBPL_SUCCESS
-
- def _get_mozlog_module(self):
- try:
- import mozlog
- except ImportError:
- self.fatal("A script class using structured logging must inherit "
- "from the MozbaseMixin to ensure that mozlog is available.")
- return mozlog
-
- def _handle_unstructured_output(self, line):
- if self.strict:
- self.critical(("Test harness output was not a valid structured log message: "
- "\n%s") % line)
- self.update_levels(TBPL_FAILURE, log.CRITICAL)
- return
- super(StructuredOutputParser, self).parse_single_line(line)
-
-
- def parse_single_line(self, line):
- """Parses a line of log output from the child process and passes
- it to mozlog to update the overall status of the run.
- Re-emits the logged line in human-readable format.
- """
- level = INFO
- tbpl_level = TBPL_SUCCESS
-
- data = None
- try:
- candidate_data = json.loads(line)
- if (isinstance(candidate_data, dict) and
- 'action' in candidate_data and candidate_data['action'] in self.log_actions):
- data = candidate_data
- except ValueError:
- pass
-
- if data is None:
- self._handle_unstructured_output(line)
- return
-
- self.handler(data)
-
- action = data["action"]
- if action == "log":
- level = getattr(log, data["level"].upper())
-
- log_data = self.formatter(data)
- if log_data is not None:
- self.log(log_data, level=level)
- self.update_levels(tbpl_level, level)
-
- def evaluate_parser(self, return_code, success_codes=None):
- success_codes = success_codes or [0]
- summary = self.handler.summarize()
-
- fail_pair = TBPL_WARNING, WARNING
- error_pair = TBPL_FAILURE, ERROR
-
- # These are warning/orange statuses.
- failure_conditions = [
- sum(summary.unexpected_statuses.values()) > 0,
- summary.action_counts.get('crash', 0) > summary.expected_statuses.get('CRASH', 0),
- summary.action_counts.get('valgrind_error', 0) > 0
- ]
- for condition in failure_conditions:
- if condition:
- self.update_levels(*fail_pair)
-
- # These are error/red statuses. A message is output here every time something
- # wouldn't otherwise be highlighted in the UI.
- required_actions = {
- 'suite_end': 'No suite end message was emitted by this harness.',
- 'test_end': 'No checks run.',
- }
- for action, diagnostic_message in required_actions.iteritems():
- if action not in summary.action_counts:
- self.log(diagnostic_message, ERROR)
- self.update_levels(*error_pair)
-
- failure_log_levels = ['ERROR', 'CRITICAL']
- for level in failure_log_levels:
- if level in summary.log_level_counts:
- self.update_levels(*error_pair)
-
- # If a superclass was used to detect errors with a regex based output parser,
- # this will be reflected in the status here.
- if self.num_errors:
- self.update_levels(*error_pair)
-
- # Harnesses typically return non-zero on test failure, so don't promote
- # to error if we already have a failing status.
- if return_code not in success_codes and self.tbpl_status == TBPL_SUCCESS:
- self.update_levels(*error_pair)
-
- return self.tbpl_status, self.worst_log_level
-
- def update_levels(self, tbpl_level, log_level):
- self.worst_log_level = self.worst_level(log_level, self.worst_log_level)
- self.tbpl_status = self.worst_level(tbpl_level, self.tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE)
-
- def print_summary(self, suite_name):
- # Summary text provided for compatibility. Counts are currently
- # in the format <pass count>/<fail count>/<todo count>,
- # <expected count>/<unexpected count>/<expected fail count> will yield the
- # expected info from a structured log (fail count from the prior implementation
- # includes unexpected passes from "todo" assertions).
- summary = self.handler.summarize()
- unexpected_count = sum(summary.unexpected_statuses.values())
- expected_count = sum(summary.expected_statuses.values())
- expected_failures = summary.expected_statuses.get('FAIL', 0)
-
- if unexpected_count:
- fail_text = '<em class="testfail">%s</em>' % unexpected_count
- else:
- fail_text = '0'
-
- text_summary = "%s/%s/%s" % (expected_count, fail_text, expected_failures)
- self.info("TinderboxPrint: %s<br/>%s\n" % (suite_name, text_summary))
-
- def append_tinderboxprint_line(self, suite_name):
- summary = self.handler.summarize()
- unexpected_count = sum(summary.unexpected_statuses.values())
- expected_count = sum(summary.expected_statuses.values())
- expected_failures = summary.expected_statuses.get('FAIL', 0)
- crashed = 0
- if 'crash' in summary.action_counts:
- crashed = summary.action_counts['crash']
- text_summary = tbox_print_summary(expected_count,
- unexpected_count,
- expected_failures,
- crashed > 0,
- False)
- self.info("TinderboxPrint: %s<br/>%s\n" % (suite_name, text_summary))
diff --git a/testing/mozharness/mozharness/mozilla/taskcluster_helper.py b/testing/mozharness/mozharness/mozilla/taskcluster_helper.py
deleted file mode 100644
index 6921b8938..000000000
--- a/testing/mozharness/mozharness/mozilla/taskcluster_helper.py
+++ /dev/null
@@ -1,274 +0,0 @@
-"""Taskcluster module. Defines a few helper functions to call into the taskcluster
- client.
-"""
-import os
-from datetime import datetime, timedelta
-from urlparse import urljoin
-
-from mozharness.base.log import LogMixin
-
-
-# Taskcluster {{{1
-class Taskcluster(LogMixin):
- """
- Helper functions to report data to Taskcluster
- """
- def __init__(self, branch, rank, client_id, access_token, log_obj,
- task_id=None):
- self.rank = rank
- self.log_obj = log_obj
-
- # Try builds use a different set of credentials which have access to the
- # buildbot-try scope.
- if branch == 'try':
- self.buildbot = 'buildbot-try'
- else:
- self.buildbot = 'buildbot'
-
- # We can't import taskcluster at the top of the script because it is
- # part of the virtualenv, so import it now. The virtualenv needs to be
- # activated before this point by the mozharness script, or else we won't
- # be able to find this module.
- import taskcluster
- taskcluster.config['credentials']['clientId'] = client_id
- taskcluster.config['credentials']['accessToken'] = access_token
- self.taskcluster_queue = taskcluster.Queue()
- self.task_id = task_id or taskcluster.slugId()
- self.put_file = taskcluster.utils.putFile
-
- def create_task(self, routes):
- curdate = datetime.utcnow()
- self.info("Taskcluster taskId: %s" % self.task_id)
- self.info("Routes: %s" % routes)
- task = self.taskcluster_queue.createTask({
- # The null-provisioner and buildbot worker type don't actually exist.
- # So this task doesn't actually run - we just need to create the task so
- # we have something to attach artifacts to.
- "provisionerId": "null-provisioner",
- "workerType": self.buildbot,
- "created": curdate,
- "deadline": curdate + timedelta(hours=1),
- "routes": routes,
- "payload": {
- },
- "extra": {
- "index": {
- "rank": self.rank,
- },
- },
- "metadata": {
- "name": "Buildbot/mozharness S3 uploader",
- "description": "Upload outputs of buildbot/mozharness builds to S3",
- "owner": "mshal@mozilla.com",
- "source": "http://hg.mozilla.org/build/mozharness/",
- }
- }, taskId=self.task_id)
- return task
-
- def claim_task(self, task):
- self.taskcluster_queue.claimTask(
- task['status']['taskId'],
- task['status']['runs'][-1]['runId'],
- {
- "workerGroup": self.buildbot,
- "workerId": self.buildbot,
- })
-
- def get_task(self, task_id):
- return self.taskcluster_queue.status(task_id)
-
- @staticmethod
- def get_mime_type(ext, default='application/octet-stream'):
- mime_types = {
- ".asc": "text/plain",
- ".checksums": "text/plain",
- ".json": "application/json",
- ".log": "text/plain",
- ".tar.bz2": "application/x-gtar",
- ".txt": "text/plain",
- ".xpi": "application/x-xpinstall",
- ".zip": "application/zip",
- }
- return mime_types.get(ext, default)
-
- @property
- def expiration(self):
- weeks = 52
- if self.buildbot == 'buildbot-try':
- weeks = 3
- return datetime.utcnow() + timedelta(weeks=weeks)
-
- def create_artifact(self, task, filename):
- mime_type = self.get_mime_type(os.path.splitext(filename)[1])
- content_length = os.path.getsize(filename)
- self.info("Uploading to S3: filename=%s mimetype=%s length=%s" % (
- filename, mime_type, content_length))
- # reclaim the task to avoid "claim-expired" errors
- self.taskcluster_queue.reclaimTask(
- task['status']['taskId'], task['status']['runs'][-1]['runId'])
- artifact = self.taskcluster_queue.createArtifact(
- task['status']['taskId'],
- task['status']['runs'][-1]['runId'],
- 'public/build/%s' % os.path.basename(filename),
- {
- "storageType": "s3",
- "expires": self.expiration,
- "contentType": mime_type,
- })
- self.put_file(filename, artifact['putUrl'], mime_type)
- return self.get_taskcluster_url(filename)
-
- def create_reference_artifact(self, task, filename, url):
- mime_type = self.get_mime_type(os.path.splitext(filename)[1])
- self.info("Create reference artifact: filename=%s mimetype=%s url=%s" %
- (filename, mime_type, url))
- # reclaim the task to avoid "claim-expired" errors
- self.taskcluster_queue.reclaimTask(
- task['status']['taskId'], task['status']['runs'][-1]['runId'])
- self.taskcluster_queue.createArtifact(
- task['status']['taskId'],
- task['status']['runs'][-1]['runId'],
- 'public/build/%s' % os.path.basename(filename),
- {
- "storageType": "reference",
- "expires": self.expiration,
- "contentType": mime_type,
- "url": url,
- })
-
- def report_completed(self, task):
- task_id = task['status']['taskId']
- run_id = task['status']['runs'][-1]['runId']
- self.info("Resolving %s, run %s. Full task:" % (task_id, run_id))
- self.info(str(task))
- self.taskcluster_queue.reportCompleted(task_id, run_id)
-
- def report_failed(self, task):
- task_id = task['status']['taskId']
- run_id = task['status']['runs'][-1]['runId']
- self.info("Resolving %s as failed, run %s. Full task:" %
- (task_id, run_id))
- self.info(str(task))
- self.taskcluster_queue.reportFailed(task_id, run_id)
-
- def get_taskcluster_url(self, filename):
- return 'https://queue.taskcluster.net/v1/task/%s/artifacts/public/build/%s' % (
- self.task_id,
- os.path.basename(filename)
- )
-
-
-# TasckClusterArtifactFinderMixin {{{1
-class TaskClusterArtifactFinderMixin(object):
- # This class depends that you have extended from the base script
- QUEUE_URL = 'https://queue.taskcluster.net/v1/task/'
- SCHEDULER_URL = 'https://scheduler.taskcluster.net/v1/task-graph/'
-
- def get_task(self, task_id):
- """ Get Task Definition """
- # Signature: task(taskId) : result
- return self.load_json_url(urljoin(self.QUEUE_URL, task_id))
-
- def get_list_latest_artifacts(self, task_id):
- """ Get Artifacts from Latest Run """
- # Signature: listLatestArtifacts(taskId) : result
-
- # Notice that this grabs the most recent run of a task since we don't
- # know the run_id. This slightly slower, however, it is more convenient
- return self.load_json_url(urljoin(self.QUEUE_URL, '{}/artifacts'.format(task_id)))
-
- def url_to_artifact(self, task_id, full_path):
- """ Return a URL for an artifact. """
- return urljoin(self.QUEUE_URL, '{}/artifacts/{}'.format(task_id, full_path))
-
- def get_inspect_graph(self, task_group_id):
- """ Inspect Task Graph """
- # Signature: inspect(taskGraphId) : result
- return self.load_json_url(urljoin(self.SCHEDULER_URL, '{}/inspect'.format(task_group_id)))
-
- def find_parent_task_id(self, task_id):
- """ Returns the task_id of the parent task associated to the given task_id."""
- # Find group id to associated to all related tasks
- task_group_id = self.get_task(task_id)['taskGroupId']
-
- # Find child task and determine on which task it depends on
- for task in self.get_inspect_graph(task_group_id)['tasks']:
- if task['taskId'] == task_id:
- parent_task_id = task['requires'][0]
-
- return parent_task_id
-
- def set_bbb_artifacts(self, task_id, properties_file_path):
- """ Find BBB artifacts through properties_file_path and set them. """
- p = self.load_json_url(
- self.url_to_artifact(task_id, properties_file_path))['properties']
-
- # Set importants artifacts for test jobs
- self.set_artifacts(
- p['packageUrl'] if p.get('packageUrl') else None,
- p['testPackagesUrl'] if p.get('testPackagesUrl') else None,
- p['symbolsUrl'] if p.get('symbolsUrl') else None
- )
-
- def set_artifacts(self, installer, tests, symbols):
- """ Sets installer, test and symbols URLs from the artifacts of BBB based task."""
- self.installer_url, self.test_url, self.symbols_url = installer, tests, symbols
- self.info('Set installer_url: %s' % self.installer_url)
- self.info('Set test_url: %s' % self.test_url)
- self.info('Set symbols_url: %s' % self.symbols_url)
-
- def set_parent_artifacts(self, child_task_id):
- """ Find and set installer_url, test_url and symbols_url by querying TaskCluster.
-
- In Buildbot Bridge's normal behaviour we can find the artifacts by inspecting
- a child's taskId, determine the task in which it depends on and find the uploaded
- artifacts.
-
- In order to support multi-tiered task graph scheduling for BBB triggered tasks,
- we remove the assumption that the task which depends on is the one from which we
- find the artifacts we need. Instead, we can set a parent_task_id which points to the
- tasks from which to retrieve the artifacts. This decouples task dependency from task
- from which to grab the artifacts.
-
- In-tree triggered BBB tasks do not use parent_task_id, once there is efforts to move
- the scheduling into tree we can make parent_task_id as the only method.
-
- """
- # Task definition
- child_task = self.get_task(child_task_id)
-
- # Case A: The parent_task_id is defined (mozci scheduling)
- if child_task['payload']['properties'].get('parent_task_id'):
- # parent_task_id is used to point to the task from which to grab artifacts
- # rather than the one we depend on
- parent_id = child_task['payload']['properties']['parent_task_id']
-
- # Find out where the parent task uploaded the build
- parent_task = self.get_task(parent_id)
-
- # Case 1: The parent task is a pure TC task
- if parent_task['extra'].get('locations'):
- # Build tasks generated under TC specify where they upload their builds
- installer_path = parent_task['extra']['locations']['build']
-
- self.set_artifacts(
- self.url_to_artifact(parent_id, installer_path),
- self.url_to_artifact(parent_id, 'public/build/test_packages.json'),
- self.url_to_artifact(parent_id, 'public/build/target.crashreporter-symbols.zip')
- )
- else:
- # Case 2: The parent task has an associated BBB task
- # graph_props.json is uploaded in buildbase.py
- self.set_bbb_artifacts(
- task_id=parent_id,
- properties_file_path='public/build/buildbot_properties.json'
- )
-
- else:
- # Case B: We need to query who the parent is since 'parent_task_id'
- # was not defined as a Buildbot property
- parent_id = self.find_parent_task_id(child_task_id)
- self.set_bbb_artifacts(
- task_id=parent_id,
- properties_file_path='public/build/buildbot_properties.json'
- )
diff --git a/testing/mozharness/mozharness/mozilla/testing/__init__.py b/testing/mozharness/mozharness/mozilla/testing/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/mozharness/mozilla/testing/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
deleted file mode 100644
index 9cb824679..000000000
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/usr/bin/env python
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import os
-import shutil
-import tempfile
-
-from mozharness.base.script import (
- PreScriptAction,
- PostScriptAction,
-)
-
-code_coverage_config_options = [
- [["--code-coverage"],
- {"action": "store_true",
- "dest": "code_coverage",
- "default": False,
- "help": "Whether test run should package and upload code coverage data."
- }],
-]
-
-
-class CodeCoverageMixin(object):
- """
- Mixin for setting GCOV_PREFIX during test execution, packaging up
- the resulting .gcda files and uploading them to blobber.
- """
- gcov_dir = None
-
- @property
- def code_coverage_enabled(self):
- try:
- if self.config.get('code_coverage'):
- return True
-
- # XXX workaround because bug 1110465 is hard
- return self.buildbot_config['properties']['stage_platform'] in ('linux64-ccov',)
- except (AttributeError, KeyError, TypeError):
- return False
-
-
- @PreScriptAction('run-tests')
- def _set_gcov_prefix(self, action):
- if not self.code_coverage_enabled:
- return
- self.gcov_dir = tempfile.mkdtemp()
- os.environ['GCOV_PREFIX'] = self.gcov_dir
-
- @PostScriptAction('run-tests')
- def _package_coverage_data(self, action, success=None):
- if not self.code_coverage_enabled:
- return
- del os.environ['GCOV_PREFIX']
-
- # TODO This is fragile, find rel_topsrcdir properly somehow
- # We need to find the path relative to the gecko topsrcdir. Use
- # some known gecko directories as a test.
- canary_dirs = ['browser', 'docshell', 'dom', 'js', 'layout', 'toolkit', 'xpcom', 'xpfe']
- rel_topsrcdir = None
- for root, dirs, files in os.walk(self.gcov_dir):
- # need to use 'any' in case no gcda data was generated in that subdir.
- if any(d in dirs for d in canary_dirs):
- rel_topsrcdir = root
- break
- else:
- # Unable to upload code coverage files. Since this is the whole
- # point of code coverage, making this fatal.
- self.fatal("Could not find relative topsrcdir in code coverage "
- "data!")
-
- dirs = self.query_abs_dirs()
- file_path = os.path.join(
- dirs['abs_blob_upload_dir'], 'code-coverage-gcda.zip')
- command = ['zip', '-r', file_path, '.']
- self.run_command(command, cwd=rel_topsrcdir)
- shutil.rmtree(self.gcov_dir)
diff --git a/testing/mozharness/mozharness/mozilla/testing/device.py b/testing/mozharness/mozharness/mozilla/testing/device.py
deleted file mode 100644
index fea43ba20..000000000
--- a/testing/mozharness/mozharness/mozilla/testing/device.py
+++ /dev/null
@@ -1,738 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-'''Interact with a device via ADB or SUT.
-
-This code is largely from
-https://hg.mozilla.org/build/tools/file/default/sut_tools
-'''
-
-import datetime
-import os
-import re
-import subprocess
-import sys
-import time
-
-from mozharness.base.errors import ADBErrorList
-from mozharness.base.log import LogMixin, DEBUG
-from mozharness.base.script import ScriptMixin
-
-
-# Device flags
-DEVICE_UNREACHABLE = 0x01
-DEVICE_NOT_CONNECTED = 0x02
-DEVICE_MISSING_SDCARD = 0x03
-DEVICE_HOST_ERROR = 0x04
-# DEVICE_UNRECOVERABLE_ERROR?
-DEVICE_NOT_REBOOTED = 0x05
-DEVICE_CANT_REMOVE_DEVROOT = 0x06
-DEVICE_CANT_REMOVE_ETC_HOSTS = 0x07
-DEVICE_CANT_SET_TIME = 0x08
-
-
-class DeviceException(Exception):
- pass
-
-
-# BaseDeviceHandler {{{1
-class BaseDeviceHandler(ScriptMixin, LogMixin):
- device_id = None
- device_root = None
- default_port = None
- device_flags = []
-
- def __init__(self, log_obj=None, config=None, script_obj=None):
- super(BaseDeviceHandler, self).__init__()
- self.config = config
- self.log_obj = log_obj
- self.script_obj = script_obj
-
- def add_device_flag(self, flag):
- if flag not in self.device_flags:
- self.device_flags.append(flag)
-
- def query_device_id(self):
- if self.device_id:
- return self.device_id
- c = self.config
- device_id = None
- if c.get('device_id'):
- device_id = c['device_id']
- elif c.get('device_ip'):
- device_id = "%s:%s" % (c['device_ip'],
- c.get('device_port', self.default_port))
- self.device_id = device_id
- return self.device_id
-
- def query_download_filename(self, file_id=None):
- pass
-
- def ping_device(self):
- pass
-
- def check_device(self):
- pass
-
- def cleanup_device(self, reboot=False):
- pass
-
- def reboot_device(self):
- pass
-
- def query_device_root(self):
- pass
-
- def wait_for_device(self, interval=60, max_attempts=20):
- pass
-
- def install_app(self, file_path):
- pass
-
-
-# ADBDeviceHandler {{{1
-class ADBDeviceHandler(BaseDeviceHandler):
- def __init__(self, **kwargs):
- super(ADBDeviceHandler, self).__init__(**kwargs)
- self.default_port = 5555
-
- def query_device_exe(self, exe_name):
- return self.query_exe(exe_name, exe_dict="device_exes")
-
- def _query_config_device_id(self):
- return BaseDeviceHandler.query_device_id(self)
-
- def query_device_id(self, auto_connect=True):
- if self.device_id:
- return self.device_id
- device_id = self._query_config_device_id()
- if device_id:
- if auto_connect:
- self.ping_device(auto_connect=True)
- else:
- self.info("Trying to find device...")
- devices = self._query_attached_devices()
- if not devices:
- self.add_device_flag(DEVICE_NOT_CONNECTED)
- self.fatal("No device connected via adb!\nUse 'adb connect' or specify a device_id or device_ip in config!")
- elif len(devices) > 1:
- self.warning("""More than one device detected; specify 'device_id' or\n'device_ip' to target a specific device!""")
- device_id = devices[0]
- self.info("Found %s." % device_id)
- self.device_id = device_id
- return self.device_id
-
- # maintenance {{{2
- def ping_device(self, auto_connect=False, silent=False):
- if auto_connect and not self._query_attached_devices():
- self.connect_device()
- if not silent:
- self.info("Determining device connectivity over adb...")
- device_id = self.query_device_id()
- adb = self.query_exe('adb')
- uptime = self.query_device_exe('uptime')
- output = self.get_output_from_command([adb, "-s", device_id,
- "shell", uptime],
- silent=silent)
- if str(output).startswith("up time:"):
- if not silent:
- self.info("Found %s." % device_id)
- return True
- elif auto_connect:
- # TODO retry?
- self.connect_device()
- return self.ping_device()
- else:
- if not silent:
- self.error("Can't find a device.")
- return False
-
- def _query_attached_devices(self):
- devices = []
- adb = self.query_exe('adb')
- output = self.get_output_from_command([adb, "devices"])
- starting_list = False
- if output is None:
- self.add_device_flag(DEVICE_HOST_ERROR)
- self.fatal("Can't get output from 'adb devices'; install the Android SDK!")
- for line in output:
- if 'adb: command not found' in line:
- self.add_device_flag(DEVICE_HOST_ERROR)
- self.fatal("Can't find adb; install the Android SDK!")
- if line.startswith("* daemon"):
- continue
- if line.startswith("List of devices"):
- starting_list = True
- continue
- # TODO somehow otherwise determine whether this is an actual
- # device?
- if starting_list:
- devices.append(re.split('\s+', line)[0])
- return devices
-
- def connect_device(self):
- self.info("Connecting device...")
- adb = self.query_exe('adb')
- cmd = [adb, "connect"]
- device_id = self._query_config_device_id()
- if device_id:
- devices = self._query_attached_devices()
- if device_id in devices:
- # TODO is this the right behavior?
- self.disconnect_device()
- cmd.append(device_id)
- # TODO error check
- self.run_command(cmd, error_list=ADBErrorList)
-
- def disconnect_device(self):
- self.info("Disconnecting device...")
- device_id = self.query_device_id()
- if device_id:
- adb = self.query_exe('adb')
- # TODO error check
- self.run_command([adb, "-s", device_id,
- "disconnect"],
- error_list=ADBErrorList)
- else:
- self.info("No device found.")
-
- def check_device(self):
- if not self.ping_device(auto_connect=True):
- self.add_device_flag(DEVICE_NOT_CONNECTED)
- self.fatal("Can't find device!")
- if self.query_device_root() is None:
- self.add_device_flag(DEVICE_NOT_CONNECTED)
- self.fatal("Can't connect to device!")
-
- def reboot_device(self):
- if not self.ping_device(auto_connect=True):
- self.add_device_flag(DEVICE_NOT_REBOOTED)
- self.error("Can't reboot disconnected device!")
- return False
- device_id = self.query_device_id()
- self.info("Rebooting device...")
- adb = self.query_exe('adb')
- cmd = [adb, "-s", device_id, "reboot"]
- self.info("Running command (in the background): %s" % cmd)
- # This won't exit until much later, but we don't need to wait.
- # However, some error checking would be good.
- p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
- time.sleep(10)
- self.disconnect_device()
- status = False
- try:
- self.wait_for_device()
- status = True
- except DeviceException:
- self.error("Can't reconnect to device!")
- if p.poll() is None:
- p.kill()
- p.wait()
- return status
-
- def cleanup_device(self, reboot=False):
- self.info("Cleaning up device.")
- c = self.config
- device_id = self.query_device_id()
- status = self.remove_device_root()
- if not status:
- self.add_device_flag(DEVICE_CANT_REMOVE_DEVROOT)
- self.fatal("Can't remove device root!")
- if c.get("enable_automation"):
- self.remove_etc_hosts()
- if c.get("device_package_name"):
- adb = self.query_exe('adb')
- killall = self.query_device_exe('killall')
- self.run_command([adb, "-s", device_id, "shell",
- killall, c["device_package_name"]],
- error_list=ADBErrorList)
- self.uninstall_app(c['device_package_name'])
- if reboot:
- self.reboot_device()
-
- # device calls {{{2
- def query_device_root(self, silent=False):
- if self.device_root:
- return self.device_root
- device_root = None
- device_id = self.query_device_id()
- adb = self.query_exe('adb')
- output = self.get_output_from_command("%s -s %s shell df" % (adb, device_id),
- silent=silent)
- # TODO this assumes we're connected; error checking?
- if output is None or ' not found' in str(output):
- self.error("Can't get output from 'adb shell df'!\n%s" % output)
- return None
- if "/mnt/sdcard" in output:
- device_root = "/mnt/sdcard/tests"
- else:
- device_root = "/data/local/tmp/tests"
- if not silent:
- self.info("Device root is %s" % str(device_root))
- self.device_root = device_root
- return self.device_root
-
- # TODO from here on down needs to be copied to Base+SUT
- def wait_for_device(self, interval=60, max_attempts=20):
- self.info("Waiting for device to come back...")
- time.sleep(interval)
- tries = 0
- while tries <= max_attempts:
- tries += 1
- self.info("Try %d" % tries)
- if self.ping_device(auto_connect=True, silent=True):
- return self.ping_device()
- time.sleep(interval)
- raise DeviceException("Remote Device Error: waiting for device timed out.")
-
- def query_device_time(self):
- device_id = self.query_device_id()
- adb = self.query_exe('adb')
- # adb shell 'date' will give a date string
- date_string = self.get_output_from_command([adb, "-s", device_id,
- "shell", "date"])
- # TODO what to do when we error?
- return date_string
-
- def set_device_time(self, device_time=None, error_level='error'):
- # adb shell date -s YYYYMMDD.hhmmss will set date
- device_id = self.query_device_id()
- if device_time is None:
- device_time = time.strftime("%Y%m%d.%H%M%S")
- self.info(self.query_device_time())
- adb = self.query_exe('adb')
- status = self.run_command([adb, "-s", device_id, "shell", "date", "-s",
- str(device_time)],
- error_list=ADBErrorList)
- self.info(self.query_device_time())
- return status
-
- def query_device_file_exists(self, file_name):
- device_id = self.query_device_id()
- adb = self.query_exe('adb')
- output = self.get_output_from_command([adb, "-s", device_id,
- "shell", "ls", "-d", file_name])
- if str(output).rstrip() == file_name:
- return True
- return False
-
- def remove_device_root(self, error_level='error'):
- device_root = self.query_device_root()
- device_id = self.query_device_id()
- if device_root is None:
- self.add_device_flag(DEVICE_UNREACHABLE)
- self.fatal("Can't connect to device!")
- adb = self.query_exe('adb')
- if self.query_device_file_exists(device_root):
- self.info("Removing device root %s." % device_root)
- self.run_command([adb, "-s", device_id, "shell", "rm",
- "-r", device_root], error_list=ADBErrorList)
- if self.query_device_file_exists(device_root):
- self.add_device_flag(DEVICE_CANT_REMOVE_DEVROOT)
- self.log("Unable to remove device root!", level=error_level)
- return False
- return True
-
- def install_app(self, file_path):
- c = self.config
- device_id = self.query_device_id()
- adb = self.query_exe('adb')
- if self._log_level_at_least(DEBUG):
- self.run_command([adb, "-s", device_id, "shell", "ps"],
- error_list=ADBErrorList)
- uptime = self.query_device_exe('uptime')
- self.run_command([adb, "-s", "shell", uptime],
- error_list=ADBErrorList)
- if not c['enable_automation']:
- # -s to install on sdcard? Needs to be config driven
- self.run_command([adb, "-s", device_id, "install", '-r',
- file_path],
- error_list=ADBErrorList)
- else:
- # A slow-booting device may not allow installs, temporarily.
- # Wait up to a few minutes if not immediately successful.
- # Note that "adb install" typically writes status messages
- # to stderr and the adb return code may not differentiate
- # successful installations from failures; instead we check
- # the command output.
- install_complete = False
- retries = 0
- while retries < 6:
- output = self.get_output_from_command([adb, "-s", device_id,
- "install", '-r',
- file_path],
- ignore_errors=True)
- if output and output.lower().find("success") >= 0:
- install_complete = True
- break
- self.warning("Failed to install %s" % file_path)
- time.sleep(30)
- retries = retries + 1
- if not install_complete:
- self.fatal("Failed to install %s!" % file_path)
-
- def uninstall_app(self, package_name, package_root="/data/data",
- error_level="error"):
- c = self.config
- device_id = self.query_device_id()
- self.info("Uninstalling %s..." % package_name)
- if self.query_device_file_exists('%s/%s' % (package_root, package_name)):
- adb = self.query_exe('adb')
- cmd = [adb, "-s", device_id, "uninstall"]
- if not c.get('enable_automation'):
- cmd.append("-k")
- cmd.append(package_name)
- status = self.run_command(cmd, error_list=ADBErrorList)
- # TODO is this the right error check?
- if status:
- self.log("Failed to uninstall %s!" % package_name,
- level=error_level)
-
- # Device-type-specific. {{{2
- def remove_etc_hosts(self, hosts_file="/system/etc/hosts"):
- c = self.config
- if c['device_type'] not in ("tegra250",):
- self.debug("No need to remove /etc/hosts on a non-Tegra250.")
- return
- device_id = self.query_device_id()
- if self.query_device_file_exists(hosts_file):
- self.info("Removing %s file." % hosts_file)
- adb = self.query_exe('adb')
- self.run_command([adb, "-s", device_id, "shell",
- "mount", "-o", "remount,rw", "-t", "yaffs2",
- "/dev/block/mtdblock3", "/system"],
- error_list=ADBErrorList)
- self.run_command([adb, "-s", device_id, "shell", "rm",
- hosts_file])
- if self.query_device_file_exists(hosts_file):
- self.add_device_flag(DEVICE_CANT_REMOVE_ETC_HOSTS)
- self.fatal("Unable to remove %s!" % hosts_file)
- else:
- self.debug("%s file doesn't exist; skipping." % hosts_file)
-
-
-# SUTDeviceHandler {{{1
-class SUTDeviceHandler(BaseDeviceHandler):
- def __init__(self, **kwargs):
- super(SUTDeviceHandler, self).__init__(**kwargs)
- self.devicemanager = None
- self.default_port = 20701
- self.default_heartbeat_port = 20700
- self.DMError = None
-
- def query_devicemanager(self):
- if self.devicemanager:
- return self.devicemanager
- c = self.config
- site_packages_path = self.script_obj.query_python_site_packages_path()
- dm_path = os.path.join(site_packages_path, 'mozdevice')
- sys.path.append(dm_path)
- try:
- from devicemanagerSUT import DeviceManagerSUT
- from devicemanagerSUT import DMError
- self.DMError = DMError
- self.devicemanager = DeviceManagerSUT(c['device_ip'])
- # TODO configurable?
- self.devicemanager.debug = c.get('devicemanager_debug_level', 0)
- except ImportError, e:
- self.fatal("Can't import DeviceManagerSUT! %s\nDid you check out talos?" % str(e))
- return self.devicemanager
-
- # maintenance {{{2
- def ping_device(self):
- #TODO writeme
- pass
-
- def check_device(self):
- self.info("Checking for device root to verify the device is alive.")
- dev_root = self.query_device_root(strict=True)
- if not dev_root:
- self.add_device_flag(DEVICE_UNREACHABLE)
- self.fatal("Can't get dev_root from devicemanager; is the device up?")
- self.info("Found a dev_root of %s." % str(dev_root))
-
- def wait_for_device(self, interval=60, max_attempts=20):
- self.info("Waiting for device to come back...")
- time.sleep(interval)
- success = False
- attempts = 0
- while attempts <= max_attempts:
- attempts += 1
- self.info("Try %d" % attempts)
- if self.query_device_root() is not None:
- success = True
- break
- time.sleep(interval)
- if not success:
- self.add_device_flag(DEVICE_UNREACHABLE)
- self.fatal("Waiting for tegra timed out.")
- else:
- self.info("Device came back.")
-
- def cleanup_device(self, reboot=False):
- c = self.config
- dev_root = self.query_device_root()
- dm = self.query_devicemanager()
- if dm.dirExists(dev_root):
- self.info("Removing dev_root %s..." % dev_root)
- try:
- dm.removeDir(dev_root)
- except self.DMError:
- self.add_device_flag(DEVICE_CANT_REMOVE_DEVROOT)
- self.fatal("Can't remove dev_root!")
- if c.get("enable_automation"):
- self.remove_etc_hosts()
- # TODO I need to abstract this uninstall as we'll need to clean
- # multiple packages off devices.
- if c.get("device_package_name"):
- if dm.dirExists('/data/data/%s' % c['device_package_name']):
- self.info("Uninstalling %s..." % c['device_package_name'])
- dm.uninstallAppAndReboot(c['device_package_name'])
- self.wait_for_device()
- elif reboot:
- self.reboot_device()
-
- # device calls {{{2
- def query_device_root(self, strict=False):
- c = self.config
- dm = self.query_devicemanager()
- dev_root = dm.getDeviceRoot()
- if strict and c.get('enable_automation'):
- if not str(dev_root).startswith("/mnt/sdcard"):
- self.add_device_flag(DEVICE_MISSING_SDCARD)
- self.fatal("dev_root from devicemanager [%s] is not correct!" %
- str(dev_root))
- if not dev_root or dev_root == "/tests":
- return None
- return dev_root
-
- def query_device_time(self):
- dm = self.query_devicemanager()
- timestamp = int(dm.getCurrentTime()) # epoch time in milliseconds
- dt = datetime.datetime.utcfromtimestamp(timestamp / 1000)
- self.info("Current device time is %s" % dt.strftime('%Y/%m/%d %H:%M:%S'))
- return dt
-
- def set_device_time(self):
- dm = self.query_devicemanager()
- s = datetime.datetime.now().strftime('%Y/%m/%d %H:%M:%S')
- self.info("Setting device time to %s" % s)
- try:
- dm.sendCMD(['settime %s' % s])
- return True
- except self.DMError, e:
- self.add_device_flag(DEVICE_CANT_SET_TIME)
- self.fatal("Exception while setting device time: %s" % str(e))
-
- def install_app(self, file_path):
- dev_root = self.query_device_root(strict=True)
- if not dev_root:
- self.add_device_flag(DEVICE_UNREACHABLE)
- # TODO wait_for_device?
- self.fatal("dev_root %s not correct!" % str(dev_root))
-
- dm = self.query_devicemanager()
-
- c = self.config
- if c.get('enable_automation'):
- self.query_device_time()
- self.set_device_time()
- self.query_device_time()
- dm.getInfo('process')
- dm.getInfo('memory')
- dm.getInfo('uptime')
-
- # This target needs to not use os.path.join due to differences with win
- # Paths vs. unix paths.
- target = "/".join([dev_root, os.path.basename(file_path)])
- self.info("Installing %s on device..." % file_path)
- dm.pushFile(file_path, target)
- # TODO screen resolution
- # TODO do something with status?
- try:
- dm.installApp(target)
- self.info('-' * 42)
- self.info("Sleeping for 90 seconds...")
- time.sleep(90)
- self.info('installApp(%s) done - gathering debug info' % target)
- try:
- self.info(repr(dm.getInfo('process')))
- self.info(repr(dm.getInfo('memory')))
- self.info(repr(dm.getInfo('uptime')))
- self.info(repr(dm.sendCMD(['exec su -c "logcat -d -v time *:W"'])))
- except Exception, e:
- self.info("Exception hit while trying to run logcat: %s" % str(e))
- self.fatal("Remote Device Error: can't run logcat")
- except self.DMError:
- self.fatal("Remote Device Error: installApp() call failed - exiting")
-
- def reboot_device(self):
- dm = self.query_devicemanager()
- # logcat?
- self.info("Rebooting device...")
- try:
- dm.reboot()
- except self.DMError:
- self.add_device_flag(DEVICE_NOT_REBOOTED)
- self.fatal("Can't reboot device!")
- self.wait_for_device()
- dm.getInfo('uptime')
-
- # device type specific {{{2
- def remove_etc_hosts(self, hosts_file="/system/etc/hosts"):
- c = self.config
- # TODO figure this out
- if c['device_type'] not in ("tegra250",) or True:
- self.debug("No need to remove /etc/hosts on a non-Tegra250.")
- return
- dm = self.query_devicemanager()
- if dm.fileExists(hosts_file):
- self.info("Removing %s file." % hosts_file)
- try:
- dm.sendCMD(['exec mount -o remount,rw -t yaffs2 /dev/block/mtdblock3 /system'])
- dm.sendCMD(['exec rm %s' % hosts_file])
- except self.DMError:
- self.add_device_flag(DEVICE_CANT_REMOVE_ETC_HOSTS)
- self.fatal("Unable to remove %s!" % hosts_file)
- if dm.fileExists(hosts_file):
- self.add_device_flag(DEVICE_CANT_REMOVE_ETC_HOSTS)
- self.fatal("Unable to remove %s!" % hosts_file)
- else:
- self.debug("%s file doesn't exist; skipping." % hosts_file)
-
-
-# SUTDeviceMozdeviceMixin {{{1
-class SUTDeviceMozdeviceMixin(SUTDeviceHandler):
- '''
- This SUT device manager class makes calls through mozdevice (from mozbase) [1]
- directly rather than calling SUT tools.
-
- [1] https://github.com/mozilla/mozbase/blob/master/mozdevice/mozdevice/devicemanagerSUT.py
- '''
- dm = None
-
- def query_devicemanager(self):
- if self.dm:
- return self.dm
- sys.path.append(self.query_python_site_packages_path())
- from mozdevice.devicemanagerSUT import DeviceManagerSUT
- self.info("Connecting to: %s" % self.mozpool_device)
- self.dm = DeviceManagerSUT(self.mozpool_device)
- # No need for 300 second SUT socket timeouts here
- self.dm.default_timeout = 30
- return self.dm
-
- def query_file(self, filename):
- dm = self.query_devicemanager()
- if not dm.fileExists(filename):
- raise Exception("Expected file (%s) not found" % filename)
-
- file_contents = dm.pullFile(filename)
- if file_contents is None:
- raise Exception("Unable to read file (%s)" % filename)
-
- return file_contents
-
- def set_device_epoch_time(self, timestamp=int(time.time())):
- dm = self.query_devicemanager()
- dm._runCmds([{'cmd': 'setutime %s' % timestamp}])
- return dm._runCmds([{'cmd': 'clok'}])
-
- def get_logcat(self):
- dm = self.query_devicemanager()
- return dm.getLogcat()
-
-
-# DeviceMixin {{{1
-DEVICE_PROTOCOL_DICT = {
- 'adb': ADBDeviceHandler,
- 'sut': SUTDeviceHandler,
-}
-
-device_config_options = [[
- ["--device-ip"],
- {"action": "store",
- "dest": "device_ip",
- "help": "Specify the IP address of the device."
- }
-], [
- ["--device-port"],
- {"action": "store",
- "dest": "device_port",
- "help": "Specify the IP port of the device."
- }
-], [
- ["--device-heartbeat-port"],
- {"action": "store",
- "dest": "device_heartbeat_port",
- "help": "Specify the heartbeat port of the SUT device."
- }
-], [
- ["--device-protocol"],
- {"action": "store",
- "type": "choice",
- "dest": "device_protocol",
- "choices": DEVICE_PROTOCOL_DICT.keys(),
- "help": "Specify the device communication protocol."
- }
-], [
- ["--device-type"],
- # A bit useless atm, but we can add new device types as we add support
- # for them.
- {"action": "store",
- "type": "choice",
- "choices": ["non-tegra", "tegra250"],
- "default": "non-tegra",
- "dest": "device_type",
- "help": "Specify the device type."
- }
-], [
- ["--devicemanager-path"],
- {"action": "store",
- "dest": "devicemanager_path",
- "help": "Specify the parent dir of devicemanagerSUT.py."
- }
-]]
-
-
-class DeviceMixin(object):
- '''BaseScript mixin, designed to interface with the device.
-
- '''
- device_handler = None
- device_root = None
-
- def query_device_handler(self):
- if self.device_handler:
- return self.device_handler
- c = self.config
- device_protocol = c.get('device_protocol')
- device_class = DEVICE_PROTOCOL_DICT.get(device_protocol)
- if not device_class:
- self.fatal("Unknown device_protocol %s; set via --device-protocol!" % str(device_protocol))
- self.device_handler = device_class(
- log_obj=self.log_obj,
- config=self.config,
- script_obj=self,
- )
- return self.device_handler
-
- def check_device(self):
- dh = self.query_device_handler()
- return dh.check_device()
-
- def cleanup_device(self, **kwargs):
- dh = self.query_device_handler()
- return dh.cleanup_device(**kwargs)
-
- def install_app(self):
- dh = self.query_device_handler()
- return dh.install_app(file_path=self.installer_path)
-
- def reboot_device(self):
- dh = self.query_device_handler()
- return dh.reboot_device()
diff --git a/testing/mozharness/mozharness/mozilla/testing/errors.py b/testing/mozharness/mozharness/mozilla/testing/errors.py
deleted file mode 100644
index 3061b5f16..000000000
--- a/testing/mozharness/mozharness/mozilla/testing/errors.py
+++ /dev/null
@@ -1,118 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""Mozilla error lists for running tests.
-
-Error lists are used to parse output in mozharness.base.log.OutputParser.
-
-Each line of output is matched against each substring or regular expression
-in the error list. On a match, we determine the 'level' of that line,
-whether IGNORE, DEBUG, INFO, WARNING, ERROR, CRITICAL, or FATAL.
-
-"""
-
-import re
-from mozharness.base.log import INFO, WARNING, ERROR
-
-# ErrorLists {{{1
-_mochitest_summary = {
- 'regex': re.compile(r'''(\d+ INFO (Passed|Failed|Todo):\ +(\d+)|\t(Passed|Failed|Todo): (\d+))'''),
- 'pass_group': "Passed",
- 'fail_group': "Failed",
- 'known_fail_group': "Todo",
-}
-
-TinderBoxPrintRe = {
- "mochitest_summary": _mochitest_summary,
- "mochitest-chrome_summary": _mochitest_summary,
- "mochitest-gl_summary": _mochitest_summary,
- "mochitest-media_summary": _mochitest_summary,
- "mochitest-plain-clipboard_summary": _mochitest_summary,
- "mochitest-plain-gpu_summary": _mochitest_summary,
- "marionette_summary": {
- 'regex': re.compile(r'''(passed|failed|todo):\ +(\d+)'''),
- 'pass_group': "passed",
- 'fail_group': "failed",
- 'known_fail_group': "todo",
- },
- "reftest_summary": {
- 'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),
- 'pass_group': "Successful",
- 'fail_group': "Unexpected",
- 'known_fail_group': "Known problems",
- },
- "crashtest_summary": {
- 'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),
- 'pass_group': "Successful",
- 'fail_group': "Unexpected",
- 'known_fail_group': "Known problems",
- },
- "xpcshell_summary": {
- 'regex': re.compile(r'''INFO \| (Passed|Failed): (\d+)'''),
- 'pass_group': "Passed",
- 'fail_group': "Failed",
- 'known_fail_group': None,
- },
- "jsreftest_summary": {
- 'regex': re.compile(r'''REFTEST INFO \| (Successful|Unexpected|Known problems): (\d+) \('''),
- 'pass_group': "Successful",
- 'fail_group': "Unexpected",
- 'known_fail_group': "Known problems",
- },
- "instrumentation_summary": _mochitest_summary,
- "cppunittest_summary": {
- 'regex': re.compile(r'''cppunittests INFO \| (Passed|Failed): (\d+)'''),
- 'pass_group': "Passed",
- 'fail_group': "Failed",
- 'known_fail_group': None,
- },
- "gtest_summary": {
- 'regex': re.compile(r'''(Passed|Failed): (\d+)'''),
- 'pass_group': "Passed",
- 'fail_group': "Failed",
- 'known_fail_group': None,
- },
- "jittest_summary": {
- 'regex': re.compile(r'''(Passed|Failed): (\d+)'''),
- 'pass_group': "Passed",
- 'fail_group': "Failed",
- 'known_fail_group': None,
- },
- "mozbase_summary": {
- 'regex': re.compile(r'''(OK)|(FAILED) \(errors=(\d+)'''),
- 'pass_group': "OK",
- 'fail_group': "FAILED",
- 'known_fail_group': None,
- },
- "mozmill_summary": {
- 'regex': re.compile(r'''INFO (Passed|Failed|Skipped): (\d+)'''),
- 'pass_group': "Passed",
- 'fail_group': "Failed",
- 'known_fail_group': "Skipped",
- },
-
- "harness_error": {
- 'full_regex': re.compile(r"(?:TEST-UNEXPECTED-FAIL|PROCESS-CRASH) \| .* \| (application crashed|missing output line for total leaks!|negative leaks caught!|\d+ bytes leaked)"),
- 'minimum_regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH)'''),
- 'retry_regex': re.compile(r'''(FAIL-SHOULD-RETRY|No space left on device|DMError|Connection to the other side was lost in a non-clean fashion|program finished with exit code 80|INFRA-ERROR|twisted.spread.pb.PBConnectionLost|_dl_open: Assertion|Timeout exceeded for _runCmd call)''')
- },
-}
-
-TestPassed = [
- {'regex': re.compile('''(TEST-INFO|TEST-KNOWN-FAIL|TEST-PASS|INFO \| )'''), 'level': INFO},
-]
-
-HarnessErrorList = [
- {'substr': 'TEST-UNEXPECTED', 'level': ERROR, },
- {'substr': 'PROCESS-CRASH', 'level': ERROR, },
-]
-
-LogcatErrorList = [
- {'substr': 'Fatal signal 11 (SIGSEGV)', 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'},
- {'substr': 'Fatal signal 7 (SIGBUS)', 'level': ERROR, 'explanation': 'This usually indicates the B2G process has crashed'},
- {'substr': '[JavaScript Error:', 'level': WARNING},
- {'substr': 'seccomp sandbox violation', 'level': ERROR, 'explanation': 'A content process has violated the system call sandbox (bug 790923)'},
-]
diff --git a/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py b/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
deleted file mode 100644
index b1874fc13..000000000
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_media_tests.py
+++ /dev/null
@@ -1,289 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** BEGIN LICENSE BLOCK *****
-
-import copy
-import os
-import re
-import urlparse
-
-from mozharness.base.log import ERROR, WARNING
-from mozharness.base.script import PreScriptAction
-from mozharness.mozilla.testing.testbase import (TestingMixin,
- testing_config_options)
-from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
-from mozharness.mozilla.vcstools import VCSToolsScript
-
-BUSTED = 'busted'
-TESTFAILED = 'testfailed'
-UNKNOWN = 'unknown'
-EXCEPTION = 'exception'
-SUCCESS = 'success'
-
-media_test_config_options = [
- [["--media-urls"],
- {"action": "store",
- "dest": "media_urls",
- "help": "Path to ini file that lists media urls for tests.",
- }],
- [["--profile"],
- {"action": "store",
- "dest": "profile",
- "default": None,
- "help": "Path to FF profile that should be used by Marionette",
- }],
- [["--test-timeout"],
- {"action": "store",
- "dest": "test_timeout",
- "default": 10000,
- "help": ("Number of seconds without output before"
- "firefox-media-tests is killed."
- "Set this based on expected time for all media to play."),
- }],
- [["--tests"],
- {"action": "store",
- "dest": "tests",
- "default": None,
- "help": ("Test(s) to run. Path to test_*.py or "
- "test manifest (*.ini)"),
- }],
- [["--e10s"],
- {"dest": "e10s",
- "action": "store_true",
- "default": False,
- "help": "Enable e10s when running marionette tests."
- }],
- [["--suite"],
- {"action": "store",
- "dest": "test_suite",
- "default": "media-tests",
- "help": "suite name",
- }],
- [['--browsermob-script'],
- {'help': 'path to the browsermob-proxy shell script or batch file',
- }],
- [['--browsermob-port'],
- {'help': 'port to run the browsermob proxy on',
- }],
- [["--allow-software-gl-layers"],
- {"action": "store_true",
- "dest": "allow_software_gl_layers",
- "default": False,
- "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor."
- }],
-] + (copy.deepcopy(testing_config_options))
-
-class JobResultParser(TestSummaryOutputParserHelper):
- """ Parses test output to determine overall result."""
- def __init__(self, **kwargs):
- super(JobResultParser, self).__init__(**kwargs)
- self.return_code = 0
- # External-resource errors that should not count as test failures
- self.exception_re = re.compile(r'^TEST-UNEXPECTED-ERROR.*'
- r'TimeoutException: Error loading page,'
- r' timed out')
- self.exceptions = []
-
- def parse_single_line(self, line):
- super(JobResultParser, self).parse_single_line(line)
- if self.exception_re.match(line):
- self.exceptions.append(line)
-
- @property
- def status(self):
- status = UNKNOWN
- if self.passed and self.failed == 0:
- status = SUCCESS
- elif self.exceptions:
- status = EXCEPTION
- elif self.failed:
- status = TESTFAILED
- elif self.return_code:
- status = BUSTED
- return status
-
-
-class FirefoxMediaTestsBase(TestingMixin, VCSToolsScript):
- job_result_parser = None
-
- error_list = [
- {'substr': 'FAILED (errors=', 'level': WARNING},
- {'substr': r'''Could not successfully complete transport of message to Gecko, socket closed''', 'level': ERROR},
- {'substr': r'''Connection to Marionette server is lost. Check gecko''', 'level': ERROR},
- {'substr': 'Timeout waiting for marionette on port', 'level': ERROR},
- {'regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH|CRASH|ERROR|FAIL)'''), 'level': ERROR},
- {'regex': re.compile(r'''(\b\w*Exception)'''), 'level': ERROR},
- {'regex': re.compile(r'''(\b\w*Error)'''), 'level': ERROR},
- ]
-
- def __init__(self, config_options=None, all_actions=None,
- default_actions=None, **kwargs):
- self.config_options = media_test_config_options + (config_options or [])
- actions = [
- 'clobber',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-media-tests',
- ]
- super(FirefoxMediaTestsBase, self).__init__(
- config_options=self.config_options,
- all_actions=all_actions or actions,
- default_actions=default_actions or actions,
- **kwargs
- )
- c = self.config
-
- self.media_urls = c.get('media_urls')
- self.profile = c.get('profile')
- self.test_timeout = int(c.get('test_timeout'))
- self.tests = c.get('tests')
- self.e10s = c.get('e10s')
- self.installer_url = c.get('installer_url')
- self.installer_path = c.get('installer_path')
- self.binary_path = c.get('binary_path')
- self.test_packages_url = c.get('test_packages_url')
- self.test_url = c.get('test_url')
- self.browsermob_script = c.get('browsermob_script')
- self.browsermob_port = c.get('browsermob_port')
-
- @PreScriptAction('create-virtualenv')
- def _pre_create_virtualenv(self, action):
- dirs = self.query_abs_dirs()
-
- media_tests_requirements = os.path.join(dirs['abs_test_install_dir'],
- 'config',
- 'external-media-tests-requirements.txt')
-
- if os.access(media_tests_requirements, os.F_OK):
- self.register_virtualenv_module(requirements=[media_tests_requirements],
- two_pass=True)
-
- def download_and_extract(self):
- """Overriding method from TestingMixin for more specific behavior.
-
- We use the test_packages_url command line argument to check where to get the
- harness, puppeteer, and tests from and how to set them up.
-
- """
- extract_dirs = ['config/*',
- 'external-media-tests/*',
- 'marionette/*',
- 'mozbase/*',
- 'puppeteer/*',
- 'tools/wptserve/*',
- ]
- super(FirefoxMediaTestsBase, self).download_and_extract(extract_dirs=extract_dirs)
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(FirefoxMediaTestsBase, self).query_abs_dirs()
- dirs = {
- 'abs_test_install_dir' : os.path.join(abs_dirs['abs_work_dir'],
- 'tests')
- }
- dirs['external-media-tests'] = os.path.join(dirs['abs_test_install_dir'],
- 'external-media-tests')
- abs_dirs.update(dirs)
- self.abs_dirs = abs_dirs
- return self.abs_dirs
-
- def _query_cmd(self):
- """ Determine how to call firefox-media-tests """
- if not self.binary_path:
- self.fatal("Binary path could not be determined. "
- "Should be set by default during 'install' action.")
- dirs = self.query_abs_dirs()
-
- import external_media_harness.runtests
-
- cmd = [
- self.query_python_path(),
- external_media_harness.runtests.__file__
- ]
-
- cmd += ['--binary', self.binary_path]
- if self.symbols_path:
- cmd += ['--symbols-path', self.symbols_path]
- if self.media_urls:
- cmd += ['--urls', self.media_urls]
- if self.profile:
- cmd += ['--profile', self.profile]
- if self.tests:
- cmd.append(self.tests)
- if not self.e10s:
- cmd.append('--disable-e10s')
- if self.browsermob_script:
- cmd += ['--browsermob-script', self.browsermob_script]
- if self.browsermob_port:
- cmd += ['--browsermob-port', self.browsermob_port]
-
- test_suite = self.config.get('test_suite')
- if test_suite not in self.config["suite_definitions"]:
- self.fatal("%s is not defined in the config!" % test_suite)
-
- test_manifest = None if test_suite != 'media-youtube-tests' else \
- os.path.join(dirs['external-media-tests'],
- 'external_media_tests',
- 'playback', 'youtube', 'manifest.ini')
- config_fmt_args = {
- 'test_manifest': test_manifest,
- }
-
- for s in self.config["suite_definitions"][test_suite]["options"]:
- cmd.append(s % config_fmt_args)
-
- return cmd
-
- def query_minidump_stackwalk(self):
- """We don't have an extracted test package available to get the manifest file.
-
- So we have to explicitely download the latest version of the manifest from the
- mozilla-central repository and feed it into the query_minidump_stackwalk() method.
-
- We can remove this whole method once our tests are part of the tree.
-
- """
- manifest_path = None
-
- if os.environ.get('MINIDUMP_STACKWALK') or self.config.get('download_minidump_stackwalk'):
- tooltool_manifest = self.query_minidump_tooltool_manifest()
- url_base = 'https://hg.mozilla.org/mozilla-central/raw-file/default/testing/'
-
- dirs = self.query_abs_dirs()
- manifest_path = os.path.join(dirs['abs_work_dir'], 'releng.manifest')
- try:
- self.download_file(urlparse.urljoin(url_base, tooltool_manifest),
- manifest_path)
- except Exception as e:
- self.fatal('Download of tooltool manifest file failed: %s' % e.message)
-
- return super(FirefoxMediaTestsBase, self).query_minidump_stackwalk(manifest=manifest_path)
-
- def run_media_tests(self):
- cmd = self._query_cmd()
- self.job_result_parser = JobResultParser(
- config=self.config,
- log_obj=self.log_obj,
- error_list=self.error_list
- )
-
- env = self.query_env()
- if self.query_minidump_stackwalk():
- env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
-
- if self.config['allow_software_gl_layers']:
- env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1'
-
- return_code = self.run_command(
- cmd,
- output_timeout=self.test_timeout,
- output_parser=self.job_result_parser,
- env=env
- )
- self.job_result_parser.return_code = return_code
- return self.job_result_parser.status
diff --git a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py b/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
deleted file mode 100644
index 684ec3a73..000000000
--- a/testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py
+++ /dev/null
@@ -1,300 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-
-import copy
-import os
-import sys
-
-from mozharness.base.log import FATAL, WARNING
-from mozharness.base.python import PostScriptRun, PreScriptAction
-from mozharness.mozilla.structuredlog import StructuredOutputParser
-from mozharness.mozilla.testing.testbase import (
- TestingMixin,
- testing_config_options,
-)
-from mozharness.mozilla.vcstools import VCSToolsScript
-
-
-# General command line arguments for Firefox ui tests
-firefox_ui_tests_config_options = [
- [["--allow-software-gl-layers"], {
- "action": "store_true",
- "dest": "allow_software_gl_layers",
- "default": False,
- "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor.",
- }],
- [['--dry-run'], {
- 'dest': 'dry_run',
- 'default': False,
- 'help': 'Only show what was going to be tested.',
- }],
- [["--e10s"], {
- 'dest': 'e10s',
- 'action': 'store_true',
- 'default': False,
- 'help': 'Enable multi-process (e10s) mode when running tests.',
- }],
- [['--symbols-path=SYMBOLS_PATH'], {
- 'dest': 'symbols_path',
- 'help': 'absolute path to directory containing breakpad '
- 'symbols, or the url of a zip file containing symbols.',
- }],
- [['--tag=TAG'], {
- 'dest': 'tag',
- 'help': 'Subset of tests to run (local, remote).',
- }],
-] + copy.deepcopy(testing_config_options)
-
-# Command line arguments for update tests
-firefox_ui_update_harness_config_options = [
- [['--update-allow-mar-channel'], {
- 'dest': 'update_allow_mar_channel',
- 'help': 'Additional MAR channel to be allowed for updates, e.g. '
- '"firefox-mozilla-beta" for updating a release build to '
- 'the latest beta build.',
- }],
- [['--update-channel'], {
- 'dest': 'update_channel',
- 'help': 'Update channel to use.',
- }],
- [['--update-direct-only'], {
- 'action': 'store_true',
- 'dest': 'update_direct_only',
- 'help': 'Only perform a direct update.',
- }],
- [['--update-fallback-only'], {
- 'action': 'store_true',
- 'dest': 'update_fallback_only',
- 'help': 'Only perform a fallback update.',
- }],
- [['--update-override-url'], {
- 'dest': 'update_override_url',
- 'help': 'Force specified URL to use for update checks.',
- }],
- [['--update-target-buildid'], {
- 'dest': 'update_target_buildid',
- 'help': 'Build ID of the updated build',
- }],
- [['--update-target-version'], {
- 'dest': 'update_target_version',
- 'help': 'Version of the updated build.',
- }],
-]
-
-firefox_ui_update_config_options = firefox_ui_update_harness_config_options \
- + copy.deepcopy(firefox_ui_tests_config_options)
-
-
-class FirefoxUITests(TestingMixin, VCSToolsScript):
-
- # Needs to be overwritten in sub classes
- cli_script = None
-
- def __init__(self, config_options=None,
- all_actions=None, default_actions=None,
- *args, **kwargs):
- config_options = config_options or firefox_ui_tests_config_options
- actions = [
- 'clobber',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-tests',
- 'uninstall',
- ]
-
- super(FirefoxUITests, self).__init__(
- config_options=config_options,
- all_actions=all_actions or actions,
- default_actions=default_actions or actions,
- *args, **kwargs)
-
- # Code which doesn't run on buildbot has to include the following properties
- self.binary_path = self.config.get('binary_path')
- self.installer_path = self.config.get('installer_path')
- self.installer_url = self.config.get('installer_url')
- self.test_packages_url = self.config.get('test_packages_url')
- self.test_url = self.config.get('test_url')
-
- if not self.test_url and not self.test_packages_url:
- self.fatal(
- 'You must use --test-url, or --test-packages-url')
-
- @PreScriptAction('create-virtualenv')
- def _pre_create_virtualenv(self, action):
- dirs = self.query_abs_dirs()
-
- requirements = os.path.join(dirs['abs_test_install_dir'],
- 'config', 'firefox_ui_requirements.txt')
- self.register_virtualenv_module(requirements=[requirements], two_pass=True)
-
- def download_and_extract(self):
- """Override method from TestingMixin for more specific behavior."""
- extract_dirs = ['config/*',
- 'firefox-ui/*',
- 'marionette/*',
- 'mozbase/*',
- 'tools/wptserve/*',
- ]
- super(FirefoxUITests, self).download_and_extract(extract_dirs=extract_dirs)
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
-
- abs_dirs = super(FirefoxUITests, self).query_abs_dirs()
- abs_tests_install_dir = os.path.join(abs_dirs['abs_work_dir'], 'tests')
-
- dirs = {
- 'abs_blob_upload_dir': os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir'),
- 'abs_test_install_dir': abs_tests_install_dir,
- 'abs_fxui_dir': os.path.join(abs_tests_install_dir, 'firefox-ui'),
- }
-
- for key in dirs:
- if key not in abs_dirs:
- abs_dirs[key] = dirs[key]
- self.abs_dirs = abs_dirs
-
- return self.abs_dirs
-
- def query_harness_args(self, extra_harness_config_options=None):
- """Collects specific test related command line arguments.
-
- Sub classes should override this method for their own specific arguments.
- """
- config_options = extra_harness_config_options or []
-
- args = []
- for option in config_options:
- dest = option[1]['dest']
- name = self.config.get(dest)
-
- if name:
- if type(name) is bool:
- args.append(option[0][0])
- else:
- args.extend([option[0][0], self.config[dest]])
-
- return args
-
- def run_test(self, binary_path, env=None, marionette_port=2828):
- """All required steps for running the tests against an installer."""
- dirs = self.query_abs_dirs()
-
- # Import the harness to retrieve the location of the cli scripts
- import firefox_ui_harness
-
- cmd = [
- self.query_python_path(),
- os.path.join(os.path.dirname(firefox_ui_harness.__file__),
- self.cli_script),
- '--binary', binary_path,
- '--address', 'localhost:{}'.format(marionette_port),
-
- # Resource files to serve via local webserver
- '--server-root', os.path.join(dirs['abs_fxui_dir'], 'resources'),
-
- # Use the work dir to get temporary data stored
- '--workspace', dirs['abs_work_dir'],
-
- # logging options
- '--gecko-log=-', # output from the gecko process redirected to stdout
- '--log-raw=-', # structured log for output parser redirected to stdout
-
- # additional reports helpful for Jenkins and inpection via Treeherder
- '--log-html', os.path.join(dirs['abs_blob_upload_dir'], 'report.html'),
- '--log-xunit', os.path.join(dirs['abs_blob_upload_dir'], 'report.xml'),
-
- # Enable tracing output to log transmission protocol
- '-vv',
- ]
-
- # Collect all pass-through harness options to the script
- cmd.extend(self.query_harness_args())
-
- # Translate deprecated --e10s flag
- if not self.config.get('e10s'):
- cmd.append('--disable-e10s')
-
- if self.symbols_url:
- cmd.extend(['--symbols-path', self.symbols_url])
-
- if self.config.get('tag'):
- cmd.extend(['--tag', self.config['tag']])
-
- parser = StructuredOutputParser(config=self.config,
- log_obj=self.log_obj,
- strict=False)
-
- # Add the default tests to run
- tests = [os.path.join(dirs['abs_fxui_dir'], 'tests', test) for test in self.default_tests]
- cmd.extend(tests)
-
- # Set further environment settings
- env = env or self.query_env()
- env.update({'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']})
- if self.query_minidump_stackwalk():
- env.update({'MINIDUMP_STACKWALK': self.minidump_stackwalk_path})
-
- if self.config['allow_software_gl_layers']:
- env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1'
-
- return_code = self.run_command(cmd,
- cwd=dirs['abs_work_dir'],
- output_timeout=300,
- output_parser=parser,
- env=env)
-
- tbpl_status, log_level = parser.evaluate_parser(return_code)
- self.buildbot_status(tbpl_status, level=log_level)
-
- return return_code
-
- @PreScriptAction('run-tests')
- def _pre_run_tests(self, action):
- if not self.installer_path and not self.installer_url:
- self.critical('Please specify an installer via --installer-path or --installer-url.')
- sys.exit(1)
-
- def run_tests(self):
- """Run all the tests"""
- return self.run_test(
- binary_path=self.binary_path,
- env=self.query_env(),
- )
-
-
-class FirefoxUIFunctionalTests(FirefoxUITests):
-
- cli_script = 'cli_functional.py'
- default_tests = [
- os.path.join('puppeteer', 'manifest.ini'),
- os.path.join('functional', 'manifest.ini'),
- ]
-
-
-class FirefoxUIUpdateTests(FirefoxUITests):
-
- cli_script = 'cli_update.py'
- default_tests = [
- os.path.join('update', 'manifest.ini')
- ]
-
- def __init__(self, config_options=None, *args, **kwargs):
- config_options = config_options or firefox_ui_update_config_options
-
- super(FirefoxUIUpdateTests, self).__init__(
- config_options=config_options,
- *args, **kwargs
- )
-
- def query_harness_args(self):
- """Collects specific update test related command line arguments."""
- return super(FirefoxUIUpdateTests, self).query_harness_args(
- firefox_ui_update_harness_config_options)
diff --git a/testing/mozharness/mozharness/mozilla/testing/mozpool.py b/testing/mozharness/mozharness/mozilla/testing/mozpool.py
deleted file mode 100644
index f9da6c190..000000000
--- a/testing/mozharness/mozharness/mozilla/testing/mozpool.py
+++ /dev/null
@@ -1,134 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-'''Interact with mozpool/lifeguard/bmm.
-'''
-
-import os
-import socket
-import sys
-
-from time import sleep
-from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_EXCEPTION
-
-#TODO - adjust these values
-MAX_RETRIES = 20
-RETRY_INTERVAL = 60
-
-# MozpoolMixin {{{1
-class MozpoolMixin(object):
- mozpool_handler = None
- mobile_imaging_format= "http://mobile-imaging"
-
- def determine_mozpool_host(self, device):
- if "mobile_imaging_format" in self.config:
- self.mobile_imaging_format = self.config["mobile_imaging_format"]
- hostname = str(self.mobile_imaging_format)[7:]
- fqdn = socket.getfqdn(hostname)
- imaging_server_fqdn = (str(self.mobile_imaging_format)).replace(hostname, fqdn)
- return imaging_server_fqdn
-
- def query_mozpool_handler(self, device=None, mozpool_api_url=None):
- if self.mozpool_handler != None:
- return self.mozpool_handler
- else:
- self.mozpool_api_url = self.determine_mozpool_host(device) if device else mozpool_api_url
- assert self.mozpool_api_url != None, \
- "query_mozpool_handler() requires either a device or mozpool_api_url!"
-
- site_packages_path = self.query_python_site_packages_path()
- mph_path = os.path.join(site_packages_path, 'mozpoolclient')
- sys.path.append(mph_path)
- sys.path.append(site_packages_path)
- try:
- from mozpoolclient import MozpoolHandler, MozpoolException, MozpoolConflictException
- self.MozpoolException = MozpoolException
- self.MozpoolConflictException = MozpoolConflictException
- self.mozpool_handler = MozpoolHandler(self.mozpool_api_url, log_obj=self)
- except ImportError, e:
- self.fatal("Can't instantiate MozpoolHandler until mozpoolclient python "
- "package is installed! (VirtualenvMixin?): \n%s" % str(e))
- return self.mozpool_handler
-
- def retrieve_b2g_device(self, b2gbase):
- mph = self.query_mozpool_handler(self.mozpool_device)
- for retry in self._retry_sleep(
- error_message="INFRA-ERROR: Could not request device '%s'" % self.mozpool_device,
- tbpl_status=TBPL_EXCEPTION):
- try:
- image = 'b2g'
- duration = 4 * 60 * 60 # request valid for 14400 seconds == 4 hours
- response = mph.request_device(self.mozpool_device, image, assignee=self.mozpool_assignee, \
- b2gbase=b2gbase, pxe_config=None, duration=duration)
- break
- except self.MozpoolConflictException:
- self.warning("Device unavailable. Retry#%i.." % retry)
- except self.MozpoolException, e:
- self.buildbot_status(TBPL_RETRY)
- self.fatal("We could not request the device: %s" % str(e))
-
- self.request_url = response['request']['url']
- self.info("Got request, url=%s" % self.request_url)
- self._wait_for_request_ready()
-
- def retrieve_android_device(self, b2gbase):
- mph = self.query_mozpool_handler(self.mozpool_device)
- for retry in self._retry_sleep(
- error_message="INFRA-ERROR: Could not request device '%s'" % self.mozpool_device,
- tbpl_status=TBPL_RETRY):
- try:
- image = 'panda-android-4.0.4_v3.3'
- duration = 4 * 60 * 60 # request valid for 14400 seconds == 4 hours
- response = mph.request_device(self.mozpool_device, image, assignee=self.mozpool_assignee, \
- b2gbase=b2gbase, pxe_config=None, duration=duration)
- break
- except self.MozpoolConflictException:
- self.warning("Device unavailable. Retry#%i.." % retry)
- except self.MozpoolException, e:
- self.buildbot_status(TBPL_RETRY)
- self.fatal("We could not request the device: %s" % str(e))
-
- self.request_url = response['request']['url']
- self.info("Got request, url=%s" % self.request_url)
- self._wait_for_request_ready()
-
- def _retry_job_and_close_request(self, message, exception=None):
- mph = self.query_mozpool_handler(self.mozpool_device)
- exception_message = str(exception) if exception!=None and str(exception) != None else ""
- self.error("%s -> %s" % (message, exception_message))
- if self.request_url:
- mph.close_request(self.request_url)
- self.buildbot_status(TBPL_RETRY)
- self.fatal(message)
-
- def _retry_sleep(self, sleep_time=RETRY_INTERVAL, max_retries=MAX_RETRIES,
- error_message=None, tbpl_status=None, fail_cb=None):
- for x in range(1, max_retries + 1):
- yield x
- sleep(sleep_time)
- if error_message:
- self.error(error_message)
- if tbpl_status:
- self.buildbot_status(tbpl_status)
- if fail_cb:
- assert callable(fail_cb)
- fail_cb()
- self.fatal('Retries limit exceeded')
-
- def _wait_for_request_ready(self):
- mph = self.query_mozpool_handler(self.mozpool_device)
- def on_fail():
- # Device is not ready after retries...
- self.info("Aborting mozpool request.")
- self.close_request()
- for retry in self._retry_sleep(sleep_time=RETRY_INTERVAL, max_retries=MAX_RETRIES,
- error_message="INFRA-ERROR: Request did not become ready in time",
- tbpl_status=TBPL_EXCEPTION, fail_cb=on_fail):
- response = mph.query_request_status(self.request_url)
- state = response['state']
- if state == 'ready':
- return
- self.info("Waiting for request 'ready' stage. Current state: '%s'" % state)
diff --git a/testing/mozharness/mozharness/mozilla/testing/talos.py b/testing/mozharness/mozharness/mozilla/testing/talos.py
deleted file mode 100755
index 73f384ce7..000000000
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ /dev/null
@@ -1,430 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""
-run talos tests in a virtualenv
-"""
-
-import os
-import pprint
-import copy
-import re
-import json
-
-import mozharness
-from mozharness.base.config import parse_config_file
-from mozharness.base.errors import PythonErrorList
-from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL
-from mozharness.base.log import INFO, WARNING
-from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
-from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
-from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.testing.errors import TinderBoxPrintRe
-from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE
-from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_FAILURE, TBPL_WARNING
-
-external_tools_path = os.path.join(
- os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
- 'external_tools',
-)
-
-TalosErrorList = PythonErrorList + [
- {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
- {'substr': r'''FAIL: Graph server unreachable''', 'level': CRITICAL},
- {'substr': r'''FAIL: Busted:''', 'level': CRITICAL},
- {'substr': r'''FAIL: failed to cleanup''', 'level': ERROR},
- {'substr': r'''erfConfigurator.py: Unknown error''', 'level': CRITICAL},
- {'substr': r'''talosError''', 'level': CRITICAL},
- {'regex': re.compile(r'''No machine_name called '.*' can be found'''), 'level': CRITICAL},
- {'substr': r"""No such file or directory: 'browser_output.txt'""",
- 'level': CRITICAL,
- 'explanation': r"""Most likely the browser failed to launch, or the test was otherwise unsuccessful in even starting."""},
-]
-
-# TODO: check for running processes on script invocation
-
-class TalosOutputParser(OutputParser):
- minidump_regex = re.compile(r'''talosError: "error executing: '(\S+) (\S+) (\S+)'"''')
- RE_PERF_DATA = re.compile(r'.*PERFHERDER_DATA:\s+(\{.*\})')
- worst_tbpl_status = TBPL_SUCCESS
-
- def __init__(self, **kwargs):
- super(TalosOutputParser, self).__init__(**kwargs)
- self.minidump_output = None
- self.found_perf_data = []
-
- def update_worst_log_and_tbpl_levels(self, log_level, tbpl_level):
- self.worst_log_level = self.worst_level(log_level,
- self.worst_log_level)
- self.worst_tbpl_status = self.worst_level(
- tbpl_level, self.worst_tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE
- )
-
- def parse_single_line(self, line):
- """ In Talos land, every line that starts with RETURN: needs to be
- printed with a TinderboxPrint:"""
- if line.startswith("RETURN:"):
- line.replace("RETURN:", "TinderboxPrint:")
- m = self.minidump_regex.search(line)
- if m:
- self.minidump_output = (m.group(1), m.group(2), m.group(3))
-
- m = self.RE_PERF_DATA.match(line)
- if m:
- self.found_perf_data.append(m.group(1))
-
- # now let's check if buildbot should retry
- harness_retry_re = TinderBoxPrintRe['harness_error']['retry_regex']
- if harness_retry_re.search(line):
- self.critical(' %s' % line)
- self.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_RETRY)
- return # skip base parse_single_line
- super(TalosOutputParser, self).parse_single_line(line)
-
-
-class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
- """
- install and run Talos tests:
- https://wiki.mozilla.org/Buildbot/Talos
- """
- config_options = [
- [["--use-talos-json"],
- {"action": "store_true",
- "dest": "use_talos_json",
- "default": False,
- "help": "Use talos config from talos.json"
- }],
- [["--suite"],
- {"action": "store",
- "dest": "suite",
- "help": "Talos suite to run (from talos json)"
- }],
- [["--branch-name"],
- {"action": "store",
- "dest": "branch",
- "help": "Graphserver branch to report to"
- }],
- [["--system-bits"],
- {"action": "store",
- "dest": "system_bits",
- "type": "choice",
- "default": "32",
- "choices": ['32', '64'],
- "help": "Testing 32 or 64 (for talos json plugins)"
- }],
- [["--add-option"],
- {"action": "extend",
- "dest": "talos_extra_options",
- "default": None,
- "help": "extra options to talos"
- }],
- [["--spsProfile"], {
- "dest": "sps_profile",
- "action": "store_true",
- "default": False,
- "help": "Whether or not to profile the test run and save the profile results"
- }],
- [["--spsProfileInterval"], {
- "dest": "sps_profile_interval",
- "type": "int",
- "default": 0,
- "help": "The interval between samples taken by the profiler (milliseconds)"
- }],
- ] + testing_config_options + copy.deepcopy(blobupload_config_options)
-
- def __init__(self, **kwargs):
- kwargs.setdefault('config_options', self.config_options)
- kwargs.setdefault('all_actions', ['clobber',
- 'read-buildbot-config',
- 'download-and-extract',
- 'populate-webroot',
- 'create-virtualenv',
- 'install',
- 'run-tests',
- ])
- kwargs.setdefault('default_actions', ['clobber',
- 'download-and-extract',
- 'populate-webroot',
- 'create-virtualenv',
- 'install',
- 'run-tests',
- ])
- kwargs.setdefault('config', {})
- super(Talos, self).__init__(**kwargs)
-
- self.workdir = self.query_abs_dirs()['abs_work_dir'] # convenience
-
- self.run_local = self.config.get('run_local')
- self.installer_url = self.config.get("installer_url")
- self.talos_json_url = self.config.get("talos_json_url")
- self.talos_json = self.config.get("talos_json")
- self.talos_json_config = self.config.get("talos_json_config")
- self.tests = None
- self.pagesets_url = None
- self.sps_profile = self.config.get('sps_profile')
- self.sps_profile_interval = self.config.get('sps_profile_interval')
-
- # We accept some configuration options from the try commit message in the format mozharness: <options>
- # Example try commit message:
- # mozharness: --spsProfile try: <stuff>
- def query_sps_profile_options(self):
- sps_results = []
- if self.buildbot_config:
- # this is inside automation
- # now let's see if we added spsProfile specs in the commit message
- try:
- junk, junk, opts = self.buildbot_config['sourcestamp']['changes'][-1]['comments'].partition('mozharness:')
- except IndexError:
- # when we don't have comments on changes (bug 1255187)
- opts = None
-
- if opts:
- # In the case of a multi-line commit message, only examine
- # the first line for mozharness options
- opts = opts.split('\n')[0]
- opts = re.sub(r'\w+:.*', '', opts).strip().split(' ')
- if "--spsProfile" in opts:
- # overwrite whatever was set here.
- self.sps_profile = True
- try:
- idx = opts.index('--spsProfileInterval')
- if len(opts) > idx + 1:
- self.sps_profile_interval = opts[idx + 1]
- except ValueError:
- pass
- # finally, if sps_profile is set, we add that to the talos options
- if self.sps_profile:
- sps_results.append('--spsProfile')
- if self.sps_profile_interval:
- sps_results.extend(
- ['--spsProfileInterval', str(self.sps_profile_interval)]
- )
- return sps_results
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(Talos, self).query_abs_dirs()
- abs_dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
- self.abs_dirs = abs_dirs
- return self.abs_dirs
-
- def query_talos_json_config(self):
- """Return the talos json config."""
- if self.talos_json_config:
- return self.talos_json_config
- if not self.talos_json:
- self.talos_json = os.path.join(self.talos_path, 'talos.json')
- self.talos_json_config = parse_config_file(self.talos_json)
- self.info(pprint.pformat(self.talos_json_config))
- return self.talos_json_config
-
- def query_pagesets_url(self):
- """Certain suites require external pagesets to be downloaded and
- extracted.
- """
- if self.pagesets_url:
- return self.pagesets_url
- if self.query_talos_json_config() and 'suite' in self.config:
- self.pagesets_url = self.talos_json_config['suites'][self.config['suite']].get('pagesets_url')
- return self.pagesets_url
-
- def talos_options(self, args=None, **kw):
- """return options to talos"""
- # binary path
- binary_path = self.binary_path or self.config.get('binary_path')
- if not binary_path:
- self.fatal("Talos requires a path to the binary. You can specify binary_path or add download-and-extract to your action list.")
-
- # talos options
- options = []
- # talos can't gather data if the process name ends with '.exe'
- if binary_path.endswith('.exe'):
- binary_path = binary_path[:-4]
- # options overwritten from **kw
- kw_options = {'executablePath': binary_path}
- if 'suite' in self.config:
- kw_options['suite'] = self.config['suite']
- if self.config.get('title'):
- kw_options['title'] = self.config['title']
- if self.config.get('branch'):
- kw_options['branchName'] = self.config['branch']
- if self.symbols_path:
- kw_options['symbolsPath'] = self.symbols_path
- kw_options.update(kw)
- # talos expects tests to be in the format (e.g.) 'ts:tp5:tsvg'
- tests = kw_options.get('activeTests')
- if tests and not isinstance(tests, basestring):
- tests = ':'.join(tests) # Talos expects this format
- kw_options['activeTests'] = tests
- for key, value in kw_options.items():
- options.extend(['--%s' % key, value])
- # configure profiling options
- options.extend(self.query_sps_profile_options())
- # extra arguments
- if args is not None:
- options += args
- if 'talos_extra_options' in self.config:
- options += self.config['talos_extra_options']
- return options
-
- def populate_webroot(self):
- """Populate the production test slaves' webroots"""
- c = self.config
-
- self.talos_path = os.path.join(
- self.query_abs_dirs()['abs_work_dir'], 'tests', 'talos'
- )
- if c.get('run_local'):
- self.talos_path = os.path.dirname(self.talos_json)
-
- src_talos_webdir = os.path.join(self.talos_path, 'talos')
-
- if self.query_pagesets_url():
- self.info('Downloading pageset...')
- dirs = self.query_abs_dirs()
- src_talos_pageset = os.path.join(src_talos_webdir, 'tests')
- archive = self.download_file(self.pagesets_url, parent_dir=dirs['abs_work_dir'])
- unzip = self.query_exe('unzip')
- unzip_cmd = [unzip, '-q', '-o', archive, '-d', src_talos_pageset]
- self.run_command(unzip_cmd, halt_on_failure=True)
-
- # Action methods. {{{1
- # clobber defined in BaseScript
- # read_buildbot_config defined in BuildbotMixin
-
- def download_and_extract(self, extract_dirs=None, suite_categories=None):
- return super(Talos, self).download_and_extract(
- suite_categories=['common', 'talos']
- )
-
- def create_virtualenv(self, **kwargs):
- """VirtualenvMixin.create_virtualenv() assuemes we're using
- self.config['virtualenv_modules']. Since we are installing
- talos from its source, we have to wrap that method here."""
- # install mozbase first, so we use in-tree versions
- if not self.run_local:
- mozbase_requirements = os.path.join(
- self.query_abs_dirs()['abs_work_dir'],
- 'tests',
- 'config',
- 'mozbase_requirements.txt'
- )
- else:
- mozbase_requirements = os.path.join(
- os.path.dirname(self.talos_path),
- 'config',
- 'mozbase_requirements.txt'
- )
- self.register_virtualenv_module(
- requirements=[mozbase_requirements],
- two_pass=True,
- editable=True,
- )
- # require pip >= 1.5 so pip will prefer .whl files to install
- super(Talos, self).create_virtualenv(
- modules=['pip>=1.5']
- )
- # talos in harness requires what else is
- # listed in talos requirements.txt file.
- self.install_module(
- requirements=[os.path.join(self.talos_path,
- 'requirements.txt')]
- )
- # install jsonschema for perfherder validation
- self.install_module(module="jsonschema")
-
- def _validate_treeherder_data(self, parser):
- # late import is required, because install is done in create_virtualenv
- import jsonschema
-
- if len(parser.found_perf_data) != 1:
- self.critical("PERFHERDER_DATA was seen %d times, expected 1."
- % len(parser.found_perf_data))
- parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING)
- return
-
- schema_path = os.path.join(external_tools_path,
- 'performance-artifact-schema.json')
- self.info("Validating PERFHERDER_DATA against %s" % schema_path)
- try:
- with open(schema_path) as f:
- schema = json.load(f)
- data = json.loads(parser.found_perf_data[0])
- jsonschema.validate(data, schema)
- except:
- self.exception("Error while validating PERFHERDER_DATA")
- parser.update_worst_log_and_tbpl_levels(WARNING, TBPL_WARNING)
-
- def run_tests(self, args=None, **kw):
- """run Talos tests"""
-
- # get talos options
- options = self.talos_options(args=args, **kw)
-
- # XXX temporary python version check
- python = self.query_python_path()
- self.run_command([python, "--version"])
- parser = TalosOutputParser(config=self.config, log_obj=self.log_obj,
- error_list=TalosErrorList)
- env = {}
- env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
- if not self.run_local:
- env['MINIDUMP_STACKWALK'] = self.query_minidump_stackwalk()
- env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
- if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
- self.mkdir_p(env['MOZ_UPLOAD_DIR'])
- env = self.query_env(partial_env=env, log_level=INFO)
- # adjust PYTHONPATH to be able to use talos as a python package
- if 'PYTHONPATH' in env:
- env['PYTHONPATH'] = self.talos_path + os.pathsep + env['PYTHONPATH']
- else:
- env['PYTHONPATH'] = self.talos_path
-
- # sets a timeout for how long talos should run without output
- output_timeout = self.config.get('talos_output_timeout', 3600)
- # run talos tests
- run_tests = os.path.join(self.talos_path, 'talos', 'run_tests.py')
-
- mozlog_opts = ['--log-tbpl-level=debug']
- if not self.run_local and 'suite' in self.config:
- fname_pattern = '%s_%%s.log' % self.config['suite']
- mozlog_opts.append('--log-errorsummary=%s'
- % os.path.join(env['MOZ_UPLOAD_DIR'],
- fname_pattern % 'errorsummary'))
- mozlog_opts.append('--log-raw=%s'
- % os.path.join(env['MOZ_UPLOAD_DIR'],
- fname_pattern % 'raw'))
-
- command = [python, run_tests] + options + mozlog_opts
- self.return_code = self.run_command(command, cwd=self.workdir,
- output_timeout=output_timeout,
- output_parser=parser,
- env=env)
- if parser.minidump_output:
- self.info("Looking at the minidump files for debugging purposes...")
- for item in parser.minidump_output:
- self.run_command(["ls", "-l", item])
-
- if self.return_code not in [0]:
- # update the worst log level and tbpl status
- log_level = ERROR
- tbpl_level = TBPL_FAILURE
- if self.return_code == 1:
- log_level = WARNING
- tbpl_level = TBPL_WARNING
- if self.return_code == 4:
- log_level = WARNING
- tbpl_level = TBPL_RETRY
-
- parser.update_worst_log_and_tbpl_levels(log_level, tbpl_level)
- else:
- if not self.sps_profile:
- self._validate_treeherder_data(parser)
-
- self.buildbot_status(parser.worst_tbpl_status,
- level=parser.worst_log_level)
diff --git a/testing/mozharness/mozharness/mozilla/testing/testbase.py b/testing/mozharness/mozharness/mozilla/testing/testbase.py
deleted file mode 100755
index 2ab2b7e43..000000000
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ /dev/null
@@ -1,862 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-import copy
-import os
-import platform
-import pprint
-import re
-import urllib2
-import json
-import socket
-
-from mozharness.base.errors import BaseErrorList
-from mozharness.base.log import FATAL, WARNING
-from mozharness.base.python import (
- ResourceMonitoringMixin,
- VirtualenvMixin,
- virtualenv_config_options,
-)
-from mozharness.mozilla.buildbot import BuildbotMixin, TBPL_WARNING
-from mozharness.mozilla.proxxy import Proxxy
-from mozharness.mozilla.structuredlog import StructuredOutputParser
-from mozharness.mozilla.taskcluster_helper import TaskClusterArtifactFinderMixin
-from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser
-from mozharness.mozilla.testing.try_tools import TryToolsMixin, try_config_options
-from mozharness.mozilla.tooltool import TooltoolMixin
-
-from mozharness.lib.python.authentication import get_credentials
-
-INSTALLER_SUFFIXES = ('.apk', # Android
- '.tar.bz2', '.tar.gz', # Linux
- '.dmg', # Mac
- '.installer-stub.exe', '.installer.exe', '.exe', '.zip', # Windows
- )
-
-# https://dxr.mozilla.org/mozilla-central/source/testing/config/tooltool-manifests
-TOOLTOOL_PLATFORM_DIR = {
- 'linux': 'linux32',
- 'linux64': 'linux64',
- 'win32': 'win32',
- 'win64': 'win32',
- 'macosx': 'macosx64',
-}
-
-
-testing_config_options = [
- [["--installer-url"],
- {"action": "store",
- "dest": "installer_url",
- "default": None,
- "help": "URL to the installer to install",
- }],
- [["--installer-path"],
- {"action": "store",
- "dest": "installer_path",
- "default": None,
- "help": "Path to the installer to install. This is set automatically if run with --download-and-extract.",
- }],
- [["--binary-path"],
- {"action": "store",
- "dest": "binary_path",
- "default": None,
- "help": "Path to installed binary. This is set automatically if run with --install.",
- }],
- [["--exe-suffix"],
- {"action": "store",
- "dest": "exe_suffix",
- "default": None,
- "help": "Executable suffix for binaries on this platform",
- }],
- [["--test-url"],
- {"action": "store",
- "dest": "test_url",
- "default": None,
- "help": "URL to the zip file containing the actual tests",
- }],
- [["--test-packages-url"],
- {"action": "store",
- "dest": "test_packages_url",
- "default": None,
- "help": "URL to a json file describing which tests archives to download",
- }],
- [["--jsshell-url"],
- {"action": "store",
- "dest": "jsshell_url",
- "default": None,
- "help": "URL to the jsshell to install",
- }],
- [["--download-symbols"],
- {"action": "store",
- "dest": "download_symbols",
- "type": "choice",
- "choices": ['ondemand', 'true'],
- "help": "Download and extract crash reporter symbols.",
- }],
-] + copy.deepcopy(virtualenv_config_options) + copy.deepcopy(try_config_options)
-
-
-# TestingMixin {{{1
-class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin,
- TaskClusterArtifactFinderMixin, TooltoolMixin, TryToolsMixin):
- """
- The steps to identify + download the proper bits for [browser] unit
- tests and Talos.
- """
-
- installer_url = None
- installer_path = None
- binary_path = None
- test_url = None
- test_packages_url = None
- symbols_url = None
- symbols_path = None
- jsshell_url = None
- minidump_stackwalk_path = None
- nodejs_path = None
- default_tools_repo = 'https://hg.mozilla.org/build/tools'
- proxxy = None
-
- def _query_proxxy(self):
- """manages the proxxy"""
- if not self.proxxy:
- self.proxxy = Proxxy(self.config, self.log_obj)
- return self.proxxy
-
- def download_proxied_file(self, url, file_name=None, parent_dir=None,
- create_parent_dir=True, error_level=FATAL,
- exit_code=3):
- proxxy = self._query_proxxy()
- return proxxy.download_proxied_file(url=url, file_name=file_name,
- parent_dir=parent_dir,
- create_parent_dir=create_parent_dir,
- error_level=error_level,
- exit_code=exit_code)
-
- def download_file(self, *args, **kwargs):
- '''
- This function helps not to use download of proxied files
- since it does not support authenticated downloads.
- This could be re-factored and fixed in bug 1087664.
- '''
- if self.config.get("developer_mode"):
- return super(TestingMixin, self).download_file(*args, **kwargs)
- else:
- return self.download_proxied_file(*args, **kwargs)
-
- def query_build_dir_url(self, file_name):
- """
- Resolve a file name to a potential url in the build upload directory where
- that file can be found.
- """
- if self.test_packages_url:
- reference_url = self.test_packages_url
- elif self.installer_url:
- reference_url = self.installer_url
- else:
- self.fatal("Can't figure out build directory urls without an installer_url "
- "or test_packages_url!")
-
- last_slash = reference_url.rfind('/')
- base_url = reference_url[:last_slash]
-
- return '%s/%s' % (base_url, file_name)
-
- def query_prefixed_build_dir_url(self, suffix):
- """Resolve a file name prefixed with platform and build details to a potential url
- in the build upload directory where that file can be found.
- """
- if self.test_packages_url:
- reference_suffixes = ['.test_packages.json']
- reference_url = self.test_packages_url
- elif self.installer_url:
- reference_suffixes = INSTALLER_SUFFIXES
- reference_url = self.installer_url
- else:
- self.fatal("Can't figure out build directory urls without an installer_url "
- "or test_packages_url!")
-
- url = None
- for reference_suffix in reference_suffixes:
- if reference_url.endswith(reference_suffix):
- url = reference_url[:-len(reference_suffix)] + suffix
- break
-
- return url
-
- def query_symbols_url(self, raise_on_failure=False):
- if self.symbols_url:
- return self.symbols_url
-
- elif self.installer_url:
- symbols_url = self.query_prefixed_build_dir_url('.crashreporter-symbols.zip')
-
- # Check if the URL exists. If not, use none to allow mozcrash to auto-check for symbols
- try:
- if symbols_url:
- self._urlopen(symbols_url, timeout=120)
- self.symbols_url = symbols_url
- except Exception as ex:
- self.warning("Cannot open symbols url %s (installer url: %s): %s" %
- (symbols_url, self.installer_url, ex))
- if raise_on_failure:
- raise
-
- # If no symbols URL can be determined let minidump_stackwalk query the symbols.
- # As of now this only works for Nightly and release builds.
- if not self.symbols_url:
- self.warning("No symbols_url found. Let minidump_stackwalk query for symbols.")
-
- return self.symbols_url
-
- def _pre_config_lock(self, rw_config):
- for i, (target_file, target_dict) in enumerate(rw_config.all_cfg_files_and_dicts):
- if 'developer_config' in target_file:
- self._developer_mode_changes(rw_config)
-
- def _developer_mode_changes(self, rw_config):
- """ This function is called when you append the config called
- developer_config.py. This allows you to run a job
- outside of the Release Engineering infrastructure.
-
- What this functions accomplishes is:
- * read-buildbot-config is removed from the list of actions
- * --installer-url is set
- * --test-url is set if needed
- * every url is substituted by another external to the
- Release Engineering network
- """
- c = self.config
- orig_config = copy.deepcopy(c)
- self.warning("When you use developer_config.py, we drop "
- "'read-buildbot-config' from the list of actions.")
- if "read-buildbot-config" in rw_config.actions:
- rw_config.actions.remove("read-buildbot-config")
- self.actions = tuple(rw_config.actions)
-
- def _replace_url(url, changes):
- for from_, to_ in changes:
- if url.startswith(from_):
- new_url = url.replace(from_, to_)
- self.info("Replacing url %s -> %s" % (url, new_url))
- return new_url
- return url
-
- if c.get("installer_url") is None:
- self.exception("You must use --installer-url with developer_config.py")
- if c.get("require_test_zip"):
- if not c.get('test_url') and not c.get('test_packages_url'):
- self.exception("You must use --test-url or --test-packages-url with developer_config.py")
-
- c["installer_url"] = _replace_url(c["installer_url"], c["replace_urls"])
- if c.get("test_url"):
- c["test_url"] = _replace_url(c["test_url"], c["replace_urls"])
- if c.get("test_packages_url"):
- c["test_packages_url"] = _replace_url(c["test_packages_url"], c["replace_urls"])
-
- for key, value in self.config.iteritems():
- if type(value) == str and value.startswith("http"):
- self.config[key] = _replace_url(value, c["replace_urls"])
-
- # Any changes to c means that we need credentials
- if not c == orig_config:
- get_credentials()
-
- def _urlopen(self, url, **kwargs):
- '''
- This function helps dealing with downloading files while outside
- of the releng network.
- '''
- # Code based on http://code.activestate.com/recipes/305288-http-basic-authentication
- def _urlopen_basic_auth(url, **kwargs):
- self.info("We want to download this file %s" % url)
- if not hasattr(self, "https_username"):
- self.info("NOTICE: Files downloaded from outside of "
- "Release Engineering network require LDAP "
- "credentials.")
-
- self.https_username, self.https_password = get_credentials()
- # This creates a password manager
- passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
- # Because we have put None at the start it will use this username/password combination from here on
- passman.add_password(None, url, self.https_username, self.https_password)
- authhandler = urllib2.HTTPBasicAuthHandler(passman)
-
- return urllib2.build_opener(authhandler).open(url, **kwargs)
-
- # If we have the developer_run flag enabled then we will switch
- # URLs to the right place and enable http authentication
- if "developer_config.py" in self.config["config_files"]:
- return _urlopen_basic_auth(url, **kwargs)
- else:
- return urllib2.urlopen(url, **kwargs)
-
- # read_buildbot_config is in BuildbotMixin.
-
- def find_artifacts_from_buildbot_changes(self):
- c = self.config
- try:
- files = self.buildbot_config['sourcestamp']['changes'][-1]['files']
- buildbot_prop_branch = self.buildbot_config['properties']['branch']
-
- # Bug 868490 - Only require exactly two files if require_test_zip;
- # otherwise accept either 1 or 2, since we'll be getting a
- # test_zip url that we don't need.
- expected_length = [1, 2, 3]
- if c.get("require_test_zip") and not self.test_url:
- expected_length = [2, 3]
- if buildbot_prop_branch.startswith('gaia-try'):
- expected_length = range(1, 1000)
- actual_length = len(files)
- if actual_length not in expected_length:
- self.fatal("Unexpected number of files in buildbot config %s.\nExpected these number(s) of files: %s, but got: %d" %
- (c['buildbot_json_path'], str(expected_length), actual_length))
- for f in files:
- if f['name'].endswith('tests.zip'): # yuk
- if not self.test_url:
- # str() because of unicode issues on mac
- self.test_url = str(f['name'])
- self.info("Found test url %s." % self.test_url)
- elif f['name'].endswith('crashreporter-symbols.zip'): # yuk
- self.symbols_url = str(f['name'])
- self.info("Found symbols url %s." % self.symbols_url)
- elif f['name'].endswith('test_packages.json'):
- self.test_packages_url = str(f['name'])
- self.info("Found a test packages url %s." % self.test_packages_url)
- elif not any(f['name'].endswith(s) for s in ('code-coverage-gcno.zip',)):
- if not self.installer_url:
- self.installer_url = str(f['name'])
- self.info("Found installer url %s." % self.installer_url)
- except IndexError, e:
- self.error(str(e))
-
- def find_artifacts_from_taskcluster(self):
- self.info("Finding installer, test and symbols from parent task. ")
- task_id = self.buildbot_config['properties']['taskId']
- self.set_parent_artifacts(task_id)
-
- def postflight_read_buildbot_config(self):
- """
- Determine which files to download from the buildprops.json file
- created via the buildbot ScriptFactory.
- """
- if self.buildbot_config:
- c = self.config
- message = "Unable to set %s from the buildbot config"
- if c.get("installer_url"):
- self.installer_url = c['installer_url']
- if c.get("test_url"):
- self.test_url = c['test_url']
- if c.get("test_packages_url"):
- self.test_packages_url = c['test_packages_url']
-
- # This supports original Buildbot to Buildbot mode
- if self.buildbot_config['sourcestamp']['changes']:
- self.find_artifacts_from_buildbot_changes()
-
- # This supports TaskCluster/BBB task to Buildbot job
- elif 'testPackagesUrl' in self.buildbot_config['properties'] and \
- 'packageUrl' in self.buildbot_config['properties']:
- self.installer_url = self.buildbot_config['properties']['packageUrl']
- self.test_packages_url = self.buildbot_config['properties']['testPackagesUrl']
-
- # This supports TaskCluster/BBB task to TaskCluster/BBB task
- elif 'taskId' in self.buildbot_config['properties']:
- self.find_artifacts_from_taskcluster()
-
- missing = []
- if not self.installer_url:
- missing.append("installer_url")
- if c.get("require_test_zip") and not self.test_url and not self.test_packages_url:
- missing.append("test_url")
- if missing:
- self.fatal("%s!" % (message % ('+'.join(missing))))
- else:
- self.fatal("self.buildbot_config isn't set after running read_buildbot_config!")
-
- def _query_binary_version(self, regex, cmd):
- output = self.get_output_from_command(cmd, silent=False)
- return regex.search(output).group(0)
-
- def preflight_download_and_extract(self):
- message = ""
- if not self.installer_url:
- message += """installer_url isn't set!
-
-You can set this by:
-
-1. specifying --installer-url URL, or
-2. running via buildbot and running the read-buildbot-config action
-
-"""
- if self.config.get("require_test_zip") and not self.test_url and not self.test_packages_url:
- message += """test_url isn't set!
-
-You can set this by:
-
-1. specifying --test-url URL, or
-2. running via buildbot and running the read-buildbot-config action
-
-"""
- if message:
- self.fatal(message + "Can't run download-and-extract... exiting")
-
- def _read_packages_manifest(self):
- dirs = self.query_abs_dirs()
- source = self.download_file(self.test_packages_url,
- parent_dir=dirs['abs_work_dir'],
- error_level=FATAL)
-
- with self.opened(os.path.realpath(source)) as (fh, err):
- package_requirements = json.load(fh)
- if not package_requirements or err:
- self.fatal("There was an error reading test package requirements from %s "
- "requirements: `%s` - error: `%s`" % (source,
- package_requirements or 'None',
- err or 'No error'))
- self.info("Using the following test package requirements:\n%s" %
- pprint.pformat(package_requirements))
- return package_requirements
-
- def _download_test_packages(self, suite_categories, extract_dirs):
- # Some platforms define more suite categories/names than others.
- # This is a difference in the convention of the configs more than
- # to how these tests are run, so we pave over these differences here.
- aliases = {
- 'mochitest-chrome': 'mochitest',
- 'mochitest-media': 'mochitest',
- 'mochitest-plain-clipboard': 'mochitest',
- 'mochitest-plain-gpu': 'mochitest',
- 'mochitest-gl': 'mochitest',
- 'jsreftest': 'reftest',
- 'crashtest': 'reftest',
- 'reftest-debug': 'reftest',
- 'jsreftest-debug': 'reftest',
- 'crashtest-debug': 'reftest',
- }
- suite_categories = [aliases.get(name, name) for name in suite_categories]
-
- dirs = self.query_abs_dirs()
- test_install_dir = dirs.get('abs_test_install_dir',
- os.path.join(dirs['abs_work_dir'], 'tests'))
- self.mkdir_p(test_install_dir)
- package_requirements = self._read_packages_manifest()
- for category in suite_categories:
- if category in package_requirements:
- target_packages = package_requirements[category]
- else:
- # If we don't harness specific requirements, assume the common zip
- # has everything we need to run tests for this suite.
- target_packages = package_requirements['common']
-
- self.info("Downloading packages: %s for test suite category: %s" %
- (target_packages, category))
- for file_name in target_packages:
- target_dir = test_install_dir
- unpack_dirs = extract_dirs
-
- if "common.tests" in file_name and isinstance(unpack_dirs, list):
- # Ensure that the following files are always getting extracted
- required_files = ["mach",
- "mozinfo.json",
- ]
- for req_file in required_files:
- if req_file not in unpack_dirs:
- self.info("Adding '{}' for extraction from common.tests zip file"
- .format(req_file))
- unpack_dirs.append(req_file)
-
- if "jsshell-" in file_name or file_name == "target.jsshell.zip":
- self.info("Special-casing the jsshell zip file")
- unpack_dirs = None
- target_dir = dirs['abs_test_bin_dir']
-
- url = self.query_build_dir_url(file_name)
- self.download_unpack(url, target_dir,
- extract_dirs=unpack_dirs)
-
- def _download_test_zip(self, extract_dirs=None):
- dirs = self.query_abs_dirs()
- test_install_dir = dirs.get('abs_test_install_dir',
- os.path.join(dirs['abs_work_dir'], 'tests'))
- self.download_unpack(self.test_url, test_install_dir,
- extract_dirs=extract_dirs)
-
- def structured_output(self, suite_category):
- """Defines whether structured logging is in use in this configuration. This
- may need to be replaced with data from a different config at the resolution
- of bug 1070041 and related bugs.
- """
- return ('structured_suites' in self.config and
- suite_category in self.config['structured_suites'])
-
- def get_test_output_parser(self, suite_category, strict=False,
- fallback_parser_class=DesktopUnittestOutputParser,
- **kwargs):
- """Derive and return an appropriate output parser, either the structured
- output parser or a fallback based on the type of logging in use as determined by
- configuration.
- """
- if not self.structured_output(suite_category):
- if fallback_parser_class is DesktopUnittestOutputParser:
- return DesktopUnittestOutputParser(suite_category=suite_category, **kwargs)
- return fallback_parser_class(**kwargs)
- self.info("Structured output parser in use for %s." % suite_category)
- return StructuredOutputParser(suite_category=suite_category, strict=strict, **kwargs)
-
- def _download_installer(self):
- file_name = None
- if self.installer_path:
- file_name = self.installer_path
- dirs = self.query_abs_dirs()
- source = self.download_file(self.installer_url,
- file_name=file_name,
- parent_dir=dirs['abs_work_dir'],
- error_level=FATAL)
- self.installer_path = os.path.realpath(source)
- self.set_buildbot_property("build_url", self.installer_url, write_to_file=True)
-
- def _download_and_extract_symbols(self):
- dirs = self.query_abs_dirs()
- if self.config.get('download_symbols') == 'ondemand':
- self.symbols_url = self.query_symbols_url()
- self.symbols_path = self.symbols_url
- return
-
- else:
- # In the case for 'ondemand', we're OK to proceed without getting a hold of the
- # symbols right this moment, however, in other cases we need to at least retry
- # before being unable to proceed (e.g. debug tests need symbols)
- self.symbols_url = self.retry(
- action=self.query_symbols_url,
- kwargs={'raise_on_failure': True},
- sleeptime=20,
- error_level=FATAL,
- error_message="We can't proceed without downloading symbols.",
- )
- if not self.symbols_path:
- self.symbols_path = os.path.join(dirs['abs_work_dir'], 'symbols')
-
- self.set_buildbot_property("symbols_url", self.symbols_url,
- write_to_file=True)
- if self.symbols_url:
- self.download_unpack(self.symbols_url, self.symbols_path)
-
- def download_and_extract(self, extract_dirs=None, suite_categories=None):
- """
- download and extract test zip / download installer
- """
- # Swap plain http for https when we're downloading from ftp
- # See bug 957502 and friends
- from_ = "http://ftp.mozilla.org"
- to_ = "https://ftp-ssl.mozilla.org"
- for attr in 'symbols_url', 'installer_url', 'test_packages_url', 'test_url':
- url = getattr(self, attr)
- if url and url.startswith(from_):
- new_url = url.replace(from_, to_)
- self.info("Replacing url %s -> %s" % (url, new_url))
- setattr(self, attr, new_url)
-
- if 'test_url' in self.config:
- # A user has specified a test_url directly, any test_packages_url will
- # be ignored.
- if self.test_packages_url:
- self.error('Test data will be downloaded from "%s", the specified test '
- ' package data at "%s" will be ignored.' %
- (self.config.get('test_url'), self.test_packages_url))
-
- self._download_test_zip(extract_dirs)
- else:
- if not self.test_packages_url:
- # The caller intends to download harness specific packages, but doesn't know
- # where the packages manifest is located. This is the case when the
- # test package manifest isn't set as a buildbot property, which is true
- # for some self-serve jobs and platforms using parse_make_upload.
- self.test_packages_url = self.query_prefixed_build_dir_url('.test_packages.json')
-
- suite_categories = suite_categories or ['common']
- self._download_test_packages(suite_categories, extract_dirs)
-
- self._download_installer()
- if self.config.get('download_symbols'):
- self._download_and_extract_symbols()
-
- # create_virtualenv is in VirtualenvMixin.
-
- def preflight_install(self):
- if not self.installer_path:
- if self.config.get('installer_path'):
- self.installer_path = self.config['installer_path']
- else:
- self.fatal("""installer_path isn't set!
-
-You can set this by:
-
-1. specifying --installer-path PATH, or
-2. running the download-and-extract action
-""")
- if not self.is_python_package_installed("mozInstall"):
- self.fatal("""Can't call install() without mozinstall!
-Did you run with --create-virtualenv? Is mozinstall in virtualenv_modules?""")
-
- def install_app(self, app=None, target_dir=None, installer_path=None):
- """ Dependent on mozinstall """
- # install the application
- cmd = self.query_exe("mozinstall", default=self.query_python_path("mozinstall"), return_type="list")
- if app:
- cmd.extend(['--app', app])
- # Remove the below when we no longer need to support mozinstall 0.3
- self.info("Detecting whether we're running mozinstall >=1.0...")
- output = self.get_output_from_command(cmd + ['-h'])
- if '--source' in output:
- cmd.append('--source')
- # End remove
- dirs = self.query_abs_dirs()
- if not target_dir:
- target_dir = dirs.get('abs_app_install_dir',
- os.path.join(dirs['abs_work_dir'],
- 'application'))
- self.mkdir_p(target_dir)
- if not installer_path:
- installer_path = self.installer_path
- cmd.extend([installer_path,
- '--destination', target_dir])
- # TODO we'll need some error checking here
- return self.get_output_from_command(cmd, halt_on_failure=True,
- fatal_exit_code=3)
-
- def install(self):
- self.binary_path = self.install_app(app=self.config.get('application'))
-
- def uninstall_app(self, install_dir=None):
- """ Dependent on mozinstall """
- # uninstall the application
- cmd = self.query_exe("mozuninstall",
- default=self.query_python_path("mozuninstall"),
- return_type="list")
- dirs = self.query_abs_dirs()
- if not install_dir:
- install_dir = dirs.get('abs_app_install_dir',
- os.path.join(dirs['abs_work_dir'],
- 'application'))
- cmd.append(install_dir)
- # TODO we'll need some error checking here
- self.get_output_from_command(cmd, halt_on_failure=True,
- fatal_exit_code=3)
-
- def uninstall(self):
- self.uninstall_app()
-
- def query_minidump_tooltool_manifest(self):
- if self.config.get('minidump_tooltool_manifest_path'):
- return self.config['minidump_tooltool_manifest_path']
-
- self.info('Minidump tooltool manifest unknown. Determining based upon '
- 'platform and architecture.')
- platform_name = self.platform_name()
-
- if platform_name:
- tooltool_path = "config/tooltool-manifests/%s/releng.manifest" % \
- TOOLTOOL_PLATFORM_DIR[platform_name]
- return tooltool_path
- else:
- self.fatal('We could not determine the minidump\'s filename.')
-
- def query_minidump_filename(self):
- if self.config.get('minidump_stackwalk_path'):
- return self.config['minidump_stackwalk_path']
-
- self.info('Minidump filename unknown. Determining based upon platform '
- 'and architecture.')
- platform_name = self.platform_name()
- if platform_name:
- minidump_filename = '%s-minidump_stackwalk' % TOOLTOOL_PLATFORM_DIR[platform_name]
- if platform_name in ('win32', 'win64'):
- minidump_filename += '.exe'
- return minidump_filename
- else:
- self.fatal('We could not determine the minidump\'s filename.')
-
- def query_nodejs_tooltool_manifest(self):
- if self.config.get('nodejs_tooltool_manifest_path'):
- return self.config['nodejs_tooltool_manifest_path']
-
- self.info('NodeJS tooltool manifest unknown. Determining based upon '
- 'platform and architecture.')
- platform_name = self.platform_name()
-
- if platform_name:
- tooltool_path = "config/tooltool-manifests/%s/nodejs.manifest" % \
- TOOLTOOL_PLATFORM_DIR[platform_name]
- return tooltool_path
- else:
- self.fatal('Could not determine nodejs manifest filename')
-
- def query_nodejs_filename(self):
- if self.config.get('nodejs_path'):
- return self.config['nodejs_path']
-
- self.fatal('Could not determine nodejs filename')
-
- def query_nodejs(self, manifest=None):
- if self.nodejs_path:
- return self.nodejs_path
-
- c = self.config
- dirs = self.query_abs_dirs();
-
- nodejs_path = self.query_nodejs_filename()
- if not self.config.get('download_nodejs'):
- self.nodejs_path = nodejs_path
- return self.nodejs_path
-
- if not manifest:
- tooltool_manifest_path = self.query_nodejs_tooltool_manifest()
- manifest = os.path.join(dirs.get('abs_test_install_dir',
- os.path.join(dirs['abs_work_dir'], 'tests')),
- tooltool_manifest_path)
-
- self.info('grabbing nodejs binary from tooltool')
- try:
- self.tooltool_fetch(
- manifest=manifest,
- output_dir=dirs['abs_work_dir'],
- cache=c.get('tooltool_cache')
- )
- except KeyError:
- self.error('missing a required key')
-
- abs_nodejs_path = os.path.join(dirs['abs_work_dir'], nodejs_path)
-
- if os.path.exists(abs_nodejs_path):
- if self.platform_name() not in ('win32', 'win64'):
- self.chmod(abs_nodejs_path, 0755)
- self.nodejs_path = abs_nodejs_path
- else:
- self.warning("nodejs path was given but couldn't be found. Tried looking in '%s'" % abs_nodejs_path)
- self.buildbot_status(TBPL_WARNING, WARNING)
-
- return self.nodejs_path
-
- def query_minidump_stackwalk(self, manifest=None):
- if self.minidump_stackwalk_path:
- return self.minidump_stackwalk_path
-
- c = self.config
- dirs = self.query_abs_dirs()
-
- # This is the path where we either download to or is already on the host
- minidump_stackwalk_path = self.query_minidump_filename()
-
- if not c.get('download_minidump_stackwalk'):
- self.minidump_stackwalk_path = minidump_stackwalk_path
- else:
- if not manifest:
- tooltool_manifest_path = self.query_minidump_tooltool_manifest()
- manifest = os.path.join(dirs.get('abs_test_install_dir',
- os.path.join(dirs['abs_work_dir'], 'tests')),
- tooltool_manifest_path)
-
- self.info('grabbing minidump binary from tooltool')
- try:
- self.tooltool_fetch(
- manifest=manifest,
- output_dir=dirs['abs_work_dir'],
- cache=c.get('tooltool_cache')
- )
- except KeyError:
- self.error('missing a required key.')
-
- abs_minidump_path = os.path.join(dirs['abs_work_dir'],
- minidump_stackwalk_path)
- if os.path.exists(abs_minidump_path):
- self.chmod(abs_minidump_path, 0755)
- self.minidump_stackwalk_path = abs_minidump_path
- else:
- self.warning("minidump stackwalk path was given but couldn't be found. "
- "Tried looking in '%s'" % abs_minidump_path)
- # don't burn the job but we should at least turn them orange so it is caught
- self.buildbot_status(TBPL_WARNING, WARNING)
-
- return self.minidump_stackwalk_path
-
- def query_options(self, *args, **kwargs):
- if "str_format_values" in kwargs:
- str_format_values = kwargs.pop("str_format_values")
- else:
- str_format_values = {}
-
- arguments = []
-
- for arg in args:
- if arg is not None:
- arguments.extend(argument % str_format_values for argument in arg)
-
- return arguments
-
- def query_tests_args(self, *args, **kwargs):
- if "str_format_values" in kwargs:
- str_format_values = kwargs.pop("str_format_values")
- else:
- str_format_values = {}
-
- arguments = []
-
- for arg in reversed(args):
- if arg:
- arguments.append("--")
- arguments.extend(argument % str_format_values for argument in arg)
- break
-
- return arguments
-
- def _run_cmd_checks(self, suites):
- if not suites:
- return
- dirs = self.query_abs_dirs()
- for suite in suites:
- # XXX platform.architecture() may give incorrect values for some
- # platforms like mac as excutable files may be universal
- # files containing multiple architectures
- # NOTE 'enabled' is only here while we have unconsolidated configs
- if not suite['enabled']:
- continue
- if suite.get('architectures'):
- arch = platform.architecture()[0]
- if arch not in suite['architectures']:
- continue
- cmd = suite['cmd']
- name = suite['name']
- self.info("Running pre test command %(name)s with '%(cmd)s'"
- % {'name': name, 'cmd': ' '.join(cmd)})
- if self.buildbot_config: # this cmd is for buildbot
- # TODO rather then checking for formatting on every string
- # in every preflight enabled cmd: find a better solution!
- # maybe I can implement WithProperties in mozharness?
- cmd = [x % (self.buildbot_config.get('properties'))
- for x in cmd]
- self.run_command(cmd,
- cwd=dirs['abs_work_dir'],
- error_list=BaseErrorList,
- halt_on_failure=suite['halt_on_failure'],
- fatal_exit_code=suite.get('fatal_exit_code', 3))
-
- def preflight_run_tests(self):
- """preflight commands for all tests"""
- c = self.config
- if c.get('run_cmd_checks_enabled'):
- self._run_cmd_checks(c.get('preflight_run_cmd_suites', []))
- elif c.get('preflight_run_cmd_suites'):
- self.warning("Proceeding without running prerun test commands."
- " These are often OS specific and disabling them may"
- " result in spurious test results!")
-
- def postflight_run_tests(self):
- """preflight commands for all tests"""
- c = self.config
- if c.get('run_cmd_checks_enabled'):
- self._run_cmd_checks(c.get('postflight_run_cmd_suites', []))
diff --git a/testing/mozharness/mozharness/mozilla/testing/try_tools.py b/testing/mozharness/mozharness/mozilla/testing/try_tools.py
deleted file mode 100644
index 3708e71db..000000000
--- a/testing/mozharness/mozharness/mozilla/testing/try_tools.py
+++ /dev/null
@@ -1,258 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-import argparse
-import os
-import re
-from collections import defaultdict
-
-from mozharness.base.script import PostScriptAction
-from mozharness.base.transfer import TransferMixin
-
-try_config_options = [
- [["--try-message"],
- {"action": "store",
- "dest": "try_message",
- "default": None,
- "help": "try syntax string to select tests to run",
- }],
-]
-
-test_flavors = {
- 'browser-chrome': {},
- 'chrome': {},
- 'devtools-chrome': {},
- 'mochitest': {},
- 'xpcshell' :{},
- 'reftest': {
- "path": lambda x: os.path.join("tests", "reftest", "tests", x)
- },
- 'crashtest': {
- "path": lambda x: os.path.join("tests", "reftest", "tests", x)
- },
- 'web-platform-tests': {
- "path": lambda x: os.path.join("tests", x.split("testing" + os.path.sep)[1])
- }
-}
-
-class TryToolsMixin(TransferMixin):
- """Utility functions for an interface between try syntax and out test harnesses.
- Requires log and script mixins."""
-
- harness_extra_args = None
- try_test_paths = {}
- known_try_arguments = {
- '--tag': ({
- 'action': 'append',
- 'dest': 'tags',
- 'default': None,
- }, (
- 'browser-chrome',
- 'chrome',
- 'devtools-chrome',
- 'marionette',
- 'mochitest',
- 'web-plaftform-tests',
- 'xpcshell',
- )),
- '--setenv': ({
- 'action': 'append',
- 'dest': 'setenv',
- 'default': [],
- 'metavar': 'NAME=VALUE',
- }, (
- 'browser-chrome',
- 'chrome',
- 'crashtest',
- 'devtools-chrome',
- 'mochitest',
- 'reftest',
- )),
- }
-
- def _extract_try_message(self):
- msg = None
- buildbot_config = self.buildbot_config or {}
- if "try_message" in self.config and self.config["try_message"]:
- msg = self.config["try_message"]
- elif 'TRY_COMMIT_MSG' in os.environ:
- msg = os.environ['TRY_COMMIT_MSG']
- elif self._is_try():
- if 'sourcestamp' in buildbot_config and buildbot_config['sourcestamp'].get('changes'):
- msg = buildbot_config['sourcestamp']['changes'][-1].get('comments')
-
- if msg is None or len(msg) == 1024:
- # This commit message was potentially truncated or not available in
- # buildbot_config (e.g. if running in TaskCluster), get the full message
- # from hg.
- props = buildbot_config.get('properties', {})
- repo_url = 'https://hg.mozilla.org/%s/'
- if 'revision' in props and 'repo_path' in props:
- rev = props['revision']
- repo_path = props['repo_path']
- else:
- # In TaskCluster we have no buildbot props, rely on env vars instead
- rev = os.environ.get('GECKO_HEAD_REV')
- repo_path = self.config.get('branch')
- if repo_path:
- repo_url = repo_url % repo_path
- else:
- repo_url = os.environ.get('GECKO_HEAD_REPOSITORY',
- repo_url % 'try')
- if not repo_url.endswith('/'):
- repo_url += '/'
-
- url = '{}json-pushes?changeset={}&full=1'.format(repo_url, rev)
-
- pushinfo = self.load_json_from_url(url)
- for k, v in pushinfo.items():
- if isinstance(v, dict) and 'changesets' in v:
- msg = v['changesets'][-1]['desc']
-
- if not msg and 'try_syntax' in buildbot_config.get('properties', {}):
- # If we don't find try syntax in the usual place, check for it in an
- # alternate property available to tools using self-serve.
- msg = buildbot_config['properties']['try_syntax']
- if not msg:
- self.warning('Try message not found.')
- return msg
-
- def _extract_try_args(self, msg):
- """ Returns a list of args from a try message, for parsing """
- if not msg:
- return None
- all_try_args = None
- for line in msg.splitlines():
- if 'try: ' in line:
- # Autoland adds quotes to try strings that will confuse our
- # args later on.
- if line.startswith('"') and line.endswith('"'):
- line = line[1:-1]
- # Allow spaces inside of [filter expressions]
- try_message = line.strip().split('try: ', 1)
- all_try_args = re.findall(r'(?:\[.*?\]|\S)+', try_message[1])
- break
- if not all_try_args:
- self.warning('Try syntax not found in: %s.' % msg )
- return all_try_args
-
- def try_message_has_flag(self, flag, message=None):
- """
- Returns True if --`flag` is present in message.
- """
- parser = argparse.ArgumentParser()
- parser.add_argument('--' + flag, action='store_true')
- message = message or self._extract_try_message()
- if not message:
- return False
- msg_list = self._extract_try_args(message)
- args, _ = parser.parse_known_args(msg_list)
- return getattr(args, flag, False)
-
- def _is_try(self):
- repo_path = None
- if self.buildbot_config and 'properties' in self.buildbot_config:
- repo_path = self.buildbot_config['properties'].get('branch')
- return (self.config.get('branch', repo_path) == 'try' or
- 'TRY_COMMIT_MSG' in os.environ)
-
- @PostScriptAction('download-and-extract')
- def set_extra_try_arguments(self, action, success=None):
- """Finds a commit message and parses it for extra arguments to pass to the test
- harness command line and test paths used to filter manifests.
-
- Extracting arguments from a commit message taken directly from the try_parser.
- """
- if not self._is_try():
- return
-
- msg = self._extract_try_message()
- if not msg:
- return
-
- all_try_args = self._extract_try_args(msg)
- if not all_try_args:
- return
-
- parser = argparse.ArgumentParser(
- description=('Parse an additional subset of arguments passed to try syntax'
- ' and forward them to the underlying test harness command.'))
-
- label_dict = {}
- def label_from_val(val):
- if val in label_dict:
- return label_dict[val]
- return '--%s' % val.replace('_', '-')
-
- for label, (opts, _) in self.known_try_arguments.iteritems():
- if 'action' in opts and opts['action'] not in ('append', 'store',
- 'store_true', 'store_false'):
- self.fatal('Try syntax does not support passing custom or store_const '
- 'arguments to the harness process.')
- if 'dest' in opts:
- label_dict[opts['dest']] = label
-
- parser.add_argument(label, **opts)
-
- parser.add_argument('--try-test-paths', nargs='*')
- (args, _) = parser.parse_known_args(all_try_args)
- self.try_test_paths = self._group_test_paths(args.try_test_paths)
- del args.try_test_paths
-
- out_args = defaultdict(list)
- # This is a pretty hacky way to echo arguments down to the harness.
- # Hopefully this can be improved once we have a configuration system
- # in tree for harnesses that relies less on a command line.
- for arg, value in vars(args).iteritems():
- if value:
- label = label_from_val(arg)
- _, flavors = self.known_try_arguments[label]
-
- for f in flavors:
- if isinstance(value, bool):
- # A store_true or store_false argument.
- out_args[f].append(label)
- elif isinstance(value, list):
- out_args[f].extend(['%s=%s' % (label, el) for el in value])
- else:
- out_args[f].append('%s=%s' % (label, value))
-
- self.harness_extra_args = dict(out_args)
-
- def _group_test_paths(self, args):
- rv = defaultdict(list)
-
- if args is None:
- return rv
-
- for item in args:
- suite, path = item.split(":", 1)
- rv[suite].append(path)
- return rv
-
- def try_args(self, flavor):
- """Get arguments, test_list derived from try syntax to apply to a command"""
- args = []
- if self.harness_extra_args:
- args = self.harness_extra_args.get(flavor, [])[:]
-
- if self.try_test_paths.get(flavor):
- self.info('TinderboxPrint: Tests will be run from the following '
- 'files: %s.' % ','.join(self.try_test_paths[flavor]))
- args.extend(['--this-chunk=1', '--total-chunks=1'])
-
- path_func = test_flavors[flavor].get("path", lambda x:x)
- tests = [path_func(item) for item in self.try_test_paths[flavor]]
- else:
- tests = []
-
- if args or tests:
- self.info('TinderboxPrint: The following arguments were forwarded from mozharness '
- 'to the test command:\nTinderboxPrint: \t%s -- %s' %
- (" ".join(args), " ".join(tests)))
-
- return args, tests
diff --git a/testing/mozharness/mozharness/mozilla/testing/unittest.py b/testing/mozharness/mozharness/mozilla/testing/unittest.py
deleted file mode 100755
index d935ff699..000000000
--- a/testing/mozharness/mozharness/mozilla/testing/unittest.py
+++ /dev/null
@@ -1,262 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-import os
-import re
-
-from mozharness.mozilla.testing.errors import TinderBoxPrintRe
-from mozharness.base.log import OutputParser, WARNING, INFO, CRITICAL, ERROR
-from mozharness.mozilla.buildbot import TBPL_WARNING, TBPL_FAILURE, TBPL_RETRY
-from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE
-
-SUITE_CATEGORIES = ['mochitest', 'reftest', 'xpcshell']
-
-
-def tbox_print_summary(pass_count, fail_count, known_fail_count=None,
- crashed=False, leaked=False):
- emphasize_fail_text = '<em class="testfail">%s</em>'
-
- if pass_count < 0 or fail_count < 0 or \
- (known_fail_count is not None and known_fail_count < 0):
- summary = emphasize_fail_text % 'T-FAIL'
- elif pass_count == 0 and fail_count == 0 and \
- (known_fail_count == 0 or known_fail_count is None):
- summary = emphasize_fail_text % 'T-FAIL'
- else:
- str_fail_count = str(fail_count)
- if fail_count > 0:
- str_fail_count = emphasize_fail_text % str_fail_count
- summary = "%d/%s" % (pass_count, str_fail_count)
- if known_fail_count is not None:
- summary += "/%d" % known_fail_count
- # Format the crash status.
- if crashed:
- summary += "&nbsp;%s" % emphasize_fail_text % "CRASH"
- # Format the leak status.
- if leaked is not False:
- summary += "&nbsp;%s" % emphasize_fail_text % (
- (leaked and "LEAK") or "L-FAIL")
- return summary
-
-
-class TestSummaryOutputParserHelper(OutputParser):
- def __init__(self, regex=re.compile(r'(passed|failed|todo): (\d+)'), **kwargs):
- self.regex = regex
- self.failed = 0
- self.passed = 0
- self.todo = 0
- self.last_line = None
- self.tbpl_status = TBPL_SUCCESS
- self.worst_log_level = INFO
- super(TestSummaryOutputParserHelper, self).__init__(**kwargs)
-
- def parse_single_line(self, line):
- super(TestSummaryOutputParserHelper, self).parse_single_line(line)
- self.last_line = line
- m = self.regex.search(line)
- if m:
- try:
- setattr(self, m.group(1), int(m.group(2)))
- except ValueError:
- # ignore bad values
- pass
-
- def evaluate_parser(self, return_code, success_codes=None):
- if return_code == 0 and self.passed > 0 and self.failed == 0:
- self.tbpl_status = TBPL_SUCCESS
- elif return_code == 10 and self.failed > 0:
- self.tbpl_status = TBPL_WARNING
- else:
- self.tbpl_status = TBPL_FAILURE
- self.worst_log_level = ERROR
-
- return (self.tbpl_status, self.worst_log_level)
-
- def print_summary(self, suite_name):
- # generate the TinderboxPrint line for TBPL
- emphasize_fail_text = '<em class="testfail">%s</em>'
- failed = "0"
- if self.passed == 0 and self.failed == 0:
- self.tsummary = emphasize_fail_text % "T-FAIL"
- else:
- if self.failed > 0:
- failed = emphasize_fail_text % str(self.failed)
- self.tsummary = "%d/%s/%d" % (self.passed, failed, self.todo)
-
- self.info("TinderboxPrint: %s<br/>%s\n" % (suite_name, self.tsummary))
-
- def append_tinderboxprint_line(self, suite_name):
- self.print_summary(suite_name)
-
-
-class DesktopUnittestOutputParser(OutputParser):
- """
- A class that extends OutputParser such that it can parse the number of
- passed/failed/todo tests from the output.
- """
-
- def __init__(self, suite_category, **kwargs):
- # worst_log_level defined already in DesktopUnittestOutputParser
- # but is here to make pylint happy
- self.worst_log_level = INFO
- super(DesktopUnittestOutputParser, self).__init__(**kwargs)
- self.summary_suite_re = TinderBoxPrintRe.get('%s_summary' % suite_category, {})
- self.harness_error_re = TinderBoxPrintRe['harness_error']['minimum_regex']
- self.full_harness_error_re = TinderBoxPrintRe['harness_error']['full_regex']
- self.harness_retry_re = TinderBoxPrintRe['harness_error']['retry_regex']
- self.fail_count = -1
- self.pass_count = -1
- # known_fail_count does not exist for some suites
- self.known_fail_count = self.summary_suite_re.get('known_fail_group') and -1
- self.crashed, self.leaked = False, False
- self.tbpl_status = TBPL_SUCCESS
-
- def parse_single_line(self, line):
- if self.summary_suite_re:
- summary_m = self.summary_suite_re['regex'].match(line) # pass/fail/todo
- if summary_m:
- message = ' %s' % line
- log_level = INFO
- # remove all the none values in groups() so this will work
- # with all suites including mochitest browser-chrome
- summary_match_list = [group for group in summary_m.groups()
- if group is not None]
- r = summary_match_list[0]
- if self.summary_suite_re['pass_group'] in r:
- if len(summary_match_list) > 1:
- self.pass_count = int(summary_match_list[-1])
- else:
- # This handles suites that either pass or report
- # number of failures. We need to set both
- # pass and fail count in the pass case.
- self.pass_count = 1
- self.fail_count = 0
- elif self.summary_suite_re['fail_group'] in r:
- self.fail_count = int(summary_match_list[-1])
- if self.fail_count > 0:
- message += '\n One or more unittests failed.'
- log_level = WARNING
- # If self.summary_suite_re['known_fail_group'] == None,
- # then r should not match it, # so this test is fine as is.
- elif self.summary_suite_re['known_fail_group'] in r:
- self.known_fail_count = int(summary_match_list[-1])
- self.log(message, log_level)
- return # skip harness check and base parse_single_line
- harness_match = self.harness_error_re.match(line)
- if harness_match:
- self.warning(' %s' % line)
- self.worst_log_level = self.worst_level(WARNING, self.worst_log_level)
- self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE)
- full_harness_match = self.full_harness_error_re.match(line)
- if full_harness_match:
- r = full_harness_match.group(1)
- if r == "application crashed":
- self.crashed = True
- elif r == "missing output line for total leaks!":
- self.leaked = None
- else:
- self.leaked = True
- return # skip base parse_single_line
- if self.harness_retry_re.search(line):
- self.critical(' %s' % line)
- self.worst_log_level = self.worst_level(CRITICAL, self.worst_log_level)
- self.tbpl_status = self.worst_level(TBPL_RETRY, self.tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE)
- return # skip base parse_single_line
- super(DesktopUnittestOutputParser, self).parse_single_line(line)
-
- def evaluate_parser(self, return_code, success_codes=None):
- success_codes = success_codes or [0]
-
- if self.num_errors: # mozharness ran into a script error
- self.tbpl_status = self.worst_level(TBPL_FAILURE, self.tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE)
-
- # I have to put this outside of parse_single_line because this checks not
- # only if fail_count was more then 0 but also if fail_count is still -1
- # (no fail summary line was found)
- if self.fail_count != 0:
- self.worst_log_level = self.worst_level(WARNING, self.worst_log_level)
- self.tbpl_status = self.worst_level(TBPL_WARNING, self.tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE)
-
- # Account for the possibility that no test summary was output.
- if self.pass_count <= 0 and self.fail_count <= 0 and \
- (self.known_fail_count is None or self.known_fail_count <= 0):
- self.error('No tests run or test summary not found')
- self.worst_log_level = self.worst_level(WARNING,
- self.worst_log_level)
- self.tbpl_status = self.worst_level(TBPL_WARNING,
- self.tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE)
-
- if return_code not in success_codes:
- self.tbpl_status = self.worst_level(TBPL_FAILURE, self.tbpl_status,
- levels=TBPL_WORST_LEVEL_TUPLE)
-
- # we can trust in parser.worst_log_level in either case
- return (self.tbpl_status, self.worst_log_level)
-
- def append_tinderboxprint_line(self, suite_name):
- # We are duplicating a condition (fail_count) from evaluate_parser and
- # parse parse_single_line but at little cost since we are not parsing
- # the log more then once. I figured this method should stay isolated as
- # it is only here for tbpl highlighted summaries and is not part of
- # buildbot evaluation or result status IIUC.
- summary = tbox_print_summary(self.pass_count,
- self.fail_count,
- self.known_fail_count,
- self.crashed,
- self.leaked)
- self.info("TinderboxPrint: %s<br/>%s\n" % (suite_name, summary))
-
-
-class EmulatorMixin(object):
- """ Currently dependent on both TooltoolMixin and TestingMixin)"""
-
- def install_emulator_from_tooltool(self, manifest_path, do_unzip=True):
- dirs = self.query_abs_dirs()
- if self.tooltool_fetch(manifest_path, output_dir=dirs['abs_work_dir'],
- cache=self.config.get("tooltool_cache", None)
- ):
- self.fatal("Unable to download emulator via tooltool!")
- if do_unzip:
- unzip = self.query_exe("unzip")
- unzip_cmd = [unzip, '-q', os.path.join(dirs['abs_work_dir'], "emulator.zip")]
- self.run_command(unzip_cmd, cwd=dirs['abs_emulator_dir'], halt_on_failure=True,
- fatal_exit_code=3)
-
- def install_emulator(self):
- dirs = self.query_abs_dirs()
- self.mkdir_p(dirs['abs_emulator_dir'])
- if self.config.get('emulator_url'):
- self.download_unpack(self.config['emulator_url'], dirs['abs_emulator_dir'])
- elif self.config.get('emulator_manifest'):
- manifest_path = self.create_tooltool_manifest(self.config['emulator_manifest'])
- do_unzip = True
- if 'unpack' in self.config['emulator_manifest']:
- do_unzip = False
- self.install_emulator_from_tooltool(manifest_path, do_unzip)
- elif self.buildbot_config:
- props = self.buildbot_config.get('properties')
- url = 'https://hg.mozilla.org/%s/raw-file/%s/b2g/test/emulator.manifest' % (
- props['repo_path'], props['revision'])
- manifest_path = self.download_file(url,
- file_name='tooltool.tt',
- parent_dir=dirs['abs_work_dir'])
- if not manifest_path:
- self.fatal("Can't download emulator manifest from %s" % url)
- self.install_emulator_from_tooltool(manifest_path)
- else:
- self.fatal("Can't get emulator; set emulator_url or emulator_manifest in the config!")
- if self.config.get('tools_manifest'):
- manifest_path = self.create_tooltool_manifest(self.config['tools_manifest'])
- do_unzip = True
- if 'unpack' in self.config['tools_manifest']:
- do_unzip = False
- self.install_emulator_from_tooltool(manifest_path, do_unzip)
diff --git a/testing/mozharness/mozharness/mozilla/tooltool.py b/testing/mozharness/mozharness/mozilla/tooltool.py
deleted file mode 100644
index 0bd98e0a2..000000000
--- a/testing/mozharness/mozharness/mozilla/tooltool.py
+++ /dev/null
@@ -1,129 +0,0 @@
-"""module for tooltool operations"""
-import os
-import sys
-
-from mozharness.base.errors import PythonErrorList
-from mozharness.base.log import ERROR, FATAL
-from mozharness.mozilla.proxxy import Proxxy
-
-TooltoolErrorList = PythonErrorList + [{
- 'substr': 'ERROR - ', 'level': ERROR
-}]
-
-
-TOOLTOOL_PY_URL = \
- "https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py"
-
-TOOLTOOL_SERVERS = [
- 'https://api.pub.build.mozilla.org/tooltool/',
-]
-
-
-class TooltoolMixin(object):
- """Mixin class for handling tooltool manifests.
- To use a tooltool server other than the Mozilla server, override
- config['tooltool_servers']. To specify a different authentication
- file than that used in releng automation,override
- config['tooltool_authentication_file']; set it to None to not pass
- any authentication information (OK for public files)
- """
- def _get_auth_file(self):
- # set the default authentication file based on platform; this
- # corresponds to where puppet puts the token
- if 'tooltool_authentication_file' in self.config:
- fn = self.config['tooltool_authentication_file']
- elif self._is_windows():
- fn = r'c:\builds\relengapi.tok'
- else:
- fn = '/builds/relengapi.tok'
-
- # if the file doesn't exist, don't pass it to tooltool (it will just
- # fail). In taskcluster, this will work OK as the relengapi-proxy will
- # take care of auth. Everywhere else, we'll get auth failures if
- # necessary.
- if os.path.exists(fn):
- return fn
-
- def tooltool_fetch(self, manifest,
- output_dir=None, privileged=False, cache=None):
- """docstring for tooltool_fetch"""
- # Use vendored tooltool.py if available.
- if self.topsrcdir:
- cmd = [
- sys.executable,
- os.path.join(self.topsrcdir, 'testing', 'docker', 'recipes',
- 'tooltool.py')
- ]
- elif self.config.get("download_tooltool"):
- cmd = [sys.executable, self._fetch_tooltool_py()]
- else:
- cmd = self.query_exe('tooltool.py', return_type='list')
-
- # get the tooltool servers from configuration
- default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS)
-
- # add slashes (bug 1155630)
- def add_slash(url):
- return url if url.endswith('/') else (url + '/')
- default_urls = [add_slash(u) for u in default_urls]
-
- # proxxy-ify
- proxxy = Proxxy(self.config, self.log_obj)
- proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
-
- for proxyied_url in proxxy_urls:
- cmd.extend(['--url', proxyied_url])
-
- # handle authentication file, if given
- auth_file = self._get_auth_file()
- if auth_file and os.path.exists(auth_file):
- cmd.extend(['--authentication-file', auth_file])
-
- cmd.extend(['fetch', '-m', manifest, '-o'])
-
- if cache:
- cmd.extend(['-c', cache])
-
- # when mock is enabled run tooltool in mock. We can't use
- # run_command_m in all cases because it won't exist unless
- # MockMixin is used on the parent class
- if self.config.get('mock_target'):
- cmd_runner = self.run_command_m
- else:
- cmd_runner = self.run_command
-
- timeout = self.config.get('tooltool_timeout', 10 * 60)
-
- self.retry(
- cmd_runner,
- args=(cmd, ),
- kwargs={'cwd': output_dir,
- 'error_list': TooltoolErrorList,
- 'privileged': privileged,
- 'output_timeout': timeout,
- },
- good_statuses=(0, ),
- error_message="Tooltool %s fetch failed!" % manifest,
- error_level=FATAL,
- )
-
- def _fetch_tooltool_py(self):
- """ Retrieve tooltool.py
- """
- dirs = self.query_abs_dirs()
- file_path = os.path.join(dirs['abs_work_dir'], "tooltool.py")
- self.download_file(TOOLTOOL_PY_URL, file_path)
- if not os.path.exists(file_path):
- self.fatal("We can't get tooltool.py")
- self.chmod(file_path, 0755)
- return file_path
-
- def create_tooltool_manifest(self, contents, path=None):
- """ Currently just creates a manifest, given the contents.
- We may want a template and individual values in the future?
- """
- if path is None:
- dirs = self.query_abs_dirs()
- path = os.path.join(dirs['abs_work_dir'], 'tooltool.tt')
- self.write_to_file(path, contents, error_level=FATAL)
- return path
diff --git a/testing/mozharness/mozharness/mozilla/updates/__init__.py b/testing/mozharness/mozharness/mozilla/updates/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/testing/mozharness/mozharness/mozilla/updates/__init__.py
+++ /dev/null
diff --git a/testing/mozharness/mozharness/mozilla/updates/balrog.py b/testing/mozharness/mozharness/mozilla/updates/balrog.py
deleted file mode 100644
index 26253283c..000000000
--- a/testing/mozharness/mozharness/mozilla/updates/balrog.py
+++ /dev/null
@@ -1,149 +0,0 @@
-from itertools import chain
-import os
-
-from mozharness.base.log import INFO
-
-
-# BalrogMixin {{{1
-class BalrogMixin(object):
- @staticmethod
- def _query_balrog_username(server_config, product=None):
- username = server_config["balrog_usernames"].get(product)
- if username:
- return username
- else:
- raise KeyError("Couldn't find balrog username.")
-
- def generate_balrog_props(self, props_path):
- self.set_buildbot_property(
- "hashType", self.config.get("hash_type", "sha512"), write_to_file=True
- )
-
- if self.buildbot_config and "properties" in self.buildbot_config:
- buildbot_properties = self.buildbot_config["properties"].items()
- else:
- buildbot_properties = []
-
- balrog_props = dict(properties=dict(chain(
- buildbot_properties,
- self.buildbot_properties.items(),
- )))
- if self.config.get('balrog_platform'):
- balrog_props["properties"]["platform"] = self.config['balrog_platform']
- if "branch" not in balrog_props["properties"]:
- balrog_props["properties"]["branch"] = self.branch
-
- self.dump_config(props_path, balrog_props)
-
- def submit_balrog_updates(self, release_type="nightly", product=None):
- c = self.config
- dirs = self.query_abs_dirs()
-
- if self.buildbot_config and "properties" in self.buildbot_config:
- product = self.buildbot_config["properties"]["product"]
-
- if product is None:
- self.fatal('There is no valid product information.')
-
- props_path = os.path.join(dirs["base_work_dir"], "balrog_props.json")
- credentials_file = os.path.join(
- dirs["base_work_dir"], c["balrog_credentials_file"]
- )
- submitter_script = os.path.join(
- dirs["abs_tools_dir"], "scripts", "updates", "balrog-submitter.py"
- )
-
- self.generate_balrog_props(props_path)
-
- cmd = [
- self.query_exe("python"),
- submitter_script,
- "--build-properties", props_path,
- "-t", release_type,
- "--credentials-file", credentials_file,
- ]
- if self._log_level_at_least(INFO):
- cmd.append("--verbose")
-
- return_codes = []
- for server in c["balrog_servers"]:
- server_args = [
- "--api-root", server["balrog_api_root"],
- "--username", self._query_balrog_username(server, product)
- ]
- if server.get("url_replacements"):
- for replacement in server["url_replacements"]:
- server_args.append("--url-replacement")
- server_args.append(",".join(replacement))
-
- self.info("Calling Balrog submission script")
- return_code = self.retry(
- self.run_command, attempts=5, args=(cmd + server_args,),
- good_statuses=(0,),
- )
- if server["ignore_failures"]:
- self.info("Ignoring result, ignore_failures set to True")
- else:
- return_codes.append(return_code)
- # return the worst (max) code
- return max(return_codes)
-
- def submit_balrog_release_pusher(self, dirs):
- product = self.buildbot_config["properties"]["product"]
- cmd = [self.query_exe("python"), os.path.join(os.path.join(dirs['abs_tools_dir'], "scripts/updates/balrog-release-pusher.py"))]
- cmd.extend(["--build-properties", os.path.join(dirs["base_work_dir"], "balrog_props.json")])
- cmd.extend(["--buildbot-configs", "https://hg.mozilla.org/build/buildbot-configs"])
- cmd.extend(["--release-config", os.path.join(dirs['build_dir'], self.config.get("release_config_file"))])
- cmd.extend(["--credentials-file", os.path.join(dirs['base_work_dir'], self.config.get("balrog_credentials_file"))])
- cmd.extend(["--release-channel", self.query_release_config()['release_channel']])
-
- return_codes = []
- for server in self.config["balrog_servers"]:
-
- server_args = [
- "--api-root", server["balrog_api_root"],
- "--username", self._query_balrog_username(server, product)
- ]
-
- self.info("Calling Balrog release pusher script")
- return_code = self.retry(
- self.run_command, args=(cmd + server_args,),
- kwargs={'cwd': dirs['abs_work_dir']},
- good_statuses=(0,),
- )
- if server["ignore_failures"]:
- self.info("Ignoring result, ignore_failures set to True")
- else:
- return_codes.append(return_code)
- # return the worst (max) code
- return max(return_codes)
-
- def lock_balrog_rules(self, rule_ids):
- c = self.config
- dirs = self.query_abs_dirs()
- submitter_script = os.path.join(
- dirs["abs_tools_dir"], "scripts", "updates",
- "balrog-nightly-locker.py"
- )
- credentials_file = os.path.join(
- dirs["base_work_dir"], c["balrog_credentials_file"]
- )
-
- cmd = [
- self.query_exe("python"),
- submitter_script,
- "--credentials-file", credentials_file,
- "--api-root", c["balrog_api_root"],
- "--username", c["balrog_username"],
- ]
- for r in rule_ids:
- cmd.extend(["-r", str(r)])
-
- if self._log_level_at_least(INFO):
- cmd.append("--verbose")
-
- cmd.append("lock")
-
- self.info("Calling Balrog rule locking script.")
- self.retry(self.run_command, attempts=5, args=cmd,
- kwargs={"halt_on_failure": True})
diff --git a/testing/mozharness/mozharness/mozilla/vcstools.py b/testing/mozharness/mozharness/mozilla/vcstools.py
deleted file mode 100644
index b73a4767d..000000000
--- a/testing/mozharness/mozharness/mozilla/vcstools.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""vcstools.py
-
-Author: Armen Zambrano G.
-"""
-import os
-
-from mozharness.base.script import PreScriptAction
-from mozharness.base.vcs.vcsbase import VCSScript
-
-VCS_TOOLS = ('gittool.py',)
-
-
-class VCSToolsScript(VCSScript):
- ''' This script allows us to fetch gittool.py if
- we're running the script on developer mode.
- '''
- @PreScriptAction('checkout')
- def _pre_checkout(self, action):
- if self.config.get('developer_mode'):
- # We put them on base_work_dir to prevent the clobber action
- # to delete them before we use them
- for vcs_tool in VCS_TOOLS:
- file_path = self.query_exe(vcs_tool)
- if not os.path.exists(file_path):
- self.download_file(
- url=self.config[vcs_tool],
- file_name=file_path,
- parent_dir=os.path.dirname(file_path),
- create_parent_dir=True,
- )
- self.chmod(file_path, 0755)
- else:
- # We simply verify that everything is in order
- # or if the user forgot to specify developer mode
- for vcs_tool in VCS_TOOLS:
- file_path = self.which(vcs_tool)
-
- if not file_path:
- file_path = self.query_exe(vcs_tool)
-
- # If the tool is specified and it is a list is
- # because we're running on Windows and we won't check
- if type(self.query_exe(vcs_tool)) is list:
- continue
-
- if file_path is None:
- self.fatal("This machine is missing %s, if this is your "
- "local machine you can use --cfg "
- "developer_config.py" % vcs_tool)
- elif not self.is_exe(file_path):
- self.critical("%s is not executable." % file_path)
diff --git a/testing/mozharness/mozinfo/__init__.py b/testing/mozharness/mozinfo/__init__.py
deleted file mode 100644
index 904dfef71..000000000
--- a/testing/mozharness/mozinfo/__init__.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-"""
-interface to transform introspected system information to a format palatable to
-Mozilla
-
-Module variables:
-
-.. attribute:: bits
-
- 32 or 64
-
-.. attribute:: isBsd
-
- Returns ``True`` if the operating system is BSD
-
-.. attribute:: isLinux
-
- Returns ``True`` if the operating system is Linux
-
-.. attribute:: isMac
-
- Returns ``True`` if the operating system is Mac
-
-.. attribute:: isWin
-
- Returns ``True`` if the operating system is Windows
-
-.. attribute:: os
-
- Operating system [``'win'``, ``'mac'``, ``'linux'``, ...]
-
-.. attribute:: processor
-
- Processor architecture [``'x86'``, ``'x86_64'``, ``'ppc'``, ...]
-
-.. attribute:: version
-
- Operating system version string. For windows, the service pack information is also included
-
-.. attribute:: info
-
- Returns information identifying the current system.
-
- * :attr:`bits`
- * :attr:`os`
- * :attr:`processor`
- * :attr:`version`
-
-"""
-
-import mozinfo
-from mozinfo import *
-__all__ = mozinfo.__all__
diff --git a/testing/mozharness/mozinfo/mozinfo.py b/testing/mozharness/mozinfo/mozinfo.py
deleted file mode 100755
index 718e1a9d7..000000000
--- a/testing/mozharness/mozinfo/mozinfo.py
+++ /dev/null
@@ -1,209 +0,0 @@
-#!/usr/bin/env python
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# TODO: it might be a good idea of adding a system name (e.g. 'Ubuntu' for
-# linux) to the information; I certainly wouldn't want anyone parsing this
-# information and having behaviour depend on it
-
-import json
-import os
-import platform
-import re
-import sys
-
-import mozfile
-
-# keep a copy of the os module since updating globals overrides this
-_os = os
-
-class unknown(object):
- """marker class for unknown information"""
- def __nonzero__(self):
- return False
- def __str__(self):
- return 'UNKNOWN'
-unknown = unknown() # singleton
-
-# get system information
-info = {'os': unknown,
- 'processor': unknown,
- 'version': unknown,
- 'bits': unknown }
-(system, node, release, version, machine, processor) = platform.uname()
-(bits, linkage) = platform.architecture()
-
-# get os information and related data
-if system in ["Microsoft", "Windows"]:
- info['os'] = 'win'
- # There is a Python bug on Windows to determine platform values
- # http://bugs.python.org/issue7860
- if "PROCESSOR_ARCHITEW6432" in os.environ:
- processor = os.environ.get("PROCESSOR_ARCHITEW6432", processor)
- else:
- processor = os.environ.get('PROCESSOR_ARCHITECTURE', processor)
- system = os.environ.get("OS", system).replace('_', ' ')
- service_pack = os.sys.getwindowsversion()[4]
- info['service_pack'] = service_pack
-elif system == "Linux":
- if hasattr(platform, "linux_distribution"):
- (distro, version, codename) = platform.linux_distribution()
- else:
- (distro, version, codename) = platform.dist()
- version = "%s %s" % (distro, version)
- if not processor:
- processor = machine
- info['os'] = 'linux'
-elif system in ['DragonFly', 'FreeBSD', 'NetBSD', 'OpenBSD']:
- info['os'] = 'bsd'
- version = sys.platform
-elif system == "Darwin":
- (release, versioninfo, machine) = platform.mac_ver()
- version = "OS X %s" % release
- info['os'] = 'mac'
-elif sys.platform in ('solaris', 'sunos5'):
- info['os'] = 'unix'
- version = sys.platform
-info['version'] = version # os version
-
-# processor type and bits
-if processor in ["i386", "i686"]:
- if bits == "32bit":
- processor = "x86"
- elif bits == "64bit":
- processor = "x86_64"
-elif processor.upper() == "AMD64":
- bits = "64bit"
- processor = "x86_64"
-elif processor == "Power Macintosh":
- processor = "ppc"
-bits = re.search('(\d+)bit', bits).group(1)
-info.update({'processor': processor,
- 'bits': int(bits),
- })
-
-# standard value of choices, for easy inspection
-choices = {'os': ['linux', 'bsd', 'win', 'mac', 'unix'],
- 'bits': [32, 64],
- 'processor': ['x86', 'x86_64', 'ppc']}
-
-
-def sanitize(info):
- """Do some sanitization of input values, primarily
- to handle universal Mac builds."""
- if "processor" in info and info["processor"] == "universal-x86-x86_64":
- # If we're running on OS X 10.6 or newer, assume 64-bit
- if release[:4] >= "10.6": # Note this is a string comparison
- info["processor"] = "x86_64"
- info["bits"] = 64
- else:
- info["processor"] = "x86"
- info["bits"] = 32
-
-# method for updating information
-def update(new_info):
- """
- Update the info.
-
- :param new_info: Either a dict containing the new info or a path/url
- to a json file containing the new info.
- """
-
- if isinstance(new_info, basestring):
- f = mozfile.load(new_info)
- new_info = json.loads(f.read())
- f.close()
-
- info.update(new_info)
- sanitize(info)
- globals().update(info)
-
- # convenience data for os access
- for os_name in choices['os']:
- globals()['is' + os_name.title()] = info['os'] == os_name
- # unix is special
- if isLinux or isBsd:
- globals()['isUnix'] = True
-
-def find_and_update_from_json(*dirs):
- """
- Find a mozinfo.json file, load it, and update the info with the
- contents.
-
- :param dirs: Directories in which to look for the file. They will be
- searched after first looking in the root of the objdir
- if the current script is being run from a Mozilla objdir.
-
- Returns the full path to mozinfo.json if it was found, or None otherwise.
- """
- # First, see if we're in an objdir
- try:
- from mozbuild.base import MozbuildObject
- build = MozbuildObject.from_environment()
- json_path = _os.path.join(build.topobjdir, "mozinfo.json")
- if _os.path.isfile(json_path):
- update(json_path)
- return json_path
- except ImportError:
- pass
-
- for d in dirs:
- d = _os.path.abspath(d)
- json_path = _os.path.join(d, "mozinfo.json")
- if _os.path.isfile(json_path):
- update(json_path)
- return json_path
-
- return None
-
-update({})
-
-# exports
-__all__ = info.keys()
-__all__ += ['is' + os_name.title() for os_name in choices['os']]
-__all__ += [
- 'info',
- 'unknown',
- 'main',
- 'choices',
- 'update',
- 'find_and_update_from_json',
- ]
-
-def main(args=None):
-
- # parse the command line
- from optparse import OptionParser
- parser = OptionParser(description=__doc__)
- for key in choices:
- parser.add_option('--%s' % key, dest=key,
- action='store_true', default=False,
- help="display choices for %s" % key)
- options, args = parser.parse_args()
-
- # args are JSON blobs to override info
- if args:
- for arg in args:
- if _os.path.exists(arg):
- string = file(arg).read()
- else:
- string = arg
- update(json.loads(string))
-
- # print out choices if requested
- flag = False
- for key, value in options.__dict__.items():
- if value is True:
- print '%s choices: %s' % (key, ' '.join([str(choice)
- for choice in choices[key]]))
- flag = True
- if flag: return
-
- # otherwise, print out all info
- for key, value in info.items():
- print '%s: %s' % (key, value)
-
-if __name__ == '__main__':
- main()
diff --git a/testing/mozharness/mozprocess/__init__.py b/testing/mozharness/mozprocess/__init__.py
deleted file mode 100644
index 6f4ae4945..000000000
--- a/testing/mozharness/mozprocess/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from processhandler import *
diff --git a/testing/mozharness/mozprocess/pid.py b/testing/mozharness/mozprocess/pid.py
deleted file mode 100755
index d1f0d9336..000000000
--- a/testing/mozharness/mozprocess/pid.py
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/usr/bin/env python
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import os
-import mozinfo
-import shlex
-import subprocess
-import sys
-
-# determine the platform-specific invocation of `ps`
-if mozinfo.isMac:
- psarg = '-Acj'
-elif mozinfo.isLinux:
- psarg = 'axwww'
-else:
- psarg = 'ax'
-
-def ps(arg=psarg):
- """
- python front-end to `ps`
- http://en.wikipedia.org/wiki/Ps_%28Unix%29
- returns a list of process dicts based on the `ps` header
- """
- retval = []
- process = subprocess.Popen(['ps', arg], stdout=subprocess.PIPE)
- stdout, _ = process.communicate()
- header = None
- for line in stdout.splitlines():
- line = line.strip()
- if header is None:
- # first line is the header
- header = line.split()
- continue
- split = line.split(None, len(header)-1)
- process_dict = dict(zip(header, split))
- retval.append(process_dict)
- return retval
-
-def running_processes(name, psarg=psarg, defunct=True):
- """
- returns a list of
- {'PID': PID of process (int)
- 'command': command line of process (list)}
- with the executable named `name`.
- - defunct: whether to return defunct processes
- """
- retval = []
- for process in ps(psarg):
- # Support for both BSD and UNIX syntax
- # `ps aux` returns COMMAND, `ps -ef` returns CMD
- try:
- command = process['COMMAND']
- except KeyError:
- command = process['CMD']
-
- command = shlex.split(command)
- if command[-1] == '<defunct>':
- command = command[:-1]
- if not command or not defunct:
- continue
- if 'STAT' in process and not defunct:
- if process['STAT'] == 'Z+':
- continue
- prog = command[0]
- basename = os.path.basename(prog)
- if basename == name:
- retval.append((int(process['PID']), command))
- return retval
-
-def get_pids(name):
- """Get all the pids matching name"""
-
- if mozinfo.isWin:
- # use the windows-specific implementation
- import wpk
- return wpk.get_pids(name)
- else:
- return [pid for pid,_ in running_processes(name)]
-
-if __name__ == '__main__':
- pids = set()
- for i in sys.argv[1:]:
- pids.update(get_pids(i))
- for i in sorted(pids):
- print i
diff --git a/testing/mozharness/mozprocess/processhandler.py b/testing/mozharness/mozprocess/processhandler.py
deleted file mode 100644
index b89e17eb0..000000000
--- a/testing/mozharness/mozprocess/processhandler.py
+++ /dev/null
@@ -1,921 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import os
-import select
-import signal
-import subprocess
-import sys
-import threading
-import time
-import traceback
-from Queue import Queue
-from datetime import datetime, timedelta
-__all__ = ['ProcessHandlerMixin', 'ProcessHandler']
-
-# Set the MOZPROCESS_DEBUG environment variable to 1 to see some debugging output
-MOZPROCESS_DEBUG = os.getenv("MOZPROCESS_DEBUG")
-
-# We dont use mozinfo because it is expensive to import, see bug 933558.
-isWin = os.name == "nt"
-isPosix = os.name == "posix" # includes MacOS X
-
-if isWin:
- import ctypes, ctypes.wintypes, msvcrt
- from ctypes import sizeof, addressof, c_ulong, byref, POINTER, WinError, c_longlong
- import winprocess
- from qijo import JobObjectAssociateCompletionPortInformation,\
- JOBOBJECT_ASSOCIATE_COMPLETION_PORT, JobObjectExtendedLimitInformation,\
- JOBOBJECT_BASIC_LIMIT_INFORMATION, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, IO_COUNTERS
-
-class ProcessHandlerMixin(object):
- """
- A class for launching and manipulating local processes.
-
- :param cmd: command to run. May be a string or a list. If specified as a list, the first element will be interpreted as the command, and all additional elements will be interpreted as arguments to that command.
- :param args: list of arguments to pass to the command (defaults to None). Must not be set when `cmd` is specified as a list.
- :param cwd: working directory for command (defaults to None).
- :param env: is the environment to use for the process (defaults to os.environ).
- :param ignore_children: causes system to ignore child processes when True, defaults to False (which tracks child processes).
- :param kill_on_timeout: when True, the process will be killed when a timeout is reached. When False, the caller is responsible for killing the process. Failure to do so could cause a call to wait() to hang indefinitely. (Defaults to True.)
- :param processOutputLine: function to be called for each line of output produced by the process (defaults to None).
- :param onTimeout: function to be called when the process times out.
- :param onFinish: function to be called when the process terminates normally without timing out.
- :param kwargs: additional keyword args to pass directly into Popen.
-
- NOTE: Child processes will be tracked by default. If for any reason
- we are unable to track child processes and ignore_children is set to False,
- then we will fall back to only tracking the root process. The fallback
- will be logged.
- """
-
- class Process(subprocess.Popen):
- """
- Represents our view of a subprocess.
- It adds a kill() method which allows it to be stopped explicitly.
- """
-
- MAX_IOCOMPLETION_PORT_NOTIFICATION_DELAY = 180
- MAX_PROCESS_KILL_DELAY = 30
-
- def __init__(self,
- args,
- bufsize=0,
- executable=None,
- stdin=None,
- stdout=None,
- stderr=None,
- preexec_fn=None,
- close_fds=False,
- shell=False,
- cwd=None,
- env=None,
- universal_newlines=False,
- startupinfo=None,
- creationflags=0,
- ignore_children=False):
-
- # Parameter for whether or not we should attempt to track child processes
- self._ignore_children = ignore_children
-
- if not self._ignore_children and not isWin:
- # Set the process group id for linux systems
- # Sets process group id to the pid of the parent process
- # NOTE: This prevents you from using preexec_fn and managing
- # child processes, TODO: Ideally, find a way around this
- def setpgidfn():
- os.setpgid(0, 0)
- preexec_fn = setpgidfn
-
- try:
- subprocess.Popen.__init__(self, args, bufsize, executable,
- stdin, stdout, stderr,
- preexec_fn, close_fds,
- shell, cwd, env,
- universal_newlines, startupinfo, creationflags)
- except OSError, e:
- print >> sys.stderr, args
- raise
-
- def __del__(self, _maxint=sys.maxint):
- if isWin:
- if self._handle:
- if hasattr(self, '_internal_poll'):
- self._internal_poll(_deadstate=_maxint)
- else:
- self.poll(_deadstate=sys.maxint)
- if self._handle or self._job or self._io_port:
- self._cleanup()
- else:
- subprocess.Popen.__del__(self)
-
- def kill(self, sig=None):
- self.returncode = 0
- if isWin:
- if not self._ignore_children and self._handle and self._job:
- winprocess.TerminateJobObject(self._job, winprocess.ERROR_CONTROL_C_EXIT)
- self.returncode = winprocess.GetExitCodeProcess(self._handle)
- elif self._handle:
- err = None
- try:
- winprocess.TerminateProcess(self._handle, winprocess.ERROR_CONTROL_C_EXIT)
- except:
- err = "Could not terminate process"
- self.returncode = winprocess.GetExitCodeProcess(self._handle)
- self._cleanup()
- if err is not None:
- raise OSError(err)
- else:
- sig = sig or signal.SIGKILL
- if not self._ignore_children:
- try:
- os.killpg(self.pid, sig)
- except BaseException, e:
- if getattr(e, "errno", None) != 3:
- # Error 3 is "no such process", which is ok
- print >> sys.stdout, "Could not kill process, could not find pid: %s, assuming it's already dead" % self.pid
- else:
- os.kill(self.pid, sig)
- self.returncode = -sig
-
- self._cleanup()
- return self.returncode
-
- def wait(self):
- """ Popen.wait
- Called to wait for a running process to shut down and return
- its exit code
- Returns the main process's exit code
- """
- # This call will be different for each OS
- self.returncode = self._wait()
- self._cleanup()
- return self.returncode
-
- """ Private Members of Process class """
-
- if isWin:
- # Redefine the execute child so that we can track process groups
- def _execute_child(self, *args_tuple):
- # workaround for bug 950894
- if sys.hexversion < 0x02070600: # prior to 2.7.6
- (args, executable, preexec_fn, close_fds,
- cwd, env, universal_newlines, startupinfo,
- creationflags, shell,
- p2cread, p2cwrite,
- c2pread, c2pwrite,
- errread, errwrite) = args_tuple
- to_close = set()
- else: # 2.7.6 and later
- (args, executable, preexec_fn, close_fds,
- cwd, env, universal_newlines, startupinfo,
- creationflags, shell, to_close,
- p2cread, p2cwrite,
- c2pread, c2pwrite,
- errread, errwrite) = args_tuple
- if not isinstance(args, basestring):
- args = subprocess.list2cmdline(args)
-
- # Always or in the create new process group
- creationflags |= winprocess.CREATE_NEW_PROCESS_GROUP
-
- if startupinfo is None:
- startupinfo = winprocess.STARTUPINFO()
-
- if None not in (p2cread, c2pwrite, errwrite):
- startupinfo.dwFlags |= winprocess.STARTF_USESTDHANDLES
- startupinfo.hStdInput = int(p2cread)
- startupinfo.hStdOutput = int(c2pwrite)
- startupinfo.hStdError = int(errwrite)
- if shell:
- startupinfo.dwFlags |= winprocess.STARTF_USESHOWWINDOW
- startupinfo.wShowWindow = winprocess.SW_HIDE
- comspec = os.environ.get("COMSPEC", "cmd.exe")
- args = comspec + " /c " + args
-
- # determine if we can create create a job
- canCreateJob = winprocess.CanCreateJobObject()
-
- # Ensure we write a warning message if we are falling back
- if not canCreateJob and not self._ignore_children:
- # We can't create job objects AND the user wanted us to
- # Warn the user about this.
- print >> sys.stderr, "ProcessManager UNABLE to use job objects to manage child processes"
-
- # set process creation flags
- creationflags |= winprocess.CREATE_SUSPENDED
- creationflags |= winprocess.CREATE_UNICODE_ENVIRONMENT
- if canCreateJob:
- creationflags |= winprocess.CREATE_BREAKAWAY_FROM_JOB
- else:
- # Since we've warned, we just log info here to inform you
- # of the consequence of setting ignore_children = True
- print "ProcessManager NOT managing child processes"
-
- # create the process
- hp, ht, pid, tid = winprocess.CreateProcess(
- executable, args,
- None, None, # No special security
- 1, # Must inherit handles!
- creationflags,
- winprocess.EnvironmentBlock(env),
- cwd, startupinfo)
- self._child_created = True
- self._handle = hp
- self._thread = ht
- self.pid = pid
- self.tid = tid
-
- if not self._ignore_children and canCreateJob:
- try:
- # We create a new job for this process, so that we can kill
- # the process and any sub-processes
- # Create the IO Completion Port
- self._io_port = winprocess.CreateIoCompletionPort()
- self._job = winprocess.CreateJobObject()
-
- # Now associate the io comp port and the job object
- joacp = JOBOBJECT_ASSOCIATE_COMPLETION_PORT(winprocess.COMPKEY_JOBOBJECT,
- self._io_port)
- winprocess.SetInformationJobObject(self._job,
- JobObjectAssociateCompletionPortInformation,
- addressof(joacp),
- sizeof(joacp)
- )
-
- # Allow subprocesses to break away from us - necessary for
- # flash with protected mode
- jbli = JOBOBJECT_BASIC_LIMIT_INFORMATION(
- c_longlong(0), # per process time limit (ignored)
- c_longlong(0), # per job user time limit (ignored)
- winprocess.JOB_OBJECT_LIMIT_BREAKAWAY_OK,
- 0, # min working set (ignored)
- 0, # max working set (ignored)
- 0, # active process limit (ignored)
- None, # affinity (ignored)
- 0, # Priority class (ignored)
- 0, # Scheduling class (ignored)
- )
-
- iocntr = IO_COUNTERS()
- jeli = JOBOBJECT_EXTENDED_LIMIT_INFORMATION(
- jbli, # basic limit info struct
- iocntr, # io_counters (ignored)
- 0, # process mem limit (ignored)
- 0, # job mem limit (ignored)
- 0, # peak process limit (ignored)
- 0) # peak job limit (ignored)
-
- winprocess.SetInformationJobObject(self._job,
- JobObjectExtendedLimitInformation,
- addressof(jeli),
- sizeof(jeli)
- )
-
- # Assign the job object to the process
- winprocess.AssignProcessToJobObject(self._job, int(hp))
-
- # It's overkill, but we use Queue to signal between threads
- # because it handles errors more gracefully than event or condition.
- self._process_events = Queue()
-
- # Spin up our thread for managing the IO Completion Port
- self._procmgrthread = threading.Thread(target = self._procmgr)
- except:
- print >> sys.stderr, """Exception trying to use job objects;
-falling back to not using job objects for managing child processes"""
- tb = traceback.format_exc()
- print >> sys.stderr, tb
- # Ensure no dangling handles left behind
- self._cleanup_job_io_port()
- else:
- self._job = None
-
- winprocess.ResumeThread(int(ht))
- if getattr(self, '_procmgrthread', None):
- self._procmgrthread.start()
- ht.Close()
-
- for i in (p2cread, c2pwrite, errwrite):
- if i is not None:
- i.Close()
-
- # Windows Process Manager - watches the IO Completion Port and
- # keeps track of child processes
- def _procmgr(self):
- if not (self._io_port) or not (self._job):
- return
-
- try:
- self._poll_iocompletion_port()
- except KeyboardInterrupt:
- raise KeyboardInterrupt
-
- def _poll_iocompletion_port(self):
- # Watch the IO Completion port for status
- self._spawned_procs = {}
- countdowntokill = 0
-
- if MOZPROCESS_DEBUG:
- print "DBG::MOZPROC Self.pid value is: %s" % self.pid
-
- while True:
- msgid = c_ulong(0)
- compkey = c_ulong(0)
- pid = c_ulong(0)
- portstatus = winprocess.GetQueuedCompletionStatus(self._io_port,
- byref(msgid),
- byref(compkey),
- byref(pid),
- 5000)
-
- # If the countdowntokill has been activated, we need to check
- # if we should start killing the children or not.
- if countdowntokill != 0:
- diff = datetime.now() - countdowntokill
- # Arbitrarily wait 3 minutes for windows to get its act together
- # Windows sometimes takes a small nap between notifying the
- # IO Completion port and actually killing the children, and we
- # don't want to mistake that situation for the situation of an unexpected
- # parent abort (which is what we're looking for here).
- if diff.seconds > self.MAX_IOCOMPLETION_PORT_NOTIFICATION_DELAY:
- print >> sys.stderr, "Parent process %s exited with children alive:" % self.pid
- print >> sys.stderr, "PIDS: %s" % ', '.join([str(i) for i in self._spawned_procs])
- print >> sys.stderr, "Attempting to kill them..."
- self.kill()
- self._process_events.put({self.pid: 'FINISHED'})
-
- if not portstatus:
- # Check to see what happened
- errcode = winprocess.GetLastError()
- if errcode == winprocess.ERROR_ABANDONED_WAIT_0:
- # Then something has killed the port, break the loop
- print >> sys.stderr, "IO Completion Port unexpectedly closed"
- break
- elif errcode == winprocess.WAIT_TIMEOUT:
- # Timeouts are expected, just keep on polling
- continue
- else:
- print >> sys.stderr, "Error Code %s trying to query IO Completion Port, exiting" % errcode
- raise WinError(errcode)
- break
-
- if compkey.value == winprocess.COMPKEY_TERMINATE.value:
- if MOZPROCESS_DEBUG:
- print "DBG::MOZPROC compkeyterminate detected"
- # Then we're done
- break
-
- # Check the status of the IO Port and do things based on it
- if compkey.value == winprocess.COMPKEY_JOBOBJECT.value:
- if msgid.value == winprocess.JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO:
- # No processes left, time to shut down
- # Signal anyone waiting on us that it is safe to shut down
- if MOZPROCESS_DEBUG:
- print "DBG::MOZPROC job object msg active processes zero"
- self._process_events.put({self.pid: 'FINISHED'})
- break
- elif msgid.value == winprocess.JOB_OBJECT_MSG_NEW_PROCESS:
- # New Process started
- # Add the child proc to our list in case our parent flakes out on us
- # without killing everything.
- if pid.value != self.pid:
- self._spawned_procs[pid.value] = 1
- if MOZPROCESS_DEBUG:
- print "DBG::MOZPROC new process detected with pid value: %s" % pid.value
- elif msgid.value == winprocess.JOB_OBJECT_MSG_EXIT_PROCESS:
- if MOZPROCESS_DEBUG:
- print "DBG::MOZPROC process id %s exited normally" % pid.value
- # One process exited normally
- if pid.value == self.pid and len(self._spawned_procs) > 0:
- # Parent process dying, start countdown timer
- countdowntokill = datetime.now()
- elif pid.value in self._spawned_procs:
- # Child Process died remove from list
- del(self._spawned_procs[pid.value])
- elif msgid.value == winprocess.JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS:
- # One process existed abnormally
- if MOZPROCESS_DEBUG:
- print "DBG::MOZPROC process id %s existed abnormally" % pid.value
- if pid.value == self.pid and len(self._spawned_procs) > 0:
- # Parent process dying, start countdown timer
- countdowntokill = datetime.now()
- elif pid.value in self._spawned_procs:
- # Child Process died remove from list
- del self._spawned_procs[pid.value]
- else:
- # We don't care about anything else
- if MOZPROCESS_DEBUG:
- print "DBG::MOZPROC We got a message %s" % msgid.value
- pass
-
- def _wait(self):
-
- # First, check to see if the process is still running
- if self._handle:
- self.returncode = winprocess.GetExitCodeProcess(self._handle)
- else:
- # Dude, the process is like totally dead!
- return self.returncode
-
- # Python 2.5 uses isAlive versus is_alive use the proper one
- threadalive = False
- if hasattr(self, "_procmgrthread"):
- if hasattr(self._procmgrthread, 'is_alive'):
- threadalive = self._procmgrthread.is_alive()
- else:
- threadalive = self._procmgrthread.isAlive()
- if self._job and threadalive:
- # Then we are managing with IO Completion Ports
- # wait on a signal so we know when we have seen the last
- # process come through.
- # We use queues to synchronize between the thread and this
- # function because events just didn't have robust enough error
- # handling on pre-2.7 versions
- err = None
- try:
- # timeout is the max amount of time the procmgr thread will wait for
- # child processes to shutdown before killing them with extreme prejudice.
- item = self._process_events.get(timeout=self.MAX_IOCOMPLETION_PORT_NOTIFICATION_DELAY +
- self.MAX_PROCESS_KILL_DELAY)
- if item[self.pid] == 'FINISHED':
- self._process_events.task_done()
- except:
- err = "IO Completion Port failed to signal process shutdown"
- # Either way, let's try to get this code
- if self._handle:
- self.returncode = winprocess.GetExitCodeProcess(self._handle)
- self._cleanup()
-
- if err is not None:
- raise OSError(err)
-
-
- else:
- # Not managing with job objects, so all we can reasonably do
- # is call waitforsingleobject and hope for the best
-
- if MOZPROCESS_DEBUG and not self._ignore_children:
- print "DBG::MOZPROC NOT USING JOB OBJECTS!!!"
- # First, make sure we have not already ended
- if self.returncode != winprocess.STILL_ACTIVE:
- self._cleanup()
- return self.returncode
-
- rc = None
- if self._handle:
- rc = winprocess.WaitForSingleObject(self._handle, -1)
-
- if rc == winprocess.WAIT_TIMEOUT:
- # The process isn't dead, so kill it
- print "Timed out waiting for process to close, attempting TerminateProcess"
- self.kill()
- elif rc == winprocess.WAIT_OBJECT_0:
- # We caught WAIT_OBJECT_0, which indicates all is well
- print "Single process terminated successfully"
- self.returncode = winprocess.GetExitCodeProcess(self._handle)
- else:
- # An error occured we should probably throw
- rc = winprocess.GetLastError()
- if rc:
- raise WinError(rc)
-
- self._cleanup()
-
- return self.returncode
-
- def _cleanup_job_io_port(self):
- """ Do the job and IO port cleanup separately because there are
- cases where we want to clean these without killing _handle
- (i.e. if we fail to create the job object in the first place)
- """
- if getattr(self, '_job') and self._job != winprocess.INVALID_HANDLE_VALUE:
- self._job.Close()
- self._job = None
- else:
- # If windows already freed our handle just set it to none
- # (saw this intermittently while testing)
- self._job = None
-
- if getattr(self, '_io_port', None) and self._io_port != winprocess.INVALID_HANDLE_VALUE:
- self._io_port.Close()
- self._io_port = None
- else:
- self._io_port = None
-
- if getattr(self, '_procmgrthread', None):
- self._procmgrthread = None
-
- def _cleanup(self):
- self._cleanup_job_io_port()
- if self._thread and self._thread != winprocess.INVALID_HANDLE_VALUE:
- self._thread.Close()
- self._thread = None
- else:
- self._thread = None
-
- if self._handle and self._handle != winprocess.INVALID_HANDLE_VALUE:
- self._handle.Close()
- self._handle = None
- else:
- self._handle = None
-
- elif isPosix:
-
- def _wait(self):
- """ Haven't found any reason to differentiate between these platforms
- so they all use the same wait callback. If it is necessary to
- craft different styles of wait, then a new _wait method
- could be easily implemented.
- """
-
- if not self._ignore_children:
- try:
- # os.waitpid return value:
- # > [...] a tuple containing its pid and exit status
- # > indication: a 16-bit number, whose low byte is the
- # > signal number that killed the process, and whose
- # > high byte is the exit status (if the signal number
- # > is zero)
- # - http://docs.python.org/2/library/os.html#os.wait
- status = os.waitpid(self.pid, 0)[1]
-
- # For consistency, format status the same as subprocess'
- # returncode attribute
- if status > 255:
- return status >> 8
- return -status
- except OSError, e:
- if getattr(e, "errno", None) != 10:
- # Error 10 is "no child process", which could indicate normal
- # close
- print >> sys.stderr, "Encountered error waiting for pid to close: %s" % e
- raise
- return 0
-
- else:
- # For non-group wait, call base class
- subprocess.Popen.wait(self)
- return self.returncode
-
- def _cleanup(self):
- pass
-
- else:
- # An unrecognized platform, we will call the base class for everything
- print >> sys.stderr, "Unrecognized platform, process groups may not be managed properly"
-
- def _wait(self):
- self.returncode = subprocess.Popen.wait(self)
- return self.returncode
-
- def _cleanup(self):
- pass
-
- def __init__(self,
- cmd,
- args=None,
- cwd=None,
- env=None,
- ignore_children = False,
- kill_on_timeout = True,
- processOutputLine=(),
- onTimeout=(),
- onFinish=(),
- **kwargs):
- self.cmd = cmd
- self.args = args
- self.cwd = cwd
- self.didTimeout = False
- self._ignore_children = ignore_children
- self._kill_on_timeout = kill_on_timeout
- self.keywordargs = kwargs
- self.outThread = None
- self.read_buffer = ''
-
- if env is None:
- env = os.environ.copy()
- self.env = env
-
- # handlers
- self.processOutputLineHandlers = list(processOutputLine)
- self.onTimeoutHandlers = list(onTimeout)
- self.onFinishHandlers = list(onFinish)
-
- # It is common for people to pass in the entire array with the cmd and
- # the args together since this is how Popen uses it. Allow for that.
- if isinstance(self.cmd, list):
- if self.args != None:
- raise TypeError("cmd and args must not both be lists")
- (self.cmd, self.args) = (self.cmd[0], self.cmd[1:])
- elif self.args is None:
- self.args = []
-
- @property
- def timedOut(self):
- """True if the process has timed out."""
- return self.didTimeout
-
- @property
- def commandline(self):
- """the string value of the command line (command + args)"""
- return subprocess.list2cmdline([self.cmd] + self.args)
-
- def run(self, timeout=None, outputTimeout=None):
- """
- Starts the process.
-
- If timeout is not None, the process will be allowed to continue for
- that number of seconds before being killed. If the process is killed
- due to a timeout, the onTimeout handler will be called.
-
- If outputTimeout is not None, the process will be allowed to continue
- for that number of seconds without producing any output before
- being killed.
- """
- self.didTimeout = False
- self.startTime = datetime.now()
-
- # default arguments
- args = dict(stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- cwd=self.cwd,
- env=self.env,
- ignore_children=self._ignore_children)
-
- # build process arguments
- args.update(self.keywordargs)
-
- # launch the process
- self.proc = self.Process([self.cmd] + self.args, **args)
-
- self.processOutput(timeout=timeout, outputTimeout=outputTimeout)
-
- def kill(self, sig=None):
- """
- Kills the managed process.
-
- If you created the process with 'ignore_children=False' (the
- default) then it will also also kill all child processes spawned by
- it. If you specified 'ignore_children=True' when creating the
- process, only the root process will be killed.
-
- Note that this does not manage any state, save any output etc,
- it immediately kills the process.
-
- :param sig: Signal used to kill the process, defaults to SIGKILL
- (has no effect on Windows)
- """
- try:
- return self.proc.kill(sig=sig)
- except AttributeError:
- # Try to print a relevant error message.
- if not self.proc:
- print >> sys.stderr, "Unable to kill Process because call to ProcessHandler constructor failed."
- else:
- raise
-
- def readWithTimeout(self, f, timeout):
- """
- Try to read a line of output from the file object *f*.
-
- *f* must be a pipe, like the *stdout* member of a subprocess.Popen
- object created with stdout=PIPE. If no output
- is received within *timeout* seconds, return a blank line.
-
- Returns a tuple (line, did_timeout), where *did_timeout* is True
- if the read timed out, and False otherwise.
- """
- # Calls a private member because this is a different function based on
- # the OS
- return self._readWithTimeout(f, timeout)
-
- def processOutputLine(self, line):
- """Called for each line of output that a process sends to stdout/stderr."""
- for handler in self.processOutputLineHandlers:
- handler(line)
-
- def onTimeout(self):
- """Called when a process times out."""
- for handler in self.onTimeoutHandlers:
- handler()
-
- def onFinish(self):
- """Called when a process finishes without a timeout."""
- for handler in self.onFinishHandlers:
- handler()
-
- def processOutput(self, timeout=None, outputTimeout=None):
- """
- Handle process output until the process terminates or times out.
-
- If timeout is not None, the process will be allowed to continue for
- that number of seconds before being killed.
-
- If outputTimeout is not None, the process will be allowed to continue
- for that number of seconds without producing any output before
- being killed.
- """
- def _processOutput():
- self.didTimeout = False
- logsource = self.proc.stdout
-
- lineReadTimeout = None
- if timeout:
- lineReadTimeout = timeout - (datetime.now() - self.startTime).seconds
- elif outputTimeout:
- lineReadTimeout = outputTimeout
-
- (lines, self.didTimeout) = self.readWithTimeout(logsource, lineReadTimeout)
- while lines != "":
- for line in lines.splitlines():
- self.processOutputLine(line.rstrip())
-
- if self.didTimeout:
- break
-
- if timeout:
- lineReadTimeout = timeout - (datetime.now() - self.startTime).seconds
- (lines, self.didTimeout) = self.readWithTimeout(logsource, lineReadTimeout)
-
- if self.didTimeout:
- if self._kill_on_timeout:
- self.proc.kill()
- self.onTimeout()
- else:
- self.onFinish()
-
- if not hasattr(self, 'proc'):
- self.run()
-
- if not self.outThread:
- self.outThread = threading.Thread(target=_processOutput)
- self.outThread.daemon = True
- self.outThread.start()
-
-
- def wait(self, timeout=None):
- """
- Waits until all output has been read and the process is
- terminated.
-
- If timeout is not None, will return after timeout seconds.
- This timeout only causes the wait function to return and
- does not kill the process.
-
- Returns the process' exit code. A None value indicates the
- process hasn't terminated yet. A negative value -N indicates
- the process was killed by signal N (Unix only).
- """
- if self.outThread:
- # Thread.join() blocks the main thread until outThread is finished
- # wake up once a second in case a keyboard interrupt is sent
- count = 0
- while self.outThread.isAlive():
- self.outThread.join(timeout=1)
- count += 1
- if timeout and count > timeout:
- return None
-
- return self.proc.wait()
-
- # TODO Remove this method when consumers have been fixed
- def waitForFinish(self, timeout=None):
- print >> sys.stderr, "MOZPROCESS WARNING: ProcessHandler.waitForFinish() is deprecated, " \
- "use ProcessHandler.wait() instead"
- return self.wait(timeout=timeout)
-
-
- ### Private methods from here on down. Thar be dragons.
-
- if isWin:
- # Windows Specific private functions are defined in this block
- PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe
- GetLastError = ctypes.windll.kernel32.GetLastError
-
- def _readWithTimeout(self, f, timeout):
- if timeout is None:
- # shortcut to allow callers to pass in "None" for no timeout.
- return (f.readline(), False)
- x = msvcrt.get_osfhandle(f.fileno())
- l = ctypes.c_long()
- done = time.time() + timeout
- while time.time() < done:
- if self.PeekNamedPipe(x, None, 0, None, ctypes.byref(l), None) == 0:
- err = self.GetLastError()
- if err == 38 or err == 109: # ERROR_HANDLE_EOF || ERROR_BROKEN_PIPE
- return ('', False)
- else:
- raise OSError("readWithTimeout got error: %d", err)
- if l.value > 0:
- # we're assuming that the output is line-buffered,
- # which is not unreasonable
- return (f.readline(), False)
- time.sleep(0.01)
- return ('', True)
-
- else:
- # Generic
- def _readWithTimeout(self, f, timeout):
- while True:
- try:
- (r, w, e) = select.select([f], [], [], timeout)
- except:
- # return a blank line
- return ('', True)
-
- if len(r) == 0:
- return ('', True)
-
- output = os.read(f.fileno(), 4096)
- if not output:
- output = self.read_buffer
- self.read_buffer = ''
- return (output, False)
- self.read_buffer += output
- if '\n' not in self.read_buffer:
- time.sleep(0.01)
- continue
- tmp = self.read_buffer.split('\n')
- lines, self.read_buffer = tmp[:-1], tmp[-1]
- real_lines = [x for x in lines if x != '']
- if not real_lines:
- time.sleep(0.01)
- continue
- break
- return ('\n'.join(lines), False)
-
- @property
- def pid(self):
- return self.proc.pid
-
-
-### default output handlers
-### these should be callables that take the output line
-
-def print_output(line):
- print line
-
-class StoreOutput(object):
- """accumulate stdout"""
-
- def __init__(self):
- self.output = []
-
- def __call__(self, line):
- self.output.append(line)
-
-class LogOutput(object):
- """pass output to a file"""
-
- def __init__(self, filename):
- self.filename = filename
- self.file = None
-
- def __call__(self, line):
- if self.file is None:
- self.file = file(self.filename, 'a')
- self.file.write(line + '\n')
- self.file.flush()
-
- def __del__(self):
- if self.file is not None:
- self.file.close()
-
-### front end class with the default handlers
-
-class ProcessHandler(ProcessHandlerMixin):
- """
- Convenience class for handling processes with default output handlers.
-
- If no processOutputLine keyword argument is specified, write all
- output to stdout. Otherwise, the function specified by this argument
- will be called for each line of output; the output will not be written
- to stdout automatically.
-
- If storeOutput==True, the output produced by the process will be saved
- as self.output.
-
- If logfile is not None, the output produced by the process will be
- appended to the given file.
- """
-
- def __init__(self, cmd, logfile=None, storeOutput=True, **kwargs):
- kwargs.setdefault('processOutputLine', [])
-
- # Print to standard output only if no outputline provided
- if not kwargs['processOutputLine']:
- kwargs['processOutputLine'].append(print_output)
-
- if logfile:
- logoutput = LogOutput(logfile)
- kwargs['processOutputLine'].append(logoutput)
-
- self.output = None
- if storeOutput:
- storeoutput = StoreOutput()
- self.output = storeoutput.output
- kwargs['processOutputLine'].append(storeoutput)
-
- ProcessHandlerMixin.__init__(self, cmd, **kwargs)
diff --git a/testing/mozharness/mozprocess/qijo.py b/testing/mozharness/mozprocess/qijo.py
deleted file mode 100644
index 1ac88430c..000000000
--- a/testing/mozharness/mozprocess/qijo.py
+++ /dev/null
@@ -1,140 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from ctypes import c_void_p, POINTER, sizeof, Structure, windll, WinError, WINFUNCTYPE, addressof, c_size_t, c_ulong
-from ctypes.wintypes import BOOL, BYTE, DWORD, HANDLE, LARGE_INTEGER
-
-LPVOID = c_void_p
-LPDWORD = POINTER(DWORD)
-SIZE_T = c_size_t
-ULONG_PTR = POINTER(c_ulong)
-
-# A ULONGLONG is a 64-bit unsigned integer.
-# Thus there are 8 bytes in a ULONGLONG.
-# XXX why not import c_ulonglong ?
-ULONGLONG = BYTE * 8
-
-class IO_COUNTERS(Structure):
- # The IO_COUNTERS struct is 6 ULONGLONGs.
- # TODO: Replace with non-dummy fields.
- _fields_ = [('dummy', ULONGLONG * 6)]
-
-class JOBOBJECT_BASIC_ACCOUNTING_INFORMATION(Structure):
- _fields_ = [('TotalUserTime', LARGE_INTEGER),
- ('TotalKernelTime', LARGE_INTEGER),
- ('ThisPeriodTotalUserTime', LARGE_INTEGER),
- ('ThisPeriodTotalKernelTime', LARGE_INTEGER),
- ('TotalPageFaultCount', DWORD),
- ('TotalProcesses', DWORD),
- ('ActiveProcesses', DWORD),
- ('TotalTerminatedProcesses', DWORD)]
-
-class JOBOBJECT_BASIC_AND_IO_ACCOUNTING_INFORMATION(Structure):
- _fields_ = [('BasicInfo', JOBOBJECT_BASIC_ACCOUNTING_INFORMATION),
- ('IoInfo', IO_COUNTERS)]
-
-# see http://msdn.microsoft.com/en-us/library/ms684147%28VS.85%29.aspx
-class JOBOBJECT_BASIC_LIMIT_INFORMATION(Structure):
- _fields_ = [('PerProcessUserTimeLimit', LARGE_INTEGER),
- ('PerJobUserTimeLimit', LARGE_INTEGER),
- ('LimitFlags', DWORD),
- ('MinimumWorkingSetSize', SIZE_T),
- ('MaximumWorkingSetSize', SIZE_T),
- ('ActiveProcessLimit', DWORD),
- ('Affinity', ULONG_PTR),
- ('PriorityClass', DWORD),
- ('SchedulingClass', DWORD)
- ]
-
-class JOBOBJECT_ASSOCIATE_COMPLETION_PORT(Structure):
- _fields_ = [('CompletionKey', c_ulong),
- ('CompletionPort', HANDLE)]
-
-# see http://msdn.microsoft.com/en-us/library/ms684156%28VS.85%29.aspx
-class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(Structure):
- _fields_ = [('BasicLimitInformation', JOBOBJECT_BASIC_LIMIT_INFORMATION),
- ('IoInfo', IO_COUNTERS),
- ('ProcessMemoryLimit', SIZE_T),
- ('JobMemoryLimit', SIZE_T),
- ('PeakProcessMemoryUsed', SIZE_T),
- ('PeakJobMemoryUsed', SIZE_T)]
-
-# These numbers below come from:
-# http://msdn.microsoft.com/en-us/library/ms686216%28v=vs.85%29.aspx
-JobObjectAssociateCompletionPortInformation = 7
-JobObjectBasicAndIoAccountingInformation = 8
-JobObjectExtendedLimitInformation = 9
-
-class JobObjectInfo(object):
- mapping = { 'JobObjectBasicAndIoAccountingInformation': 8,
- 'JobObjectExtendedLimitInformation': 9,
- 'JobObjectAssociateCompletionPortInformation': 7
- }
- structures = {
- 7: JOBOBJECT_ASSOCIATE_COMPLETION_PORT,
- 8: JOBOBJECT_BASIC_AND_IO_ACCOUNTING_INFORMATION,
- 9: JOBOBJECT_EXTENDED_LIMIT_INFORMATION
- }
- def __init__(self, _class):
- if isinstance(_class, basestring):
- assert _class in self.mapping, 'Class should be one of %s; you gave %s' % (self.mapping, _class)
- _class = self.mapping[_class]
- assert _class in self.structures, 'Class should be one of %s; you gave %s' % (self.structures, _class)
- self.code = _class
- self.info = self.structures[_class]()
-
-
-QueryInformationJobObjectProto = WINFUNCTYPE(
- BOOL, # Return type
- HANDLE, # hJob
- DWORD, # JobObjectInfoClass
- LPVOID, # lpJobObjectInfo
- DWORD, # cbJobObjectInfoLength
- LPDWORD # lpReturnLength
- )
-
-QueryInformationJobObjectFlags = (
- (1, 'hJob'),
- (1, 'JobObjectInfoClass'),
- (1, 'lpJobObjectInfo'),
- (1, 'cbJobObjectInfoLength'),
- (1, 'lpReturnLength', None)
- )
-
-_QueryInformationJobObject = QueryInformationJobObjectProto(
- ('QueryInformationJobObject', windll.kernel32),
- QueryInformationJobObjectFlags
- )
-
-class SubscriptableReadOnlyStruct(object):
- def __init__(self, struct):
- self._struct = struct
-
- def _delegate(self, name):
- result = getattr(self._struct, name)
- if isinstance(result, Structure):
- return SubscriptableReadOnlyStruct(result)
- return result
-
- def __getitem__(self, name):
- match = [fname for fname, ftype in self._struct._fields_
- if fname == name]
- if match:
- return self._delegate(name)
- raise KeyError(name)
-
- def __getattr__(self, name):
- return self._delegate(name)
-
-def QueryInformationJobObject(hJob, JobObjectInfoClass):
- jobinfo = JobObjectInfo(JobObjectInfoClass)
- result = _QueryInformationJobObject(
- hJob=hJob,
- JobObjectInfoClass=jobinfo.code,
- lpJobObjectInfo=addressof(jobinfo.info),
- cbJobObjectInfoLength=sizeof(jobinfo.info)
- )
- if not result:
- raise WinError()
- return SubscriptableReadOnlyStruct(jobinfo.info)
diff --git a/testing/mozharness/mozprocess/winprocess.py b/testing/mozharness/mozprocess/winprocess.py
deleted file mode 100644
index 6f3afc8de..000000000
--- a/testing/mozharness/mozprocess/winprocess.py
+++ /dev/null
@@ -1,457 +0,0 @@
-# A module to expose various thread/process/job related structures and
-# methods from kernel32
-#
-# The MIT License
-#
-# Copyright (c) 2003-2004 by Peter Astrand <astrand@lysator.liu.se>
-#
-# Additions and modifications written by Benjamin Smedberg
-# <benjamin@smedbergs.us> are Copyright (c) 2006 by the Mozilla Foundation
-# <http://www.mozilla.org/>
-#
-# More Modifications
-# Copyright (c) 2006-2007 by Mike Taylor <bear@code-bear.com>
-# Copyright (c) 2007-2008 by Mikeal Rogers <mikeal@mozilla.com>
-#
-# By obtaining, using, and/or copying this software and/or its
-# associated documentation, you agree that you have read, understood,
-# and will comply with the following terms and conditions:
-#
-# Permission to use, copy, modify, and distribute this software and
-# its associated documentation for any purpose and without fee is
-# hereby granted, provided that the above copyright notice appears in
-# all copies, and that both that copyright notice and this permission
-# notice appear in supporting documentation, and that the name of the
-# author not be used in advertising or publicity pertaining to
-# distribution of the software without specific, written prior
-# permission.
-#
-# THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
-# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
-# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR
-# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
-# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
-# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
-# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
-from ctypes import c_void_p, POINTER, sizeof, Structure, Union, windll, WinError, WINFUNCTYPE, c_ulong
-from ctypes.wintypes import BOOL, BYTE, DWORD, HANDLE, LPCWSTR, LPWSTR, UINT, WORD, ULONG
-from qijo import QueryInformationJobObject
-
-LPVOID = c_void_p
-LPBYTE = POINTER(BYTE)
-LPDWORD = POINTER(DWORD)
-LPBOOL = POINTER(BOOL)
-LPULONG = POINTER(c_ulong)
-
-def ErrCheckBool(result, func, args):
- """errcheck function for Windows functions that return a BOOL True
- on success"""
- if not result:
- raise WinError()
- return args
-
-
-# AutoHANDLE
-
-class AutoHANDLE(HANDLE):
- """Subclass of HANDLE which will call CloseHandle() on deletion."""
-
- CloseHandleProto = WINFUNCTYPE(BOOL, HANDLE)
- CloseHandle = CloseHandleProto(("CloseHandle", windll.kernel32))
- CloseHandle.errcheck = ErrCheckBool
-
- def Close(self):
- if self.value and self.value != HANDLE(-1).value:
- self.CloseHandle(self)
- self.value = 0
-
- def __del__(self):
- self.Close()
-
- def __int__(self):
- return self.value
-
-def ErrCheckHandle(result, func, args):
- """errcheck function for Windows functions that return a HANDLE."""
- if not result:
- raise WinError()
- return AutoHANDLE(result)
-
-# PROCESS_INFORMATION structure
-
-class PROCESS_INFORMATION(Structure):
- _fields_ = [("hProcess", HANDLE),
- ("hThread", HANDLE),
- ("dwProcessID", DWORD),
- ("dwThreadID", DWORD)]
-
- def __init__(self):
- Structure.__init__(self)
-
- self.cb = sizeof(self)
-
-LPPROCESS_INFORMATION = POINTER(PROCESS_INFORMATION)
-
-# STARTUPINFO structure
-
-class STARTUPINFO(Structure):
- _fields_ = [("cb", DWORD),
- ("lpReserved", LPWSTR),
- ("lpDesktop", LPWSTR),
- ("lpTitle", LPWSTR),
- ("dwX", DWORD),
- ("dwY", DWORD),
- ("dwXSize", DWORD),
- ("dwYSize", DWORD),
- ("dwXCountChars", DWORD),
- ("dwYCountChars", DWORD),
- ("dwFillAttribute", DWORD),
- ("dwFlags", DWORD),
- ("wShowWindow", WORD),
- ("cbReserved2", WORD),
- ("lpReserved2", LPBYTE),
- ("hStdInput", HANDLE),
- ("hStdOutput", HANDLE),
- ("hStdError", HANDLE)
- ]
-LPSTARTUPINFO = POINTER(STARTUPINFO)
-
-SW_HIDE = 0
-
-STARTF_USESHOWWINDOW = 0x01
-STARTF_USESIZE = 0x02
-STARTF_USEPOSITION = 0x04
-STARTF_USECOUNTCHARS = 0x08
-STARTF_USEFILLATTRIBUTE = 0x10
-STARTF_RUNFULLSCREEN = 0x20
-STARTF_FORCEONFEEDBACK = 0x40
-STARTF_FORCEOFFFEEDBACK = 0x80
-STARTF_USESTDHANDLES = 0x100
-
-# EnvironmentBlock
-
-class EnvironmentBlock:
- """An object which can be passed as the lpEnv parameter of CreateProcess.
- It is initialized with a dictionary."""
-
- def __init__(self, dict):
- if not dict:
- self._as_parameter_ = None
- else:
- values = ["%s=%s" % (key, value)
- for (key, value) in dict.iteritems()]
- values.append("")
- self._as_parameter_ = LPCWSTR("\0".join(values))
-
-# Error Messages we need to watch for go here
-# See: http://msdn.microsoft.com/en-us/library/ms681388%28v=vs.85%29.aspx
-ERROR_ABANDONED_WAIT_0 = 735
-
-# GetLastError()
-GetLastErrorProto = WINFUNCTYPE(DWORD # Return Type
- )
-GetLastErrorFlags = ()
-GetLastError = GetLastErrorProto(("GetLastError", windll.kernel32), GetLastErrorFlags)
-
-# CreateProcess()
-
-CreateProcessProto = WINFUNCTYPE(BOOL, # Return type
- LPCWSTR, # lpApplicationName
- LPWSTR, # lpCommandLine
- LPVOID, # lpProcessAttributes
- LPVOID, # lpThreadAttributes
- BOOL, # bInheritHandles
- DWORD, # dwCreationFlags
- LPVOID, # lpEnvironment
- LPCWSTR, # lpCurrentDirectory
- LPSTARTUPINFO, # lpStartupInfo
- LPPROCESS_INFORMATION # lpProcessInformation
- )
-
-CreateProcessFlags = ((1, "lpApplicationName", None),
- (1, "lpCommandLine"),
- (1, "lpProcessAttributes", None),
- (1, "lpThreadAttributes", None),
- (1, "bInheritHandles", True),
- (1, "dwCreationFlags", 0),
- (1, "lpEnvironment", None),
- (1, "lpCurrentDirectory", None),
- (1, "lpStartupInfo"),
- (2, "lpProcessInformation"))
-
-def ErrCheckCreateProcess(result, func, args):
- ErrCheckBool(result, func, args)
- # return a tuple (hProcess, hThread, dwProcessID, dwThreadID)
- pi = args[9]
- return AutoHANDLE(pi.hProcess), AutoHANDLE(pi.hThread), pi.dwProcessID, pi.dwThreadID
-
-CreateProcess = CreateProcessProto(("CreateProcessW", windll.kernel32),
- CreateProcessFlags)
-CreateProcess.errcheck = ErrCheckCreateProcess
-
-# flags for CreateProcess
-CREATE_BREAKAWAY_FROM_JOB = 0x01000000
-CREATE_DEFAULT_ERROR_MODE = 0x04000000
-CREATE_NEW_CONSOLE = 0x00000010
-CREATE_NEW_PROCESS_GROUP = 0x00000200
-CREATE_NO_WINDOW = 0x08000000
-CREATE_SUSPENDED = 0x00000004
-CREATE_UNICODE_ENVIRONMENT = 0x00000400
-
-# Flags for IOCompletion ports (some of these would probably be defined if
-# we used the win32 extensions for python, but we don't want to do that if we
-# can help it.
-INVALID_HANDLE_VALUE = HANDLE(-1) # From winbase.h
-
-# Self Defined Constants for IOPort <--> Job Object communication
-COMPKEY_TERMINATE = c_ulong(0)
-COMPKEY_JOBOBJECT = c_ulong(1)
-
-# flags for job limit information
-# see http://msdn.microsoft.com/en-us/library/ms684147%28VS.85%29.aspx
-JOB_OBJECT_LIMIT_BREAKAWAY_OK = 0x00000800
-JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK = 0x00001000
-
-# Flags for Job Object Completion Port Message IDs from winnt.h
-# See also: http://msdn.microsoft.com/en-us/library/ms684141%28v=vs.85%29.aspx
-JOB_OBJECT_MSG_END_OF_JOB_TIME = 1
-JOB_OBJECT_MSG_END_OF_PROCESS_TIME = 2
-JOB_OBJECT_MSG_ACTIVE_PROCESS_LIMIT = 3
-JOB_OBJECT_MSG_ACTIVE_PROCESS_ZERO = 4
-JOB_OBJECT_MSG_NEW_PROCESS = 6
-JOB_OBJECT_MSG_EXIT_PROCESS = 7
-JOB_OBJECT_MSG_ABNORMAL_EXIT_PROCESS = 8
-JOB_OBJECT_MSG_PROCESS_MEMORY_LIMIT = 9
-JOB_OBJECT_MSG_JOB_MEMORY_LIMIT = 10
-
-# See winbase.h
-DEBUG_ONLY_THIS_PROCESS = 0x00000002
-DEBUG_PROCESS = 0x00000001
-DETACHED_PROCESS = 0x00000008
-
-# GetQueuedCompletionPortStatus - http://msdn.microsoft.com/en-us/library/aa364986%28v=vs.85%29.aspx
-GetQueuedCompletionStatusProto = WINFUNCTYPE(BOOL, # Return Type
- HANDLE, # Completion Port
- LPDWORD, # Msg ID
- LPULONG, # Completion Key
- LPULONG, # PID Returned from the call (may be null)
- DWORD) # milliseconds to wait
-GetQueuedCompletionStatusFlags = ((1, "CompletionPort", INVALID_HANDLE_VALUE),
- (1, "lpNumberOfBytes", None),
- (1, "lpCompletionKey", None),
- (1, "lpPID", None),
- (1, "dwMilliseconds", 0))
-GetQueuedCompletionStatus = GetQueuedCompletionStatusProto(("GetQueuedCompletionStatus",
- windll.kernel32),
- GetQueuedCompletionStatusFlags)
-
-# CreateIOCompletionPort
-# Note that the completion key is just a number, not a pointer.
-CreateIoCompletionPortProto = WINFUNCTYPE(HANDLE, # Return Type
- HANDLE, # File Handle
- HANDLE, # Existing Completion Port
- c_ulong, # Completion Key
- DWORD # Number of Threads
- )
-CreateIoCompletionPortFlags = ((1, "FileHandle", INVALID_HANDLE_VALUE),
- (1, "ExistingCompletionPort", 0),
- (1, "CompletionKey", c_ulong(0)),
- (1, "NumberOfConcurrentThreads", 0))
-CreateIoCompletionPort = CreateIoCompletionPortProto(("CreateIoCompletionPort",
- windll.kernel32),
- CreateIoCompletionPortFlags)
-CreateIoCompletionPort.errcheck = ErrCheckHandle
-
-# SetInformationJobObject
-SetInformationJobObjectProto = WINFUNCTYPE(BOOL, # Return Type
- HANDLE, # Job Handle
- DWORD, # Type of Class next param is
- LPVOID, # Job Object Class
- DWORD # Job Object Class Length
- )
-SetInformationJobObjectProtoFlags = ((1, "hJob", None),
- (1, "JobObjectInfoClass", None),
- (1, "lpJobObjectInfo", None),
- (1, "cbJobObjectInfoLength", 0))
-SetInformationJobObject = SetInformationJobObjectProto(("SetInformationJobObject",
- windll.kernel32),
- SetInformationJobObjectProtoFlags)
-SetInformationJobObject.errcheck = ErrCheckBool
-
-# CreateJobObject()
-CreateJobObjectProto = WINFUNCTYPE(HANDLE, # Return type
- LPVOID, # lpJobAttributes
- LPCWSTR # lpName
- )
-
-CreateJobObjectFlags = ((1, "lpJobAttributes", None),
- (1, "lpName", None))
-
-CreateJobObject = CreateJobObjectProto(("CreateJobObjectW", windll.kernel32),
- CreateJobObjectFlags)
-CreateJobObject.errcheck = ErrCheckHandle
-
-# AssignProcessToJobObject()
-
-AssignProcessToJobObjectProto = WINFUNCTYPE(BOOL, # Return type
- HANDLE, # hJob
- HANDLE # hProcess
- )
-AssignProcessToJobObjectFlags = ((1, "hJob"),
- (1, "hProcess"))
-AssignProcessToJobObject = AssignProcessToJobObjectProto(
- ("AssignProcessToJobObject", windll.kernel32),
- AssignProcessToJobObjectFlags)
-AssignProcessToJobObject.errcheck = ErrCheckBool
-
-# GetCurrentProcess()
-# because os.getPid() is way too easy
-GetCurrentProcessProto = WINFUNCTYPE(HANDLE # Return type
- )
-GetCurrentProcessFlags = ()
-GetCurrentProcess = GetCurrentProcessProto(
- ("GetCurrentProcess", windll.kernel32),
- GetCurrentProcessFlags)
-GetCurrentProcess.errcheck = ErrCheckHandle
-
-# IsProcessInJob()
-try:
- IsProcessInJobProto = WINFUNCTYPE(BOOL, # Return type
- HANDLE, # Process Handle
- HANDLE, # Job Handle
- LPBOOL # Result
- )
- IsProcessInJobFlags = ((1, "ProcessHandle"),
- (1, "JobHandle", HANDLE(0)),
- (2, "Result"))
- IsProcessInJob = IsProcessInJobProto(
- ("IsProcessInJob", windll.kernel32),
- IsProcessInJobFlags)
- IsProcessInJob.errcheck = ErrCheckBool
-except AttributeError:
- # windows 2k doesn't have this API
- def IsProcessInJob(process):
- return False
-
-
-# ResumeThread()
-
-def ErrCheckResumeThread(result, func, args):
- if result == -1:
- raise WinError()
-
- return args
-
-ResumeThreadProto = WINFUNCTYPE(DWORD, # Return type
- HANDLE # hThread
- )
-ResumeThreadFlags = ((1, "hThread"),)
-ResumeThread = ResumeThreadProto(("ResumeThread", windll.kernel32),
- ResumeThreadFlags)
-ResumeThread.errcheck = ErrCheckResumeThread
-
-# TerminateProcess()
-
-TerminateProcessProto = WINFUNCTYPE(BOOL, # Return type
- HANDLE, # hProcess
- UINT # uExitCode
- )
-TerminateProcessFlags = ((1, "hProcess"),
- (1, "uExitCode", 127))
-TerminateProcess = TerminateProcessProto(
- ("TerminateProcess", windll.kernel32),
- TerminateProcessFlags)
-TerminateProcess.errcheck = ErrCheckBool
-
-# TerminateJobObject()
-
-TerminateJobObjectProto = WINFUNCTYPE(BOOL, # Return type
- HANDLE, # hJob
- UINT # uExitCode
- )
-TerminateJobObjectFlags = ((1, "hJob"),
- (1, "uExitCode", 127))
-TerminateJobObject = TerminateJobObjectProto(
- ("TerminateJobObject", windll.kernel32),
- TerminateJobObjectFlags)
-TerminateJobObject.errcheck = ErrCheckBool
-
-# WaitForSingleObject()
-
-WaitForSingleObjectProto = WINFUNCTYPE(DWORD, # Return type
- HANDLE, # hHandle
- DWORD, # dwMilliseconds
- )
-WaitForSingleObjectFlags = ((1, "hHandle"),
- (1, "dwMilliseconds", -1))
-WaitForSingleObject = WaitForSingleObjectProto(
- ("WaitForSingleObject", windll.kernel32),
- WaitForSingleObjectFlags)
-
-# http://msdn.microsoft.com/en-us/library/ms681381%28v=vs.85%29.aspx
-INFINITE = -1
-WAIT_TIMEOUT = 0x0102
-WAIT_OBJECT_0 = 0x0
-WAIT_ABANDONED = 0x0080
-
-# http://msdn.microsoft.com/en-us/library/ms683189%28VS.85%29.aspx
-STILL_ACTIVE = 259
-
-# Used when we terminate a process.
-ERROR_CONTROL_C_EXIT = 0x23c
-
-# GetExitCodeProcess()
-
-GetExitCodeProcessProto = WINFUNCTYPE(BOOL, # Return type
- HANDLE, # hProcess
- LPDWORD, # lpExitCode
- )
-GetExitCodeProcessFlags = ((1, "hProcess"),
- (2, "lpExitCode"))
-GetExitCodeProcess = GetExitCodeProcessProto(
- ("GetExitCodeProcess", windll.kernel32),
- GetExitCodeProcessFlags)
-GetExitCodeProcess.errcheck = ErrCheckBool
-
-def CanCreateJobObject():
- currentProc = GetCurrentProcess()
- if IsProcessInJob(currentProc):
- jobinfo = QueryInformationJobObject(HANDLE(0), 'JobObjectExtendedLimitInformation')
- limitflags = jobinfo['BasicLimitInformation']['LimitFlags']
- return bool(limitflags & JOB_OBJECT_LIMIT_BREAKAWAY_OK) or bool(limitflags & JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK)
- else:
- return True
-
-### testing functions
-
-def parent():
- print 'Starting parent'
- currentProc = GetCurrentProcess()
- if IsProcessInJob(currentProc):
- print >> sys.stderr, "You should not be in a job object to test"
- sys.exit(1)
- assert CanCreateJobObject()
- print 'File: %s' % __file__
- command = [sys.executable, __file__, '-child']
- print 'Running command: %s' % command
- process = Popen(command)
- process.kill()
- code = process.returncode
- print 'Child code: %s' % code
- assert code == 127
-
-def child():
- print 'Starting child'
- currentProc = GetCurrentProcess()
- injob = IsProcessInJob(currentProc)
- print "Is in a job?: %s" % injob
- can_create = CanCreateJobObject()
- print 'Can create job?: %s' % can_create
- process = Popen('c:\\windows\\notepad.exe')
- assert process._job
- jobinfo = QueryInformationJobObject(process._job, 'JobObjectExtendedLimitInformation')
- print 'Job info: %s' % jobinfo
- limitflags = jobinfo['BasicLimitInformation']['LimitFlags']
- print 'LimitFlags: %s' % limitflags
- process.kill()
diff --git a/testing/mozharness/mozprocess/wpk.py b/testing/mozharness/mozprocess/wpk.py
deleted file mode 100644
index a86f9bf22..000000000
--- a/testing/mozharness/mozprocess/wpk.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from ctypes import sizeof, windll, addressof, c_wchar, create_unicode_buffer
-from ctypes.wintypes import DWORD, HANDLE
-
-PROCESS_TERMINATE = 0x0001
-PROCESS_QUERY_INFORMATION = 0x0400
-PROCESS_VM_READ = 0x0010
-
-def get_pids(process_name):
- BIG_ARRAY = DWORD * 4096
- processes = BIG_ARRAY()
- needed = DWORD()
-
- pids = []
- result = windll.psapi.EnumProcesses(processes,
- sizeof(processes),
- addressof(needed))
- if not result:
- return pids
-
- num_results = needed.value / sizeof(DWORD)
-
- for i in range(num_results):
- pid = processes[i]
- process = windll.kernel32.OpenProcess(PROCESS_QUERY_INFORMATION |
- PROCESS_VM_READ,
- 0, pid)
- if process:
- module = HANDLE()
- result = windll.psapi.EnumProcessModules(process,
- addressof(module),
- sizeof(module),
- addressof(needed))
- if result:
- name = create_unicode_buffer(1024)
- result = windll.psapi.GetModuleBaseNameW(process, module,
- name, len(name))
- # TODO: This might not be the best way to
- # match a process name; maybe use a regexp instead.
- if name.value.startswith(process_name):
- pids.append(pid)
- windll.kernel32.CloseHandle(module)
- windll.kernel32.CloseHandle(process)
-
- return pids
-
-def kill_pid(pid):
- process = windll.kernel32.OpenProcess(PROCESS_TERMINATE, 0, pid)
- if process:
- windll.kernel32.TerminateProcess(process, 0)
- windll.kernel32.CloseHandle(process)
diff --git a/testing/mozharness/requirements.txt b/testing/mozharness/requirements.txt
deleted file mode 100644
index 9314abfee..000000000
--- a/testing/mozharness/requirements.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-# These packages are needed for mozharness unit tests.
-# Output from 'pip freeze'; we may be able to use other versions of the below packages.
-Cython==0.14.1
-Fabric==1.6.0
-coverage==3.6
-distribute==0.6.35
-dulwich==0.8.7
-hg-git==0.4.0
-logilab-astng==0.24.2
-logilab-common==0.59.0
-mercurial ~> 4.1.3
-mock==1.0.1
-nose==1.2.1
-ordereddict==1.1
-paramiko ~> 1.17.6
-pycrypto > 2.6.1
-pyflakes==0.6.1
-pylint==0.27.0
-simplejson==2.1.1
-unittest2==0.5.1
-virtualenv==1.5.1
-wsgiref==0.1.2
-urllib3==1.9.1
-google-api-python-client==1.5.1
-oauth2client==1.4.2
diff --git a/testing/mozharness/scripts/android_emulator_unittest.py b/testing/mozharness/scripts/android_emulator_unittest.py
deleted file mode 100644
index 84c5447ed..000000000
--- a/testing/mozharness/scripts/android_emulator_unittest.py
+++ /dev/null
@@ -1,733 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-import copy
-import datetime
-import glob
-import os
-import re
-import sys
-import signal
-import socket
-import subprocess
-import telnetlib
-import time
-import tempfile
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozprocess import ProcessHandler
-
-from mozharness.base.log import FATAL
-from mozharness.base.script import BaseScript, PreScriptAction, PostScriptAction
-from mozharness.base.vcs.vcsbase import VCSMixin
-from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
-from mozharness.mozilla.mozbase import MozbaseMixin
-from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
-from mozharness.mozilla.testing.unittest import EmulatorMixin
-
-
-class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin, BaseScript, MozbaseMixin):
- config_options = [[
- ["--test-suite"],
- {"action": "store",
- "dest": "test_suite",
- }
- ], [
- ["--adb-path"],
- {"action": "store",
- "dest": "adb_path",
- "default": None,
- "help": "Path to adb",
- }
- ], [
- ["--total-chunk"],
- {"action": "store",
- "dest": "total_chunks",
- "default": None,
- "help": "Number of total chunks",
- }
- ], [
- ["--this-chunk"],
- {"action": "store",
- "dest": "this_chunk",
- "default": None,
- "help": "Number of this chunk",
- }
- ]] + copy.deepcopy(testing_config_options) + \
- copy.deepcopy(blobupload_config_options)
-
- error_list = [
- ]
-
- virtualenv_requirements = [
- ]
-
- virtualenv_modules = [
- ]
-
- app_name = None
-
- def __init__(self, require_config_file=False):
- super(AndroidEmulatorTest, self).__init__(
- config_options=self.config_options,
- all_actions=['clobber',
- 'read-buildbot-config',
- 'setup-avds',
- 'start-emulator',
- 'download-and-extract',
- 'create-virtualenv',
- 'verify-emulator',
- 'install',
- 'run-tests',
- ],
- default_actions=['clobber',
- 'start-emulator',
- 'download-and-extract',
- 'create-virtualenv',
- 'verify-emulator',
- 'install',
- 'run-tests',
- ],
- require_config_file=require_config_file,
- config={
- 'virtualenv_modules': self.virtualenv_modules,
- 'virtualenv_requirements': self.virtualenv_requirements,
- 'require_test_zip': True,
- # IP address of the host as seen from the emulator
- 'remote_webserver': '10.0.2.2',
- }
- )
-
- # these are necessary since self.config is read only
- c = self.config
- abs_dirs = self.query_abs_dirs()
- self.adb_path = self.query_exe('adb')
- self.installer_url = c.get('installer_url')
- self.installer_path = c.get('installer_path')
- self.test_url = c.get('test_url')
- self.test_packages_url = c.get('test_packages_url')
- self.test_manifest = c.get('test_manifest')
- self.minidump_stackwalk_path = c.get("minidump_stackwalk_path")
- self.emulator = c.get('emulator')
- self.test_suite = c.get('test_suite')
- self.this_chunk = c.get('this_chunk')
- self.total_chunks = c.get('total_chunks')
- if self.test_suite not in self.config["suite_definitions"]:
- # accept old-style test suite name like "mochitest-3"
- m = re.match("(.*)-(\d*)", self.test_suite)
- if m:
- self.test_suite = m.group(1)
- if self.this_chunk is None:
- self.this_chunk = m.group(2)
- self.sdk_level = None
- self.xre_path = None
-
- def _query_tests_dir(self):
- dirs = self.query_abs_dirs()
- try:
- test_dir = self.config["suite_definitions"][self.test_suite]["testsdir"]
- except:
- test_dir = self.test_suite
- return os.path.join(dirs['abs_test_install_dir'], test_dir)
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(AndroidEmulatorTest, self).query_abs_dirs()
- dirs = {}
- dirs['abs_test_install_dir'] = os.path.join(
- abs_dirs['abs_work_dir'], 'tests')
- dirs['abs_xre_dir'] = os.path.join(
- abs_dirs['abs_work_dir'], 'hostutils')
- dirs['abs_modules_dir'] = os.path.join(
- dirs['abs_test_install_dir'], 'modules')
- dirs['abs_blob_upload_dir'] = os.path.join(
- abs_dirs['abs_work_dir'], 'blobber_upload_dir')
- dirs['abs_emulator_dir'] = abs_dirs['abs_work_dir']
- dirs['abs_mochitest_dir'] = os.path.join(
- dirs['abs_test_install_dir'], 'mochitest')
- dirs['abs_marionette_dir'] = os.path.join(
- dirs['abs_test_install_dir'], 'marionette', 'harness', 'marionette_harness')
- dirs['abs_marionette_tests_dir'] = os.path.join(
- dirs['abs_test_install_dir'], 'marionette', 'tests', 'testing',
- 'marionette', 'harness', 'marionette_harness', 'tests')
- dirs['abs_avds_dir'] = self.config.get("avds_dir", "/home/cltbld/.android")
-
- for key in dirs.keys():
- if key not in abs_dirs:
- abs_dirs[key] = dirs[key]
- self.abs_dirs = abs_dirs
- return self.abs_dirs
-
- @PreScriptAction('create-virtualenv')
- def _pre_create_virtualenv(self, action):
- dirs = self.query_abs_dirs()
- requirements = None
- if os.path.isdir(dirs['abs_mochitest_dir']):
- # mochitest is the only thing that needs this
- requirements = os.path.join(dirs['abs_mochitest_dir'],
- 'websocketprocessbridge',
- 'websocketprocessbridge_requirements.txt')
- elif self.test_suite == 'marionette':
- requirements = os.path.join(dirs['abs_test_install_dir'],
- 'config', 'marionette_requirements.txt')
- if requirements:
- self.register_virtualenv_module(requirements=[requirements],
- two_pass=True)
-
- def _launch_emulator(self):
- env = self.query_env()
-
- # Set $LD_LIBRARY_PATH to self.dirs['abs_work_dir'] so that
- # the emulator picks up the symlink to libGL.so.1 that we
- # constructed in start_emulator.
- env['LD_LIBRARY_PATH'] = self.abs_dirs['abs_work_dir']
-
- # Set environment variables to help emulator find the AVD.
- # In newer versions of the emulator, ANDROID_AVD_HOME should
- # point to the 'avd' directory.
- # For older versions of the emulator, ANDROID_SDK_HOME should
- # point to the directory containing the '.android' directory
- # containing the 'avd' directory.
- avd_home_dir = self.abs_dirs['abs_avds_dir']
- env['ANDROID_AVD_HOME'] = os.path.join(avd_home_dir, 'avd')
- env['ANDROID_SDK_HOME'] = os.path.abspath(os.path.join(avd_home_dir, '..'))
-
- command = [
- "emulator", "-avd", self.emulator["name"],
- "-port", str(self.emulator["emulator_port"]),
- ]
- if "emulator_extra_args" in self.config:
- command += self.config["emulator_extra_args"].split()
-
- tmp_file = tempfile.NamedTemporaryFile(mode='w')
- tmp_stdout = open(tmp_file.name, 'w')
- self.info("Created temp file %s." % tmp_file.name)
- self.info("Trying to start the emulator with this command: %s" % ' '.join(command))
- proc = subprocess.Popen(command, stdout=tmp_stdout, stderr=tmp_stdout, env=env)
- return {
- "process": proc,
- "tmp_file": tmp_file,
- }
-
- def _retry(self, max_attempts, interval, func, description, max_time = 0):
- '''
- Execute func until it returns True, up to max_attempts times, waiting for
- interval seconds between each attempt. description is logged on each attempt.
- If max_time is specified, no further attempts will be made once max_time
- seconds have elapsed; this provides some protection for the case where
- the run-time for func is long or highly variable.
- '''
- status = False
- attempts = 0
- if max_time > 0:
- end_time = datetime.datetime.now() + datetime.timedelta(seconds = max_time)
- else:
- end_time = None
- while attempts < max_attempts and not status:
- if (end_time is not None) and (datetime.datetime.now() > end_time):
- self.info("Maximum retry run-time of %d seconds exceeded; remaining attempts abandoned" % max_time)
- break
- if attempts != 0:
- self.info("Sleeping %d seconds" % interval)
- time.sleep(interval)
- attempts += 1
- self.info(">> %s: Attempt #%d of %d" % (description, attempts, max_attempts))
- status = func()
- return status
-
- def _run_with_timeout(self, timeout, cmd):
- timeout_cmd = ['timeout', '%s' % timeout] + cmd
- return self._run_proc(timeout_cmd)
-
- def _run_proc(self, cmd):
- self.info('Running %s' % subprocess.list2cmdline(cmd))
- p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- out, err = p.communicate()
- if out:
- self.info('%s' % str(out.strip()))
- if err:
- self.info('stderr: %s' % str(err.strip()))
- return out
-
- def _telnet_cmd(self, telnet, command):
- telnet.write('%s\n' % command)
- result = telnet.read_until('OK', 10)
- self.info('%s: %s' % (command, result))
- return result
-
- def _verify_adb(self):
- self.info('Verifying adb connectivity')
- self._run_with_timeout(180, [self.adb_path, 'wait-for-device'])
- return True
-
- def _verify_adb_device(self):
- out = self._run_with_timeout(30, [self.adb_path, 'devices'])
- if (self.emulator['device_id'] in out) and ("device" in out):
- return True
- return False
-
- def _is_boot_completed(self):
- boot_cmd = [self.adb_path, '-s', self.emulator['device_id'],
- 'shell', 'getprop', 'sys.boot_completed']
- out = self._run_with_timeout(30, boot_cmd)
- if out.strip() == '1':
- return True
- return False
-
- def _telnet_to_emulator(self):
- port = self.emulator["emulator_port"]
- telnet_ok = False
- try:
- tn = telnetlib.Telnet('localhost', port, 10)
- if tn is not None:
- self.info('Connected to port %d' % port)
- res = tn.read_until('OK', 10)
- self.info(res)
- self._telnet_cmd(tn, 'avd status')
- self._telnet_cmd(tn, 'redir list')
- self._telnet_cmd(tn, 'network status')
- tn.write('quit\n')
- tn.read_all()
- telnet_ok = True
- else:
- self.warning('Unable to connect to port %d' % port)
- except socket.error, e:
- self.info('Trying again after socket error: %s' % str(e))
- pass
- except EOFError:
- self.info('Trying again after EOF')
- pass
- except:
- self.info('Trying again after unexpected exception')
- pass
- finally:
- if tn is not None:
- tn.close()
- return telnet_ok
-
- def _verify_emulator(self):
- adb_ok = self._verify_adb()
- if not adb_ok:
- self.warning('Unable to communicate with adb')
- return False
- adb_device_ok = self._retry(4, 30, self._verify_adb_device, "Verify emulator visible to adb")
- if not adb_device_ok:
- self.warning('Unable to communicate with emulator via adb')
- return False
- boot_ok = self._retry(30, 10, self._is_boot_completed, "Verify Android boot completed", max_time = 330)
- if not boot_ok:
- self.warning('Unable to verify Android boot completion')
- return False
- telnet_ok = self._retry(4, 30, self._telnet_to_emulator, "Verify telnet to emulator")
- if not telnet_ok:
- self.warning('Unable to telnet to emulator on port %d' % self.emulator["emulator_port"])
- return False
- return True
-
- def _verify_emulator_and_restart_on_fail(self):
- emulator_ok = self._verify_emulator()
- if not emulator_ok:
- self._dump_host_state()
- self._screenshot("emulator-startup-screenshot-")
- self._kill_processes(self.config["emulator_process_name"])
- self._run_proc(['ps', '-ef'])
- self._dump_emulator_log()
- # remove emulator tmp files
- for dir in glob.glob("/tmp/android-*"):
- self.rmtree(dir)
- self._restart_adbd()
- time.sleep(5)
- self.emulator_proc = self._launch_emulator()
- return emulator_ok
-
- def _install_fennec_apk(self):
- install_ok = False
- if int(self.sdk_level) >= 23:
- cmd = [self.adb_path, '-s', self.emulator['device_id'], 'install', '-r', '-g', self.installer_path]
- else:
- cmd = [self.adb_path, '-s', self.emulator['device_id'], 'install', '-r', self.installer_path]
- out = self._run_with_timeout(300, cmd)
- if 'Success' in out:
- install_ok = True
- return install_ok
-
- def _dump_host_state(self):
- self._run_proc(['ps', '-ef'])
- self._run_proc(['netstat', '-a', '-p', '-n', '-t', '-u'])
-
- def _dump_emulator_log(self):
- self.info("##### %s emulator log begins" % self.emulator["name"])
- output = self.read_from_file(self.emulator_proc["tmp_file"].name, verbose=False)
- if output:
- self.info(output)
- self.info("##### %s emulator log ends" % self.emulator["name"])
-
- def _kill_processes(self, process_name):
- p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE)
- out, err = p.communicate()
- self.info("Let's kill every process called %s" % process_name)
- for line in out.splitlines():
- if process_name in line:
- pid = int(line.split(None, 1)[0])
- self.info("Killing pid %d." % pid)
- os.kill(pid, signal.SIGKILL)
-
- def _restart_adbd(self):
- self._run_with_timeout(30, [self.adb_path, 'kill-server'])
- self._run_with_timeout(30, [self.adb_path, 'start-server'])
-
- def _screenshot(self, prefix):
- """
- Save a screenshot of the entire screen to the blob upload directory.
- """
- dirs = self.query_abs_dirs()
- utility = os.path.join(self.xre_path, "screentopng")
- if not os.path.exists(utility):
- self.warning("Unable to take screenshot: %s does not exist" % utility)
- return
- try:
- tmpfd, filename = tempfile.mkstemp(prefix=prefix, suffix='.png',
- dir=dirs['abs_blob_upload_dir'])
- os.close(tmpfd)
- self.info("Taking screenshot with %s; saving to %s" % (utility, filename))
- subprocess.call([utility, filename], env=self.query_env())
- except OSError, err:
- self.warning("Failed to take screenshot: %s" % err.strerror)
-
- def _query_package_name(self):
- if self.app_name is None:
- #find appname from package-name.txt - assumes download-and-extract has completed successfully
- apk_dir = self.abs_dirs['abs_work_dir']
- self.apk_path = os.path.join(apk_dir, self.installer_path)
- unzip = self.query_exe("unzip")
- package_path = os.path.join(apk_dir, 'package-name.txt')
- unzip_cmd = [unzip, '-q', '-o', self.apk_path]
- self.run_command(unzip_cmd, cwd=apk_dir, halt_on_failure=True)
- self.app_name = str(self.read_from_file(package_path, verbose=True)).rstrip()
- return self.app_name
-
- def preflight_install(self):
- # in the base class, this checks for mozinstall, but we don't use it
- pass
-
- def _build_command(self):
- c = self.config
- dirs = self.query_abs_dirs()
-
- if self.test_suite not in self.config["suite_definitions"]:
- self.fatal("Key '%s' not defined in the config!" % self.test_suite)
-
- cmd = [
- self.query_python_path('python'),
- '-u',
- os.path.join(
- self._query_tests_dir(),
- self.config["suite_definitions"][self.test_suite]["run_filename"]
- ),
- ]
-
- raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
- '%s_raw.log' % self.test_suite)
-
- error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
- '%s_errorsummary.log' % self.test_suite)
- str_format_values = {
- 'app': self._query_package_name(),
- 'remote_webserver': c['remote_webserver'],
- 'xre_path': self.xre_path,
- 'utility_path': self.xre_path,
- 'http_port': self.emulator['http_port'],
- 'ssl_port': self.emulator['ssl_port'],
- 'certs_path': os.path.join(dirs['abs_work_dir'], 'tests/certs'),
- # TestingMixin._download_and_extract_symbols() will set
- # self.symbols_path when downloading/extracting.
- 'symbols_path': self.symbols_path,
- 'modules_dir': dirs['abs_modules_dir'],
- 'installer_path': self.installer_path,
- 'raw_log_file': raw_log_file,
- 'error_summary_file': error_summary_file,
- 'dm_trans': c['device_manager'],
- # marionette options
- 'address': c.get('marionette_address'),
- 'gecko_log': os.path.join(dirs["abs_blob_upload_dir"], 'gecko.log'),
- 'test_manifest': os.path.join(
- dirs['abs_marionette_tests_dir'],
- self.config.get('marionette_test_manifest', '')
- ),
- }
- for option in self.config["suite_definitions"][self.test_suite]["options"]:
- opt = option.split('=')[0]
- # override configured chunk options with script args, if specified
- if opt == '--this-chunk' and self.this_chunk is not None:
- continue
- if opt == '--total-chunks' and self.total_chunks is not None:
- continue
- cmd.extend([option % str_format_values])
-
- if self.this_chunk is not None:
- cmd.extend(['--this-chunk', self.this_chunk])
- if self.total_chunks is not None:
- cmd.extend(['--total-chunks', self.total_chunks])
-
- try_options, try_tests = self.try_args(self.test_suite)
- cmd.extend(try_options)
- cmd.extend(self.query_tests_args(
- self.config["suite_definitions"][self.test_suite].get("tests"),
- None,
- try_tests))
-
- return cmd
-
- def _get_repo_url(self, path):
- """
- Return a url for a file (typically a tooltool manifest) in this hg repo
- and using this revision (or mozilla-central/default if repo/rev cannot
- be determined).
-
- :param path specifies the directory path to the file of interest.
- """
- if 'GECKO_HEAD_REPOSITORY' in os.environ and 'GECKO_HEAD_REV' in os.environ:
- # probably taskcluster
- repo = os.environ['GECKO_HEAD_REPOSITORY']
- revision = os.environ['GECKO_HEAD_REV']
- elif self.buildbot_config and 'properties' in self.buildbot_config:
- # probably buildbot
- repo = 'https://hg.mozilla.org/%s' % self.buildbot_config['properties']['repo_path']
- revision = self.buildbot_config['properties']['revision']
- else:
- # something unexpected!
- repo = 'https://hg.mozilla.org/mozilla-central'
- revision = 'default'
- self.warning('Unable to find repo/revision for manifest; using mozilla-central/default')
- url = '%s/raw-file/%s/%s' % (
- repo,
- revision,
- path)
- return url
-
- def _tooltool_fetch(self, url, dir):
- c = self.config
-
- manifest_path = self.download_file(
- url,
- file_name='releng.manifest',
- parent_dir=dir
- )
-
- if not os.path.exists(manifest_path):
- self.fatal("Could not retrieve manifest needed to retrieve "
- "artifacts from %s" % manifest_path)
-
- self.tooltool_fetch(manifest_path,
- output_dir=dir,
- cache=c.get("tooltool_cache", None))
-
- ##########################################
- ### Actions for AndroidEmulatorTest ###
- ##########################################
- def setup_avds(self):
- '''
- If tooltool cache mechanism is enabled, the cached version is used by
- the fetch command. If the manifest includes an "unpack" field, tooltool
- will unpack all compressed archives mentioned in the manifest.
- '''
- c = self.config
- dirs = self.query_abs_dirs()
-
- # FIXME
- # Clobbering and re-unpacking would not be needed if we had a way to
- # check whether the unpacked content already present match the
- # contents of the tar ball
- self.rmtree(dirs['abs_avds_dir'])
- self.mkdir_p(dirs['abs_avds_dir'])
- if 'avd_url' in c:
- # Intended for experimental setups to evaluate an avd prior to
- # tooltool deployment.
- url = c['avd_url']
- self.download_unpack(url, dirs['abs_avds_dir'])
- else:
- url = self._get_repo_url(c["tooltool_manifest_path"])
- self._tooltool_fetch(url, dirs['abs_avds_dir'])
-
- avd_home_dir = self.abs_dirs['abs_avds_dir']
- if avd_home_dir != "/home/cltbld/.android":
- # Modify the downloaded avds to point to the right directory.
- cmd = [
- 'bash', '-c',
- 'sed -i "s|/home/cltbld/.android|%s|" %s/test-*.ini' %
- (avd_home_dir, os.path.join(avd_home_dir, 'avd'))
- ]
- proc = ProcessHandler(cmd)
- proc.run()
- proc.wait()
-
- def start_emulator(self):
- '''
- Starts the emulator
- '''
- if 'emulator_url' in self.config or 'emulator_manifest' in self.config or 'tools_manifest' in self.config:
- self.install_emulator()
-
- if not os.path.isfile(self.adb_path):
- self.fatal("The adb binary '%s' is not a valid file!" % self.adb_path)
- self._restart_adbd()
-
- if not self.config.get("developer_mode"):
- # We kill compiz because it sometimes prevents us from starting the emulator
- self._kill_processes("compiz")
- self._kill_processes("xpcshell")
-
- # We add a symlink for libGL.so because the emulator dlopen()s it by that name
- # even though the installed library on most systems without dev packages is
- # libGL.so.1
- linkfile = os.path.join(self.abs_dirs['abs_work_dir'], "libGL.so")
- self.info("Attempting to establish symlink for %s" % linkfile)
- try:
- os.unlink(linkfile)
- except OSError:
- pass
- for libdir in ["/usr/lib/x86_64-linux-gnu/mesa",
- "/usr/lib/i386-linux-gnu/mesa",
- "/usr/lib/mesa"]:
- libfile = os.path.join(libdir, "libGL.so.1")
- if os.path.exists(libfile):
- self.info("Symlinking %s -> %s" % (linkfile, libfile))
- self.mkdir_p(self.abs_dirs['abs_work_dir'])
- os.symlink(libfile, linkfile)
- break
- self.emulator_proc = self._launch_emulator()
-
- def verify_emulator(self):
- '''
- Check to see if the emulator can be contacted via adb and telnet.
- If any communication attempt fails, kill the emulator, re-launch, and re-check.
- '''
- self.mkdir_p(self.query_abs_dirs()['abs_blob_upload_dir'])
- max_restarts = 5
- emulator_ok = self._retry(max_restarts, 10, self._verify_emulator_and_restart_on_fail, "Check emulator")
- if not emulator_ok:
- self.fatal('INFRA-ERROR: Unable to start emulator after %d attempts' % max_restarts)
- # Start logcat for the emulator. The adb process runs until the
- # corresponding emulator is killed. Output is written directly to
- # the blobber upload directory so that it is uploaded automatically
- # at the end of the job.
- logcat_filename = 'logcat-%s.log' % self.emulator["device_id"]
- logcat_path = os.path.join(self.abs_dirs['abs_blob_upload_dir'], logcat_filename)
- logcat_cmd = '%s -s %s logcat -v threadtime Trace:S StrictMode:S ExchangeService:S > %s &' % \
- (self.adb_path, self.emulator["device_id"], logcat_path)
- self.info(logcat_cmd)
- os.system(logcat_cmd)
- # Get a post-boot emulator process list for diagnostics
- ps_cmd = [self.adb_path, '-s', self.emulator["device_id"], 'shell', 'ps']
- self._run_with_timeout(30, ps_cmd)
-
- def download_and_extract(self):
- """
- Download and extract fennec APK, tests.zip, host utils (if required).
- """
- super(AndroidEmulatorTest, self).download_and_extract(suite_categories=[self.test_suite])
- dirs = self.query_abs_dirs()
- self.rmtree(dirs['abs_xre_dir'])
- self.mkdir_p(dirs['abs_xre_dir'])
- if self.config["hostutils_manifest_path"]:
- url = self._get_repo_url(self.config["hostutils_manifest_path"])
- self._tooltool_fetch(url, dirs['abs_xre_dir'])
- for p in glob.glob(os.path.join(dirs['abs_xre_dir'], 'host-utils-*')):
- if os.path.isdir(p) and os.path.isfile(os.path.join(p, 'xpcshell')):
- self.xre_path = p
- if not self.xre_path:
- self.fatal("xre path not found in %s" % dirs['abs_xre_dir'])
- else:
- self.fatal("configure hostutils_manifest_path!")
-
- def install(self):
- """
- Install APKs on the emulator
- """
- assert self.installer_path is not None, \
- "Either add installer_path to the config or use --installer-path."
- install_needed = self.config["suite_definitions"][self.test_suite].get("install")
- if install_needed == False:
- self.info("Skipping apk installation for %s" % self.test_suite)
- return
-
- self.sdk_level = self._run_with_timeout(30, [self.adb_path, '-s', self.emulator['device_id'],
- 'shell', 'getprop', 'ro.build.version.sdk'])
-
- # Install Fennec
- install_ok = self._retry(3, 30, self._install_fennec_apk, "Install Fennec APK")
- if not install_ok:
- self.fatal('INFRA-ERROR: Failed to install %s on %s' % (self.installer_path, self.emulator["name"]))
-
- self.info("Finished installing apps for %s" % self.emulator["name"])
-
- def run_tests(self):
- """
- Run the tests
- """
- cmd = self._build_command()
-
- try:
- cwd = self._query_tests_dir()
- except:
- self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite)
- env = self.query_env()
- if self.query_minidump_stackwalk():
- env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
- env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
- env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
-
- self.info("Running on %s the command %s" % (self.emulator["name"], subprocess.list2cmdline(cmd)))
- self.info("##### %s log begins" % self.test_suite)
-
- # TinderBoxPrintRe does not know about the '-debug' categories
- aliases = {
- 'reftest-debug': 'reftest',
- 'jsreftest-debug': 'jsreftest',
- 'crashtest-debug': 'crashtest',
- }
- suite_category = aliases.get(self.test_suite, self.test_suite)
- parser = self.get_test_output_parser(
- suite_category,
- config=self.config,
- log_obj=self.log_obj,
- error_list=self.error_list)
- self.run_command(cmd, cwd=cwd, env=env, output_parser=parser)
- tbpl_status, log_level = parser.evaluate_parser(0)
- parser.append_tinderboxprint_line(self.test_suite)
-
- self.info("##### %s log ends" % self.test_suite)
- self._dump_emulator_log()
- self.buildbot_status(tbpl_status, level=log_level)
-
- @PostScriptAction('run-tests')
- def stop_emulator(self, action, success=None):
- '''
- Report emulator health, then make sure that the emulator has been stopped
- '''
- self._verify_emulator()
- self._kill_processes(self.config["emulator_process_name"])
-
- def upload_blobber_files(self):
- '''
- Override BlobUploadMixin.upload_blobber_files to ensure emulator is killed
- first (if the emulator is still running, logcat may still be running, which
- may lock the blob upload directory, causing a hang).
- '''
- if self.config.get('blob_upload_branch'):
- # Except on interactive workers, we want the emulator to keep running
- # after the script is finished. So only kill it if blobber would otherwise
- # have run anyway (it doesn't get run on interactive workers).
- self._kill_processes(self.config["emulator_process_name"])
- super(AndroidEmulatorTest, self).upload_blobber_files()
-
-if __name__ == '__main__':
- emulatorTest = AndroidEmulatorTest()
- emulatorTest.run_and_exit()
diff --git a/testing/mozharness/scripts/bouncer_submitter.py b/testing/mozharness/scripts/bouncer_submitter.py
deleted file mode 100755
index eaa43e851..000000000
--- a/testing/mozharness/scripts/bouncer_submitter.py
+++ /dev/null
@@ -1,192 +0,0 @@
-#!/usr/bin/env python
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import os
-import sys
-
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.script import BaseScript
-from mozharness.mozilla.bouncer.submitter import BouncerSubmitterMixin
-from mozharness.mozilla.buildbot import BuildbotMixin
-from mozharness.mozilla.purge import PurgeMixin
-
-
-class BouncerSubmitter(BaseScript, PurgeMixin, BouncerSubmitterMixin, BuildbotMixin):
- config_options = [
- [["--repo"], {
- "dest": "repo",
- "help": "Specify source repo, e.g. releases/mozilla-beta",
- }],
- [["--revision"], {
- "dest": "revision",
- "help": "Source revision/tag used to fetch shipped-locales",
- }],
- [["--version"], {
- "dest": "version",
- "help": "Current version",
- }],
- [["--previous-version"], {
- "dest": "prev_versions",
- "action": "extend",
- "help": "Previous version(s)",
- }],
- [["--build-number"], {
- "dest": "build_number",
- "help": "Build number of version",
- }],
- [["--bouncer-api-prefix"], {
- "dest": "bouncer-api-prefix",
- "help": "Bouncer admin API URL prefix",
- }],
- [["--credentials-file"], {
- "dest": "credentials_file",
- "help": "File containing Bouncer credentials",
- }],
- ]
-
- def __init__(self, require_config_file=True):
- BaseScript.__init__(self,
- config_options=self.config_options,
- require_config_file=require_config_file,
- # other stuff
- all_actions=[
- 'clobber',
- 'download-shipped-locales',
- 'submit',
- ],
- default_actions=[
- 'clobber',
- 'download-shipped-locales',
- 'submit',
- ],
- config={
- 'buildbot_json_path' : 'buildprops.json'
- }
- )
- self.locales = None
- self.credentials = None
-
- def _pre_config_lock(self, rw_config):
- super(BouncerSubmitter, self)._pre_config_lock(rw_config)
-
- #override properties from buildbot properties here as defined by taskcluster properties
- self.read_buildbot_config()
-
- #check if release promotion is true first before overwriting these properties
- if self.buildbot_config["properties"].get("release_promotion"):
- for prop in ['product', 'version', 'build_number', 'revision', 'bouncer_submitter_config', ]:
- if self.buildbot_config["properties"].get(prop):
- self.info("Overriding %s with %s" % (prop, self.buildbot_config["properties"].get(prop)))
- self.config[prop] = self.buildbot_config["properties"].get(prop)
- if self.buildbot_config["properties"].get("partial_versions"):
- self.config["prev_versions"] = self.buildbot_config["properties"].get("partial_versions").split(", ")
-
- for opt in ["version", "credentials_file", "bouncer-api-prefix"]:
- if opt not in self.config:
- self.fatal("%s must be specified" % opt)
- if self.need_shipped_locales():
- for opt in ["shipped-locales-url", "repo", "revision"]:
- if opt not in self.config:
- self.fatal("%s must be specified" % opt)
-
- def need_shipped_locales(self):
- return any(e.get("add-locales") for e in
- self.config["products"].values())
-
- def query_shipped_locales_path(self):
- dirs = self.query_abs_dirs()
- return os.path.join(dirs["abs_work_dir"], "shipped-locales")
-
- def download_shipped_locales(self):
- if not self.need_shipped_locales():
- self.info("No need to download shipped-locales")
- return
-
- replace_dict = {"revision": self.config["revision"],
- "repo": self.config["repo"]}
- url = self.config["shipped-locales-url"] % replace_dict
- dirs = self.query_abs_dirs()
- self.mkdir_p(dirs["abs_work_dir"])
- if not self.download_file(url=url,
- file_name=self.query_shipped_locales_path()):
- self.fatal("Unable to fetch shipped-locales from %s" % url)
- # populate the list
- self.load_shipped_locales()
-
- def load_shipped_locales(self):
- if self.locales:
- return self.locales
- content = self.read_from_file(self.query_shipped_locales_path())
- locales = []
- for line in content.splitlines():
- locale = line.split()[0]
- if locale:
- locales.append(locale)
- self.locales = locales
- return self.locales
-
- def submit(self):
- subs = {
- "version": self.config["version"]
- }
- if self.config.get("build_number"):
- subs["build_number"] = self.config["build_number"]
-
- for product, pr_config in sorted(self.config["products"].items()):
- product_name = pr_config["product-name"] % subs
- if self.product_exists(product_name):
- self.warning("Product %s already exists. Skipping..." %
- product_name)
- continue
- self.info("Adding %s..." % product)
- self.api_add_product(
- product_name=product_name,
- add_locales=pr_config.get("add-locales"),
- ssl_only=pr_config.get("ssl-only"))
- self.info("Adding paths...")
- for platform, pl_config in sorted(pr_config["paths"].items()):
- bouncer_platform = pl_config["bouncer-platform"]
- path = pl_config["path"] % subs
- self.info("%s (%s): %s" % (platform, bouncer_platform, path))
- self.api_add_location(product_name, bouncer_platform, path)
-
- # Add partial updates
- if "partials" in self.config and self.config.get("prev_versions"):
- self.submit_partials()
-
- def submit_partials(self):
- subs = {
- "version": self.config["version"]
- }
- if self.config.get("build_number"):
- subs["build_number"] = self.config["build_number"]
- prev_versions = self.config.get("prev_versions")
- for product, part_config in sorted(self.config["partials"].items()):
- product_name_tmpl = part_config["product-name"]
- for prev_version in prev_versions:
- prev_version, prev_build_number = prev_version.split("build")
- subs["prev_version"] = prev_version
- subs["prev_build_number"] = prev_build_number
- product_name = product_name_tmpl % subs
- if self.product_exists(product_name):
- self.warning("Product %s already exists. Skipping..." %
- product_name)
- continue
- self.info("Adding partial updates for %s" % product_name)
- self.api_add_product(
- product_name=product_name,
- add_locales=part_config.get("add-locales"),
- ssl_only=part_config.get("ssl-only"))
- for platform, pl_config in sorted(part_config["paths"].items()):
- bouncer_platform = pl_config["bouncer-platform"]
- path = pl_config["path"] % subs
- self.info("%s (%s): %s" % (platform, bouncer_platform, path))
- self.api_add_location(product_name, bouncer_platform, path)
-
-
-if __name__ == '__main__':
- myScript = BouncerSubmitter()
- myScript.run_and_exit()
diff --git a/testing/mozharness/scripts/configtest.py b/testing/mozharness/scripts/configtest.py
deleted file mode 100755
index 5db684f0a..000000000
--- a/testing/mozharness/scripts/configtest.py
+++ /dev/null
@@ -1,142 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""configtest.py
-
-Verify the .json and .py files in the configs/ directory are well-formed.
-Further tests to verify validity would be desirable.
-
-This is also a good example script to look at to understand mozharness.
-"""
-
-import os
-import pprint
-import sys
-try:
- import simplejson as json
-except ImportError:
- import json
-
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.script import BaseScript
-
-# ConfigTest {{{1
-class ConfigTest(BaseScript):
- config_options = [[
- ["--test-file",],
- {"action": "extend",
- "dest": "test_files",
- "help": "Specify which config files to test"
- }
- ]]
-
- def __init__(self, require_config_file=False):
- self.config_files = []
- BaseScript.__init__(self, config_options=self.config_options,
- all_actions=['list-config-files',
- 'test-json-configs',
- 'test-python-configs',
- 'summary',
- ],
- default_actions=['test-json-configs',
- 'test-python-configs',
- 'summary',
- ],
- require_config_file=require_config_file)
-
- def query_config_files(self):
- """This query method, much like others, caches its runtime
- settings in self.VAR so we don't have to figure out config_files
- multiple times.
- """
- if self.config_files:
- return self.config_files
- c = self.config
- if 'test_files' in c:
- self.config_files = c['test_files']
- return self.config_files
- self.debug("No --test-file(s) specified; defaulting to crawling the configs/ directory.")
- config_files = []
- for root, dirs, files in os.walk(os.path.join(sys.path[0], "..",
- "configs")):
- for name in files:
- # Hardcode =P
- if name.endswith(".json") or name.endswith(".py"):
- if not name.startswith("test_malformed"):
- config_files.append(os.path.join(root, name))
- self.config_files = config_files
- return self.config_files
-
- def list_config_files(self):
- """ Non-default action that is mainly here to demonstrate how
- non-default actions work in a mozharness script.
- """
- config_files = self.query_config_files()
- for config_file in config_files:
- self.info(config_file)
-
- def test_json_configs(self):
- """ Currently only "is this well-formed json?"
-
- """
- config_files = self.query_config_files()
- filecount = [0, 0]
- for config_file in config_files:
- if config_file.endswith(".json"):
- filecount[0] += 1
- self.info("Testing %s." % config_file)
- contents = self.read_from_file(config_file, verbose=False)
- try:
- json.loads(contents)
- except ValueError:
- self.add_summary("%s is invalid json." % config_file,
- level="error")
- self.error(pprint.pformat(sys.exc_info()[1]))
- else:
- self.info("Good.")
- filecount[1] += 1
- if filecount[0]:
- self.add_summary("%d of %d json config files were good." %
- (filecount[1], filecount[0]))
- else:
- self.add_summary("No json config files to test.")
-
- def test_python_configs(self):
- """Currently only "will this give me a config dictionary?"
-
- """
- config_files = self.query_config_files()
- filecount = [0, 0]
- for config_file in config_files:
- if config_file.endswith(".py"):
- filecount[0] += 1
- self.info("Testing %s." % config_file)
- global_dict = {}
- local_dict = {}
- try:
- execfile(config_file, global_dict, local_dict)
- except:
- self.add_summary("%s is invalid python." % config_file,
- level="error")
- self.error(pprint.pformat(sys.exc_info()[1]))
- else:
- if 'config' in local_dict and isinstance(local_dict['config'], dict):
- self.info("Good.")
- filecount[1] += 1
- else:
- self.add_summary("%s is valid python, but doesn't create a config dictionary." %
- config_file, level="error")
- if filecount[0]:
- self.add_summary("%d of %d python config files were good." %
- (filecount[1], filecount[0]))
- else:
- self.add_summary("No python config files to test.")
-
-# __main__ {{{1
-if __name__ == '__main__':
- config_test = ConfigTest()
- config_test.run_and_exit()
diff --git a/testing/mozharness/scripts/desktop_l10n.py b/testing/mozharness/scripts/desktop_l10n.py
deleted file mode 100755
index 0626ce35b..000000000
--- a/testing/mozharness/scripts/desktop_l10n.py
+++ /dev/null
@@ -1,1152 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""desktop_l10n.py
-
-This script manages Desktop repacks for nightly builds.
-"""
-import os
-import re
-import sys
-import time
-import shlex
-import subprocess
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.errors import BaseErrorList, MakefileErrorList
-from mozharness.base.script import BaseScript
-from mozharness.base.transfer import TransferMixin
-from mozharness.base.vcs.vcsbase import VCSMixin
-from mozharness.mozilla.buildbot import BuildbotMixin
-from mozharness.mozilla.purge import PurgeMixin
-from mozharness.mozilla.building.buildbase import MakeUploadOutputParser
-from mozharness.mozilla.l10n.locales import LocalesMixin
-from mozharness.mozilla.mar import MarMixin
-from mozharness.mozilla.mock import MockMixin
-from mozharness.mozilla.release import ReleaseMixin
-from mozharness.mozilla.signing import SigningMixin
-from mozharness.mozilla.updates.balrog import BalrogMixin
-from mozharness.mozilla.taskcluster_helper import Taskcluster
-from mozharness.base.python import VirtualenvMixin
-from mozharness.mozilla.mock import ERROR_MSGS
-
-try:
- import simplejson as json
- assert json
-except ImportError:
- import json
-
-
-# needed by _map
-SUCCESS = 0
-FAILURE = 1
-
-SUCCESS_STR = "Success"
-FAILURE_STR = "Failed"
-
-# when running get_output_form_command, pymake has some extra output
-# that needs to be filtered out
-PyMakeIgnoreList = [
- re.compile(r'''.*make\.py(?:\[\d+\])?: Entering directory'''),
- re.compile(r'''.*make\.py(?:\[\d+\])?: Leaving directory'''),
-]
-
-
-# mandatory configuration options, without them, this script will not work
-# it's a list of values that are already known before starting a build
-configuration_tokens = ('branch',
- 'platform',
- 'update_platform',
- 'update_channel',
- 'ssh_key_dir',
- 'stage_product',
- 'upload_environment',
- )
-# some other values such as "%(version)s", "%(buildid)s", ...
-# are defined at run time and they cannot be enforced in the _pre_config_lock
-# phase
-runtime_config_tokens = ('buildid', 'version', 'locale', 'from_buildid',
- 'abs_objdir', 'abs_merge_dir', 'revision',
- 'to_buildid', 'en_us_binary_url', 'mar_tools_url',
- 'post_upload_extra', 'who')
-
-# DesktopSingleLocale {{{1
-class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
- VCSMixin, SigningMixin, PurgeMixin, BaseScript,
- BalrogMixin, MarMixin, VirtualenvMixin, TransferMixin):
- """Manages desktop repacks"""
- config_options = [[
- ['--balrog-config', ],
- {"action": "extend",
- "dest": "config_files",
- "type": "string",
- "help": "Specify the balrog configuration file"}
- ], [
- ['--branch-config', ],
- {"action": "extend",
- "dest": "config_files",
- "type": "string",
- "help": "Specify the branch configuration file"}
- ], [
- ['--environment-config', ],
- {"action": "extend",
- "dest": "config_files",
- "type": "string",
- "help": "Specify the environment (staging, production, ...) configuration file"}
- ], [
- ['--platform-config', ],
- {"action": "extend",
- "dest": "config_files",
- "type": "string",
- "help": "Specify the platform configuration file"}
- ], [
- ['--locale', ],
- {"action": "extend",
- "dest": "locales",
- "type": "string",
- "help": "Specify the locale(s) to sign and update. Optionally pass"
- " revision separated by colon, en-GB:default."}
- ], [
- ['--locales-file', ],
- {"action": "store",
- "dest": "locales_file",
- "type": "string",
- "help": "Specify a file to determine which locales to sign and update"}
- ], [
- ['--tag-override', ],
- {"action": "store",
- "dest": "tag_override",
- "type": "string",
- "help": "Override the tags set for all repos"}
- ], [
- ['--revision', ],
- {"action": "store",
- "dest": "revision",
- "type": "string",
- "help": "Override the gecko revision to use (otherwise use buildbot supplied"
- " value, or en-US revision) "}
- ], [
- ['--user-repo-override', ],
- {"action": "store",
- "dest": "user_repo_override",
- "type": "string",
- "help": "Override the user repo path for all repos"}
- ], [
- ['--release-config-file', ],
- {"action": "store",
- "dest": "release_config_file",
- "type": "string",
- "help": "Specify the release config file to use"}
- ], [
- ['--this-chunk', ],
- {"action": "store",
- "dest": "this_locale_chunk",
- "type": "int",
- "help": "Specify which chunk of locales to run"}
- ], [
- ['--total-chunks', ],
- {"action": "store",
- "dest": "total_locale_chunks",
- "type": "int",
- "help": "Specify the total number of chunks of locales"}
- ], [
- ['--en-us-installer-url', ],
- {"action": "store",
- "dest": "en_us_installer_url",
- "type": "string",
- "help": "Specify the url of the en-us binary"}
- ], [
- ["--disable-mock"], {
- "dest": "disable_mock",
- "action": "store_true",
- "help": "do not run under mock despite what gecko-config says"}
- ]]
-
- def __init__(self, require_config_file=True):
- # fxbuild style:
- buildscript_kwargs = {
- 'all_actions': [
- "clobber",
- "pull",
- "clone-locales",
- "list-locales",
- "setup",
- "repack",
- "taskcluster-upload",
- "funsize-props",
- "submit-to-balrog",
- "summary",
- ],
- 'config': {
- "buildbot_json_path": "buildprops.json",
- "ignore_locales": ["en-US"],
- "locales_dir": "browser/locales",
- "update_mar_dir": "dist/update",
- "buildid_section": "App",
- "buildid_option": "BuildID",
- "application_ini": "application.ini",
- "log_name": "single_locale",
- "clobber_file": 'CLOBBER',
- "appName": "Firefox",
- "hashType": "sha512",
- "taskcluster_credentials_file": "oauth.txt",
- 'virtualenv_modules': [
- 'requests==2.8.1',
- 'PyHawk-with-a-single-extra-commit==0.1.5',
- 'taskcluster==0.0.26',
- ],
- 'virtualenv_path': 'venv',
- },
- }
- #
-
- LocalesMixin.__init__(self)
- BaseScript.__init__(
- self,
- config_options=self.config_options,
- require_config_file=require_config_file,
- **buildscript_kwargs
- )
-
- self.buildid = None
- self.make_ident_output = None
- self.bootstrap_env = None
- self.upload_env = None
- self.revision = None
- self.enUS_revision = None
- self.version = None
- self.upload_urls = {}
- self.locales_property = {}
- self.package_urls = {}
- self.pushdate = None
- # upload_files is a dictionary of files to upload, keyed by locale.
- self.upload_files = {}
-
- if 'mock_target' in self.config:
- self.enable_mock()
-
- def _pre_config_lock(self, rw_config):
- """replaces 'configuration_tokens' with their values, before the
- configuration gets locked. If some of the configuration_tokens
- are not present, stops the execution of the script"""
- # since values as branch, platform are mandatory, can replace them in
- # in the configuration before it is locked down
- # mandatory tokens
- for token in configuration_tokens:
- if token not in self.config:
- self.fatal('No %s in configuration!' % token)
-
- # all the important tokens are present in our configuration
- for token in configuration_tokens:
- # token_string '%(branch)s'
- token_string = ''.join(('%(', token, ')s'))
- # token_value => ash
- token_value = self.config[token]
- for element in self.config:
- # old_value => https://hg.mozilla.org/projects/%(branch)s
- old_value = self.config[element]
- # new_value => https://hg.mozilla.org/projects/ash
- new_value = self.__detokenise_element(self.config[element],
- token_string,
- token_value)
- if new_value and new_value != old_value:
- msg = "%s: replacing %s with %s" % (element,
- old_value,
- new_value)
- self.debug(msg)
- self.config[element] = new_value
-
- # now, only runtime_config_tokens should be present in config
- # we should parse self.config and fail if any other we spot any
- # other token
- tokens_left = set(self._get_configuration_tokens(self.config))
- unknown_tokens = set(tokens_left) - set(runtime_config_tokens)
- if unknown_tokens:
- msg = ['unknown tokens in configuration:']
- for t in unknown_tokens:
- msg.append(t)
- self.fatal(' '.join(msg))
- self.info('configuration looks ok')
-
- self.read_buildbot_config()
- if not self.buildbot_config:
- self.warning("Skipping buildbot properties overrides")
- return
- props = self.buildbot_config["properties"]
- for prop in ['mar_tools_url']:
- if props.get(prop):
- self.info("Overriding %s with %s" % (prop, props[prop]))
- self.config[prop] = props.get(prop)
-
- def _get_configuration_tokens(self, iterable):
- """gets a list of tokens in iterable"""
- regex = re.compile('%\(\w+\)s')
- results = []
- try:
- for element in iterable:
- if isinstance(iterable, str):
- # this is a string, look for tokens
- # self.debug("{0}".format(re.findall(regex, element)))
- tokens = re.findall(regex, iterable)
- for token in tokens:
- # clean %(branch)s => branch
- # remove %(
- token_name = token.partition('%(')[2]
- # remove )s
- token_name = token_name.partition(')s')[0]
- results.append(token_name)
- break
-
- elif isinstance(iterable, (list, tuple)):
- results.extend(self._get_configuration_tokens(element))
-
- elif isinstance(iterable, dict):
- results.extend(self._get_configuration_tokens(iterable[element]))
-
- except TypeError:
- # element is a int/float/..., nothing to do here
- pass
-
- # remove duplicates, and return results
-
- return list(set(results))
-
- def __detokenise_element(self, config_option, token, value):
- """reads config_options and returns a version of the same config_option
- replacing token with value recursively"""
- # config_option is a string, let's replace token with value
- if isinstance(config_option, str):
- # if token does not appear in this string,
- # nothing happens and the original value is returned
- return config_option.replace(token, value)
- # it's a dictionary
- elif isinstance(config_option, dict):
- # replace token for each element of this dictionary
- for element in config_option:
- config_option[element] = self.__detokenise_element(
- config_option[element], token, value)
- return config_option
- # it's a list
- elif isinstance(config_option, list):
- # create a new list and append the replaced elements
- new_list = []
- for element in config_option:
- new_list.append(self.__detokenise_element(element, token, value))
- return new_list
- elif isinstance(config_option, tuple):
- # create a new list and append the replaced elements
- new_list = []
- for element in config_option:
- new_list.append(self.__detokenise_element(element, token, value))
- return tuple(new_list)
- else:
- # everything else, bool, number, ...
- return config_option
-
- # Helper methods {{{2
- def query_bootstrap_env(self):
- """returns the env for repacks"""
- if self.bootstrap_env:
- return self.bootstrap_env
- config = self.config
- replace_dict = self.query_abs_dirs()
-
- replace_dict['en_us_binary_url'] = config.get('en_us_binary_url')
- self.read_buildbot_config()
- # Override en_us_binary_url if packageUrl is passed as a property from
- # the en-US build
- if self.buildbot_config["properties"].get("packageUrl"):
- packageUrl = self.buildbot_config["properties"]["packageUrl"]
- # trim off the filename, the build system wants a directory
- packageUrl = packageUrl.rsplit('/', 1)[0]
- self.info("Overriding en_us_binary_url with %s" % packageUrl)
- replace_dict['en_us_binary_url'] = str(packageUrl)
- # Override en_us_binary_url if passed as a buildbot property
- if self.buildbot_config["properties"].get("en_us_binary_url"):
- self.info("Overriding en_us_binary_url with %s" %
- self.buildbot_config["properties"]["en_us_binary_url"])
- replace_dict['en_us_binary_url'] = \
- str(self.buildbot_config["properties"]["en_us_binary_url"])
- bootstrap_env = self.query_env(partial_env=config.get("bootstrap_env"),
- replace_dict=replace_dict)
- if 'MOZ_SIGNING_SERVERS' in os.environ:
- sign_cmd = self.query_moz_sign_cmd(formats=None)
- sign_cmd = subprocess.list2cmdline(sign_cmd)
- # windows fix
- bootstrap_env['MOZ_SIGN_CMD'] = sign_cmd.replace('\\', '\\\\\\\\')
- for binary in self._mar_binaries():
- # "mar -> MAR" and 'mar.exe -> MAR' (windows)
- name = binary.replace('.exe', '')
- name = name.upper()
- binary_path = os.path.join(self._mar_tool_dir(), binary)
- # windows fix...
- if binary.endswith('.exe'):
- binary_path = binary_path.replace('\\', '\\\\\\\\')
- bootstrap_env[name] = binary_path
- if 'LOCALE_MERGEDIR' in bootstrap_env:
- # windows fix
- bootstrap_env['LOCALE_MERGEDIR'] = bootstrap_env['LOCALE_MERGEDIR'].replace('\\', '\\\\\\\\')
- if self.query_is_nightly():
- bootstrap_env["IS_NIGHTLY"] = "yes"
- self.bootstrap_env = bootstrap_env
- return self.bootstrap_env
-
- def _query_upload_env(self):
- """returns the environment used for the upload step"""
- if self.upload_env:
- return self.upload_env
- config = self.config
-
- replace_dict = {
- 'buildid': self._query_buildid(),
- 'version': self.query_version(),
- 'post_upload_extra': ' '.join(config.get('post_upload_extra', [])),
- 'upload_environment': config['upload_environment'],
- }
- if config['branch'] == 'try':
- replace_dict.update({
- 'who': self.query_who(),
- 'revision': self._query_revision(),
- })
- upload_env = self.query_env(partial_env=config.get("upload_env"),
- replace_dict=replace_dict)
- # check if there are any extra option from the platform configuration
- # and append them to the env
-
- if 'upload_env_extra' in config:
- for extra in config['upload_env_extra']:
- upload_env[extra] = config['upload_env_extra'][extra]
-
- self.upload_env = upload_env
- return self.upload_env
-
- def query_l10n_env(self):
- l10n_env = self._query_upload_env().copy()
- # both upload_env and bootstrap_env define MOZ_SIGN_CMD
- # the one from upload_env is taken from os.environ, the one from
- # bootstrap_env is set with query_moz_sign_cmd()
- # we need to use the value provided my query_moz_sign_cmd or make upload
- # will fail (signtool.py path is wrong)
- l10n_env.update(self.query_bootstrap_env())
- return l10n_env
-
- def _query_make_ident_output(self):
- """Get |make ident| output from the objdir.
- Only valid after setup is run.
- """
- if self.make_ident_output:
- return self.make_ident_output
- dirs = self.query_abs_dirs()
- self.make_ident_output = self._get_output_from_make(
- target=["ident"],
- cwd=dirs['abs_locales_dir'],
- env=self.query_bootstrap_env())
- return self.make_ident_output
-
- def _query_buildid(self):
- """Get buildid from the objdir.
- Only valid after setup is run.
- """
- if self.buildid:
- return self.buildid
- r = re.compile(r"buildid (\d+)")
- output = self._query_make_ident_output()
- for line in output.splitlines():
- match = r.match(line)
- if match:
- self.buildid = match.groups()[0]
- return self.buildid
-
- def _query_revision(self):
- """ Get the gecko revision in this order of precedence
- * cached value
- * command line arg --revision (development, taskcluster)
- * buildbot properties (try with buildbot forced build)
- * buildbot change (try with buildbot scheduler)
- * from the en-US build (m-c & m-a)
-
- This will fail the last case if the build hasn't been pulled yet.
- """
- if self.revision:
- return self.revision
-
- self.read_buildbot_config()
- config = self.config
- revision = None
- if config.get("revision"):
- revision = config["revision"]
- elif 'revision' in self.buildbot_properties:
- revision = self.buildbot_properties['revision']
- elif (self.buildbot_config and
- self.buildbot_config.get('sourcestamp', {}).get('revision')):
- revision = self.buildbot_config['sourcestamp']['revision']
- elif self.buildbot_config and self.buildbot_config.get('revision'):
- revision = self.buildbot_config['revision']
- elif config.get("update_gecko_source_to_enUS", True):
- revision = self._query_enUS_revision()
-
- if not revision:
- self.fatal("Can't determine revision!")
- self.revision = str(revision)
- return self.revision
-
- def _query_enUS_revision(self):
- """Get revision from the objdir.
- Only valid after setup is run.
- """
- if self.enUS_revision:
- return self.enUS_revision
- r = re.compile(r"^(gecko|fx)_revision ([0-9a-f]+\+?)$")
- output = self._query_make_ident_output()
- for line in output.splitlines():
- match = r.match(line)
- if match:
- self.enUS_revision = match.groups()[1]
- return self.enUS_revision
-
- def _query_make_variable(self, variable, make_args=None,
- exclude_lines=PyMakeIgnoreList):
- """returns the value of make echo-variable-<variable>
- it accepts extra make arguements (make_args)
- it also has an exclude_lines from the output filer
- exclude_lines defaults to PyMakeIgnoreList because
- on windows, pymake writes extra output lines that need
- to be filtered out.
- """
- dirs = self.query_abs_dirs()
- make_args = make_args or []
- exclude_lines = exclude_lines or []
- target = ["echo-variable-%s" % variable] + make_args
- cwd = dirs['abs_locales_dir']
- raw_output = self._get_output_from_make(target, cwd=cwd,
- env=self.query_bootstrap_env())
- # we want to log all the messages from make/pymake and
- # exlcude some messages from the output ("Entering directory...")
- output = []
- for line in raw_output.split("\n"):
- discard = False
- for element in exclude_lines:
- if element.match(line):
- discard = True
- continue
- if not discard:
- output.append(line.strip())
- output = " ".join(output).strip()
- self.info('echo-variable-%s: %s' % (variable, output))
- return output
-
- def query_version(self):
- """Gets the version from the objdir.
- Only valid after setup is run."""
- if self.version:
- return self.version
- config = self.config
- if config.get('release_config_file'):
- release_config = self.query_release_config()
- self.version = release_config['version']
- else:
- self.version = self._query_make_variable("MOZ_APP_VERSION")
- return self.version
-
- def _map(self, func, items):
- """runs func for any item in items, calls the add_failure() for each
- error. It assumes that function returns 0 when successful.
- returns a two element tuple with (success_count, total_count)"""
- success_count = 0
- total_count = len(items)
- name = func.__name__
- for item in items:
- result = func(item)
- if result == SUCCESS:
- # success!
- success_count += 1
- else:
- # func failed...
- message = 'failure: %s(%s)' % (name, item)
- self._add_failure(item, message)
- return (success_count, total_count)
-
- def _add_failure(self, locale, message, **kwargs):
- """marks current step as failed"""
- self.locales_property[locale] = FAILURE_STR
- prop_key = "%s_failure" % locale
- prop_value = self.query_buildbot_property(prop_key)
- if prop_value:
- prop_value = "%s %s" % (prop_value, message)
- else:
- prop_value = message
- self.set_buildbot_property(prop_key, prop_value, write_to_file=True)
- BaseScript.add_failure(self, locale, message=message, **kwargs)
-
- def query_failed_locales(self):
- return [l for l, res in self.locales_property.items() if
- res == FAILURE_STR]
-
- def summary(self):
- """generates a summary"""
- BaseScript.summary(self)
- # TODO we probably want to make this configurable on/off
- locales = self.query_locales()
- for locale in locales:
- self.locales_property.setdefault(locale, SUCCESS_STR)
- self.set_buildbot_property("locales",
- json.dumps(self.locales_property),
- write_to_file=True)
-
- # Actions {{{2
- def clobber(self):
- """clobber"""
- dirs = self.query_abs_dirs()
- clobber_dirs = (dirs['abs_objdir'], dirs['abs_upload_dir'])
- PurgeMixin.clobber(self, always_clobber_dirs=clobber_dirs)
-
- def pull(self):
- """pulls source code"""
- config = self.config
- dirs = self.query_abs_dirs()
- repos = []
- # replace dictionary for repos
- # we need to interpolate some values:
- # branch, branch_repo
- # and user_repo_override if exists
- replace_dict = {}
- if config.get("user_repo_override"):
- replace_dict['user_repo_override'] = config['user_repo_override']
- # this is OK so early because we get it from buildbot, or
- # the command line for local dev
- replace_dict['revision'] = self._query_revision()
-
- for repository in config['repos']:
- current_repo = {}
- for key, value in repository.iteritems():
- try:
- current_repo[key] = value % replace_dict
- except TypeError:
- # pass through non-interpolables, like booleans
- current_repo[key] = value
- except KeyError:
- self.error('not all the values in "{0}" can be replaced. Check your configuration'.format(value))
- raise
- repos.append(current_repo)
- self.info("repositories: %s" % repos)
- self.vcs_checkout_repos(repos, parent_dir=dirs['abs_work_dir'],
- tag_override=config.get('tag_override'))
-
- def clone_locales(self):
- self.pull_locale_source()
-
- def setup(self):
- """setup step"""
- dirs = self.query_abs_dirs()
- self._run_tooltool()
- self._copy_mozconfig()
- self._mach_configure()
- self._run_make_in_config_dir()
- self.make_wget_en_US()
- self.make_unpack_en_US()
- self.download_mar_tools()
-
- # on try we want the source we already have, otherwise update to the
- # same as the en-US binary
- if self.config.get("update_gecko_source_to_enUS", True):
- revision = self._query_enUS_revision()
- # TODO do this through VCSMixin instead of hardcoding hg
- # self.update(dest=dirs["abs_mozilla_dir"], revision=revision)
- hg = self.query_exe("hg")
- self.run_command([hg, "update", "-r", revision],
- cwd=dirs["abs_mozilla_dir"],
- env=self.query_bootstrap_env(),
- error_list=BaseErrorList,
- halt_on_failure=True, fatal_exit_code=3)
- # if checkout updates CLOBBER file with a newer timestamp,
- # next make -f client.mk configure will delete archives
- # downloaded with make wget_en_US, so just touch CLOBBER file
- _clobber_file = self._clobber_file()
- if os.path.exists(_clobber_file):
- self._touch_file(_clobber_file)
- # and again...
- # thanks to the last hg update, we can be on different firefox 'version'
- # than the one on default,
- self._mach_configure()
- self._run_make_in_config_dir()
-
- def _run_make_in_config_dir(self):
- """this step creates nsinstall, needed my make_wget_en_US()
- """
- dirs = self.query_abs_dirs()
- config_dir = os.path.join(dirs['abs_objdir'], 'config')
- env = self.query_bootstrap_env()
- return self._make(target=['export'], cwd=config_dir, env=env)
-
- def _clobber_file(self):
- """returns the full path of the clobber file"""
- config = self.config
- dirs = self.query_abs_dirs()
- return os.path.join(dirs['abs_objdir'], config.get('clobber_file'))
-
- def _copy_mozconfig(self):
- """copies the mozconfig file into abs_mozilla_dir/.mozconfig
- and logs the content
- """
- config = self.config
- dirs = self.query_abs_dirs()
- mozconfig = config['mozconfig']
- src = os.path.join(dirs['abs_work_dir'], mozconfig)
- dst = os.path.join(dirs['abs_mozilla_dir'], '.mozconfig')
- self.copyfile(src, dst)
- self.read_from_file(dst, verbose=True)
-
- def _mach(self, target, env, halt_on_failure=True, output_parser=None):
- dirs = self.query_abs_dirs()
- mach = self._get_mach_executable()
- return self.run_command(mach + target,
- halt_on_failure=True,
- env=env,
- cwd=dirs['abs_mozilla_dir'],
- output_parser=None)
-
- def _mach_configure(self):
- """calls mach configure"""
- env = self.query_bootstrap_env()
- target = ["configure"]
- return self._mach(target=target, env=env)
-
- def _get_mach_executable(self):
- python = self.query_exe('python2.7')
- return [python, 'mach']
-
- def _get_make_executable(self):
- config = self.config
- dirs = self.query_abs_dirs()
- if config.get('enable_mozmake'): # e.g. windows
- make = r"/".join([dirs['abs_mozilla_dir'], 'mozmake.exe'])
- # mysterious subprocess errors, let's try to fix this path...
- make = make.replace('\\', '/')
- make = [make]
- else:
- make = ['make']
- return make
-
- def _make(self, target, cwd, env, error_list=MakefileErrorList,
- halt_on_failure=True, output_parser=None):
- """Runs make. Returns the exit code"""
- make = self._get_make_executable()
- if target:
- make = make + target
- return self.run_command(make,
- cwd=cwd,
- env=env,
- error_list=error_list,
- halt_on_failure=halt_on_failure,
- output_parser=output_parser)
-
- def _get_output_from_make(self, target, cwd, env, halt_on_failure=True, ignore_errors=False):
- """runs make and returns the output of the command"""
- make = self._get_make_executable()
- return self.get_output_from_command(make + target,
- cwd=cwd,
- env=env,
- silent=True,
- halt_on_failure=halt_on_failure,
- ignore_errors=ignore_errors)
-
- def make_unpack_en_US(self):
- """wrapper for make unpack"""
- config = self.config
- dirs = self.query_abs_dirs()
- env = self.query_bootstrap_env()
- cwd = os.path.join(dirs['abs_objdir'], config['locales_dir'])
- return self._make(target=["unpack"], cwd=cwd, env=env)
-
- def make_wget_en_US(self):
- """wrapper for make wget-en-US"""
- env = self.query_bootstrap_env()
- dirs = self.query_abs_dirs()
- cwd = dirs['abs_locales_dir']
- return self._make(target=["wget-en-US"], cwd=cwd, env=env)
-
- def make_upload(self, locale):
- """wrapper for make upload command"""
- config = self.config
- env = self.query_l10n_env()
- dirs = self.query_abs_dirs()
- buildid = self._query_buildid()
- replace_dict = {
- 'buildid': buildid,
- 'branch': config['branch']
- }
- try:
- env['POST_UPLOAD_CMD'] = config['base_post_upload_cmd'] % replace_dict
- except KeyError:
- # no base_post_upload_cmd in configuration, just skip it
- pass
- target = ['upload', 'AB_CD=%s' % (locale)]
- cwd = dirs['abs_locales_dir']
- parser = MakeUploadOutputParser(config=self.config,
- log_obj=self.log_obj)
- retval = self._make(target=target, cwd=cwd, env=env,
- halt_on_failure=False, output_parser=parser)
- if locale not in self.package_urls:
- self.package_urls[locale] = {}
- self.package_urls[locale].update(parser.matches)
- if retval == SUCCESS:
- self.info('Upload successful (%s)' % locale)
- ret = SUCCESS
- else:
- self.error('failed to upload %s' % locale)
- ret = FAILURE
- return ret
-
- def set_upload_files(self, locale):
- # The tree doesn't have a good way of exporting the list of files
- # created during locale generation, but we can grab them by echoing the
- # UPLOAD_FILES variable for each locale.
- env = self.query_l10n_env()
- target = ['echo-variable-UPLOAD_FILES', 'echo-variable-CHECKSUM_FILES',
- 'AB_CD=%s' % locale]
- dirs = self.query_abs_dirs()
- cwd = dirs['abs_locales_dir']
- # Bug 1242771 - echo-variable-UPLOAD_FILES via mozharness fails when stderr is found
- # we should ignore stderr as unfortunately it's expected when parsing for values
- output = self._get_output_from_make(target=target, cwd=cwd, env=env,
- ignore_errors=True)
- self.info('UPLOAD_FILES is "%s"' % output)
- files = shlex.split(output)
- if not files:
- self.error('failed to get upload file list for locale %s' % locale)
- return FAILURE
-
- self.upload_files[locale] = [
- os.path.abspath(os.path.join(cwd, f)) for f in files
- ]
- return SUCCESS
-
- def make_installers(self, locale):
- """wrapper for make installers-(locale)"""
- env = self.query_l10n_env()
- self._copy_mozconfig()
- dirs = self.query_abs_dirs()
- cwd = os.path.join(dirs['abs_locales_dir'])
- target = ["installers-%s" % locale,
- "LOCALE_MERGEDIR=%s" % env["LOCALE_MERGEDIR"], ]
- return self._make(target=target, cwd=cwd,
- env=env, halt_on_failure=False)
-
- def repack_locale(self, locale):
- """wraps the logic for compare locale, make installers and generating
- complete updates."""
-
- if self.run_compare_locales(locale) != SUCCESS:
- self.error("compare locale %s failed" % (locale))
- return FAILURE
-
- # compare locale succeeded, run make installers
- if self.make_installers(locale) != SUCCESS:
- self.error("make installers-%s failed" % (locale))
- return FAILURE
-
- # now try to upload the artifacts
- if self.make_upload(locale):
- self.error("make upload for locale %s failed!" % (locale))
- return FAILURE
-
- # set_upload_files() should be called after make upload, to make sure
- # we have all files in place (checksums, etc)
- if self.set_upload_files(locale):
- self.error("failed to get list of files to upload for locale %s" % locale)
- return FAILURE
-
- return SUCCESS
-
- def repack(self):
- """creates the repacks and udpates"""
- self._map(self.repack_locale, self.query_locales())
-
- def _query_objdir(self):
- """returns objdir name from configuration"""
- return self.config['objdir']
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(DesktopSingleLocale, self).query_abs_dirs()
- for directory in abs_dirs:
- value = abs_dirs[directory]
- abs_dirs[directory] = value
- dirs = {}
- dirs['abs_tools_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tools')
- for key in dirs.keys():
- if key not in abs_dirs:
- abs_dirs[key] = dirs[key]
- self.abs_dirs = abs_dirs
- return self.abs_dirs
-
- def submit_to_balrog(self):
- """submit to balrog"""
- if not self.config.get("balrog_servers"):
- self.info("balrog_servers not set; skipping balrog submission.")
- return
- self.info("Reading buildbot build properties...")
- self.read_buildbot_config()
- # get platform, appName and hashType from configuration
- # common values across different locales
- config = self.config
- platform = config["platform"]
- hashType = config['hashType']
- appName = config['appName']
- branch = config['branch']
- # values from configuration
- self.set_buildbot_property("branch", branch)
- self.set_buildbot_property("appName", appName)
- # it's hardcoded to sha512 in balrog.py
- self.set_buildbot_property("hashType", hashType)
- self.set_buildbot_property("platform", platform)
- # values common to the current repacks
- self.set_buildbot_property("buildid", self._query_buildid())
- self.set_buildbot_property("appVersion", self.query_version())
-
- # submit complete mar to balrog
- # clean up buildbot_properties
- self._map(self.submit_repack_to_balrog, self.query_locales())
-
- def submit_repack_to_balrog(self, locale):
- """submit a single locale to balrog"""
- # check if locale has been uploaded, if not just return a FAILURE
- if locale not in self.package_urls:
- self.error("%s is not present in package_urls. Did you run make upload?" % locale)
- return FAILURE
-
- if not self.query_is_nightly():
- # remove this check when we extend this script to non-nightly builds
- self.fatal("Not a nightly build")
- return FAILURE
-
- # complete mar file
- c_marfile = self._query_complete_mar_filename(locale)
- c_mar_url = self._query_complete_mar_url(locale)
-
- # Set other necessary properties for Balrog submission. None need to
- # be passed back to buildbot, so we won't write them to the properties
- # files
- # Locale is hardcoded to en-US, for silly reasons
- # The Balrog submitter translates this platform into a build target
- # via https://github.com/mozilla/build-tools/blob/master/lib/python/release/platforms.py#L23
- self.set_buildbot_property("completeMarSize", self.query_filesize(c_marfile))
- self.set_buildbot_property("completeMarHash", self.query_sha512sum(c_marfile))
- self.set_buildbot_property("completeMarUrl", c_mar_url)
- self.set_buildbot_property("locale", locale)
- if "partialInfo" in self.package_urls[locale]:
- self.set_buildbot_property("partialInfo",
- self.package_urls[locale]["partialInfo"])
- ret = FAILURE
- try:
- result = self.submit_balrog_updates()
- self.info("balrog return code: %s" % (result))
- if result == 0:
- ret = SUCCESS
- except Exception as error:
- self.error("submit repack to balrog failed: %s" % (str(error)))
- return ret
-
- def _query_complete_mar_filename(self, locale):
- """returns the full path to a localized complete mar file"""
- config = self.config
- version = self.query_version()
- complete_mar_name = config['localized_mar'] % {'version': version,
- 'locale': locale}
- return os.path.join(self._update_mar_dir(), complete_mar_name)
-
- def _query_complete_mar_url(self, locale):
- """returns the complete mar url taken from self.package_urls[locale]
- this value is available only after make_upload"""
- if "complete_mar_url" in self.config:
- return self.config["complete_mar_url"]
- if "completeMarUrl" in self.package_urls[locale]:
- return self.package_urls[locale]["completeMarUrl"]
- # url = self.config.get("update", {}).get("mar_base_url")
- # if url:
- # url += os.path.basename(self.query_marfile_path())
- # return url.format(branch=self.query_branch())
- self.fatal("Couldn't find complete mar url in config or package_urls")
-
- def _update_mar_dir(self):
- """returns the full path of the update/ directory"""
- return self._mar_dir('update_mar_dir')
-
- def _mar_binaries(self):
- """returns a tuple with mar and mbsdiff paths"""
- config = self.config
- return (config['mar'], config['mbsdiff'])
-
- def _mar_dir(self, dirname):
- """returns the full path of dirname;
- dirname is an entry in configuration"""
- dirs = self.query_abs_dirs()
- return os.path.join(dirs['abs_objdir'], self.config[dirname])
-
- # TODO: replace with ToolToolMixin
- def _get_tooltool_auth_file(self):
- # set the default authentication file based on platform; this
- # corresponds to where puppet puts the token
- if 'tooltool_authentication_file' in self.config:
- fn = self.config['tooltool_authentication_file']
- elif self._is_windows():
- fn = r'c:\builds\relengapi.tok'
- else:
- fn = '/builds/relengapi.tok'
-
- # if the file doesn't exist, don't pass it to tooltool (it will just
- # fail). In taskcluster, this will work OK as the relengapi-proxy will
- # take care of auth. Everywhere else, we'll get auth failures if
- # necessary.
- if os.path.exists(fn):
- return fn
-
- def _run_tooltool(self):
- config = self.config
- dirs = self.query_abs_dirs()
- if not config.get('tooltool_manifest_src'):
- return self.warning(ERROR_MSGS['tooltool_manifest_undetermined'])
- fetch_script_path = os.path.join(dirs['abs_tools_dir'],
- 'scripts/tooltool/tooltool_wrapper.sh')
- tooltool_manifest_path = os.path.join(dirs['abs_mozilla_dir'],
- config['tooltool_manifest_src'])
- cmd = [
- 'sh',
- fetch_script_path,
- tooltool_manifest_path,
- config['tooltool_url'],
- config['tooltool_bootstrap'],
- ]
- cmd.extend(config['tooltool_script'])
- auth_file = self._get_tooltool_auth_file()
- if auth_file and os.path.exists(auth_file):
- cmd.extend(['--authentication-file', auth_file])
- cache = config['bootstrap_env'].get('TOOLTOOL_CACHE')
- if cache:
- cmd.extend(['-c', cache])
- self.info(str(cmd))
- self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True)
-
- def funsize_props(self):
- """Set buildbot properties required to trigger funsize tasks
- responsible to generate partial updates for successfully generated locales"""
- locales = self.query_locales()
- funsize_info = {
- 'locales': locales,
- 'branch': self.config['branch'],
- 'appName': self.config['appName'],
- 'platform': self.config['platform'],
- 'completeMarUrls': {locale: self._query_complete_mar_url(locale) for locale in locales},
- }
- self.info('funsize info: %s' % funsize_info)
- self.set_buildbot_property('funsize_info', json.dumps(funsize_info),
- write_to_file=True)
-
- def taskcluster_upload(self):
- auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file'])
- credentials = {}
- execfile(auth, credentials)
- client_id = credentials.get('taskcluster_clientId')
- access_token = credentials.get('taskcluster_accessToken')
- if not client_id or not access_token:
- self.warning('Skipping S3 file upload: No taskcluster credentials.')
- return
-
- # We need to activate the virtualenv so that we can import taskcluster
- # (and its dependent modules, like requests and hawk). Normally we
- # could create the virtualenv as an action, but due to some odd
- # dependencies with query_build_env() being called from build(), which
- # is necessary before the virtualenv can be created.
- self.disable_mock()
- self.create_virtualenv()
- self.enable_mock()
- self.activate_virtualenv()
-
- branch = self.config['branch']
- revision = self._query_revision()
- repo = self.query_l10n_repo()
- if not repo:
- self.fatal("Unable to determine repository for querying the push info.")
- pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hg')
- pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate))
-
- routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'],
- 'testing/mozharness/configs/routes.json')
- with open(routes_json) as f:
- contents = json.load(f)
- templates = contents['l10n']
-
- # Release promotion creates a special task to accumulate all artifacts
- # under the same task
- artifacts_task = None
- self.read_buildbot_config()
- if "artifactsTaskId" in self.buildbot_config.get("properties", {}):
- artifacts_task_id = self.buildbot_config["properties"]["artifactsTaskId"]
- artifacts_tc = Taskcluster(
- branch=branch, rank=pushinfo.pushdate, client_id=client_id,
- access_token=access_token, log_obj=self.log_obj,
- task_id=artifacts_task_id)
- artifacts_task = artifacts_tc.get_task(artifacts_task_id)
- artifacts_tc.claim_task(artifacts_task)
-
- for locale, files in self.upload_files.iteritems():
- self.info("Uploading files to S3 for locale '%s': %s" % (locale, files))
- routes = []
- for template in templates:
- fmt = {
- 'index': self.config.get('taskcluster_index', 'index.garbage.staging'),
- 'project': branch,
- 'head_rev': revision,
- 'pushdate': pushdate,
- 'year': pushdate[0:4],
- 'month': pushdate[4:6],
- 'day': pushdate[6:8],
- 'build_product': self.config['stage_product'],
- 'build_name': self.query_build_name(),
- 'build_type': self.query_build_type(),
- 'locale': locale,
- }
- fmt.update(self.buildid_to_dict(self._query_buildid()))
- routes.append(template.format(**fmt))
-
- self.info('Using routes: %s' % routes)
- tc = Taskcluster(branch,
- pushinfo.pushdate, # Use pushdate as the rank
- client_id,
- access_token,
- self.log_obj,
- )
- task = tc.create_task(routes)
- tc.claim_task(task)
-
- for upload_file in files:
- # Create an S3 artifact for each file that gets uploaded. We also
- # check the uploaded file against the property conditions so that we
- # can set the buildbot config with the correct URLs for package
- # locations.
- artifact_url = tc.create_artifact(task, upload_file)
- if artifacts_task:
- artifacts_tc.create_reference_artifact(
- artifacts_task, upload_file, artifact_url)
-
- tc.report_completed(task)
-
- if artifacts_task:
- if not self.query_failed_locales():
- artifacts_tc.report_completed(artifacts_task)
- else:
- # If some locales fail, we want to mark the artifacts
- # task failed, so a retry can reuse the same task ID
- artifacts_tc.report_failed(artifacts_task)
-
-
-# main {{{
-if __name__ == '__main__':
- single_locale = DesktopSingleLocale()
- single_locale.run_and_exit()
diff --git a/testing/mozharness/scripts/desktop_partner_repacks.py b/testing/mozharness/scripts/desktop_partner_repacks.py
deleted file mode 100755
index ff07dffc8..000000000
--- a/testing/mozharness/scripts/desktop_partner_repacks.py
+++ /dev/null
@@ -1,198 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""desktop_partner_repacks.py
-
-This script manages Desktop partner repacks for beta/release builds.
-"""
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.script import BaseScript
-from mozharness.mozilla.buildbot import BuildbotMixin
-from mozharness.mozilla.purge import PurgeMixin
-from mozharness.mozilla.release import ReleaseMixin
-from mozharness.base.python import VirtualenvMixin
-from mozharness.base.log import FATAL
-
-
-# DesktopPartnerRepacks {{{1
-class DesktopPartnerRepacks(ReleaseMixin, BuildbotMixin, PurgeMixin,
- BaseScript, VirtualenvMixin):
- """Manages desktop partner repacks"""
- actions = [
- "clobber",
- "create-virtualenv",
- "activate-virtualenv",
- "setup",
- "repack",
- "summary",
- ]
- config_options = [
- [["--version", "-v"], {
- "dest": "version",
- "help": "Version of Firefox to repack",
- }],
- [["--build-number", "-n"], {
- "dest": "build_number",
- "help": "Build number of Firefox to repack",
- }],
- [["--platform"], {
- "dest": "platform",
- "help": "Platform to repack (e.g. linux64, macosx64, ...)",
- }],
- [["--partner", "-p"], {
- "dest": "partner",
- "help": "Limit repackaging to partners matching this string",
- }],
- [["--s3cfg"], {
- "dest": "s3cfg",
- "help": "Configuration file for uploading to S3 using s3cfg",
- }],
- [["--hgroot"], {
- "dest": "hgroot",
- "help": "Use a different hg server for retrieving files",
- }],
- [["--hgrepo"], {
- "dest": "hgrepo",
- "help": "Use a different base repo for retrieving files",
- }],
- [["--require-buildprops"], {
- "action": "store_true",
- "dest": "require_buildprops",
- "default": False,
- "help": "Read in config options (like partner) from the buildbot properties file."
- }],
- ]
-
- def __init__(self):
- # fxbuild style:
- buildscript_kwargs = {
- 'all_actions': DesktopPartnerRepacks.actions,
- 'default_actions': DesktopPartnerRepacks.actions,
- 'config': {
- 'buildbot_json_path': 'buildprops.json',
- "log_name": "partner-repacks",
- "hashType": "sha512",
- 'virtualenv_modules': [
- 'requests==2.2.1',
- 'PyHawk-with-a-single-extra-commit==0.1.5',
- 'taskcluster==0.0.15',
- 's3cmd==1.6.0',
- ],
- 'virtualenv_path': 'venv',
- 'workdir': 'partner-repacks',
- },
- }
- #
-
- BaseScript.__init__(
- self,
- config_options=self.config_options,
- **buildscript_kwargs
- )
-
-
- def _pre_config_lock(self, rw_config):
- self.read_buildbot_config()
- if not self.buildbot_config:
- self.warning("Skipping buildbot properties overrides")
- else:
- if self.config.get('require_buildprops', False) is True:
- if not self.buildbot_config:
- self.fatal("Unable to load properties from file: %s" % self.config.get('buildbot_json_path'))
- props = self.buildbot_config["properties"]
- for prop in ['version', 'build_number', 'revision', 'repo_file', 'repack_manifests_url', 'partner']:
- if props.get(prop):
- self.info("Overriding %s with %s" % (prop, props[prop]))
- self.config[prop] = props.get(prop)
-
- if 'version' not in self.config:
- self.fatal("Version (-v) not supplied.")
- if 'build_number' not in self.config:
- self.fatal("Build number (-n) not supplied.")
- if 'repo_file' not in self.config:
- self.fatal("repo_file not supplied.")
- if 'repack_manifests_url' not in self.config:
- self.fatal("repack_manifests_url not supplied.")
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(DesktopPartnerRepacks, self).query_abs_dirs()
- for directory in abs_dirs:
- value = abs_dirs[directory]
- abs_dirs[directory] = value
- dirs = {}
- dirs['abs_repo_dir'] = os.path.join(abs_dirs['abs_work_dir'], '.repo')
- dirs['abs_partners_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'partners')
- dirs['abs_scripts_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'scripts')
- for key in dirs.keys():
- if key not in abs_dirs:
- abs_dirs[key] = dirs[key]
- self.abs_dirs = abs_dirs
- return self.abs_dirs
-
- # Actions {{{
- def _repo_cleanup(self):
- self.rmtree(self.query_abs_dirs()['abs_repo_dir'])
- self.rmtree(self.query_abs_dirs()['abs_partners_dir'])
- self.rmtree(self.query_abs_dirs()['abs_scripts_dir'])
-
- def _repo_init(self, repo):
- status = self.run_command([repo, "init", "--no-repo-verify",
- "-u", self.config['repack_manifests_url']],
- cwd=self.query_abs_dirs()['abs_work_dir'])
- if status:
- return status
- return self.run_command([repo, "sync"],
- cwd=self.query_abs_dirs()['abs_work_dir'])
-
- def setup(self):
- """setup step"""
- repo = self.download_file(self.config['repo_file'],
- file_name='repo',
- parent_dir=self.query_abs_dirs()['abs_work_dir'],
- error_level=FATAL)
- if not os.path.exists(repo):
- self.fatal("Unable to download repo tool.")
- self.chmod(repo, 0755)
- self.retry(self._repo_init,
- args=(repo,),
- error_level=FATAL,
- cleanup=self._repo_cleanup(),
- good_statuses=[0],
- sleeptime=5)
-
- def repack(self):
- """creates the repacks"""
- python = self.query_exe("python2.7")
- repack_cmd = [python, "partner-repacks.py",
- "-v", self.config['version'],
- "-n", str(self.config['build_number'])]
- if self.config.get('platform'):
- repack_cmd.extend(["--platform", self.config['platform']])
- if self.config.get('partner'):
- repack_cmd.extend(["--partner", self.config['partner']])
- if self.config.get('s3cfg'):
- repack_cmd.extend(["--s3cfg", self.config['s3cfg']])
- if self.config.get('hgroot'):
- repack_cmd.extend(["--hgroot", self.config['hgroot']])
- if self.config.get('hgrepo'):
- repack_cmd.extend(["--repo", self.config['hgrepo']])
- if self.config.get('revision'):
- repack_cmd.extend(["--tag", self.config["revision"]])
-
- return self.run_command(repack_cmd,
- cwd=self.query_abs_dirs()['abs_scripts_dir'])
-
-# main {{{
-if __name__ == '__main__':
- partner_repacks = DesktopPartnerRepacks()
- partner_repacks.run_and_exit()
diff --git a/testing/mozharness/scripts/desktop_unittest.py b/testing/mozharness/scripts/desktop_unittest.py
deleted file mode 100755
index b2e754567..000000000
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ /dev/null
@@ -1,742 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""desktop_unittest.py
-The goal of this is to extract desktop unittesting from buildbot's factory.py
-
-author: Jordan Lund
-"""
-
-import os
-import re
-import sys
-import copy
-import shutil
-import glob
-import imp
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.errors import BaseErrorList
-from mozharness.base.log import INFO, ERROR
-from mozharness.base.script import PreScriptAction
-from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
-from mozharness.mozilla.buildbot import TBPL_EXCEPTION
-from mozharness.mozilla.mozbase import MozbaseMixin
-from mozharness.mozilla.structuredlog import StructuredOutputParser
-from mozharness.mozilla.testing.errors import HarnessErrorList
-from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser
-from mozharness.mozilla.testing.codecoverage import (
- CodeCoverageMixin,
- code_coverage_config_options
-)
-from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
-
-SUITE_CATEGORIES = ['gtest', 'cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell', 'mozbase', 'mozmill']
-SUITE_DEFAULT_E10S = ['mochitest', 'reftest']
-
-
-# DesktopUnittest {{{1
-class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMixin, CodeCoverageMixin):
- config_options = [
- [['--mochitest-suite', ], {
- "action": "extend",
- "dest": "specified_mochitest_suites",
- "type": "string",
- "help": "Specify which mochi suite to run. "
- "Suites are defined in the config file.\n"
- "Examples: 'all', 'plain1', 'plain5', 'chrome', or 'a11y'"}
- ],
- [['--reftest-suite', ], {
- "action": "extend",
- "dest": "specified_reftest_suites",
- "type": "string",
- "help": "Specify which reftest suite to run. "
- "Suites are defined in the config file.\n"
- "Examples: 'all', 'crashplan', or 'jsreftest'"}
- ],
- [['--xpcshell-suite', ], {
- "action": "extend",
- "dest": "specified_xpcshell_suites",
- "type": "string",
- "help": "Specify which xpcshell suite to run. "
- "Suites are defined in the config file\n."
- "Examples: 'xpcshell'"}
- ],
- [['--cppunittest-suite', ], {
- "action": "extend",
- "dest": "specified_cppunittest_suites",
- "type": "string",
- "help": "Specify which cpp unittest suite to run. "
- "Suites are defined in the config file\n."
- "Examples: 'cppunittest'"}
- ],
- [['--gtest-suite', ], {
- "action": "extend",
- "dest": "specified_gtest_suites",
- "type": "string",
- "help": "Specify which gtest suite to run. "
- "Suites are defined in the config file\n."
- "Examples: 'gtest'"}
- ],
- [['--jittest-suite', ], {
- "action": "extend",
- "dest": "specified_jittest_suites",
- "type": "string",
- "help": "Specify which jit-test suite to run. "
- "Suites are defined in the config file\n."
- "Examples: 'jittest'"}
- ],
- [['--mozbase-suite', ], {
- "action": "extend",
- "dest": "specified_mozbase_suites",
- "type": "string",
- "help": "Specify which mozbase suite to run. "
- "Suites are defined in the config file\n."
- "Examples: 'mozbase'"}
- ],
- [['--mozmill-suite', ], {
- "action": "extend",
- "dest": "specified_mozmill_suites",
- "type": "string",
- "help": "Specify which mozmill suite to run. "
- "Suites are defined in the config file\n."
- "Examples: 'mozmill'"}
- ],
- [['--run-all-suites', ], {
- "action": "store_true",
- "dest": "run_all_suites",
- "default": False,
- "help": "This will run all suites that are specified "
- "in the config file. You do not need to specify "
- "any other suites.\nBeware, this may take a while ;)"}
- ],
- [['--e10s', ], {
- "action": "store_true",
- "dest": "e10s",
- "default": False,
- "help": "Run tests with multiple processes."}
- ],
- [['--strict-content-sandbox', ], {
- "action": "store_true",
- "dest": "strict_content_sandbox",
- "default": False,
- "help": "Run tests with a more strict content sandbox (Windows only)."}
- ],
- [['--no-random', ], {
- "action": "store_true",
- "dest": "no_random",
- "default": False,
- "help": "Run tests with no random intermittents and bisect in case of real failure."}
- ],
- [["--total-chunks"], {
- "action": "store",
- "dest": "total_chunks",
- "help": "Number of total chunks"}
- ],
- [["--this-chunk"], {
- "action": "store",
- "dest": "this_chunk",
- "help": "Number of this chunk"}
- ],
- [["--allow-software-gl-layers"], {
- "action": "store_true",
- "dest": "allow_software_gl_layers",
- "default": False,
- "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor."}
- ],
- ] + copy.deepcopy(testing_config_options) + \
- copy.deepcopy(blobupload_config_options) + \
- copy.deepcopy(code_coverage_config_options)
-
- def __init__(self, require_config_file=True):
- # abs_dirs defined already in BaseScript but is here to make pylint happy
- self.abs_dirs = None
- super(DesktopUnittest, self).__init__(
- config_options=self.config_options,
- all_actions=[
- 'clobber',
- 'read-buildbot-config',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'stage-files',
- 'run-tests',
- ],
- require_config_file=require_config_file,
- config={'require_test_zip': True})
-
- c = self.config
- self.global_test_options = []
- self.installer_url = c.get('installer_url')
- self.test_url = c.get('test_url')
- self.test_packages_url = c.get('test_packages_url')
- self.symbols_url = c.get('symbols_url')
- # this is so mozinstall in install() doesn't bug out if we don't run
- # the download_and_extract action
- self.installer_path = c.get('installer_path')
- self.binary_path = c.get('binary_path')
- self.abs_app_dir = None
- self.abs_res_dir = None
-
- # Construct an identifier to be used to identify Perfherder data
- # for resource monitoring recording. This attempts to uniquely
- # identify this test invocation configuration.
- perfherder_parts = []
- perfherder_options = []
- suites = (
- ('specified_mochitest_suites', 'mochitest'),
- ('specified_reftest_suites', 'reftest'),
- ('specified_xpcshell_suites', 'xpcshell'),
- ('specified_cppunittest_suites', 'cppunit'),
- ('specified_gtest_suites', 'gtest'),
- ('specified_jittest_suites', 'jittest'),
- ('specified_mozbase_suites', 'mozbase'),
- ('specified_mozmill_suites', 'mozmill'),
- )
- for s, prefix in suites:
- if s in c:
- perfherder_parts.append(prefix)
- perfherder_parts.extend(c[s])
-
- if 'this_chunk' in c:
- perfherder_parts.append(c['this_chunk'])
-
- if c['e10s']:
- perfherder_options.append('e10s')
-
- self.resource_monitor_perfherder_id = ('.'.join(perfherder_parts),
- perfherder_options)
-
- # helper methods {{{2
- def _pre_config_lock(self, rw_config):
- super(DesktopUnittest, self)._pre_config_lock(rw_config)
- c = self.config
- if not c.get('run_all_suites'):
- return # configs are valid
- for category in SUITE_CATEGORIES:
- specific_suites = c.get('specified_%s_suites' % (category))
- if specific_suites:
- if specific_suites != 'all':
- self.fatal("Config options are not valid. Please ensure"
- " that if the '--run-all-suites' flag was enabled,"
- " then do not specify to run only specific suites "
- "like:\n '--mochitest-suite browser-chrome'")
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(DesktopUnittest, self).query_abs_dirs()
-
- c = self.config
- dirs = {}
- dirs['abs_app_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'application')
- dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
- dirs['abs_test_extensions_dir'] = os.path.join(dirs['abs_test_install_dir'], 'extensions')
- dirs['abs_test_bin_dir'] = os.path.join(dirs['abs_test_install_dir'], 'bin')
- dirs['abs_test_bin_plugins_dir'] = os.path.join(dirs['abs_test_bin_dir'],
- 'plugins')
- dirs['abs_test_bin_components_dir'] = os.path.join(dirs['abs_test_bin_dir'],
- 'components')
- dirs['abs_mochitest_dir'] = os.path.join(dirs['abs_test_install_dir'], "mochitest")
- dirs['abs_reftest_dir'] = os.path.join(dirs['abs_test_install_dir'], "reftest")
- dirs['abs_xpcshell_dir'] = os.path.join(dirs['abs_test_install_dir'], "xpcshell")
- dirs['abs_cppunittest_dir'] = os.path.join(dirs['abs_test_install_dir'], "cppunittest")
- dirs['abs_gtest_dir'] = os.path.join(dirs['abs_test_install_dir'], "gtest")
- dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
- dirs['abs_jittest_dir'] = os.path.join(dirs['abs_test_install_dir'], "jit-test", "jit-test")
- dirs['abs_mozbase_dir'] = os.path.join(dirs['abs_test_install_dir'], "mozbase")
- dirs['abs_mozmill_dir'] = os.path.join(dirs['abs_test_install_dir'], "mozmill")
-
- if os.path.isabs(c['virtualenv_path']):
- dirs['abs_virtualenv_dir'] = c['virtualenv_path']
- else:
- dirs['abs_virtualenv_dir'] = os.path.join(abs_dirs['abs_work_dir'],
- c['virtualenv_path'])
- abs_dirs.update(dirs)
- self.abs_dirs = abs_dirs
-
- return self.abs_dirs
-
- def query_abs_app_dir(self):
- """We can't set this in advance, because OSX install directories
- change depending on branding and opt/debug.
- """
- if self.abs_app_dir:
- return self.abs_app_dir
- if not self.binary_path:
- self.fatal("Can't determine abs_app_dir (binary_path not set!)")
- self.abs_app_dir = os.path.dirname(self.binary_path)
- return self.abs_app_dir
-
- def query_abs_res_dir(self):
- """The directory containing resources like plugins and extensions. On
- OSX this is Contents/Resources, on all other platforms its the same as
- the app dir.
-
- As with the app dir, we can't set this in advance, because OSX install
- directories change depending on branding and opt/debug.
- """
- if self.abs_res_dir:
- return self.abs_res_dir
-
- abs_app_dir = self.query_abs_app_dir()
- if self._is_darwin():
- res_subdir = self.config.get("mac_res_subdir", "Resources")
- self.abs_res_dir = os.path.join(os.path.dirname(abs_app_dir), res_subdir)
- else:
- self.abs_res_dir = abs_app_dir
- return self.abs_res_dir
-
- @PreScriptAction('create-virtualenv')
- def _pre_create_virtualenv(self, action):
- dirs = self.query_abs_dirs()
-
- self.register_virtualenv_module(name='pip>=1.5')
- self.register_virtualenv_module('psutil==3.1.1', method='pip')
- self.register_virtualenv_module(name='mock')
- self.register_virtualenv_module(name='simplejson')
-
- requirements_files = [
- os.path.join(dirs['abs_test_install_dir'],
- 'config',
- 'marionette_requirements.txt')]
-
- if os.path.isdir(dirs['abs_mochitest_dir']):
- # mochitest is the only thing that needs this
- requirements_files.append(
- os.path.join(dirs['abs_mochitest_dir'],
- 'websocketprocessbridge',
- 'websocketprocessbridge_requirements.txt'))
-
- for requirements_file in requirements_files:
- self.register_virtualenv_module(requirements=[requirements_file],
- two_pass=True)
-
- def _query_symbols_url(self):
- """query the full symbols URL based upon binary URL"""
- # may break with name convention changes but is one less 'input' for script
- if self.symbols_url:
- return self.symbols_url
-
- symbols_url = None
- self.info("finding symbols_url based upon self.installer_url")
- if self.installer_url:
- for ext in ['.zip', '.dmg', '.tar.bz2']:
- if ext in self.installer_url:
- symbols_url = self.installer_url.replace(
- ext, '.crashreporter-symbols.zip')
- if not symbols_url:
- self.fatal("self.installer_url was found but symbols_url could \
- not be determined")
- else:
- self.fatal("self.installer_url was not found in self.config")
- self.info("setting symbols_url as %s" % (symbols_url))
- self.symbols_url = symbols_url
- return self.symbols_url
-
- def _query_abs_base_cmd(self, suite_category, suite):
- if self.binary_path:
- c = self.config
- dirs = self.query_abs_dirs()
- run_file = c['run_file_names'][suite_category]
- base_cmd = [self.query_python_path('python'), '-u']
- base_cmd.append(os.path.join(dirs["abs_%s_dir" % suite_category], run_file))
- abs_app_dir = self.query_abs_app_dir()
- abs_res_dir = self.query_abs_res_dir()
-
- raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
- '%s_raw.log' % suite)
-
- error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
- '%s_errorsummary.log' % suite)
- str_format_values = {
- 'binary_path': self.binary_path,
- 'symbols_path': self._query_symbols_url(),
- 'abs_app_dir': abs_app_dir,
- 'abs_res_dir': abs_res_dir,
- 'raw_log_file': raw_log_file,
- 'error_summary_file': error_summary_file,
- 'gtest_dir': os.path.join(dirs['abs_test_install_dir'],
- 'gtest'),
- }
-
- # TestingMixin._download_and_extract_symbols() will set
- # self.symbols_path when downloading/extracting.
- if self.symbols_path:
- str_format_values['symbols_path'] = self.symbols_path
-
- if suite_category in SUITE_DEFAULT_E10S and not c['e10s']:
- base_cmd.append('--disable-e10s')
- elif suite_category not in SUITE_DEFAULT_E10S and c['e10s']:
- base_cmd.append('--e10s')
-
- if c.get('strict_content_sandbox'):
- if suite_category == "mochitest":
- base_cmd.append('--strict-content-sandbox')
- else:
- self.fatal("--strict-content-sandbox only works with mochitest suites.")
-
- if c.get('total_chunks') and c.get('this_chunk'):
- base_cmd.extend(['--total-chunks', c['total_chunks'],
- '--this-chunk', c['this_chunk']])
-
- if c['no_random']:
- if suite_category == "mochitest":
- base_cmd.append('--bisect-chunk=default')
- else:
- self.warning("--no-random does not currently work with suites other than mochitest.")
-
- # set pluginsPath
- abs_res_plugins_dir = os.path.join(abs_res_dir, 'plugins')
- str_format_values['test_plugin_path'] = abs_res_plugins_dir
-
- if suite_category not in c["suite_definitions"]:
- self.fatal("'%s' not defined in the config!")
-
- if suite in ('browser-chrome-coverage', 'xpcshell-coverage', 'mochitest-devtools-chrome-coverage'):
- base_cmd.append('--jscov-dir-prefix=%s' %
- dirs['abs_blob_upload_dir'])
-
- options = c["suite_definitions"][suite_category]["options"]
- if options:
- for option in options:
- option = option % str_format_values
- if not option.endswith('None'):
- base_cmd.append(option)
- if self.structured_output(
- suite_category,
- self._query_try_flavor(suite_category, suite)
- ):
- base_cmd.append("--log-raw=-")
- return base_cmd
- else:
- self.warning("Suite options for %s could not be determined."
- "\nIf you meant to have options for this suite, "
- "please make sure they are specified in your "
- "config under %s_options" %
- (suite_category, suite_category))
-
- return base_cmd
- else:
- self.fatal("'binary_path' could not be determined.\n This should "
- "be like '/path/build/application/firefox/firefox'"
- "\nIf you are running this script without the 'install' "
- "action (where binary_path is set), please ensure you are"
- " either:\n(1) specifying it in the config file under "
- "binary_path\n(2) specifying it on command line with the"
- " '--binary-path' flag")
-
- def _query_specified_suites(self, category):
- # logic goes: if at least one '--{category}-suite' was given,
- # then run only that(those) given suite(s). Elif no suites were
- # specified and the --run-all-suites flag was given,
- # run all {category} suites. Anything else, run no suites.
- c = self.config
- all_suites = c.get('all_%s_suites' % (category))
- specified_suites = c.get('specified_%s_suites' % (category)) # list
- suites = None
-
- if specified_suites:
- if 'all' in specified_suites:
- # useful if you want a quick way of saying run all suites
- # of a specific category.
- suites = all_suites
- else:
- # suites gets a dict of everything from all_suites where a key
- # is also in specified_suites
- suites = dict((key, all_suites.get(key)) for key in
- specified_suites if key in all_suites.keys())
- else:
- if c.get('run_all_suites'): # needed if you dont specify any suites
- suites = all_suites
-
- return suites
-
- def _query_try_flavor(self, category, suite):
- flavors = {
- "mochitest": [("plain.*", "mochitest"),
- ("browser-chrome.*", "browser-chrome"),
- ("mochitest-devtools-chrome.*", "devtools-chrome"),
- ("chrome", "chrome"),
- ("jetpack.*", "jetpack")],
- "xpcshell": [("xpcshell", "xpcshell")],
- "reftest": [("reftest", "reftest"),
- ("crashtest", "crashtest")]
- }
- for suite_pattern, flavor in flavors.get(category, []):
- if re.compile(suite_pattern).match(suite):
- return flavor
-
- def structured_output(self, suite_category, flavor=None):
- unstructured_flavors = self.config.get('unstructured_flavors')
- if not unstructured_flavors:
- return False
- if suite_category not in unstructured_flavors:
- return True
- if not unstructured_flavors.get(suite_category) or flavor in unstructured_flavors.get(suite_category):
- return False
- return True
-
- def get_test_output_parser(self, suite_category, flavor=None, strict=False,
- **kwargs):
- if not self.structured_output(suite_category, flavor):
- return DesktopUnittestOutputParser(suite_category=suite_category, **kwargs)
- self.info("Structured output parser in use for %s." % suite_category)
- return StructuredOutputParser(suite_category=suite_category, strict=strict, **kwargs)
-
- # Actions {{{2
-
- # clobber defined in BaseScript, deletes mozharness/build if exists
- # read_buildbot_config is in BuildbotMixin.
- # postflight_read_buildbot_config is in TestingMixin.
- # preflight_download_and_extract is in TestingMixin.
- # create_virtualenv is in VirtualenvMixin.
- # preflight_install is in TestingMixin.
- # install is in TestingMixin.
- # upload_blobber_files is in BlobUploadMixin
-
- @PreScriptAction('download-and-extract')
- def _pre_download_and_extract(self, action):
- """Abort if --artifact try syntax is used with compiled-code tests"""
- if not self.try_message_has_flag('artifact'):
- return
- self.info('Artifact build requested in try syntax.')
- rejected = []
- compiled_code_suites = [
- "cppunit",
- "gtest",
- "jittest",
- ]
- for category in SUITE_CATEGORIES:
- suites = self._query_specified_suites(category) or []
- for suite in suites:
- if any([suite.startswith(c) for c in compiled_code_suites]):
- rejected.append(suite)
- break
- if rejected:
- self.buildbot_status(TBPL_EXCEPTION)
- self.fatal("There are specified suites that are incompatible with "
- "--artifact try syntax flag: {}".format(', '.join(rejected)),
- exit_code=self.return_code)
-
-
- def download_and_extract(self):
- """
- download and extract test zip / download installer
- optimizes which subfolders to extract from tests zip
- """
- c = self.config
-
- extract_dirs = None
- if c['specific_tests_zip_dirs']:
- extract_dirs = list(c['minimum_tests_zip_dirs'])
- for category in c['specific_tests_zip_dirs'].keys():
- if c['run_all_suites'] or self._query_specified_suites(category) \
- or 'run-tests' not in self.actions:
- extract_dirs.extend(c['specific_tests_zip_dirs'][category])
-
- if c.get('run_all_suites'):
- target_categories = SUITE_CATEGORIES
- else:
- target_categories = [cat for cat in SUITE_CATEGORIES
- if self._query_specified_suites(cat) is not None]
- super(DesktopUnittest, self).download_and_extract(extract_dirs=extract_dirs,
- suite_categories=target_categories)
-
- def stage_files(self):
- for category in SUITE_CATEGORIES:
- suites = self._query_specified_suites(category)
- stage = getattr(self, '_stage_{}'.format(category), None)
- if suites and stage:
- stage(suites)
-
- def _stage_files(self, bin_name=None):
- dirs = self.query_abs_dirs()
- abs_app_dir = self.query_abs_app_dir()
-
- # For mac these directories are in Contents/Resources, on other
- # platforms abs_res_dir will point to abs_app_dir.
- abs_res_dir = self.query_abs_res_dir()
- abs_res_components_dir = os.path.join(abs_res_dir, 'components')
- abs_res_plugins_dir = os.path.join(abs_res_dir, 'plugins')
- abs_res_extensions_dir = os.path.join(abs_res_dir, 'extensions')
-
- if bin_name:
- self.info('copying %s to %s' % (os.path.join(dirs['abs_test_bin_dir'],
- bin_name), os.path.join(abs_app_dir, bin_name)))
- shutil.copy2(os.path.join(dirs['abs_test_bin_dir'], bin_name),
- os.path.join(abs_app_dir, bin_name))
-
- self.copytree(dirs['abs_test_bin_components_dir'],
- abs_res_components_dir,
- overwrite='overwrite_if_exists')
- self.mkdir_p(abs_res_plugins_dir)
- self.copytree(dirs['abs_test_bin_plugins_dir'],
- abs_res_plugins_dir,
- overwrite='overwrite_if_exists')
- if os.path.isdir(dirs['abs_test_extensions_dir']):
- self.mkdir_p(abs_res_extensions_dir)
- self.copytree(dirs['abs_test_extensions_dir'],
- abs_res_extensions_dir,
- overwrite='overwrite_if_exists')
-
- def _stage_xpcshell(self, suites):
- self._stage_files(self.config['xpcshell_name'])
-
- def _stage_cppunittest(self, suites):
- abs_res_dir = self.query_abs_res_dir()
- dirs = self.query_abs_dirs()
- abs_cppunittest_dir = dirs['abs_cppunittest_dir']
-
- # move manifest and js fils to resources dir, where tests expect them
- files = glob.glob(os.path.join(abs_cppunittest_dir, '*.js'))
- files.extend(glob.glob(os.path.join(abs_cppunittest_dir, '*.manifest')))
- for f in files:
- self.move(f, abs_res_dir)
-
- def _stage_gtest(self, suites):
- abs_res_dir = self.query_abs_res_dir()
- abs_app_dir = self.query_abs_app_dir()
- dirs = self.query_abs_dirs()
- abs_gtest_dir = dirs['abs_gtest_dir']
- dirs['abs_test_bin_dir'] = os.path.join(dirs['abs_test_install_dir'], 'bin')
-
- files = glob.glob(os.path.join(dirs['abs_test_bin_plugins_dir'], 'gmp-*'))
- files.append(os.path.join(abs_gtest_dir, 'dependentlibs.list.gtest'))
- for f in files:
- self.move(f, abs_res_dir)
-
- self.copytree(os.path.join(abs_gtest_dir, 'gtest_bin'),
- os.path.join(abs_app_dir))
-
- def _stage_mozmill(self, suites):
- self._stage_files()
- dirs = self.query_abs_dirs()
- modules = ['jsbridge', 'mozmill']
- for module in modules:
- self.install_module(module=os.path.join(dirs['abs_mozmill_dir'],
- 'resources',
- module))
-
- # pull defined in VCSScript.
- # preflight_run_tests defined in TestingMixin.
-
- def run_tests(self):
- for category in SUITE_CATEGORIES:
- self._run_category_suites(category)
-
- def get_timeout_for_category(self, suite_category):
- if suite_category == 'cppunittest':
- return 2500
- return self.config["suite_definitions"][suite_category].get('run_timeout', 1000)
-
- def _run_category_suites(self, suite_category):
- """run suite(s) to a specific category"""
- dirs = self.query_abs_dirs()
- suites = self._query_specified_suites(suite_category)
- abs_app_dir = self.query_abs_app_dir()
- abs_res_dir = self.query_abs_res_dir()
-
- if suites:
- self.info('#### Running %s suites' % suite_category)
- for suite in suites:
- abs_base_cmd = self._query_abs_base_cmd(suite_category, suite)
- cmd = abs_base_cmd[:]
- replace_dict = {
- 'abs_app_dir': abs_app_dir,
-
- # Mac specific, but points to abs_app_dir on other
- # platforms.
- 'abs_res_dir': abs_res_dir,
- }
- options_list = []
- env = {}
- if isinstance(suites[suite], dict):
- options_list = suites[suite].get('options', [])
- tests_list = suites[suite].get('tests', [])
- env = copy.deepcopy(suites[suite].get('env', {}))
- else:
- options_list = suites[suite]
- tests_list = []
-
- flavor = self._query_try_flavor(suite_category, suite)
- try_options, try_tests = self.try_args(flavor)
-
- cmd.extend(self.query_options(options_list,
- try_options,
- str_format_values=replace_dict))
- cmd.extend(self.query_tests_args(tests_list,
- try_tests,
- str_format_values=replace_dict))
-
- suite_name = suite_category + '-' + suite
- tbpl_status, log_level = None, None
- error_list = BaseErrorList + HarnessErrorList
- parser = self.get_test_output_parser(suite_category,
- flavor=flavor,
- config=self.config,
- error_list=error_list,
- log_obj=self.log_obj)
-
- if suite_category == "reftest":
- ref_formatter = imp.load_source(
- "ReftestFormatter",
- os.path.abspath(
- os.path.join(dirs["abs_reftest_dir"], "output.py")))
- parser.formatter = ref_formatter.ReftestFormatter()
-
- if self.query_minidump_stackwalk():
- env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
- if self.query_nodejs():
- env['MOZ_NODE_PATH'] = self.nodejs_path
- env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
- env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
- if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
- self.mkdir_p(env['MOZ_UPLOAD_DIR'])
-
- if self.config['allow_software_gl_layers']:
- env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1'
-
- env = self.query_env(partial_env=env, log_level=INFO)
- cmd_timeout = self.get_timeout_for_category(suite_category)
- return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'],
- output_timeout=cmd_timeout,
- output_parser=parser,
- env=env)
-
- # mochitest, reftest, and xpcshell suites do not return
- # appropriate return codes. Therefore, we must parse the output
- # to determine what the tbpl_status and worst_log_level must
- # be. We do this by:
- # 1) checking to see if our mozharness script ran into any
- # errors itself with 'num_errors' <- OutputParser
- # 2) if num_errors is 0 then we look in the subclassed 'parser'
- # findings for harness/suite errors <- DesktopUnittestOutputParser
- # 3) checking to see if the return code is in success_codes
-
- success_codes = None
- if self._is_windows() and suite_category != 'gtest':
- # bug 1120644
- success_codes = [0, 1]
-
- tbpl_status, log_level = parser.evaluate_parser(return_code,
- success_codes=success_codes)
- parser.append_tinderboxprint_line(suite_name)
-
- self.buildbot_status(tbpl_status, level=log_level)
- self.log("The %s suite: %s ran with return status: %s" %
- (suite_category, suite, tbpl_status), level=log_level)
- else:
- self.debug('There were no suites to run for %s' % suite_category)
-
-
-# main {{{1
-if __name__ == '__main__':
- desktop_unittest = DesktopUnittest()
- desktop_unittest.run_and_exit()
diff --git a/testing/mozharness/scripts/firefox_media_tests_buildbot.py b/testing/mozharness/scripts/firefox_media_tests_buildbot.py
deleted file mode 100644
index 17b830f0f..000000000
--- a/testing/mozharness/scripts/firefox_media_tests_buildbot.py
+++ /dev/null
@@ -1,122 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** BEGIN LICENSE BLOCK *****
-"""firefox_media_tests_buildbot.py
-
-Author: Maja Frydrychowicz
-"""
-import copy
-import glob
-import os
-import sys
-
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.log import DEBUG, ERROR, INFO
-from mozharness.base.script import PostScriptAction
-from mozharness.mozilla.blob_upload import (
- BlobUploadMixin,
- blobupload_config_options
-)
-from mozharness.mozilla.buildbot import (
- TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
-)
-from mozharness.mozilla.testing.firefox_media_tests import (
- FirefoxMediaTestsBase, TESTFAILED, SUCCESS
-)
-
-
-class FirefoxMediaTestsBuildbot(FirefoxMediaTestsBase, BlobUploadMixin):
-
- def __init__(self):
- config_options = copy.deepcopy(blobupload_config_options)
- super(FirefoxMediaTestsBuildbot, self).__init__(
- config_options=config_options,
- all_actions=['clobber',
- 'read-buildbot-config',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-media-tests',
- ],
- )
-
- def run_media_tests(self):
- status = super(FirefoxMediaTestsBuildbot, self).run_media_tests()
- if status == SUCCESS:
- tbpl_status = TBPL_SUCCESS
- else:
- tbpl_status = TBPL_FAILURE
- if status == TESTFAILED:
- tbpl_status = TBPL_WARNING
- self.buildbot_status(tbpl_status)
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(FirefoxMediaTestsBuildbot, self).query_abs_dirs()
- dirs = {
- 'abs_blob_upload_dir': os.path.join(abs_dirs['abs_work_dir'],
- 'blobber_upload_dir')
- }
- abs_dirs.update(dirs)
- self.abs_dirs = abs_dirs
- return self.abs_dirs
-
- def _query_cmd(self):
- cmd = super(FirefoxMediaTestsBuildbot, self)._query_cmd()
- dirs = self.query_abs_dirs()
- # configure logging
- blob_upload_dir = dirs.get('abs_blob_upload_dir')
- cmd += ['--gecko-log', os.path.join(blob_upload_dir, 'gecko.log')]
- cmd += ['--log-html', os.path.join(blob_upload_dir, 'media_tests.html')]
- cmd += ['--log-mach', os.path.join(blob_upload_dir, 'media_tests_mach.log')]
- return cmd
-
- @PostScriptAction('run-media-tests')
- def _collect_uploads(self, action, success=None):
- """ Copy extra (log) files to blob upload dir. """
- dirs = self.query_abs_dirs()
- log_dir = dirs.get('abs_log_dir')
- blob_upload_dir = dirs.get('abs_blob_upload_dir')
- if not log_dir or not blob_upload_dir:
- return
- self.mkdir_p(blob_upload_dir)
- # Move firefox-media-test screenshots into log_dir
- screenshots_dir = os.path.join(dirs['base_work_dir'],
- 'screenshots')
- log_screenshots_dir = os.path.join(log_dir, 'screenshots')
- if os.access(log_screenshots_dir, os.F_OK):
- self.rmtree(log_screenshots_dir)
- if os.access(screenshots_dir, os.F_OK):
- self.move(screenshots_dir, log_screenshots_dir)
-
- # logs to upload: broadest level (info), error, screenshots
- uploads = glob.glob(os.path.join(log_screenshots_dir, '*'))
- log_files = self.log_obj.log_files
- log_level = self.log_obj.log_level
-
- def append_path(filename, dir=log_dir):
- if filename:
- uploads.append(os.path.join(dir, filename))
-
- append_path(log_files.get(ERROR))
- # never upload debug logs
- if log_level == DEBUG:
- append_path(log_files.get(INFO))
- else:
- append_path(log_files.get(log_level))
- # in case of SimpleFileLogger
- append_path(log_files.get('default'))
- for f in uploads:
- if os.access(f, os.F_OK):
- dest = os.path.join(blob_upload_dir, os.path.basename(f))
- self.copyfile(f, dest)
-
-
-if __name__ == '__main__':
- media_test = FirefoxMediaTestsBuildbot()
- media_test.run_and_exit()
diff --git a/testing/mozharness/scripts/firefox_media_tests_jenkins.py b/testing/mozharness/scripts/firefox_media_tests_jenkins.py
deleted file mode 100755
index e35655257..000000000
--- a/testing/mozharness/scripts/firefox_media_tests_jenkins.py
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** BEGIN LICENSE BLOCK *****
-"""firefox_media_tests_jenkins.py
-
-Author: Syd Polk
-"""
-import os
-import sys
-
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.mozilla.testing.firefox_media_tests import (
- FirefoxMediaTestsBase
-)
-
-
-class FirefoxMediaTestsJenkins(FirefoxMediaTestsBase):
-
- def __init__(self):
- super(FirefoxMediaTestsJenkins, self).__init__(
- all_actions=['clobber',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-media-tests',
- ],
- )
-
- def _query_cmd(self):
- cmd = super(FirefoxMediaTestsJenkins, self)._query_cmd()
-
- dirs = self.query_abs_dirs()
-
- # configure logging
- log_dir = dirs.get('abs_log_dir')
- cmd += ['--gecko-log', os.path.join(log_dir, 'gecko.log')]
- cmd += ['--log-html', os.path.join(log_dir, 'media_tests.html')]
- cmd += ['--log-mach', os.path.join(log_dir, 'media_tests_mach.log')]
-
- return cmd
-
-if __name__ == '__main__':
- media_test = FirefoxMediaTestsJenkins()
- media_test.run_and_exit()
diff --git a/testing/mozharness/scripts/firefox_media_tests_taskcluster.py b/testing/mozharness/scripts/firefox_media_tests_taskcluster.py
deleted file mode 100644
index 7a0121dca..000000000
--- a/testing/mozharness/scripts/firefox_media_tests_taskcluster.py
+++ /dev/null
@@ -1,110 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** BEGIN LICENSE BLOCK *****
-"""firefox_media_tests_taskcluster.py
-
-Adapted from firefox_media_tests_buildbot.py
-
-Author: Bryce Van Dyk
-"""
-import copy
-import glob
-import os
-import sys
-
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.log import DEBUG, ERROR, INFO
-from mozharness.base.script import PostScriptAction
-from mozharness.mozilla.blob_upload import (
- BlobUploadMixin,
- blobupload_config_options
-)
-from mozharness.mozilla.testing.firefox_media_tests import (
- FirefoxMediaTestsBase, TESTFAILED, SUCCESS
-)
-
-
-class FirefoxMediaTestsTaskcluster(FirefoxMediaTestsBase):
-
- def __init__(self):
- config_options = copy.deepcopy(blobupload_config_options)
- super(FirefoxMediaTestsTaskcluster, self).__init__(
- config_options=config_options,
- all_actions=['clobber',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-media-tests',
- ],
- )
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(FirefoxMediaTestsTaskcluster, self).query_abs_dirs()
- dirs = {
- 'abs_blob_upload_dir': os.path.join(abs_dirs['abs_work_dir'],
- 'blobber_upload_dir')
- }
- abs_dirs.update(dirs)
- self.abs_dirs = abs_dirs
- return self.abs_dirs
-
- def _query_cmd(self):
- cmd = super(FirefoxMediaTestsTaskcluster, self)._query_cmd()
- dirs = self.query_abs_dirs()
- # configure logging
- blob_upload_dir = dirs.get('abs_blob_upload_dir')
- cmd += ['--gecko-log', os.path.join(blob_upload_dir, 'gecko.log')]
- cmd += ['--log-html', os.path.join(blob_upload_dir, 'media_tests.html')]
- cmd += ['--log-mach', os.path.join(blob_upload_dir, 'media_tests_mach.log')]
- return cmd
-
- @PostScriptAction('run-media-tests')
- def _collect_uploads(self, action, success=None):
- """ Copy extra (log) files to blob upload dir. """
- dirs = self.query_abs_dirs()
- log_dir = dirs.get('abs_log_dir')
- blob_upload_dir = dirs.get('abs_blob_upload_dir')
- if not log_dir or not blob_upload_dir:
- return
- self.mkdir_p(blob_upload_dir)
- # Move firefox-media-test screenshots into log_dir
- screenshots_dir = os.path.join(dirs['base_work_dir'],
- 'screenshots')
- log_screenshots_dir = os.path.join(log_dir, 'screenshots')
- if os.access(log_screenshots_dir, os.F_OK):
- self.rmtree(log_screenshots_dir)
- if os.access(screenshots_dir, os.F_OK):
- self.move(screenshots_dir, log_screenshots_dir)
-
- # logs to upload: broadest level (info), error, screenshots
- uploads = glob.glob(os.path.join(log_screenshots_dir, '*'))
- log_files = self.log_obj.log_files
- log_level = self.log_obj.log_level
-
- def append_path(filename, dir=log_dir):
- if filename:
- uploads.append(os.path.join(dir, filename))
-
- append_path(log_files.get(ERROR))
- # never upload debug logs
- if log_level == DEBUG:
- append_path(log_files.get(INFO))
- else:
- append_path(log_files.get(log_level))
- # in case of SimpleFileLogger
- append_path(log_files.get('default'))
- for f in uploads:
- if os.access(f, os.F_OK):
- dest = os.path.join(blob_upload_dir, os.path.basename(f))
- self.copyfile(f, dest)
-
-
-if __name__ == '__main__':
- media_test = FirefoxMediaTestsTaskcluster()
- media_test.run_and_exit()
diff --git a/testing/mozharness/scripts/firefox_ui_tests/functional.py b/testing/mozharness/scripts/firefox_ui_tests/functional.py
deleted file mode 100755
index 58048ad33..000000000
--- a/testing/mozharness/scripts/firefox_ui_tests/functional.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.mozilla.testing.firefox_ui_tests import FirefoxUIFunctionalTests
-
-
-if __name__ == '__main__':
- myScript = FirefoxUIFunctionalTests()
- myScript.run_and_exit()
diff --git a/testing/mozharness/scripts/firefox_ui_tests/update.py b/testing/mozharness/scripts/firefox_ui_tests/update.py
deleted file mode 100755
index c8f5842b7..000000000
--- a/testing/mozharness/scripts/firefox_ui_tests/update.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.mozilla.testing.firefox_ui_tests import FirefoxUIUpdateTests
-
-
-if __name__ == '__main__':
- myScript = FirefoxUIUpdateTests()
- myScript.run_and_exit()
diff --git a/testing/mozharness/scripts/firefox_ui_tests/update_release.py b/testing/mozharness/scripts/firefox_ui_tests/update_release.py
deleted file mode 100755
index f1ec81646..000000000
--- a/testing/mozharness/scripts/firefox_ui_tests/update_release.py
+++ /dev/null
@@ -1,323 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-
-import copy
-import os
-import pprint
-import sys
-import urllib
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.base.python import PreScriptAction
-from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING, EXIT_STATUS_DICT
-from mozharness.mozilla.testing.firefox_ui_tests import (
- FirefoxUIUpdateTests,
- firefox_ui_update_config_options
-)
-
-
-# Command line arguments for release update tests
-firefox_ui_update_release_config_options = [
- [['--build-number'], {
- 'dest': 'build_number',
- 'help': 'Build number of release, eg: 2',
- }],
- [['--limit-locales'], {
- 'dest': 'limit_locales',
- 'default': -1,
- 'type': int,
- 'help': 'Limit the number of locales to run.',
- }],
- [['--release-update-config'], {
- 'dest': 'release_update_config',
- 'help': 'Name of the release update verification config file to use.',
- }],
- [['--this-chunk'], {
- 'dest': 'this_chunk',
- 'default': 1,
- 'help': 'What chunk of locales to process.',
- }],
- [['--tools-repo'], {
- 'dest': 'tools_repo',
- 'default': 'http://hg.mozilla.org/build/tools',
- 'help': 'Which tools repo to check out',
- }],
- [['--tools-tag'], {
- 'dest': 'tools_tag',
- 'help': 'Which revision/tag to use for the tools repository.',
- }],
- [['--total-chunks'], {
- 'dest': 'total_chunks',
- 'default': 1,
- 'help': 'Total chunks to dive the locales into.',
- }],
-] + copy.deepcopy(firefox_ui_update_config_options)
-
-
-class ReleaseFirefoxUIUpdateTests(FirefoxUIUpdateTests):
-
- def __init__(self):
- all_actions = [
- 'clobber',
- 'checkout',
- 'create-virtualenv',
- 'query_minidump_stackwalk',
- 'read-release-update-config',
- 'run-tests',
- ]
-
- super(ReleaseFirefoxUIUpdateTests, self).__init__(
- all_actions=all_actions,
- default_actions=all_actions,
- config_options=firefox_ui_update_release_config_options,
- append_env_variables_from_configs=True,
- )
-
- self.tools_repo = self.config.get('tools_repo')
- self.tools_tag = self.config.get('tools_tag')
-
- assert self.tools_repo and self.tools_tag, \
- 'Without the "--tools-tag" we can\'t clone the releng\'s tools repository.'
-
- self.limit_locales = int(self.config.get('limit_locales'))
-
- # This will be a list containing one item per release based on configs
- # from tools/release/updates/*cfg
- self.releases = None
-
- def checkout(self):
- """
- We checkout the tools repository and update to the right branch
- for it.
- """
- dirs = self.query_abs_dirs()
-
- super(ReleaseFirefoxUIUpdateTests, self).checkout()
-
- self.vcs_checkout(
- repo=self.tools_repo,
- dest=dirs['abs_tools_dir'],
- branch=self.tools_tag,
- vcs='hg'
- )
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
-
- abs_dirs = super(ReleaseFirefoxUIUpdateTests, self).query_abs_dirs()
- dirs = {
- 'abs_tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'tools'),
- }
-
- for key in dirs:
- if key not in abs_dirs:
- abs_dirs[key] = dirs[key]
- self.abs_dirs = abs_dirs
-
- return self.abs_dirs
-
- def read_release_update_config(self):
- '''
- Builds a testing matrix based on an update verification configuration
- file under the tools repository (release/updates/*.cfg).
-
- Each release info line of the update verification files look similar to the following.
-
- NOTE: This shows each pair of information as a new line but in reality
- there is one white space separting them. We only show the values we care for.
-
- release="38.0"
- platform="Linux_x86_64-gcc3"
- build_id="20150429135941"
- locales="ach af ... zh-TW"
- channel="beta-localtest"
- from="/firefox/releases/38.0b9/linux-x86_64/%locale%/firefox-38.0b9.tar.bz2"
- ftp_server_from="http://archive.mozilla.org/pub"
-
- We will store this information in self.releases as a list of releases.
-
- NOTE: We will talk of full and quick releases. Full release info normally contains a subset
- of all locales (except for the most recent releases). A quick release has all locales,
- however, it misses the fields 'from' and 'ftp_server_from'.
- Both pairs of information complement each other but differ in such manner.
- '''
- dirs = self.query_abs_dirs()
- assert os.path.exists(dirs['abs_tools_dir']), \
- 'Without the tools/ checkout we can\'t use releng\'s config parser.'
-
- if self.config.get('release_update_config'):
- # The config file is part of the tools repository. Make sure that if specified
- # we force a revision of that repository to be set.
- if self.tools_tag is None:
- self.fatal('Make sure to specify the --tools-tag')
-
- self.release_update_config = self.config['release_update_config']
-
- # Import the config parser
- sys.path.insert(1, os.path.join(dirs['abs_tools_dir'], 'lib', 'python'))
- from release.updates.verify import UpdateVerifyConfig
-
- uvc = UpdateVerifyConfig()
- config_file = os.path.join(dirs['abs_tools_dir'], 'release', 'updates',
- self.config['release_update_config'])
- uvc.read(config_file)
- if not hasattr(self, 'update_channel'):
- self.update_channel = uvc.channel
-
- # Filter out any releases that are less than Gecko 38
- uvc.releases = [r for r in uvc.releases
- if int(r['release'].split('.')[0]) >= 38]
-
- temp_releases = []
- for rel_info in uvc.releases:
- # This is the full release info
- if 'from' in rel_info and rel_info['from'] is not None:
- # Let's find the associated quick release which contains the remaining locales
- # for all releases except for the most recent release which contain all locales
- quick_release = uvc.getRelease(build_id=rel_info['build_id'], from_path=None)
- if quick_release != {}:
- rel_info['locales'] = sorted(rel_info['locales'] + quick_release['locales'])
- temp_releases.append(rel_info)
-
- uvc.releases = temp_releases
- chunked_config = uvc.getChunk(
- chunks=int(self.config['total_chunks']),
- thisChunk=int(self.config['this_chunk'])
- )
-
- self.releases = chunked_config.releases
-
- @PreScriptAction('run-tests')
- def _pre_run_tests(self, action):
- assert ('release_update_config' in self.config or
- self.installer_url or self.installer_path), \
- 'Either specify --update-verify-config, --installer-url or --installer-path.'
-
- def run_tests(self):
- dirs = self.query_abs_dirs()
-
- # We don't want multiple outputs of the same environment information. To prevent
- # that, we can't make it an argument of run_command and have to print it on our own.
- self.info('Using env: {}'.format(pprint.pformat(self.query_env())))
-
- results = {}
-
- locales_counter = 0
- for rel_info in sorted(self.releases, key=lambda release: release['build_id']):
- build_id = rel_info['build_id']
- results[build_id] = {}
-
- self.info('About to run {buildid} {path} - {num_locales} locales'.format(
- buildid=build_id,
- path=rel_info['from'],
- num_locales=len(rel_info['locales'])
- ))
-
- # Each locale gets a fresh port to avoid address in use errors in case of
- # tests that time out unexpectedly.
- marionette_port = 2827
- for locale in rel_info['locales']:
- locales_counter += 1
- self.info('Running {buildid} {locale}'.format(buildid=build_id,
- locale=locale))
-
- if self.limit_locales > -1 and locales_counter > self.limit_locales:
- self.info('We have reached the limit of locales we were intending to run')
- break
-
- if self.config['dry_run']:
- continue
-
- # Determine from where to download the file
- installer_url = '{server}/{fragment}'.format(
- server=rel_info['ftp_server_from'],
- fragment=urllib.quote(rel_info['from'].replace('%locale%', locale))
- )
- installer_path = self.download_file(
- url=installer_url,
- parent_dir=dirs['abs_work_dir']
- )
-
- binary_path = self.install_app(app=self.config.get('application'),
- installer_path=installer_path)
-
- marionette_port += 1
-
- retcode = self.run_test(
- binary_path=binary_path,
- env=self.query_env(avoid_host_env=True),
- marionette_port=marionette_port,
- )
-
- self.uninstall_app()
-
- # Remove installer which is not needed anymore
- self.info('Removing {}'.format(installer_path))
- os.remove(installer_path)
-
- if retcode:
- self.warning('FAIL: {} has failed.'.format(sys.argv[0]))
-
- base_cmd = 'python {command} --firefox-ui-branch {branch} ' \
- '--release-update-config {config} --tools-tag {tag}'.format(
- command=sys.argv[0],
- branch=self.firefox_ui_branch,
- config=self.release_update_config,
- tag=self.tools_tag
- )
-
- for config in self.config['config_files']:
- base_cmd += ' --cfg {}'.format(config)
-
- if self.symbols_url:
- base_cmd += ' --symbols-path {}'.format(self.symbols_url)
-
- base_cmd += ' --installer-url {}'.format(installer_url)
-
- self.info('You can run the *specific* locale on the same machine with:')
- self.info(base_cmd)
-
- self.info('You can run the *specific* locale on *your* machine with:')
- self.info('{} --cfg developer_config.py'.format(base_cmd))
-
- results[build_id][locale] = retcode
-
- self.info('Completed {buildid} {locale} with return code: {retcode}'.format(
- buildid=build_id,
- locale=locale,
- retcode=retcode))
-
- if self.limit_locales > -1 and locales_counter > self.limit_locales:
- break
-
- # Determine which locales have failed and set scripts exit code
- exit_status = TBPL_SUCCESS
- for build_id in sorted(results.keys()):
- failed_locales = []
- for locale in sorted(results[build_id].keys()):
- if results[build_id][locale] != 0:
- failed_locales.append(locale)
-
- if failed_locales:
- if exit_status == TBPL_SUCCESS:
- self.info('\nSUMMARY - Failed locales for {}:'.format(self.cli_script))
- self.info('====================================================')
- exit_status = TBPL_WARNING
-
- self.info(build_id)
- self.info(' {}'.format(', '.join(failed_locales)))
-
- self.return_code = EXIT_STATUS_DICT[exit_status]
-
-
-if __name__ == '__main__':
- myScript = ReleaseFirefoxUIUpdateTests()
- myScript.run_and_exit()
diff --git a/testing/mozharness/scripts/fx_desktop_build.py b/testing/mozharness/scripts/fx_desktop_build.py
deleted file mode 100755
index 40f20442c..000000000
--- a/testing/mozharness/scripts/fx_desktop_build.py
+++ /dev/null
@@ -1,235 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""fx_desktop_build.py.
-
-script harness to build nightly firefox within Mozilla's build environment
-and developer machines alike
-
-author: Jordan Lund
-
-"""
-
-import copy
-import pprint
-import sys
-import os
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-import mozharness.base.script as script
-from mozharness.mozilla.building.buildbase import BUILD_BASE_CONFIG_OPTIONS, \
- BuildingConfig, BuildOptionParser, BuildScript
-from mozharness.base.config import parse_config_file
-from mozharness.mozilla.testing.try_tools import TryToolsMixin, try_config_options
-
-
-class FxDesktopBuild(BuildScript, TryToolsMixin, object):
- def __init__(self):
- buildscript_kwargs = {
- 'config_options': BUILD_BASE_CONFIG_OPTIONS + copy.deepcopy(try_config_options),
- 'all_actions': [
- 'get-secrets',
- 'clobber',
- 'clone-tools',
- 'checkout-sources',
- 'setup-mock',
- 'build',
- 'upload-files', # upload from BB to TC
- 'sendchange',
- 'check-test',
- 'valgrind-test',
- 'package-source',
- 'generate-source-signing-manifest',
- 'multi-l10n',
- 'generate-build-stats',
- 'update',
- ],
- 'require_config_file': True,
- # Default configuration
- 'config': {
- 'is_automation': True,
- "pgo_build": False,
- "debug_build": False,
- "pgo_platforms": ['linux', 'linux64', 'win32', 'win64'],
- # nightly stuff
- "nightly_build": False,
- 'balrog_credentials_file': 'oauth.txt',
- 'taskcluster_credentials_file': 'oauth.txt',
- 'periodic_clobber': 168,
- # hg tool stuff
- "tools_repo": "https://hg.mozilla.org/build/tools",
- # Seed all clones with mozilla-unified. This ensures subsequent
- # jobs have a minimal `hg pull`.
- "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
- "repo_base": "https://hg.mozilla.org",
- 'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
- "graph_selector": "/server/collect.cgi",
- # only used for make uploadsymbols
- 'old_packages': [
- "%(objdir)s/dist/firefox-*",
- "%(objdir)s/dist/fennec*",
- "%(objdir)s/dist/seamonkey*",
- "%(objdir)s/dist/thunderbird*",
- "%(objdir)s/dist/install/sea/*.exe"
- ],
- 'stage_product': 'firefox',
- 'platform_supports_post_upload_to_latest': True,
- 'build_resources_path': '%(abs_src_dir)s/obj-firefox/.mozbuild/build_resources.json',
- 'nightly_promotion_branches': ['mozilla-central', 'mozilla-aurora'],
-
- # try will overwrite these
- 'clone_with_purge': False,
- 'clone_by_revision': False,
- 'tinderbox_build_dir': None,
- 'to_tinderbox_dated': True,
- 'release_to_try_builds': False,
- 'include_post_upload_builddir': False,
- 'use_clobberer': True,
-
- 'stage_username': 'ffxbld',
- 'stage_ssh_key': 'ffxbld_rsa',
- 'virtualenv_modules': [
- 'requests==2.8.1',
- 'PyHawk-with-a-single-extra-commit==0.1.5',
- 'taskcluster==0.0.26',
- ],
- 'virtualenv_path': 'venv',
- #
-
- },
- 'ConfigClass': BuildingConfig,
- }
- super(FxDesktopBuild, self).__init__(**buildscript_kwargs)
-
- def _pre_config_lock(self, rw_config):
- """grab buildbot props if we are running this in automation"""
- super(FxDesktopBuild, self)._pre_config_lock(rw_config)
- c = self.config
- if c['is_automation']:
- # parse buildbot config and add it to self.config
- self.info("We are running this in buildbot, grab the build props")
- self.read_buildbot_config()
- ###
- if c.get('stage_platform'):
- platform_for_log_url = c['stage_platform']
- if c.get('pgo_build'):
- platform_for_log_url += '-pgo'
- # postrun.py uses stage_platform buildbot prop as part of the log url
- self.set_buildbot_property('stage_platform',
- platform_for_log_url,
- write_to_file=True)
- else:
- self.fatal("'stage_platform' not determined and is required in your config")
-
- if self.try_message_has_flag('artifact'):
- self.info('Artifact build requested in try syntax.')
- variant = 'artifact'
- if c.get('build_variant') in ['debug', 'cross-debug']:
- variant = 'debug-artifact'
- self._update_build_variant(rw_config, variant)
-
- # helpers
- def _update_build_variant(self, rw_config, variant='artifact'):
- """ Intended for use in _pre_config_lock """
- c = self.config
- variant_cfg_path, _ = BuildOptionParser.find_variant_cfg_path(
- '--custom-build-variant-cfg',
- variant,
- rw_config.config_parser
- )
- if not variant_cfg_path:
- self.fatal('Could not find appropriate config file for variant %s' % variant)
- # Update other parts of config to keep dump-config accurate
- # Only dump-config is affected because most config info is set during
- # initial parsing
- variant_cfg_dict = parse_config_file(variant_cfg_path)
- rw_config.all_cfg_files_and_dicts.append((variant_cfg_path, variant_cfg_dict))
- c.update({
- 'build_variant': variant,
- 'config_files': c['config_files'] + [variant_cfg_path]
- })
-
- self.info("Updating self.config with the following from {}:".format(variant_cfg_path))
- self.info(pprint.pformat(variant_cfg_dict))
- c.update(variant_cfg_dict)
- c['forced_artifact_build'] = True
- # Bug 1231320 adds MOZHARNESS_ACTIONS in TaskCluster tasks to override default_actions
- # We don't want that when forcing an artifact build.
- if rw_config.volatile_config['actions']:
- self.info("Updating volatile_config to include default_actions "
- "from {}.".format(variant_cfg_path))
- # add default actions in correct order
- combined_actions = []
- for a in rw_config.all_actions:
- if a in c['default_actions'] or a in rw_config.volatile_config['actions']:
- combined_actions.append(a)
- rw_config.volatile_config['actions'] = combined_actions
- self.info("Actions in volatile_config are now: {}".format(
- rw_config.volatile_config['actions'])
- )
- # replace rw_config as well to set actions as in BaseScript
- rw_config.set_config(c, overwrite=True)
- rw_config.update_actions()
- self.actions = tuple(rw_config.actions)
- self.all_actions = tuple(rw_config.all_actions)
-
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- c = self.config
- abs_dirs = super(FxDesktopBuild, self).query_abs_dirs()
- if not c.get('app_ini_path'):
- self.fatal('"app_ini_path" is needed in your config for this '
- 'script.')
-
- dirs = {
- # BuildFactories in factory.py refer to a 'build' dir on the slave.
- # This contains all the source code/objdir to compile. However,
- # there is already a build dir in mozharness for every mh run. The
- # 'build' that factory refers to I named: 'src' so
- # there is a seperation in mh. for example, rather than having
- # '{mozharness_repo}/build/build/', I have '{
- # mozharness_repo}/build/src/'
- 'abs_src_dir': os.path.join(abs_dirs['abs_work_dir'],
- 'src'),
- 'abs_obj_dir': os.path.join(abs_dirs['abs_work_dir'],
- 'src',
- self._query_objdir()),
- 'abs_tools_dir': os.path.join(abs_dirs['abs_work_dir'], 'tools'),
- 'abs_app_ini_path': c['app_ini_path'] % {
- 'obj_dir': os.path.join(abs_dirs['abs_work_dir'],
- 'src',
- self._query_objdir())
- },
- }
- abs_dirs.update(dirs)
- self.abs_dirs = abs_dirs
- return self.abs_dirs
-
- # Actions {{{2
- # read_buildbot_config in BuildingMixin
- # clobber in BuildingMixin -> PurgeMixin
- # if Linux config:
- # reset_mock in BuildingMixing -> MockMixin
- # setup_mock in BuildingMixing (overrides MockMixin.mock_setup)
-
- def set_extra_try_arguments(self, action, success=None):
- """ Override unneeded method from TryToolsMixin """
- pass
-
- @script.PreScriptRun
- def suppress_windows_modal_dialogs(self, *args, **kwargs):
- if self._is_windows():
- # Suppress Windows modal dialogs to avoid hangs
- import ctypes
- ctypes.windll.kernel32.SetErrorMode(0x8001)
-
-if __name__ == '__main__':
- fx_desktop_build = FxDesktopBuild()
- fx_desktop_build.run_and_exit()
diff --git a/testing/mozharness/scripts/gaia_build_integration.py b/testing/mozharness/scripts/gaia_build_integration.py
deleted file mode 100755
index 32d188ffd..000000000
--- a/testing/mozharness/scripts/gaia_build_integration.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.mozilla.testing.gaia_test import GaiaTest
-from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
-
-
-class GaiaBuildIntegrationTest(GaiaTest):
-
- def __init__(self, require_config_file=False):
- GaiaTest.__init__(self, require_config_file)
-
- def run_tests(self):
- """
- Run the integration test suite.
- """
- dirs = self.query_abs_dirs()
-
- self.node_setup()
-
- output_parser = TestSummaryOutputParserHelper(
- config=self.config, log_obj=self.log_obj, error_list=self.error_list)
-
- cmd = [
- 'make',
- 'build-test-integration',
- 'REPORTER=mocha-tbpl-reporter',
- 'NODE_MODULES_SRC=npm-cache',
- 'VIRTUALENV_EXISTS=1',
- 'TRY_ENV=1'
- ]
-
- # for Mulet
- if 'firefox' in self.binary_path:
- cmd += ['RUNTIME=%s' % self.binary_path]
-
- code = self.run_command(cmd, cwd=dirs['abs_gaia_dir'],
- output_parser=output_parser,
- output_timeout=600)
-
- output_parser.print_summary('gaia-build-integration-tests')
- self.publish(code)
-
-if __name__ == '__main__':
- gaia_build_integration_test = GaiaBuildIntegrationTest()
- gaia_build_integration_test.run_and_exit()
diff --git a/testing/mozharness/scripts/gaia_build_unit.py b/testing/mozharness/scripts/gaia_build_unit.py
deleted file mode 100755
index c16ce99fa..000000000
--- a/testing/mozharness/scripts/gaia_build_unit.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.mozilla.testing.gaia_test import GaiaTest
-from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
-
-
-class GaiaBuildUnitTest(GaiaTest):
-
- def __init__(self, require_config_file=False):
- GaiaTest.__init__(self, require_config_file)
-
- def run_tests(self):
- """
- Run the gaia build unit test suite.
- """
- dirs = self.query_abs_dirs()
-
- self.node_setup()
-
- output_parser = TestSummaryOutputParserHelper(
- config=self.config, log_obj=self.log_obj, error_list=self.error_list)
-
- cmd = [
- 'make',
- 'build-test-unit',
- 'REPORTER=mocha-tbpl-reporter',
- 'NODE_MODULES_SRC=npm-cache',
- 'VIRTUALENV_EXISTS=1',
- 'TRY_ENV=1'
- ]
-
- # for Mulet
- if 'firefox' in self.binary_path:
- cmd += ['RUNTIME=%s' % self.binary_path]
-
- code = self.run_command(cmd, cwd=dirs['abs_gaia_dir'],
- output_parser=output_parser,
- output_timeout=330)
-
- output_parser.print_summary('gaia-build-unit-tests')
- self.publish(code)
-
-if __name__ == '__main__':
- gaia_build_unit_test = GaiaBuildUnitTest()
- gaia_build_unit_test.run_and_exit()
diff --git a/testing/mozharness/scripts/gaia_integration.py b/testing/mozharness/scripts/gaia_integration.py
deleted file mode 100644
index 3edb8b964..000000000
--- a/testing/mozharness/scripts/gaia_integration.py
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.mozilla.testing.gaia_test import GaiaTest
-from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
-
-
-class GaiaIntegrationTest(GaiaTest):
-
- def __init__(self, require_config_file=False):
- GaiaTest.__init__(self, require_config_file)
-
- def run_tests(self):
- """
- Run the integration test suite.
- """
- dirs = self.query_abs_dirs()
-
- self.node_setup()
-
- output_parser = TestSummaryOutputParserHelper(
- config=self.config, log_obj=self.log_obj, error_list=self.error_list)
-
- # Bug 1046694 - add environment variables which govern test chunking
- env = {}
- if self.config.get('this_chunk') and self.config.get('total_chunks'):
- env["PART"] = self.config.get('this_chunk')
- env["NBPARTS"] = self.config.get('total_chunks')
- env = self.query_env(partial_env=env)
-
- # Bug 1137884 - marionette-js-runner needs to know about virtualenv
- gaia_runner_service = (
- dirs['abs_gaia_dir'] +
- '/node_modules/marionette-js-runner/host/python/runner-service')
- # Check whether python package is around since there exist versions
- # of gaia that depend on versions of marionette-js-runner without
- # the python stuff.
- if os.path.exists(gaia_runner_service):
- self.install_module('gaia-runner-service', gaia_runner_service)
- env['VIRTUALENV_PATH'] = self.query_virtualenv_path()
- env['HOST_LOG'] = os.path.join(dirs['abs_log_dir'], 'gecko_output.log')
-
- cmd = [
- 'make',
- 'test-integration',
- 'REPORTER=mocha-tbpl-reporter',
- 'TEST_MANIFEST=./shared/test/integration/tbpl-manifest.json',
- 'NODE_MODULE_SRC=npm-cache',
- 'VIRTUALENV_EXISTS=1'
- ]
-
- # for Mulet
- if 'firefox' in self.binary_path:
- cmd += ['RUNTIME=%s' % self.binary_path]
-
- code = self.run_command(cmd, cwd=dirs['abs_gaia_dir'], env=env,
- output_parser=output_parser,
- output_timeout=330)
-
- output_parser.print_summary('gaia-integration-tests')
- self.publish(code, passed=output_parser.passed, failed=output_parser.failed)
-
-if __name__ == '__main__':
- gaia_integration_test = GaiaIntegrationTest()
- gaia_integration_test.run_and_exit()
diff --git a/testing/mozharness/scripts/gaia_linter.py b/testing/mozharness/scripts/gaia_linter.py
deleted file mode 100755
index e4441b92b..000000000
--- a/testing/mozharness/scripts/gaia_linter.py
+++ /dev/null
@@ -1,148 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-import os
-import re
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.log import OutputParser, ERROR
-from mozharness.mozilla.testing.gaia_test import GaiaTest
-
-
-class GaiaLinterOutputParser(OutputParser):
-
- JSHINT_START = "Running jshint..."
- JSHINT_DONE = "xfailed)"
- JSHINT_ERROR = re.compile('(.+): (.*?) \(ERROR\)')
-
- LAST_FILE = re.compile('----- FILE : (.*?) -----')
-
- GJSLINT_START = "Running gjslint..."
- GJSLINT_ERROR = re.compile('Line (\d+), E:(\d+):')
-
- GENERAL_ERRORS = (re.compile('make(.*?)\*\*\*(.*?)Error'),)
-
- def __init__(self, **kwargs):
- self.base_dir = kwargs.pop('base_dir')
- super(GaiaLinterOutputParser, self).__init__(**kwargs)
- self.in_jshint = False
- self.in_gjslint = False
- self.last_file = 'unknown'
-
- def log_error(self, message, filename=None):
- if not filename:
- self.log('TEST-UNEXPECTED-FAIL | make lint | %s' % message)
- else:
- path = filename
- if self.base_dir in path:
- path = os.path.relpath(filename, self.base_dir)
- self.log('TEST-UNEXPECTED-FAIL | %s | %s' % (path, message),
- level=ERROR)
- self.num_errors += 1
- self.worst_log_level = self.worst_level(ERROR,
- self.worst_log_level)
-
- def parse_single_line(self, line):
- if not self.in_jshint:
- if self.JSHINT_START in line:
- self.in_jshint = True
- self.in_gjslint = False
- else:
- if self.JSHINT_DONE in line:
- self.in_jshint = False
-
- if not self.in_gjslint:
- if self.GJSLINT_START in line:
- self.in_gjslint = True
-
- if self.in_jshint:
- m = self.JSHINT_ERROR.search(line)
- if m:
- self.log_error(m.groups()[1], m.groups()[0])
-
- if self.in_gjslint:
- m = self.LAST_FILE.search(line)
- if m:
- self.last_file = m.groups()[0]
-
- m = self.GJSLINT_ERROR.search(line)
- if m:
- self.log_error(line, self.last_file)
-
- for an_error in self.GENERAL_ERRORS:
- if an_error.search(line):
- self.log_error(line)
-
- if self.log_output:
- self.info(' %s' % line)
-
- def evaluate_parser(self):
- # generate the TinderboxPrint line for TBPL
- if self.num_errors:
- self.tsummary = '<em class="testfail">%d errors</em>' % self.num_errors
- else:
- self.tsummary = "0 errors"
-
- def print_summary(self, suite_name):
- self.evaluate_parser()
- self.info("TinderboxPrint: %s: %s\n" % (suite_name, self.tsummary))
-
-
-class GaiaIntegrationTest(GaiaTest):
-
- virtualenv_modules = ['closure_linter==2.3.13',
- 'python-gflags',
- ]
-
- def __init__(self, require_config_file=False):
- GaiaTest.__init__(self, require_config_file)
-
- def run_tests(self):
- """
- Run the integration test suite.
- """
- dirs = self.query_abs_dirs()
-
- # Copy the b2g desktop we built to the gaia directory so that it
- # gets used by the marionette-js-runner.
- self.copytree(
- os.path.join(os.path.dirname(self.binary_path)),
- os.path.join(dirs['abs_gaia_dir'], 'b2g'),
- overwrite='clobber'
- )
-
- cmd = [
- 'make',
- 'lint',
- 'NODE_MODULES_SRC=npm-cache',
- 'VIRTUALENV_EXISTS=1'
- ]
-
- # for Mulet
- if 'firefox' in self.binary_path:
- cmd += ['RUNTIME=%s' % self.binary_path]
-
- self.make_node_modules()
-
- output_parser = GaiaLinterOutputParser(
- base_dir=dirs['abs_gaia_dir'],
- config=self.config,
- log_obj=self.log_obj)
-
- code = self.run_command(cmd, cwd=dirs['abs_gaia_dir'],
- output_parser=output_parser,
- output_timeout=600)
-
- output_parser.print_summary('gaia-lint')
- self.publish(code)
-
-if __name__ == '__main__':
- gaia_integration_test = GaiaIntegrationTest()
- gaia_integration_test.run_and_exit()
diff --git a/testing/mozharness/scripts/gaia_unit.py b/testing/mozharness/scripts/gaia_unit.py
deleted file mode 100755
index 660643b74..000000000
--- a/testing/mozharness/scripts/gaia_unit.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-import os
-import sys
-import glob
-import subprocess
-import json
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.mozilla.testing.gaia_test import GaiaTest
-from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
-
-
-class GaiaUnitTest(GaiaTest):
- def __init__(self, require_config_file=False):
- GaiaTest.__init__(self, require_config_file)
-
- def pull(self, **kwargs):
- GaiaTest.pull(self, **kwargs)
-
- def run_tests(self):
- """
- Run the unit test suite.
- """
- dirs = self.query_abs_dirs()
-
- self.make_node_modules()
-
- # make the gaia profile
- self.make_gaia(dirs['abs_gaia_dir'],
- self.config.get('xre_path'),
- xre_url=self.config.get('xre_url'),
- debug=True)
-
- # build the testrunner command arguments
- python = self.query_python_path('python')
- cmd = [python, '-u', os.path.join(dirs['abs_runner_dir'],
- 'gaia_unit_test',
- 'main.py')]
- executable = 'firefox'
- if 'b2g' in self.binary_path:
- executable = 'b2g-bin'
-
- profile = os.path.join(dirs['abs_gaia_dir'], 'profile-debug')
- binary = os.path.join(os.path.dirname(self.binary_path), executable)
- cmd.extend(self._build_arg('--binary', binary))
- cmd.extend(self._build_arg('--profile', profile))
- cmd.extend(self._build_arg('--symbols-path', self.symbols_path))
- cmd.extend(self._build_arg('--browser-arg', self.config.get('browser_arg')))
-
- # Add support for chunking
- if self.config.get('total_chunks') and self.config.get('this_chunk'):
- chunker = [ os.path.join(dirs['abs_gaia_dir'], 'bin', 'chunk'),
- self.config.get('total_chunks'), self.config.get('this_chunk') ]
-
- disabled_tests = []
- disabled_manifest = os.path.join(dirs['abs_runner_dir'],
- 'gaia_unit_test',
- 'disabled.json')
- with open(disabled_manifest, 'r') as m:
- try:
- disabled_tests = json.loads(m.read())
- except:
- print "Error while decoding disabled.json; please make sure this file has valid JSON syntax."
- sys.exit(1)
-
- # Construct a list of all tests
- unit_tests = []
- for path in ('apps', 'tv_apps'):
- test_root = os.path.join(dirs['abs_gaia_dir'], path)
- full_paths = glob.glob(os.path.join(test_root, '*/test/unit/*_test.js'))
- unit_tests += map(lambda x: os.path.relpath(x, test_root), full_paths)
-
- # Remove the tests that are disabled
- active_unit_tests = filter(lambda x: x not in disabled_tests, unit_tests)
-
- # Chunk the list as requested
- tests_to_run = subprocess.check_output(chunker + active_unit_tests).strip().split(' ')
-
- cmd.extend(tests_to_run)
-
- output_parser = TestSummaryOutputParserHelper(config=self.config,
- log_obj=self.log_obj,
- error_list=self.error_list)
-
- upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
- if not os.path.isdir(upload_dir):
- self.mkdir_p(upload_dir)
-
- env = self.query_env()
- env['MOZ_UPLOAD_DIR'] = upload_dir
- # I don't like this output_timeout hardcode, but bug 920153
- code = self.run_command(cmd, env=env,
- output_parser=output_parser,
- output_timeout=1760)
-
- output_parser.print_summary('gaia-unit-tests')
- self.publish(code)
-
-if __name__ == '__main__':
- gaia_unit_test = GaiaUnitTest()
- gaia_unit_test.run_and_exit()
diff --git a/testing/mozharness/scripts/marionette.py b/testing/mozharness/scripts/marionette.py
deleted file mode 100755
index b7f9c2765..000000000
--- a/testing/mozharness/scripts/marionette.py
+++ /dev/null
@@ -1,358 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-
-import copy
-import os
-import re
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.errors import TarErrorList
-from mozharness.base.log import INFO, ERROR, WARNING
-from mozharness.base.script import PreScriptAction
-from mozharness.base.transfer import TransferMixin
-from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
-from mozharness.mozilla.testing.errors import LogcatErrorList
-from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
-from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
-from mozharness.mozilla.structuredlog import StructuredOutputParser
-
-# TODO: we could remove emulator specific code after B2G ICS emulator buildbot
-# builds is turned off, Bug 1209180.
-
-
-class MarionetteTest(TestingMixin, MercurialScript, BlobUploadMixin, TransferMixin):
- config_options = [[
- ["--application"],
- {"action": "store",
- "dest": "application",
- "default": None,
- "help": "application name of binary"
- }
- ], [
- ["--app-arg"],
- {"action": "store",
- "dest": "app_arg",
- "default": None,
- "help": "Optional command-line argument to pass to the browser"
- }
- ], [
- ["--marionette-address"],
- {"action": "store",
- "dest": "marionette_address",
- "default": None,
- "help": "The host:port of the Marionette server running inside Gecko. Unused for emulator testing",
- }
- ], [
- ["--emulator"],
- {"action": "store",
- "type": "choice",
- "choices": ['arm', 'x86'],
- "dest": "emulator",
- "default": None,
- "help": "Use an emulator for testing",
- }
- ], [
- ["--test-manifest"],
- {"action": "store",
- "dest": "test_manifest",
- "default": "unit-tests.ini",
- "help": "Path to test manifest to run relative to the Marionette "
- "tests directory",
- }
- ], [
- ["--total-chunks"],
- {"action": "store",
- "dest": "total_chunks",
- "help": "Number of total chunks",
- }
- ], [
- ["--this-chunk"],
- {"action": "store",
- "dest": "this_chunk",
- "help": "Number of this chunk",
- }
- ], [
- ["--e10s"],
- {"action": "store_true",
- "dest": "e10s",
- "default": False,
- "help": "Run tests with multiple processes. (Desktop builds only)",
- }
- ], [
- ["--allow-software-gl-layers"],
- {"action": "store_true",
- "dest": "allow_software_gl_layers",
- "default": False,
- "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor."
- }
- ]] + copy.deepcopy(testing_config_options) \
- + copy.deepcopy(blobupload_config_options)
-
- error_list = [
- {'substr': 'FAILED (errors=', 'level': WARNING},
- {'substr': r'''Could not successfully complete transport of message to Gecko, socket closed''', 'level': ERROR},
- {'substr': r'''Connection to Marionette server is lost. Check gecko''', 'level': ERROR},
- {'substr': 'Timeout waiting for marionette on port', 'level': ERROR},
- {'regex': re.compile(r'''(TEST-UNEXPECTED|PROCESS-CRASH)'''), 'level': ERROR},
- {'regex': re.compile(r'''(\b((?!Marionette|TestMarionette|NoSuchElement|XPathLookup|NoSuchWindow|StaleElement|ScriptTimeout|ElementNotVisible|NoSuchFrame|InvalidResponse|Javascript|Timeout|InvalidElementState|NoAlertPresent|InvalidCookieDomain|UnableToSetCookie|InvalidSelector|MoveTargetOutOfBounds)\w*)Exception)'''), 'level': ERROR},
- ]
-
- repos = []
-
- def __init__(self, require_config_file=False):
- super(MarionetteTest, self).__init__(
- config_options=self.config_options,
- all_actions=['clobber',
- 'read-buildbot-config',
- 'pull',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-tests'],
- default_actions=['clobber',
- 'pull',
- 'download-and-extract',
- 'create-virtualenv',
- 'install',
- 'run-tests'],
- require_config_file=require_config_file,
- config={'require_test_zip': True})
-
- # these are necessary since self.config is read only
- c = self.config
- self.installer_url = c.get('installer_url')
- self.installer_path = c.get('installer_path')
- self.binary_path = c.get('binary_path')
- self.test_url = c.get('test_url')
- self.test_packages_url = c.get('test_packages_url')
-
- if c.get('structured_output'):
- self.parser_class = StructuredOutputParser
- else:
- self.parser_class = TestSummaryOutputParserHelper
-
- def _pre_config_lock(self, rw_config):
- super(MarionetteTest, self)._pre_config_lock(rw_config)
- if not self.config.get('emulator') and not self.config.get('marionette_address'):
- self.fatal("You need to specify a --marionette-address for non-emulator tests! (Try --marionette-address localhost:2828 )")
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(MarionetteTest, self).query_abs_dirs()
- dirs = {}
- dirs['abs_test_install_dir'] = os.path.join(
- abs_dirs['abs_work_dir'], 'tests')
- dirs['abs_marionette_dir'] = os.path.join(
- dirs['abs_test_install_dir'], 'marionette', 'harness', 'marionette_harness')
- dirs['abs_marionette_tests_dir'] = os.path.join(
- dirs['abs_test_install_dir'], 'marionette', 'tests', 'testing',
- 'marionette', 'harness', 'marionette_harness', 'tests')
- dirs['abs_gecko_dir'] = os.path.join(
- abs_dirs['abs_work_dir'], 'gecko')
- dirs['abs_emulator_dir'] = os.path.join(
- abs_dirs['abs_work_dir'], 'emulator')
-
- dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
-
- for key in dirs.keys():
- if key not in abs_dirs:
- abs_dirs[key] = dirs[key]
- self.abs_dirs = abs_dirs
- return self.abs_dirs
-
- @PreScriptAction('create-virtualenv')
- def _configure_marionette_virtualenv(self, action):
- dirs = self.query_abs_dirs()
- requirements = os.path.join(dirs['abs_test_install_dir'],
- 'config',
- 'marionette_requirements.txt')
- if os.access(requirements, os.F_OK):
- self.register_virtualenv_module(requirements=[requirements],
- two_pass=True)
- else:
- # XXX Bug 879765: Dependent modules need to be listed before parent
- # modules, otherwise they will get installed from the pypi server.
- # XXX Bug 908356: This block can be removed as soon as the
- # in-tree requirements files propagate to all active trees.
- mozbase_dir = os.path.join('tests', 'mozbase')
- self.register_virtualenv_module(
- 'manifestparser', os.path.join(mozbase_dir, 'manifestdestiny'))
- for m in ('mozfile', 'mozlog', 'mozinfo', 'moznetwork', 'mozhttpd',
- 'mozcrash', 'mozinstall', 'mozdevice', 'mozprofile',
- 'mozprocess', 'mozrunner'):
- self.register_virtualenv_module(
- m, os.path.join(mozbase_dir, m))
-
- self.register_virtualenv_module(
- 'marionette', os.path.join('tests', 'marionette'))
-
- def _get_options_group(self, is_emulator):
- """
- Determine which in tree options group to use and return the
- appropriate key.
- """
- platform = 'emulator' if is_emulator else 'desktop'
- # Currently running marionette on an emulator means webapi
- # tests. This method will need to change if this does.
- testsuite = 'webapi' if is_emulator else 'marionette'
- return '{}_{}'.format(testsuite, platform)
-
- def download_and_extract(self):
- super(MarionetteTest, self).download_and_extract()
-
- if self.config.get('emulator'):
- dirs = self.query_abs_dirs()
-
- self.mkdir_p(dirs['abs_emulator_dir'])
- tar = self.query_exe('tar', return_type='list')
- self.run_command(tar + ['zxf', self.installer_path],
- cwd=dirs['abs_emulator_dir'],
- error_list=TarErrorList,
- halt_on_failure=True, fatal_exit_code=3)
-
- def install(self):
- if self.config.get('emulator'):
- self.info("Emulator tests; skipping.")
- else:
- super(MarionetteTest, self).install()
-
- def run_tests(self):
- """
- Run the Marionette tests
- """
- dirs = self.query_abs_dirs()
-
- raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
- 'marionette_raw.log')
- error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
- 'marionette_errorsummary.log')
- html_report_file = os.path.join(dirs['abs_blob_upload_dir'],
- 'report.html')
-
- config_fmt_args = {
- # emulator builds require a longer timeout
- 'timeout': 60000 if self.config.get('emulator') else 10000,
- 'profile': os.path.join(dirs['abs_work_dir'], 'profile'),
- 'xml_output': os.path.join(dirs['abs_work_dir'], 'output.xml'),
- 'html_output': os.path.join(dirs['abs_blob_upload_dir'], 'output.html'),
- 'logcat_dir': dirs['abs_work_dir'],
- 'emulator': 'arm',
- 'symbols_path': self.symbols_path,
- 'binary': self.binary_path,
- 'address': self.config.get('marionette_address'),
- 'raw_log_file': raw_log_file,
- 'error_summary_file': error_summary_file,
- 'html_report_file': html_report_file,
- 'gecko_log': dirs["abs_blob_upload_dir"],
- 'this_chunk': self.config.get('this_chunk', 1),
- 'total_chunks': self.config.get('total_chunks', 1)
- }
-
- self.info("The emulator type: %s" % config_fmt_args["emulator"])
- # build the marionette command arguments
- python = self.query_python_path('python')
-
- cmd = [python, '-u', os.path.join(dirs['abs_marionette_dir'],
- 'runtests.py')]
-
- manifest = os.path.join(dirs['abs_marionette_tests_dir'],
- self.config['test_manifest'])
-
- if self.config.get('app_arg'):
- config_fmt_args['app_arg'] = self.config['app_arg']
-
- if not self.config['e10s']:
- cmd.append('--disable-e10s')
-
- cmd.append('--gecko-log=%s' % os.path.join(dirs["abs_blob_upload_dir"],
- 'gecko.log'))
-
- if self.config.get("structured_output"):
- cmd.append("--log-raw=-")
-
- options_group = self._get_options_group(self.config.get('emulator'))
-
- if options_group not in self.config["suite_definitions"]:
- self.fatal("%s is not defined in the config!" % options_group)
-
- for s in self.config["suite_definitions"][options_group]["options"]:
- cmd.append(s % config_fmt_args)
-
- if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
- # Make sure that the logging directory exists
- self.fatal("Could not create blobber upload directory")
-
- cmd.append(manifest)
-
- try_options, try_tests = self.try_args("marionette")
- cmd.extend(self.query_tests_args(try_tests,
- str_format_values=config_fmt_args))
-
- env = {}
- if self.query_minidump_stackwalk():
- env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
- env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
- env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
-
- if self.config['allow_software_gl_layers']:
- env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1'
-
- if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
- self.mkdir_p(env['MOZ_UPLOAD_DIR'])
- env = self.query_env(partial_env=env)
-
- marionette_parser = self.parser_class(config=self.config,
- log_obj=self.log_obj,
- error_list=self.error_list,
- strict=False)
- return_code = self.run_command(cmd, env=env,
- output_timeout=1000,
- output_parser=marionette_parser)
- level = INFO
- tbpl_status, log_level = marionette_parser.evaluate_parser(
- return_code=return_code)
- marionette_parser.append_tinderboxprint_line("marionette")
-
- qemu = os.path.join(dirs['abs_work_dir'], 'qemu.log')
- if os.path.isfile(qemu):
- self.copyfile(qemu, os.path.join(dirs['abs_blob_upload_dir'],
- 'qemu.log'))
-
- # dump logcat output if there were failures
- if self.config.get('emulator'):
- if marionette_parser.failed != "0" or 'T-FAIL' in marionette_parser.tsummary:
- logcat = os.path.join(dirs['abs_work_dir'], 'emulator-5554.log')
- if os.access(logcat, os.F_OK):
- self.info('dumping logcat')
- self.run_command(['cat', logcat], error_list=LogcatErrorList)
- else:
- self.info('no logcat file found')
- else:
- # .. or gecko.log if it exists
- gecko_log = os.path.join(self.config['base_work_dir'], 'gecko.log')
- if os.access(gecko_log, os.F_OK):
- self.info('dumping gecko.log')
- self.run_command(['cat', gecko_log])
- self.rmtree(gecko_log)
- else:
- self.info('gecko.log not found')
-
- marionette_parser.print_summary('marionette')
-
- self.log("Marionette exited with return code %s: %s" % (return_code, tbpl_status),
- level=level)
- self.buildbot_status(tbpl_status)
-
-
-if __name__ == '__main__':
- marionetteTest = MarionetteTest()
- marionetteTest.run_and_exit()
diff --git a/testing/mozharness/scripts/marionette_harness_tests.py b/testing/mozharness/scripts/marionette_harness_tests.py
deleted file mode 100644
index 0811bef9c..000000000
--- a/testing/mozharness/scripts/marionette_harness_tests.py
+++ /dev/null
@@ -1,141 +0,0 @@
-#!/usr/bin/env python
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-import copy
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.python import PreScriptAction
-from mozharness.base.python import (
- VirtualenvMixin,
- virtualenv_config_options,
-)
-from mozharness.base.script import BaseScript
-from mozharness.mozilla.buildbot import (
- BuildbotMixin, TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE,
- TBPL_EXCEPTION
-)
-
-marionette_harness_tests_config_options = [
- [['--tests'], {
- 'dest': 'test_path',
- 'default': None,
- 'help': 'Path to test_*.py or directory relative to src root.',
- }],
- [['--src-dir'], {
- 'dest': 'rel_src_dir',
- 'default': None,
- 'help': 'Path to hg.mo source checkout relative to work dir.',
- }],
-
-] + copy.deepcopy(virtualenv_config_options)
-
-marionette_harness_tests_config = {
- "find_links": [
- "http://pypi.pub.build.mozilla.org/pub",
- ],
- "pip_index": False,
- # relative to workspace
- "rel_src_dir": os.path.join("build", "src"),
-}
-
-class MarionetteHarnessTests(VirtualenvMixin, BuildbotMixin, BaseScript):
-
- def __init__(self, config_options=None,
- all_actions=None, default_actions=None,
- *args, **kwargs):
- config_options = config_options or marionette_harness_tests_config_options
- actions = [
- 'clobber',
- 'create-virtualenv',
- 'run-tests',
- ]
- super(MarionetteHarnessTests, self).__init__(
- config_options=config_options,
- all_actions=all_actions or actions,
- default_actions=default_actions or actions,
- config=marionette_harness_tests_config,
- *args, **kwargs)
-
- @PreScriptAction('create-virtualenv')
- def _pre_create_virtualenv(self, action):
- dirs = self.query_abs_dirs()
- c = self.config
- requirements = os.path.join(
- dirs['abs_src_dir'],
- 'testing', 'config',
- 'marionette_harness_test_requirements.txt'
- )
- self.register_virtualenv_module(
- requirements=[requirements],
- two_pass=True
- )
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- c = self.config
- abs_dirs = super(MarionetteHarnessTests, self).query_abs_dirs()
- dirs = {
- 'abs_src_dir': os.path.abspath(
- os.path.join(abs_dirs['base_work_dir'], c['rel_src_dir'])
- ),
- }
-
- for key in dirs:
- if key not in abs_dirs:
- abs_dirs[key] = dirs[key]
- self.abs_dirs = abs_dirs
-
- return self.abs_dirs
-
- def _get_pytest_status(self, code):
- """
- Translate pytest exit code to TH status
-
- Based on https://github.com/pytest-dev/pytest/blob/master/_pytest/main.py#L21-L26
- """
- if code == 0:
- return TBPL_SUCCESS
- elif code == 1:
- return TBPL_WARNING
- elif 1 < code < 6:
- self.error("pytest returned exit code: %s" % code)
- return TBPL_FAILURE
- else:
- return TBPL_EXCEPTION
-
- def run_tests(self):
- """Run all the tests"""
- dirs = self.query_abs_dirs()
- test_relpath = self.config.get(
- 'test_path',
- os.path.join('testing', 'marionette',
- 'harness', 'marionette_harness', 'tests',
- 'harness_unit')
- )
- test_path = os.path.join(dirs['abs_src_dir'], test_relpath)
- self.activate_virtualenv()
- import pytest
- command = ['-p', 'no:terminalreporter', # disable pytest logging
- test_path]
- logs = {}
- for fmt in ['tbpl', 'mach', 'raw']:
- logs[fmt] = os.path.join(dirs['abs_log_dir'],
- 'mn-harness_{}.log'.format(fmt))
- command.extend(['--log-'+fmt, logs[fmt]])
- self.info('Calling pytest.main with the following arguments: %s' % command)
- status = self._get_pytest_status(pytest.main(command))
- self.read_from_file(logs['tbpl'])
- for log in logs.values():
- self.copy_to_upload_dir(log, dest='logs/')
- self.buildbot_status(status)
-
-
-if __name__ == '__main__':
- script = MarionetteHarnessTests()
- script.run_and_exit()
diff --git a/testing/mozharness/scripts/merge_day/gecko_migration.py b/testing/mozharness/scripts/merge_day/gecko_migration.py
deleted file mode 100755
index 7208630e0..000000000
--- a/testing/mozharness/scripts/merge_day/gecko_migration.py
+++ /dev/null
@@ -1,545 +0,0 @@
-#!/usr/bin/env python
-# lint_ignore=E501
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-""" gecko_migration.py
-
-Merge day script for gecko (mozilla-central -> mozilla-aurora,
-mozilla-aurora -> mozilla-beta, mozilla-beta -> mozilla-release).
-
-Ported largely from
-http://hg.mozilla.org/build/tools/file/084bc4e2fc76/release/beta2release.py
-and
-http://hg.mozilla.org/build/tools/file/084bc4e2fc76/release/merge_helper.py
-"""
-
-import os
-import pprint
-import subprocess
-import sys
-from getpass import getpass
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.base.errors import HgErrorList
-from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
-from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.selfserve import SelfServeMixin
-from mozharness.mozilla.updates.balrog import BalrogMixin
-from mozharness.mozilla.buildbot import BuildbotMixin
-from mozharness.mozilla.repo_manupulation import MercurialRepoManipulationMixin
-
-VALID_MIGRATION_BEHAVIORS = (
- "beta_to_release", "aurora_to_beta", "central_to_aurora", "release_to_esr",
- "bump_second_digit",
-)
-
-
-# GeckoMigration {{{1
-class GeckoMigration(MercurialScript, BalrogMixin, VirtualenvMixin,
- SelfServeMixin, BuildbotMixin,
- MercurialRepoManipulationMixin):
- config_options = [
- [['--hg-user', ], {
- "action": "store",
- "dest": "hg_user",
- "type": "string",
- "default": "ffxbld <release@mozilla.com>",
- "help": "Specify what user to use to commit to hg.",
- }],
- [['--balrog-api-root', ], {
- "action": "store",
- "dest": "balrog_api_root",
- "type": "string",
- "help": "Specify Balrog API root URL.",
- }],
- [['--balrog-username', ], {
- "action": "store",
- "dest": "balrog_username",
- "type": "string",
- "help": "Specify what user to connect to Balrog with.",
- }],
- [['--balrog-credentials-file', ], {
- "action": "store",
- "dest": "balrog_credentials_file",
- "type": "string",
- "help": "The file containing the Balrog credentials.",
- }],
- [['--remove-locale', ], {
- "action": "extend",
- "dest": "remove_locales",
- "type": "string",
- "help": "Comma separated list of locales to remove from the 'to' repo.",
- }],
- ]
- gecko_repos = None
-
- def __init__(self, require_config_file=True):
- super(GeckoMigration, self).__init__(
- config_options=virtualenv_config_options + self.config_options,
- all_actions=[
- 'clobber',
- 'create-virtualenv',
- 'clean-repos',
- 'pull',
- 'lock-update-paths',
- 'migrate',
- 'bump_second_digit',
- 'commit-changes',
- 'push',
- 'trigger-builders',
- ],
- default_actions=[
- 'clean-repos',
- 'pull',
- 'migrate',
- ],
- require_config_file=require_config_file
- )
- self.run_sanity_check()
-
-# Helper methods {{{1
- def run_sanity_check(self):
- """ Verify the configs look sane before proceeding.
- """
- message = ""
- if self.config['migration_behavior'] not in VALID_MIGRATION_BEHAVIORS:
- message += "%s must be one of %s!\n" % (self.config['migration_behavior'], VALID_MIGRATION_BEHAVIORS)
- if self.config['migration_behavior'] == 'beta_to_release':
- if self.config.get("require_remove_locales") and not self.config.get("remove_locales") and 'migrate' in self.actions:
- message += "You must specify --remove-locale!\n"
- else:
- if self.config.get("require_remove_locales") or self.config.get("remove_locales"):
- self.warning("--remove-locale isn't valid unless you're using beta_to_release migration_behavior!\n")
- if message:
- self.fatal(message)
-
- def query_abs_dirs(self):
- """ Allow for abs_from_dir and abs_to_dir
- """
- if self.abs_dirs:
- return self.abs_dirs
- dirs = super(GeckoMigration, self).query_abs_dirs()
- self.abs_dirs['abs_tools_dir'] = os.path.join(
- dirs['abs_work_dir'], 'tools'
- )
- self.abs_dirs['abs_tools_lib_dir'] = os.path.join(
- dirs['abs_work_dir'], 'tools', 'lib', 'python'
- )
- for k in ('from', 'to'):
- url = self.config.get("%s_repo_url" % k)
- if url:
- dir_name = self.get_filename_from_url(url)
- self.info("adding %s" % dir_name)
- self.abs_dirs['abs_%s_dir' % k] = os.path.join(
- dirs['abs_work_dir'], dir_name
- )
- return self.abs_dirs
-
- def query_repos(self):
- """ Build a list of repos to clone.
- """
- if self.gecko_repos:
- return self.gecko_repos
- self.info("Building gecko_repos list...")
- dirs = self.query_abs_dirs()
- self.gecko_repos = []
- for k in ('from', 'to'):
- repo_key = "%s_repo_url" % k
- url = self.config.get(repo_key)
- if url:
- self.gecko_repos.append({
- "repo": url,
- "branch": self.config.get("%s_repo_branch" % (k,), "default"),
- "dest": dirs['abs_%s_dir' % k],
- "vcs": "hg",
- # "hg" vcs uses robustcheckout extension requires the use of a share
- # but having a share breaks migration logic when merging repos.
- # Solution: tell hg vcs to create a unique share directory for each
- # gecko repo. see mozharness/base/vcs/mercurial.py for implementation
- "use_vcs_unique_share": True,
- })
- else:
- self.warning("Skipping %s" % repo_key)
- self.info(pprint.pformat(self.gecko_repos))
- return self.gecko_repos
-
- def query_commit_dirs(self):
- dirs = self.query_abs_dirs()
- commit_dirs = [dirs['abs_to_dir']]
- if self.config['migration_behavior'] == 'central_to_aurora':
- commit_dirs.append(dirs['abs_from_dir'])
- return commit_dirs
-
- def query_commit_message(self):
- return "Update configs. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release ba=release"
-
- def query_push_dirs(self):
- dirs = self.query_abs_dirs()
- return dirs.get('abs_from_dir'), dirs.get('abs_to_dir')
-
- def query_push_args(self, cwd):
- if cwd == self.query_abs_dirs()['abs_to_dir'] and \
- self.config['migration_behavior'] == 'beta_to_release':
- return ['--new-branch', '-r', '.']
- else:
- return ['-r', '.']
-
- def query_from_revision(self):
- """ Shortcut to get the revision for the from repo
- """
- dirs = self.query_abs_dirs()
- return self.query_hg_revision(dirs['abs_from_dir'])
-
- def query_to_revision(self):
- """ Shortcut to get the revision for the to repo
- """
- dirs = self.query_abs_dirs()
- return self.query_hg_revision(dirs['abs_to_dir'])
-
- def hg_merge_via_debugsetparents(self, cwd, old_head, new_head,
- preserve_tags=True, user=None):
- """ Merge 2 heads avoiding non-fastforward commits
- """
- hg = self.query_exe('hg', return_type='list')
- cmd = hg + ['debugsetparents', new_head, old_head]
- self.run_command(cmd, cwd=cwd, error_list=HgErrorList,
- halt_on_failure=True)
- self.hg_commit(
- cwd,
- message="Merge old head via |hg debugsetparents %s %s|. "
- "CLOSED TREE DONTBUILD a=release" % (new_head, old_head),
- user=user
- )
- if preserve_tags:
- # I don't know how to do this elegantly.
- # I'm reverting .hgtags to old_head, then appending the new tags
- # from new_head to .hgtags, and hoping nothing goes wrong.
- # I'd rather not write patch files from scratch, so this seems
- # like a slightly more complex but less objectionable method?
- self.info("Trying to preserve tags from before debugsetparents...")
- dirs = self.query_abs_dirs()
- patch_file = os.path.join(dirs['abs_work_dir'], 'patch_file')
- self.run_command(
- subprocess.list2cmdline(hg + ['diff', '-r', old_head, '.hgtags', '-U9', '>', patch_file]),
- cwd=cwd,
- )
- self.run_command(
- ['patch', '-R', '-p1', '-i', patch_file],
- cwd=cwd,
- halt_on_failure=True,
- )
- tag_diff = self.read_from_file(patch_file)
- with self.opened(os.path.join(cwd, '.hgtags'), open_mode='a') as (fh, err):
- if err:
- self.fatal("Can't append to .hgtags!")
- for n, line in enumerate(tag_diff.splitlines()):
- # The first 4 lines of a patch are headers, so we ignore them.
- if n < 5:
- continue
- # Even after that, the only lines we really care about are
- # additions to the file.
- # TODO: why do we only care about additions? I couldn't
- # figure that out by reading this code.
- if not line.startswith('+'):
- continue
- line = line.replace('+', '')
- (changeset, tag) = line.split(' ')
- if len(changeset) != 40:
- continue
- fh.write("%s\n" % line)
- out = self.get_output_from_command(['hg', 'status', '.hgtags'],
- cwd=cwd)
- if out:
- self.hg_commit(
- cwd,
- message="Preserve old tags after debugsetparents. "
- "CLOSED TREE DONTBUILD a=release",
- user=user,
- )
- else:
- self.info(".hgtags file is identical, no need to commit")
-
- def remove_locales(self, file_name, locales):
- """ Remove locales from shipped-locales (m-r only)
- """
- contents = self.read_from_file(file_name)
- new_contents = ""
- for line in contents.splitlines():
- locale = line.split()[0]
- if locale not in locales:
- new_contents += "%s\n" % line
- else:
- self.info("Removed locale: %s" % locale)
- self.write_to_file(file_name, new_contents)
-
- def touch_clobber_file(self, cwd):
- clobber_file = os.path.join(cwd, 'CLOBBER')
- contents = self.read_from_file(clobber_file)
- new_contents = ""
- for line in contents.splitlines():
- line = line.strip()
- if line.startswith("#") or line == '':
- new_contents += "%s\n" % line
- new_contents += "Merge day clobber"
- self.write_to_file(clobber_file, new_contents)
-
- def bump_version(self, cwd, curr_version, next_version, curr_suffix,
- next_suffix, bump_major=False):
- """ Bump versions (m-c, m-a, m-b).
-
- At some point we may want to unhardcode these filenames into config
- """
- curr_weave_version = str(int(curr_version) + 2)
- next_weave_version = str(int(curr_weave_version) + 1)
- for f in self.config["version_files"]:
- from_ = "%s.0%s" % (curr_version, curr_suffix)
- to = "%s.0%s%s" % (next_version, next_suffix, f["suffix"])
- self.replace(os.path.join(cwd, f["file"]), from_, to)
-
- # only applicable for m-c
- if bump_major:
- self.replace(
- os.path.join(cwd, "xpcom/components/Module.h"),
- "static const unsigned int kVersion = %s;" % curr_version,
- "static const unsigned int kVersion = %s;" % next_version
- )
- self.replace(
- os.path.join(cwd, "services/sync/moz.build"),
- "DEFINES['weave_version'] = '1.%s.0'" % curr_weave_version,
- "DEFINES['weave_version'] = '1.%s.0'" % next_weave_version
- )
-
- # Branch-specific workflow helper methods {{{1
- def central_to_aurora(self, end_tag):
- """ mozilla-central -> mozilla-aurora behavior.
-
- We could have all of these individually toggled by flags, but
- by separating into workflow methods we can be more precise about
- what happens in each workflow, while allowing for things like
- staging beta user repo migrations.
- """
- dirs = self.query_abs_dirs()
- self.info("Reverting locales")
- hg = self.query_exe("hg", return_type="list")
- for f in self.config["locale_files"]:
- self.run_command(
- hg + ["revert", "-r", end_tag, f],
- cwd=dirs['abs_to_dir'],
- error_list=HgErrorList,
- halt_on_failure=True,
- )
- next_ma_version = self.get_version(dirs['abs_to_dir'])[0]
- self.bump_version(dirs['abs_to_dir'], next_ma_version, next_ma_version, "a1", "a2")
- self.apply_replacements()
- # bump m-c version
- curr_mc_version = self.get_version(dirs['abs_from_dir'])[0]
- next_mc_version = str(int(curr_mc_version) + 1)
- self.bump_version(
- dirs['abs_from_dir'], curr_mc_version, next_mc_version, "a1", "a1",
- bump_major=True
- )
- # touch clobber files
- self.touch_clobber_file(dirs['abs_from_dir'])
- self.touch_clobber_file(dirs['abs_to_dir'])
-
- def aurora_to_beta(self, *args, **kwargs):
- """ mozilla-aurora -> mozilla-beta behavior.
-
- We could have all of these individually toggled by flags, but
- by separating into workflow methods we can be more precise about
- what happens in each workflow, while allowing for things like
- staging beta user repo migrations.
- """
- dirs = self.query_abs_dirs()
- mb_version = self.get_version(dirs['abs_to_dir'])[0]
- self.bump_version(dirs['abs_to_dir'], mb_version, mb_version, "a2", "")
- self.apply_replacements()
- self.touch_clobber_file(dirs['abs_to_dir'])
- # TODO mozconfig diffing
- # The build/tools version only checks the mozconfigs from hgweb, so
- # can't help pre-push. The in-tree mozconfig diffing requires a mach
- # virtualenv to be installed. If we want this sooner we can put this
- # in the push action; otherwise we may just wait until we have in-tree
- # mozconfig checking.
-
- def beta_to_release(self, *args, **kwargs):
- """ mozilla-beta -> mozilla-release behavior.
-
- We could have all of these individually toggled by flags, but
- by separating into workflow methods we can be more precise about
- what happens in each workflow, while allowing for things like
- staging beta user repo migrations.
- """
- dirs = self.query_abs_dirs()
- # Reset display_version.txt
- for f in self.config["copy_files"]:
- self.copyfile(
- os.path.join(dirs['abs_to_dir'], f["src"]),
- os.path.join(dirs['abs_to_dir'], f["dst"]))
-
- self.apply_replacements()
- if self.config.get("remove_locales"):
- self.remove_locales(
- os.path.join(dirs['abs_to_dir'], "browser/locales/shipped-locales"),
- self.config['remove_locales']
- )
- self.touch_clobber_file(dirs['abs_to_dir'])
-
- def release_to_esr(self, *args, **kwargs):
- """ mozilla-release -> mozilla-esrNN behavior. """
- dirs = self.query_abs_dirs()
- for to_transplant in self.config.get("transplant_patches", []):
- self.transplant(repo=to_transplant["repo"],
- changeset=to_transplant["changeset"],
- cwd=dirs['abs_to_dir'])
- self.apply_replacements()
- self.touch_clobber_file(dirs['abs_to_dir'])
-
- def apply_replacements(self):
- dirs = self.query_abs_dirs()
- for f, from_, to in self.config["replacements"]:
- self.replace(os.path.join(dirs['abs_to_dir'], f), from_, to)
-
- def transplant(self, repo, changeset, cwd):
- """Transplant a Mercurial changeset from a remote repository."""
- hg = self.query_exe("hg", return_type="list")
- cmd = hg + ["--config", "extensions.transplant=", "transplant",
- "--source", repo, changeset]
- self.info("Transplanting %s from %s" % (changeset, repo))
- status = self.run_command(
- cmd,
- cwd=cwd,
- error_list=HgErrorList,
- )
- if status != 0:
- self.fatal("Cannot transplant %s from %s properly" %
- (changeset, repo))
-
- def pull_from_repo(self, from_dir, to_dir, revision=None, branch=None):
- """ Pull from one repo to another. """
- hg = self.query_exe("hg", return_type="list")
- cmd = hg + ["pull"]
- if revision:
- cmd.extend(["-r", revision])
- cmd.append(from_dir)
- self.run_command(
- cmd,
- cwd=to_dir,
- error_list=HgErrorList,
- halt_on_failure=True,
- )
- cmd = hg + ["update", "-C"]
- if branch or revision:
- cmd.extend(["-r", branch or revision])
- self.run_command(
- cmd,
- cwd=to_dir,
- error_list=HgErrorList,
- halt_on_failure=True,
- )
-
-# Actions {{{1
- def bump_second_digit(self, *args, **kwargs):
- """Bump second digit.
-
- ESR need only the second digit bumped as a part of merge day."""
- dirs = self.query_abs_dirs()
- version = self.get_version(dirs['abs_to_dir'])
- curr_version = ".".join(version)
- next_version = list(version)
- # bump the second digit
- next_version[1] = str(int(next_version[1]) + 1)
- # Take major+minor and append '0' accordng to Firefox version schema.
- # 52.0 will become 52.1.0, not 52.1
- next_version = ".".join(next_version[:2] + ['0'])
- for f in self.config["version_files"]:
- self.replace(os.path.join(dirs['abs_to_dir'], f["file"]),
- curr_version, next_version + f["suffix"])
- self.touch_clobber_file(dirs['abs_to_dir'])
-
- def pull(self):
- """ Pull tools first, then clone the gecko repos
- """
- repos = [{
- "repo": self.config["tools_repo_url"],
- "branch": self.config["tools_repo_branch"],
- "dest": "tools",
- "vcs": "hg",
- }] + self.query_repos()
- super(GeckoMigration, self).pull(repos=repos)
-
- def lock_update_paths(self):
- self.lock_balrog_rules(self.config["balrog_rules_to_lock"])
-
- def migrate(self):
- """ Perform the migration.
- """
- dirs = self.query_abs_dirs()
- from_fx_major_version = self.get_version(dirs['abs_from_dir'])[0]
- to_fx_major_version = self.get_version(dirs['abs_to_dir'])[0]
- base_from_rev = self.query_from_revision()
- base_to_rev = self.query_to_revision()
- base_tag = self.config['base_tag'] % {'major_version': from_fx_major_version}
- end_tag = self.config['end_tag'] % {'major_version': to_fx_major_version}
- self.hg_tag(
- dirs['abs_from_dir'], base_tag, user=self.config['hg_user'],
- revision=base_from_rev,
- )
- new_from_rev = self.query_from_revision()
- self.info("New revision %s" % new_from_rev)
- pull_revision = None
- if not self.config.get("pull_all_branches"):
- pull_revision = new_from_rev
- self.pull_from_repo(
- dirs['abs_from_dir'], dirs['abs_to_dir'],
- revision=pull_revision,
- branch="default",
- )
- if self.config.get("requires_head_merge") is not False:
- self.hg_merge_via_debugsetparents(
- dirs['abs_to_dir'], old_head=base_to_rev, new_head=new_from_rev,
- user=self.config['hg_user'],
- )
- self.hg_tag(
- dirs['abs_to_dir'], end_tag, user=self.config['hg_user'],
- revision=base_to_rev, force=True,
- )
- # Call beta_to_release etc.
- if not hasattr(self, self.config['migration_behavior']):
- self.fatal("Don't know how to proceed with migration_behavior %s !" % self.config['migration_behavior'])
- getattr(self, self.config['migration_behavior'])(end_tag=end_tag)
- self.info("Verify the diff, and apply any manual changes, such as disabling features, and --commit-changes")
-
- def trigger_builders(self):
- """Triggers builders that should be run directly after a merge.
- There are two different types of things we trigger:
- 1) Nightly builds ("post_merge_nightly_branches" in the config).
- These are triggered with buildapi's nightly build endpoint to avoid
- duplicating all of the nightly builder names into the gecko
- migration mozharness configs. (Which would surely get out of date
- very quickly).
- 2) Arbitrary builders ("post_merge_builders"). These are additional
- builders to trigger that aren't part of the nightly builder set.
- Previous example: hg bundle generation builders.
- """
- dirs = self.query_abs_dirs()
- branch = self.config["to_repo_url"].rstrip("/").split("/")[-1]
- revision = self.query_to_revision()
- # Horrible hack because our internal buildapi interface doesn't let us
- # actually do anything. Need to use the public one w/ auth.
- username = raw_input("LDAP Username: ")
- password = getpass(prompt="LDAP Password: ")
- auth = (username, password)
- for builder in self.config["post_merge_builders"]:
- self.trigger_arbitrary_job(builder, branch, revision, auth)
- for nightly_branch in self.config["post_merge_nightly_branches"]:
- nightly_revision = self.query_hg_revision(os.path.join(dirs["abs_work_dir"], nightly_branch))
- self.trigger_nightly_builds(nightly_branch, nightly_revision, auth)
-
-# __main__ {{{1
-if __name__ == '__main__':
- GeckoMigration().run_and_exit()
diff --git a/testing/mozharness/scripts/mobile_l10n.py b/testing/mozharness/scripts/mobile_l10n.py
deleted file mode 100755
index cbac6fa67..000000000
--- a/testing/mozharness/scripts/mobile_l10n.py
+++ /dev/null
@@ -1,714 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""mobile_l10n.py
-
-This currently supports nightly and release single locale repacks for
-Android. This also creates nightly updates.
-"""
-
-from copy import deepcopy
-import os
-import re
-import subprocess
-import sys
-import time
-import shlex
-
-try:
- import simplejson as json
- assert json
-except ImportError:
- import json
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.errors import BaseErrorList, MakefileErrorList
-from mozharness.base.log import OutputParser
-from mozharness.base.transfer import TransferMixin
-from mozharness.mozilla.buildbot import BuildbotMixin
-from mozharness.mozilla.purge import PurgeMixin
-from mozharness.mozilla.release import ReleaseMixin
-from mozharness.mozilla.signing import MobileSigningMixin
-from mozharness.mozilla.tooltool import TooltoolMixin
-from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.l10n.locales import LocalesMixin
-from mozharness.mozilla.mock import MockMixin
-from mozharness.mozilla.updates.balrog import BalrogMixin
-from mozharness.base.python import VirtualenvMixin
-from mozharness.mozilla.taskcluster_helper import Taskcluster
-
-
-# MobileSingleLocale {{{1
-class MobileSingleLocale(MockMixin, LocalesMixin, ReleaseMixin,
- MobileSigningMixin, TransferMixin, TooltoolMixin,
- BuildbotMixin, PurgeMixin, MercurialScript, BalrogMixin,
- VirtualenvMixin):
- config_options = [[
- ['--locale', ],
- {"action": "extend",
- "dest": "locales",
- "type": "string",
- "help": "Specify the locale(s) to sign and update"
- }
- ], [
- ['--locales-file', ],
- {"action": "store",
- "dest": "locales_file",
- "type": "string",
- "help": "Specify a file to determine which locales to sign and update"
- }
- ], [
- ['--tag-override', ],
- {"action": "store",
- "dest": "tag_override",
- "type": "string",
- "help": "Override the tags set for all repos"
- }
- ], [
- ['--user-repo-override', ],
- {"action": "store",
- "dest": "user_repo_override",
- "type": "string",
- "help": "Override the user repo path for all repos"
- }
- ], [
- ['--release-config-file', ],
- {"action": "store",
- "dest": "release_config_file",
- "type": "string",
- "help": "Specify the release config file to use"
- }
- ], [
- ['--key-alias', ],
- {"action": "store",
- "dest": "key_alias",
- "type": "choice",
- "default": "nightly",
- "choices": ["nightly", "release"],
- "help": "Specify the signing key alias"
- }
- ], [
- ['--this-chunk', ],
- {"action": "store",
- "dest": "this_locale_chunk",
- "type": "int",
- "help": "Specify which chunk of locales to run"
- }
- ], [
- ['--total-chunks', ],
- {"action": "store",
- "dest": "total_locale_chunks",
- "type": "int",
- "help": "Specify the total number of chunks of locales"
- }
- ], [
- ["--disable-mock"],
- {"dest": "disable_mock",
- "action": "store_true",
- "help": "do not run under mock despite what gecko-config says",
- }
- ], [
- ['--revision', ],
- {"action": "store",
- "dest": "revision",
- "type": "string",
- "help": "Override the gecko revision to use (otherwise use buildbot supplied"
- " value, or en-US revision) "}
- ]]
-
- def __init__(self, require_config_file=True):
- buildscript_kwargs = {
- 'all_actions': [
- "clobber",
- "pull",
- "clone-locales",
- "list-locales",
- "setup",
- "repack",
- "validate-repacks-signed",
- "upload-repacks",
- "create-virtualenv",
- "taskcluster-upload",
- "submit-to-balrog",
- "summary",
- ],
- 'config': {
- 'taskcluster_credentials_file': 'oauth.txt',
- 'virtualenv_modules': [
- 'requests==2.8.1',
- 'PyHawk-with-a-single-extra-commit==0.1.5',
- 'taskcluster==0.0.26',
- ],
- 'virtualenv_path': 'venv',
- },
- }
- LocalesMixin.__init__(self)
- MercurialScript.__init__(
- self,
- config_options=self.config_options,
- require_config_file=require_config_file,
- **buildscript_kwargs
- )
- self.base_package_name = None
- self.buildid = None
- self.make_ident_output = None
- self.repack_env = None
- self.revision = None
- self.upload_env = None
- self.version = None
- self.upload_urls = {}
- self.locales_property = {}
-
- # Helper methods {{{2
- def query_repack_env(self):
- if self.repack_env:
- return self.repack_env
- c = self.config
- replace_dict = {}
- if c.get('release_config_file'):
- rc = self.query_release_config()
- replace_dict = {
- 'version': rc['version'],
- 'buildnum': rc['buildnum']
- }
- repack_env = self.query_env(partial_env=c.get("repack_env"),
- replace_dict=replace_dict)
- if c.get('base_en_us_binary_url') and c.get('release_config_file'):
- rc = self.query_release_config()
- repack_env['EN_US_BINARY_URL'] = c['base_en_us_binary_url'] % replace_dict
- if 'MOZ_SIGNING_SERVERS' in os.environ:
- repack_env['MOZ_SIGN_CMD'] = subprocess.list2cmdline(self.query_moz_sign_cmd(formats=['jar']))
- self.repack_env = repack_env
- return self.repack_env
-
- def query_l10n_env(self):
- return self.query_env()
-
- def query_upload_env(self):
- if self.upload_env:
- return self.upload_env
- c = self.config
- replace_dict = {
- 'buildid': self.query_buildid(),
- 'version': self.query_version(),
- }
- replace_dict.update(c)
-
- # Android l10n builds use a non-standard location for l10n files. Other
- # builds go to 'mozilla-central-l10n', while android builds add part of
- # the platform name as well, like 'mozilla-central-android-api-15-l10n'.
- # So we override the branch with something that contains the platform
- # name.
- replace_dict['branch'] = c['upload_branch']
- replace_dict['post_upload_extra'] = ' '.join(c.get('post_upload_extra', []))
-
- upload_env = self.query_env(partial_env=c.get("upload_env"),
- replace_dict=replace_dict)
- if 'MOZ_SIGNING_SERVERS' in os.environ:
- upload_env['MOZ_SIGN_CMD'] = subprocess.list2cmdline(self.query_moz_sign_cmd())
- if self.query_is_release_or_beta():
- upload_env['MOZ_PKG_VERSION'] = '%(version)s' % replace_dict
- self.upload_env = upload_env
- return self.upload_env
-
- def _query_make_ident_output(self):
- """Get |make ident| output from the objdir.
- Only valid after setup is run.
- """
- if self.make_ident_output:
- return self.make_ident_output
- env = self.query_repack_env()
- dirs = self.query_abs_dirs()
- output = self.get_output_from_command_m(["make", "ident"],
- cwd=dirs['abs_locales_dir'],
- env=env,
- silent=True,
- halt_on_failure=True)
- parser = OutputParser(config=self.config, log_obj=self.log_obj,
- error_list=MakefileErrorList)
- parser.add_lines(output)
- self.make_ident_output = output
- return output
-
- def query_buildid(self):
- """Get buildid from the objdir.
- Only valid after setup is run.
- """
- if self.buildid:
- return self.buildid
- r = re.compile("buildid (\d+)")
- output = self._query_make_ident_output()
- for line in output.splitlines():
- m = r.match(line)
- if m:
- self.buildid = m.groups()[0]
- return self.buildid
-
- def query_revision(self):
- """Get revision from the objdir.
- Only valid after setup is run.
- """
- if self.revision:
- return self.revision
- r = re.compile(r"gecko_revision ([0-9a-f]+\+?)")
- output = self._query_make_ident_output()
- for line in output.splitlines():
- m = r.match(line)
- if m:
- self.revision = m.groups()[0]
- return self.revision
-
- def _query_make_variable(self, variable, make_args=None):
- make = self.query_exe('make')
- env = self.query_repack_env()
- dirs = self.query_abs_dirs()
- if make_args is None:
- make_args = []
- # TODO error checking
- output = self.get_output_from_command_m(
- [make, "echo-variable-%s" % variable] + make_args,
- cwd=dirs['abs_locales_dir'], silent=True,
- env=env
- )
- parser = OutputParser(config=self.config, log_obj=self.log_obj,
- error_list=MakefileErrorList)
- parser.add_lines(output)
- return output.strip()
-
- def query_base_package_name(self):
- """Get the package name from the objdir.
- Only valid after setup is run.
- """
- if self.base_package_name:
- return self.base_package_name
- self.base_package_name = self._query_make_variable(
- "PACKAGE",
- make_args=['AB_CD=%(locale)s']
- )
- return self.base_package_name
-
- def query_version(self):
- """Get the package name from the objdir.
- Only valid after setup is run.
- """
- if self.version:
- return self.version
- c = self.config
- if c.get('release_config_file'):
- rc = self.query_release_config()
- self.version = rc['version']
- else:
- self.version = self._query_make_variable("MOZ_APP_VERSION")
- return self.version
-
- def query_upload_url(self, locale):
- if locale in self.upload_urls:
- return self.upload_urls[locale]
- else:
- self.error("Can't determine the upload url for %s!" % locale)
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(MobileSingleLocale, self).query_abs_dirs()
-
- dirs = {
- 'abs_tools_dir':
- os.path.join(abs_dirs['base_work_dir'], 'tools'),
- 'build_dir':
- os.path.join(abs_dirs['base_work_dir'], 'build'),
- }
-
- abs_dirs.update(dirs)
- self.abs_dirs = abs_dirs
- return self.abs_dirs
-
- def add_failure(self, locale, message, **kwargs):
- self.locales_property[locale] = "Failed"
- prop_key = "%s_failure" % locale
- prop_value = self.query_buildbot_property(prop_key)
- if prop_value:
- prop_value = "%s %s" % (prop_value, message)
- else:
- prop_value = message
- self.set_buildbot_property(prop_key, prop_value, write_to_file=True)
- MercurialScript.add_failure(self, locale, message=message, **kwargs)
-
- def summary(self):
- MercurialScript.summary(self)
- # TODO we probably want to make this configurable on/off
- locales = self.query_locales()
- for locale in locales:
- self.locales_property.setdefault(locale, "Success")
- self.set_buildbot_property("locales", json.dumps(self.locales_property), write_to_file=True)
-
- # Actions {{{2
- def clobber(self):
- self.read_buildbot_config()
- dirs = self.query_abs_dirs()
- c = self.config
- objdir = os.path.join(dirs['abs_work_dir'], c['mozilla_dir'],
- c['objdir'])
- super(MobileSingleLocale, self).clobber(always_clobber_dirs=[objdir])
-
- def pull(self):
- c = self.config
- dirs = self.query_abs_dirs()
- repos = []
- replace_dict = {}
- if c.get("user_repo_override"):
- replace_dict['user_repo_override'] = c['user_repo_override']
- # deepcopy() needed because of self.config lock bug :(
- for repo_dict in deepcopy(c['repos']):
- repo_dict['repo'] = repo_dict['repo'] % replace_dict
- repos.append(repo_dict)
- else:
- repos = c['repos']
- self.vcs_checkout_repos(repos, parent_dir=dirs['abs_work_dir'],
- tag_override=c.get('tag_override'))
-
- def clone_locales(self):
- self.pull_locale_source()
-
- # list_locales() is defined in LocalesMixin.
-
- def _setup_configure(self, buildid=None):
- c = self.config
- dirs = self.query_abs_dirs()
- env = self.query_repack_env()
- make = self.query_exe("make")
- if self.run_command_m([make, "-f", "client.mk", "configure"],
- cwd=dirs['abs_mozilla_dir'],
- env=env,
- error_list=MakefileErrorList):
- self.fatal("Configure failed!")
-
- # Run 'make export' in objdir/config to get nsinstall
- self.run_command_m([make, 'export'],
- cwd=os.path.join(dirs['abs_objdir'], 'config'),
- env=env,
- error_list=MakefileErrorList,
- halt_on_failure=True)
-
- # Run 'make buildid.h' in objdir/ to get the buildid.h file
- cmd = [make, 'buildid.h']
- if buildid:
- cmd.append('MOZ_BUILD_DATE=%s' % str(buildid))
- self.run_command_m(cmd,
- cwd=dirs['abs_objdir'],
- env=env,
- error_list=MakefileErrorList,
- halt_on_failure=True)
-
- def setup(self):
- c = self.config
- dirs = self.query_abs_dirs()
- mozconfig_path = os.path.join(dirs['abs_mozilla_dir'], '.mozconfig')
- self.copyfile(os.path.join(dirs['abs_work_dir'], c['mozconfig']),
- mozconfig_path)
- # TODO stop using cat
- cat = self.query_exe("cat")
- make = self.query_exe("make")
- self.run_command_m([cat, mozconfig_path])
- env = self.query_repack_env()
- if self.config.get("tooltool_config"):
- self.tooltool_fetch(
- self.config['tooltool_config']['manifest'],
- output_dir=self.config['tooltool_config']['output_dir'] % self.query_abs_dirs(),
- )
- self._setup_configure()
- self.run_command_m([make, "wget-en-US"],
- cwd=dirs['abs_locales_dir'],
- env=env,
- error_list=MakefileErrorList,
- halt_on_failure=True)
- self.run_command_m([make, "unpack"],
- cwd=dirs['abs_locales_dir'],
- env=env,
- error_list=MakefileErrorList,
- halt_on_failure=True)
-
- # on try we want the source we already have, otherwise update to the
- # same as the en-US binary
- if self.config.get("update_gecko_source_to_enUS", True):
- revision = self.query_revision()
- if not revision:
- self.fatal("Can't determine revision!")
- hg = self.query_exe("hg")
- # TODO do this through VCSMixin instead of hardcoding hg
- self.run_command_m([hg, "update", "-r", revision],
- cwd=dirs["abs_mozilla_dir"],
- env=env,
- error_list=BaseErrorList,
- halt_on_failure=True)
- self.set_buildbot_property('revision', revision, write_to_file=True)
- # Configure again since the hg update may have invalidated it.
- buildid = self.query_buildid()
- self._setup_configure(buildid=buildid)
-
- def repack(self):
- # TODO per-locale logs and reporting.
- dirs = self.query_abs_dirs()
- locales = self.query_locales()
- make = self.query_exe("make")
- repack_env = self.query_repack_env()
- success_count = total_count = 0
- for locale in locales:
- total_count += 1
- self.enable_mock()
- result = self.run_compare_locales(locale)
- self.disable_mock()
- if result:
- self.add_failure(locale, message="%s failed in compare-locales!" % locale)
- continue
- if self.run_command_m([make, "installers-%s" % locale],
- cwd=dirs['abs_locales_dir'],
- env=repack_env,
- error_list=MakefileErrorList,
- halt_on_failure=False):
- self.add_failure(locale, message="%s failed in make installers-%s!" % (locale, locale))
- continue
- success_count += 1
- self.summarize_success_count(success_count, total_count,
- message="Repacked %d of %d binaries successfully.")
-
- def validate_repacks_signed(self):
- c = self.config
- dirs = self.query_abs_dirs()
- locales = self.query_locales()
- base_package_name = self.query_base_package_name()
- base_package_dir = os.path.join(dirs['abs_objdir'], 'dist')
- repack_env = self.query_repack_env()
- success_count = total_count = 0
- for locale in locales:
- total_count += 1
- signed_path = os.path.join(base_package_dir,
- base_package_name % {'locale': locale})
- # We need to wrap what this function does with mock, since
- # MobileSigningMixin doesn't know about mock
- self.enable_mock()
- status = self.verify_android_signature(
- signed_path,
- script=c['signature_verification_script'],
- env=repack_env,
- key_alias=c['key_alias'],
- )
- self.disable_mock()
- if status:
- self.add_failure(locale, message="Errors verifying %s binary!" % locale)
- # No need to rm because upload is per-locale
- continue
- success_count += 1
- self.summarize_success_count(success_count, total_count,
- message="Validated signatures on %d of %d binaries successfully.")
-
- def taskcluster_upload(self):
- auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file'])
- credentials = {}
- execfile(auth, credentials)
- client_id = credentials.get('taskcluster_clientId')
- access_token = credentials.get('taskcluster_accessToken')
- if not client_id or not access_token:
- self.warning('Skipping S3 file upload: No taskcluster credentials.')
- return
-
- self.activate_virtualenv()
-
- dirs = self.query_abs_dirs()
- locales = self.query_locales()
- make = self.query_exe("make")
- upload_env = self.query_upload_env()
- cwd = dirs['abs_locales_dir']
- branch = self.config['branch']
- revision = self.query_revision()
- repo = self.query_l10n_repo()
- pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hg')
- pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate))
- routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'],
- 'testing/mozharness/configs/routes.json')
- with open(routes_json) as routes_file:
- contents = json.load(routes_file)
- templates = contents['l10n']
-
- for locale in locales:
- output = self.get_output_from_command_m(
- "%s echo-variable-UPLOAD_FILES AB_CD=%s" % (make, locale),
- cwd=cwd,
- env=upload_env,
- )
- files = shlex.split(output)
- abs_files = [os.path.abspath(os.path.join(cwd, f)) for f in files]
-
- routes = []
- fmt = {
- 'index': self.config.get('taskcluster_index', 'index.garbage.staging'),
- 'project': branch,
- 'head_rev': revision,
- 'pushdate': pushdate,
- 'year': pushdate[0:4],
- 'month': pushdate[4:6],
- 'day': pushdate[6:8],
- 'build_product': self.config['stage_product'],
- 'build_name': self.query_build_name(),
- 'build_type': self.query_build_type(),
- 'locale': locale,
- }
- for template in templates:
- routes.append(template.format(**fmt))
-
- self.info('Using routes: %s' % routes)
- tc = Taskcluster(branch,
- pushinfo.pushdate, # Use pushdate as the rank
- client_id,
- access_token,
- self.log_obj,
- )
- task = tc.create_task(routes)
- tc.claim_task(task)
-
- for upload_file in abs_files:
- tc.create_artifact(task, upload_file)
- tc.report_completed(task)
-
- def upload_repacks(self):
- c = self.config
- dirs = self.query_abs_dirs()
- locales = self.query_locales()
- make = self.query_exe("make")
- base_package_name = self.query_base_package_name()
- version = self.query_version()
- upload_env = self.query_upload_env()
- success_count = total_count = 0
- buildnum = None
- if c.get('release_config_file'):
- rc = self.query_release_config()
- buildnum = rc['buildnum']
- for locale in locales:
- if self.query_failure(locale):
- self.warning("Skipping previously failed locale %s." % locale)
- continue
- total_count += 1
- if c.get('base_post_upload_cmd'):
- upload_env['POST_UPLOAD_CMD'] = c['base_post_upload_cmd'] % {'version': version, 'locale': locale, 'buildnum': str(buildnum), 'post_upload_extra': ' '.join(c.get('post_upload_extra', []))}
- output = self.get_output_from_command_m(
- # Ugly hack to avoid |make upload| stderr from showing up
- # as get_output_from_command errors
- "%s upload AB_CD=%s 2>&1" % (make, locale),
- cwd=dirs['abs_locales_dir'],
- env=upload_env,
- silent=True
- )
- parser = OutputParser(config=self.config, log_obj=self.log_obj,
- error_list=MakefileErrorList)
- parser.add_lines(output)
- if parser.num_errors:
- self.add_failure(locale, message="%s failed in make upload!" % (locale))
- continue
- package_name = base_package_name % {'locale': locale}
- r = re.compile("(http.*%s)" % package_name)
- for line in output.splitlines():
- m = r.match(line)
- if m:
- self.upload_urls[locale] = m.groups()[0]
- self.info("Found upload url %s" % self.upload_urls[locale])
- success_count += 1
- self.summarize_success_count(success_count, total_count,
- message="Make Upload for %d of %d locales successful.")
-
- def checkout_tools(self):
- dirs = self.query_abs_dirs()
-
- # We need hg.m.o/build/tools checked out
- self.info("Checking out tools")
- repos = [{
- 'repo': self.config['tools_repo'],
- 'vcs': "hg",
- 'branch': "default",
- 'dest': dirs['abs_tools_dir'],
- }]
- rev = self.vcs_checkout(**repos[0])
- self.set_buildbot_property("tools_revision", rev, write_to_file=True)
-
- def query_apkfile_path(self,locale):
-
- dirs = self.query_abs_dirs()
- apkdir = os.path.join(dirs['abs_objdir'], 'dist')
- r = r"(\.)" + re.escape(locale) + r"(\.*)"
-
- apks = []
- for f in os.listdir(apkdir):
- if f.endswith(".apk") and re.search(r, f):
- apks.append(f)
- if len(apks) == 0:
- self.fatal("Found no apks files in %s, don't know what to do:\n%s" % (apkdir, apks), exit_code=1)
-
- return os.path.join(apkdir, apks[0])
-
- def query_is_release_or_beta(self):
-
- return bool(self.config.get("is_release_or_beta"))
-
- def submit_to_balrog(self):
-
- if not self.query_is_nightly() and not self.query_is_release_or_beta():
- self.info("Not a nightly or release build, skipping balrog submission.")
- return
-
- if not self.config.get("balrog_servers"):
- self.info("balrog_servers not set; skipping balrog submission.")
- return
-
- self.checkout_tools()
-
- dirs = self.query_abs_dirs()
- locales = self.query_locales()
- balrogReady = True
- for locale in locales:
- apk_url = self.query_upload_url(locale)
- if not apk_url:
- self.add_failure(locale, message="Failed to detect %s url in make upload!" % (locale))
- balrogReady = False
- continue
- if not balrogReady:
- return self.fatal(message="Not all repacks successful, abort without submitting to balrog")
-
- for locale in locales:
- apkfile = self.query_apkfile_path(locale)
- apk_url = self.query_upload_url(locale)
-
- # Set other necessary properties for Balrog submission. None need to
- # be passed back to buildbot, so we won't write them to the properties
- #files.
- self.set_buildbot_property("locale", locale)
-
- self.set_buildbot_property("appVersion", self.query_version())
- # The Balrog submitter translates this platform into a build target
- # via https://github.com/mozilla/build-tools/blob/master/lib/python/release/platforms.py#L23
- self.set_buildbot_property("platform", self.buildbot_config["properties"]["platform"])
- #TODO: Is there a better way to get this?
-
- self.set_buildbot_property("appName", "Fennec")
- # TODO: don't hardcode
- self.set_buildbot_property("hashType", "sha512")
- self.set_buildbot_property("completeMarSize", self.query_filesize(apkfile))
- self.set_buildbot_property("completeMarHash", self.query_sha512sum(apkfile))
- self.set_buildbot_property("completeMarUrl", apk_url)
- self.set_buildbot_property("isOSUpdate", False)
- self.set_buildbot_property("buildid", self.query_buildid())
-
- if self.query_is_nightly():
- self.submit_balrog_updates(release_type="nightly")
- else:
- self.submit_balrog_updates(release_type="release")
- if not self.query_is_nightly():
- self.submit_balrog_release_pusher(dirs)
-
-# main {{{1
-if __name__ == '__main__':
- single_locale = MobileSingleLocale()
- single_locale.run_and_exit()
diff --git a/testing/mozharness/scripts/mobile_partner_repack.py b/testing/mozharness/scripts/mobile_partner_repack.py
deleted file mode 100755
index 8d99f825a..000000000
--- a/testing/mozharness/scripts/mobile_partner_repack.py
+++ /dev/null
@@ -1,327 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""mobile_partner_repack.py
-
-"""
-
-from copy import deepcopy
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.errors import ZipErrorList
-from mozharness.base.log import FATAL
-from mozharness.base.transfer import TransferMixin
-from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.l10n.locales import LocalesMixin
-from mozharness.mozilla.release import ReleaseMixin
-from mozharness.mozilla.signing import MobileSigningMixin
-
-SUPPORTED_PLATFORMS = ["android"]
-
-
-# MobilePartnerRepack {{{1
-class MobilePartnerRepack(LocalesMixin, ReleaseMixin, MobileSigningMixin,
- TransferMixin, MercurialScript):
- config_options = [[
- ['--locale', ],
- {"action": "extend",
- "dest": "locales",
- "type": "string",
- "help": "Specify the locale(s) to repack"
- }
- ], [
- ['--partner', ],
- {"action": "extend",
- "dest": "partners",
- "type": "string",
- "help": "Specify the partner(s) to repack"
- }
- ], [
- ['--locales-file', ],
- {"action": "store",
- "dest": "locales_file",
- "type": "string",
- "help": "Specify a json file to determine which locales to repack"
- }
- ], [
- ['--tag-override', ],
- {"action": "store",
- "dest": "tag_override",
- "type": "string",
- "help": "Override the tags set for all repos"
- }
- ], [
- ['--platform', ],
- {"action": "extend",
- "dest": "platforms",
- "type": "choice",
- "choices": SUPPORTED_PLATFORMS,
- "help": "Specify the platform(s) to repack"
- }
- ], [
- ['--user-repo-override', ],
- {"action": "store",
- "dest": "user_repo_override",
- "type": "string",
- "help": "Override the user repo path for all repos"
- }
- ], [
- ['--release-config-file', ],
- {"action": "store",
- "dest": "release_config_file",
- "type": "string",
- "help": "Specify the release config file to use"
- }
- ], [
- ['--version', ],
- {"action": "store",
- "dest": "version",
- "type": "string",
- "help": "Specify the current version"
- }
- ], [
- ['--buildnum', ],
- {"action": "store",
- "dest": "buildnum",
- "type": "int",
- "default": 1,
- "metavar": "INT",
- "help": "Specify the current release build num (e.g. build1, build2)"
- }
- ]]
-
- def __init__(self, require_config_file=True):
- self.release_config = {}
- LocalesMixin.__init__(self)
- MercurialScript.__init__(
- self,
- config_options=self.config_options,
- all_actions=[
- "passphrase",
- "clobber",
- "pull",
- "download",
- "repack",
- "upload-unsigned-bits",
- "sign",
- "upload-signed-bits",
- "summary",
- ],
- require_config_file=require_config_file
- )
-
- # Helper methods {{{2
- def add_failure(self, platform, locale, **kwargs):
- s = "%s:%s" % (platform, locale)
- if 'message' in kwargs:
- kwargs['message'] = kwargs['message'] % {'platform': platform, 'locale': locale}
- super(MobilePartnerRepack, self).add_failure(s, **kwargs)
-
- def query_failure(self, platform, locale):
- s = "%s:%s" % (platform, locale)
- return super(MobilePartnerRepack, self).query_failure(s)
-
- # Actions {{{2
-
- def pull(self):
- c = self.config
- dirs = self.query_abs_dirs()
- repos = []
- replace_dict = {}
- if c.get("user_repo_override"):
- replace_dict['user_repo_override'] = c['user_repo_override']
- # deepcopy() needed because of self.config lock bug :(
- for repo_dict in deepcopy(c['repos']):
- repo_dict['repo'] = repo_dict['repo'] % replace_dict
- repos.append(repo_dict)
- else:
- repos = c['repos']
- self.vcs_checkout_repos(repos, parent_dir=dirs['abs_work_dir'],
- tag_override=c.get('tag_override'))
-
- def download(self):
- c = self.config
- rc = self.query_release_config()
- dirs = self.query_abs_dirs()
- locales = self.query_locales()
- replace_dict = {
- 'buildnum': rc['buildnum'],
- 'version': rc['version'],
- }
- success_count = total_count = 0
- for platform in c['platforms']:
- base_installer_name = c['installer_base_names'][platform]
- base_url = c['download_base_url'] + '/' + \
- c['download_unsigned_base_subdir'] + '/' + \
- base_installer_name
- replace_dict['platform'] = platform
- for locale in locales:
- replace_dict['locale'] = locale
- url = base_url % replace_dict
- installer_name = base_installer_name % replace_dict
- parent_dir = '%s/original/%s/%s' % (dirs['abs_work_dir'],
- platform, locale)
- file_path = '%s/%s' % (parent_dir, installer_name)
- self.mkdir_p(parent_dir)
- total_count += 1
- if not self.download_file(url, file_path):
- self.add_failure(platform, locale,
- message="Unable to download %(platform)s:%(locale)s installer!")
- else:
- success_count += 1
- self.summarize_success_count(success_count, total_count,
- message="Downloaded %d of %d installers successfully.")
-
- def _repack_apk(self, partner, orig_path, repack_path):
- """ Repack the apk with a partner update channel.
- Returns True for success, None for failure
- """
- dirs = self.query_abs_dirs()
- zip_bin = self.query_exe("zip")
- unzip_bin = self.query_exe("unzip")
- file_name = os.path.basename(orig_path)
- tmp_dir = os.path.join(dirs['abs_work_dir'], 'tmp')
- tmp_file = os.path.join(tmp_dir, file_name)
- tmp_prefs_dir = os.path.join(tmp_dir, 'defaults', 'pref')
- # Error checking for each step.
- # Ignoring the mkdir_p()s since the subsequent copyfile()s will
- # error out if unsuccessful.
- if self.rmtree(tmp_dir):
- return
- self.mkdir_p(tmp_prefs_dir)
- if self.copyfile(orig_path, tmp_file):
- return
- if self.write_to_file(os.path.join(tmp_prefs_dir, 'partner.js'),
- 'pref("app.partner.%s", "%s");' % (partner, partner)
- ) is None:
- return
- if self.run_command([unzip_bin, '-q', file_name, 'omni.ja'],
- error_list=ZipErrorList,
- return_type='num_errors',
- cwd=tmp_dir):
- self.error("Can't extract omni.ja from %s!" % file_name)
- return
- if self.run_command([zip_bin, '-9r', 'omni.ja',
- 'defaults/pref/partner.js'],
- error_list=ZipErrorList,
- return_type='num_errors',
- cwd=tmp_dir):
- self.error("Can't add partner.js to omni.ja!")
- return
- if self.run_command([zip_bin, '-9r', file_name, 'omni.ja'],
- error_list=ZipErrorList,
- return_type='num_errors',
- cwd=tmp_dir):
- self.error("Can't re-add omni.ja to %s!" % file_name)
- return
- if self.unsign_apk(tmp_file):
- return
- repack_dir = os.path.dirname(repack_path)
- self.mkdir_p(repack_dir)
- if self.copyfile(tmp_file, repack_path):
- return
- return True
-
- def repack(self):
- c = self.config
- rc = self.query_release_config()
- dirs = self.query_abs_dirs()
- locales = self.query_locales()
- success_count = total_count = 0
- for platform in c['platforms']:
- for locale in locales:
- installer_name = c['installer_base_names'][platform] % {'version': rc['version'], 'locale': locale}
- if self.query_failure(platform, locale):
- self.warning("%s:%s had previous issues; skipping!" % (platform, locale))
- continue
- original_path = '%s/original/%s/%s/%s' % (dirs['abs_work_dir'], platform, locale, installer_name)
- for partner in c['partner_config'].keys():
- repack_path = '%s/unsigned/partner-repacks/%s/%s/%s/%s' % (dirs['abs_work_dir'], partner, platform, locale, installer_name)
- total_count += 1
- if self._repack_apk(partner, original_path, repack_path):
- success_count += 1
- else:
- self.add_failure(platform, locale,
- message="Unable to repack %(platform)s:%(locale)s installer!")
- self.summarize_success_count(success_count, total_count,
- message="Repacked %d of %d installers successfully.")
-
- def _upload(self, dir_name="unsigned/partner-repacks"):
- c = self.config
- dirs = self.query_abs_dirs()
- local_path = os.path.join(dirs['abs_work_dir'], dir_name)
- rc = self.query_release_config()
- replace_dict = {
- 'buildnum': rc['buildnum'],
- 'version': rc['version'],
- }
- remote_path = '%s/%s' % (c['ftp_upload_base_dir'] % replace_dict, dir_name)
- if self.rsync_upload_directory(local_path, c['ftp_ssh_key'],
- c['ftp_user'], c['ftp_server'],
- remote_path):
- self.return_code += 1
-
- def upload_unsigned_bits(self):
- self._upload()
-
- # passphrase() in AndroidSigningMixin
- # verify_passphrases() in AndroidSigningMixin
-
- def preflight_sign(self):
- if 'passphrase' not in self.actions:
- self.passphrase()
- self.verify_passphrases()
-
- def sign(self):
- c = self.config
- rc = self.query_release_config()
- dirs = self.query_abs_dirs()
- locales = self.query_locales()
- success_count = total_count = 0
- for platform in c['platforms']:
- for locale in locales:
- installer_name = c['installer_base_names'][platform] % {'version': rc['version'], 'locale': locale}
- if self.query_failure(platform, locale):
- self.warning("%s:%s had previous issues; skipping!" % (platform, locale))
- continue
- for partner in c['partner_config'].keys():
- unsigned_path = '%s/unsigned/partner-repacks/%s/%s/%s/%s' % (dirs['abs_work_dir'], partner, platform, locale, installer_name)
- signed_dir = '%s/partner-repacks/%s/%s/%s' % (dirs['abs_work_dir'], partner, platform, locale)
- signed_path = "%s/%s" % (signed_dir, installer_name)
- total_count += 1
- self.info("Signing %s %s." % (platform, locale))
- if not os.path.exists(unsigned_path):
- self.error("Missing apk %s!" % unsigned_path)
- continue
- if self.sign_apk(unsigned_path, c['keystore'],
- self.store_passphrase, self.key_passphrase,
- c['key_alias']) != 0:
- self.add_summary("Unable to sign %s:%s apk!" % (platform, locale), level=FATAL)
- else:
- self.mkdir_p(signed_dir)
- if self.align_apk(unsigned_path, signed_path):
- self.add_failure(platform, locale,
- message="Unable to align %(platform)s%(locale)s apk!")
- self.rmtree(signed_dir)
- else:
- success_count += 1
- self.summarize_success_count(success_count, total_count,
- message="Signed %d of %d apks successfully.")
-
- # TODO verify signatures.
-
- def upload_signed_bits(self):
- self._upload(dir_name="partner-repacks")
-
-
-# main {{{1
-if __name__ == '__main__':
- mobile_partner_repack = MobilePartnerRepack()
- mobile_partner_repack.run_and_exit()
diff --git a/testing/mozharness/scripts/multil10n.py b/testing/mozharness/scripts/multil10n.py
deleted file mode 100755
index c89caf7c6..000000000
--- a/testing/mozharness/scripts/multil10n.py
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""multil10n.py
-
-"""
-
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.mozilla.l10n.multi_locale_build import MultiLocaleBuild
-
-if __name__ == '__main__':
- multi_locale_build = MultiLocaleBuild()
- multi_locale_build.run_and_exit()
diff --git a/testing/mozharness/scripts/openh264_build.py b/testing/mozharness/scripts/openh264_build.py
deleted file mode 100644
index 072d102d5..000000000
--- a/testing/mozharness/scripts/openh264_build.py
+++ /dev/null
@@ -1,250 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-import sys
-import os
-import glob
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-# import the guts
-from mozharness.base.vcs.vcsbase import VCSScript
-from mozharness.base.log import ERROR
-from mozharness.base.transfer import TransferMixin
-from mozharness.mozilla.mock import MockMixin
-
-
-class OpenH264Build(MockMixin, TransferMixin, VCSScript):
- all_actions = [
- 'clobber',
- 'checkout-sources',
- 'build',
- 'test',
- 'package',
- 'upload',
- ]
-
- default_actions = [
- 'checkout-sources',
- 'build',
- 'test',
- 'package',
- ]
-
- config_options = [
- [["--repo"], {
- "dest": "repo",
- "help": "OpenH264 repository to use",
- "default": "https://github.com/cisco/openh264.git"
- }],
- [["--rev"], {
- "dest": "revision",
- "help": "revision to checkout",
- "default": "master"
- }],
- [["--debug"], {
- "dest": "debug_build",
- "action": "store_true",
- "help": "Do a debug build",
- }],
- [["--64"], {
- "dest": "64bit",
- "action": "store_true",
- "help": "Do a 64-bit build",
- "default": True,
- }],
- [["--32"], {
- "dest": "64bit",
- "action": "store_false",
- "help": "Do a 32-bit build",
- }],
- [["--os"], {
- "dest": "operating_system",
- "help": "Specify the operating system to build for",
- }],
- [["--use-mock"], {
- "dest": "use_mock",
- "help": "use mock to set up build environment",
- "action": "store_true",
- "default": False,
- }],
- [["--use-yasm"], {
- "dest": "use_yasm",
- "help": "use yasm instead of nasm",
- "action": "store_true",
- "default": False,
- }],
- ]
-
- def __init__(self, require_config_file=False, config={},
- all_actions=all_actions,
- default_actions=default_actions):
-
- # Default configuration
- default_config = {
- 'debug_build': False,
- 'mock_target': 'mozilla-centos6-x86_64',
- 'mock_packages': ['make', 'git', 'nasm', 'glibc-devel.i686', 'libstdc++-devel.i686', 'zip', 'yasm'],
- 'mock_files': [],
- 'upload_ssh_key': os.path.expanduser("~/.ssh/ffxbld_rsa"),
- 'upload_ssh_user': 'ffxbld',
- 'upload_ssh_host': 'stage.mozilla.org',
- 'upload_path_base': '/home/ffxbld/openh264',
- 'use_yasm': False,
- }
- default_config.update(config)
-
- VCSScript.__init__(
- self,
- config_options=self.config_options,
- require_config_file=require_config_file,
- config=default_config,
- all_actions=all_actions,
- default_actions=default_actions,
- )
-
- if self.config['use_mock']:
- self.setup_mock()
- self.enable_mock()
-
- def query_package_name(self):
- if self.config['64bit']:
- bits = '64'
- else:
- bits = '32'
-
- version = self.config['revision']
-
- if sys.platform == 'linux2':
- if self.config.get('operating_system') == 'android':
- return 'openh264-android-{version}.zip'.format(version=version, bits=bits)
- else:
- return 'openh264-linux{bits}-{version}.zip'.format(version=version, bits=bits)
- elif sys.platform == 'darwin':
- return 'openh264-macosx{bits}-{version}.zip'.format(version=version, bits=bits)
- elif sys.platform == 'win32':
- return 'openh264-win{bits}-{version}.zip'.format(version=version, bits=bits)
- self.fatal("can't determine platform")
-
- def query_make_params(self):
- retval = []
- if self.config['debug_build']:
- retval.append('BUILDTYPE=Debug')
-
- if self.config['64bit']:
- retval.append('ENABLE64BIT=Yes')
- else:
- retval.append('ENABLE64BIT=No')
-
- if "operating_system" in self.config:
- retval.append("OS=%s" % self.config['operating_system'])
-
- if self.config['use_yasm']:
- retval.append('ASM=yasm')
-
- return retval
-
- def query_upload_ssh_key(self):
- return self.config['upload_ssh_key']
-
- def query_upload_ssh_host(self):
- return self.config['upload_ssh_host']
-
- def query_upload_ssh_user(self):
- return self.config['upload_ssh_user']
-
- def query_upload_ssh_path(self):
- return "%s/%s" % (self.config['upload_path_base'], self.config['revision'])
-
- def run_make(self, target):
- cmd = ['make', target] + self.query_make_params()
- dirs = self.query_abs_dirs()
- repo_dir = os.path.join(dirs['abs_work_dir'], 'src')
- return self.run_command(cmd, cwd=repo_dir)
-
- def checkout_sources(self):
- repo = self.config['repo']
- rev = self.config['revision']
-
- dirs = self.query_abs_dirs()
- repo_dir = os.path.join(dirs['abs_work_dir'], 'src')
-
- repos = [
- {'vcs': 'gittool', 'repo': repo, 'dest': repo_dir, 'revision': rev},
- ]
-
- # self.vcs_checkout already retries, so no need to wrap it in
- # self.retry. We set the error_level to ERROR to prevent it going fatal
- # so we can do our own handling here.
- retval = self.vcs_checkout_repos(repos, error_level=ERROR)
- if not retval:
- self.rmtree(repo_dir)
- self.fatal("Automation Error: couldn't clone repo", exit_code=4)
-
- # Checkout gmp-api
- # TODO: Nothing here updates it yet, or enforces versions!
- if not os.path.exists(os.path.join(repo_dir, 'gmp-api')):
- retval = self.run_make('gmp-bootstrap')
- if retval != 0:
- self.fatal("couldn't bootstrap gmp")
- else:
- self.info("skipping gmp bootstrap - we have it locally")
-
- # Checkout gtest
- # TODO: Requires svn!
- if not os.path.exists(os.path.join(repo_dir, 'gtest')):
- retval = self.run_make('gtest-bootstrap')
- if retval != 0:
- self.fatal("couldn't bootstrap gtest")
- else:
- self.info("skipping gtest bootstrap - we have it locally")
-
- return retval
-
- def build(self):
- retval = self.run_make('plugin')
- if retval != 0:
- self.fatal("couldn't build plugin")
-
- def package(self):
- dirs = self.query_abs_dirs()
- srcdir = os.path.join(dirs['abs_work_dir'], 'src')
- package_name = self.query_package_name()
- package_file = os.path.join(dirs['abs_work_dir'], package_name)
- if os.path.exists(package_file):
- os.unlink(package_file)
- to_package = [os.path.basename(f) for f in glob.glob(os.path.join(srcdir, "*gmpopenh264*"))]
- cmd = ['zip', package_file] + to_package
- retval = self.run_command(cmd, cwd=srcdir)
- if retval != 0:
- self.fatal("couldn't make package")
- self.copy_to_upload_dir(package_file)
-
- def upload(self):
- if self.config['use_mock']:
- self.disable_mock()
- dirs = self.query_abs_dirs()
- self.rsync_upload_directory(
- dirs['abs_upload_dir'],
- self.query_upload_ssh_key(),
- self.query_upload_ssh_user(),
- self.query_upload_ssh_host(),
- self.query_upload_ssh_path(),
- )
- if self.config['use_mock']:
- self.enable_mock()
-
- def test(self):
- retval = self.run_make('test')
- if retval != 0:
- self.fatal("test failures")
-
-
-# main {{{1
-if __name__ == '__main__':
- myScript = OpenH264Build()
- myScript.run_and_exit()
diff --git a/testing/mozharness/scripts/release/antivirus.py b/testing/mozharness/scripts/release/antivirus.py
deleted file mode 100644
index a28c3fd2f..000000000
--- a/testing/mozharness/scripts/release/antivirus.py
+++ /dev/null
@@ -1,192 +0,0 @@
-from multiprocessing.pool import ThreadPool
-import os
-import re
-import sys
-import shutil
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
-from mozharness.base.script import BaseScript
-
-
-class AntivirusScan(BaseScript, VirtualenvMixin):
- config_options = [
- [["--product"], {
- "dest": "product",
- "help": "Product being released, eg: firefox, thunderbird",
- }],
- [["--version"], {
- "dest": "version",
- "help": "Version of release, eg: 39.0b5",
- }],
- [["--build-number"], {
- "dest": "build_number",
- "help": "Build number of release, eg: 2",
- }],
- [["--bucket-name"], {
- "dest": "bucket_name",
- "help": "S3 Bucket to retrieve files from",
- }],
- [["--exclude"], {
- "dest": "excludes",
- "action": "append",
- "help": "List of filename patterns to exclude. See script source for default",
- }],
- [["-d", "--download-parallelization"], {
- "dest": "download_parallelization",
- "default": 6,
- "type": "int",
- "help": "Number of concurrent file downloads",
- }],
- [["-s", "--scan-parallelization"], {
- "dest": "scan_parallelization",
- "default": 4,
- "type": "int",
- "help": "Number of concurrent file scans",
- }],
- [["--tools-repo"], {
- "dest": "tools_repo",
- "default": "https://hg.mozilla.org/build/tools",
- }],
- [["--tools-revision"], {
- "dest": "tools_revision",
- "help": "Revision of tools repo to use when downloading extract_and_run_command.py",
- }],
- ] + virtualenv_config_options
-
- DEFAULT_EXCLUDES = [
- r"^.*tests.*$",
- r"^.*crashreporter.*$",
- r"^.*\.zip(\.asc)?$",
- r"^.*\.log$",
- r"^.*\.txt$",
- r"^.*\.asc$",
- r"^.*/partner-repacks.*$",
- r"^.*.checksums(\.asc)?$",
- r"^.*/logs/.*$",
- r"^.*/jsshell.*$",
- r"^.*json$",
- r"^.*/host.*$",
- r"^.*/mar-tools/.*$",
- r"^.*contrib.*"
- ]
- CACHE_DIR = 'cache'
-
- def __init__(self):
- BaseScript.__init__(self,
- config_options=self.config_options,
- require_config_file=False,
- config={
- "virtualenv_modules": [
- "boto",
- "redo",
- "mar",
- ],
- "virtualenv_path": "venv",
- },
- all_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "get-extract-script",
- "get-files",
- "scan-files",
- "cleanup-cache",
- ],
- default_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "get-extract-script",
- "get-files",
- "scan-files",
- "cleanup-cache",
- ],
- )
- self.excludes = self.config.get('excludes', self.DEFAULT_EXCLUDES)
- self.dest_dir = self.CACHE_DIR
-
- def _get_candidates_prefix(self):
- return "pub/{}/candidates/{}-candidates/build{}/".format(
- self.config['product'],
- self.config["version"],
- self.config["build_number"]
- )
-
- def _matches_exclude(self, keyname):
- for exclude in self.excludes:
- if re.search(exclude, keyname):
- return True
- return False
-
- def get_extract_script(self):
- """Gets a copy of extract_and_run_command.py from tools, and the supporting mar.py,
- so that we can unpack various files for clam to scan them."""
- remote_file = "{}/raw-file/{}/stage/extract_and_run_command.py".format(self.config["tools_repo"],
- self.config["tools_revision"])
- self.download_file(remote_file, file_name="extract_and_run_command.py")
-
- def get_files(self):
- """Pull the candidate files down from S3 for scanning, using parallel requests"""
- from boto.s3.connection import S3Connection
- from boto.exception import S3CopyError, S3ResponseError
- from redo import retry
- from httplib import HTTPException
-
- # suppress boto debug logging, it's too verbose with --loglevel=debug
- import logging
- logging.getLogger('boto').setLevel(logging.INFO)
-
- self.info("Connecting to S3")
- conn = S3Connection(anon=True)
- self.info("Getting bucket {}".format(self.config["bucket_name"]))
- bucket = conn.get_bucket(self.config["bucket_name"])
-
- if os.path.exists(self.dest_dir):
- self.info('Emptying {}'.format(self.dest_dir))
- shutil.rmtree(self.dest_dir)
- os.makedirs(self.dest_dir)
-
- def worker(item):
- source, destination = item
-
- self.info("Downloading {} to {}".format(source, destination))
- key = bucket.get_key(source)
- return retry(key.get_contents_to_filename,
- args=(destination, ),
- sleeptime=30, max_sleeptime=150,
- retry_exceptions=(S3CopyError, S3ResponseError,
- IOError, HTTPException))
-
- def find_release_files():
- candidates_prefix = self._get_candidates_prefix()
- self.info("Getting key names from candidates")
- for key in bucket.list(prefix=candidates_prefix):
- keyname = key.name
- if self._matches_exclude(keyname):
- self.debug("Excluding {}".format(keyname))
- else:
- destination = os.path.join(self.dest_dir, keyname.replace(candidates_prefix, ''))
- dest_dir = os.path.dirname(destination)
- if not os.path.isdir(dest_dir):
- os.makedirs(dest_dir)
- yield (keyname, destination)
-
- pool = ThreadPool(self.config["download_parallelization"])
- pool.map(worker, find_release_files())
-
- def scan_files(self):
- """Scan the files we've collected. We do the download and scan concurrently to make
- it easier to have a coherent log afterwards. Uses the venv python."""
- self.run_command([self.query_python_path(), 'extract_and_run_command.py',
- '-j{}'.format(self.config['scan_parallelization']),
- 'clamdscan', '-m', '--no-summary', '--', self.dest_dir])
-
- def cleanup_cache(self):
- """If we have simultaneous releases in flight an av slave may end up doing another
- av job before being recycled, and we need to make sure the full disk is available."""
- shutil.rmtree(self.dest_dir)
-
-
-if __name__ == "__main__":
- myScript = AntivirusScan()
- myScript.run_and_exit()
diff --git a/testing/mozharness/scripts/release/beet_mover.py b/testing/mozharness/scripts/release/beet_mover.py
deleted file mode 100755
index 0f3c18753..000000000
--- a/testing/mozharness/scripts/release/beet_mover.py
+++ /dev/null
@@ -1,371 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""beet_mover.py.
-
-downloads artifacts, scans them and uploads them to s3
-"""
-import hashlib
-import sys
-import os
-import pprint
-import re
-from os import listdir
-from os.path import isfile, join
-import sh
-import redo
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-from mozharness.base.log import FATAL
-from mozharness.base.python import VirtualenvMixin
-from mozharness.base.script import BaseScript
-from mozharness.mozilla.aws import pop_aws_auth_from_env
-import mozharness
-import mimetypes
-
-
-def get_hash(content, hash_type="md5"):
- h = hashlib.new(hash_type)
- h.update(content)
- return h.hexdigest()
-
-
-CONFIG_OPTIONS = [
- [["--template"], {
- "dest": "template",
- "help": "Specify jinja2 template file",
- }],
- [['--locale', ], {
- "action": "extend",
- "dest": "locales",
- "type": "string",
- "help": "Specify the locale(s) to upload."}],
- [["--platform"], {
- "dest": "platform",
- "help": "Specify the platform of the build",
- }],
- [["--version"], {
- "dest": "version",
- "help": "full release version based on gecko and tag/stage identifier. e.g. '44.0b1'"
- }],
- [["--app-version"], {
- "dest": "app_version",
- "help": "numbered version based on gecko. e.g. '44.0'"
- }],
- [["--partial-version"], {
- "dest": "partial_version",
- "help": "the partial version the mar is based off of"
- }],
- [["--artifact-subdir"], {
- "dest": "artifact_subdir",
- "default": 'build',
- "help": "subdir location for taskcluster artifacts after public/ base.",
- }],
- [["--build-num"], {
- "dest": "build_num",
- "help": "the release build identifier"
- }],
- [["--taskid"], {
- "dest": "taskid",
- "help": "taskcluster task id to download artifacts from",
- }],
- [["--bucket"], {
- "dest": "bucket",
- "help": "s3 bucket to move beets to.",
- }],
- [["--product"], {
- "dest": "product",
- "help": "product for which artifacts are beetmoved",
- }],
- [["--exclude"], {
- "dest": "excludes",
- "action": "append",
- "help": "List of filename patterns to exclude. See script source for default",
- }],
- [["-s", "--scan-parallelization"], {
- "dest": "scan_parallelization",
- "default": 4,
- "type": "int",
- "help": "Number of concurrent file scans",
- }],
-]
-
-DEFAULT_EXCLUDES = [
- r"^.*tests.*$",
- r"^.*crashreporter.*$",
- r"^.*\.zip(\.asc)?$",
- r"^.*\.log$",
- r"^.*\.txt$",
- r"^.*\.asc$",
- r"^.*/partner-repacks.*$",
- r"^.*.checksums(\.asc)?$",
- r"^.*/logs/.*$",
- r"^.*/jsshell.*$",
- r"^.*json$",
- r"^.*/host.*$",
- r"^.*/mar-tools/.*$",
- r"^.*contrib.*"
-]
-CACHE_DIR = 'cache'
-
-MIME_MAP = {
- '': 'text/plain',
- '.asc': 'text/plain',
- '.beet': 'text/plain',
- '.bundle': 'application/octet-stream',
- '.bz2': 'application/octet-stream',
- '.checksums': 'text/plain',
- '.dmg': 'application/x-iso9660-image',
- '.mar': 'application/octet-stream',
- '.xpi': 'application/x-xpinstall'
-}
-
-HASH_FORMATS = ["sha512", "sha256"]
-
-
-class BeetMover(BaseScript, VirtualenvMixin, object):
- def __init__(self, aws_creds):
- beetmover_kwargs = {
- 'config_options': CONFIG_OPTIONS,
- 'all_actions': [
- # 'clobber',
- 'create-virtualenv',
- 'activate-virtualenv',
- 'generate-candidates-manifest',
- 'refresh-antivirus',
- 'verify-bits', # beets
- 'download-bits', # beets
- 'scan-bits', # beets
- 'upload-bits', # beets
- ],
- 'require_config_file': False,
- # Default configuration
- 'config': {
- # base index url where to find taskcluster artifact based on taskid
- "artifact_base_url": 'https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/{subdir}',
- "virtualenv_modules": [
- "boto",
- "PyYAML",
- "Jinja2",
- "redo",
- "cryptography==2.0.3",
- "mar",
- ],
- "virtualenv_path": "venv",
- },
- }
- #todo do excludes need to be configured via command line for specific builds?
- super(BeetMover, self).__init__(**beetmover_kwargs)
-
- c = self.config
- self.manifest = {}
- # assigned in _post_create_virtualenv
- self.virtualenv_imports = None
- self.bucket = c['bucket']
- if not all(aws_creds):
- self.fatal('credentials must be passed in env: "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"')
- self.aws_key_id, self.aws_secret_key = aws_creds
- # if excludes is set from command line, use it otherwise use defaults
- self.excludes = self.config.get('excludes', DEFAULT_EXCLUDES)
- dirs = self.query_abs_dirs()
- self.dest_dir = os.path.join(dirs['abs_work_dir'], CACHE_DIR)
- self.mime_fix()
-
- def activate_virtualenv(self):
- """
- activates virtualenv and adds module imports to a instance wide namespace.
-
- creating and activating a virtualenv onto the currently executing python interpreter is a
- bit black magic. Rather than having import statements added in various places within the
- script, we import them here immediately after we activate the newly created virtualenv
- """
- VirtualenvMixin.activate_virtualenv(self)
-
- import boto
- import yaml
- import jinja2
- self.virtualenv_imports = {
- 'boto': boto,
- 'yaml': yaml,
- 'jinja2': jinja2,
- }
- self.log("activated virtualenv with the modules: {}".format(str(self.virtualenv_imports)))
-
- def _get_template_vars(self):
- return {
- "platform": self.config['platform'],
- "locales": self.config.get('locales'),
- "version": self.config['version'],
- "app_version": self.config.get('app_version', ''),
- "partial_version": self.config.get('partial_version', ''),
- "build_num": self.config['build_num'],
- # keep the trailing slash
- "s3_prefix": 'pub/{prod}/candidates/{ver}-candidates/{n}/'.format(
- prod=self.config['product'], ver=self.config['version'],
- n=self.config['build_num']
- ),
- "artifact_base_url": self.config['artifact_base_url'].format(
- taskid=self.config['taskid'], subdir=self.config['artifact_subdir']
- )
- }
-
- def generate_candidates_manifest(self):
- """
- generates and outputs a manifest that maps expected Taskcluster artifact names
- to release deliverable names
- """
- self.log('generating manifest from {}...'.format(self.config['template']))
- template_dir, template_file = os.path.split(os.path.abspath(self.config['template']))
- jinja2 = self.virtualenv_imports['jinja2']
- yaml = self.virtualenv_imports['yaml']
-
- jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir),
- undefined=jinja2.StrictUndefined)
- template = jinja_env.get_template(template_file)
- self.manifest = yaml.safe_load(template.render(**self._get_template_vars()))
-
- self.log("manifest generated:")
- self.log(pprint.pformat(self.manifest['mapping']))
-
- def verify_bits(self):
- """
- inspects each artifact and verifies that they were created by trustworthy tasks
- """
- # TODO
- self.log('skipping verification. unimplemented...')
-
- def refresh_antivirus(self):
- self.info("Refreshing clamav db...")
- try:
- redo.retry(lambda:
- sh.freshclam("--stdout", "--verbose", _timeout=300,
- _err_to_out=True))
- self.info("Done.")
- except sh.ErrorReturnCode:
- self.warning("Freshclam failed, skipping DB update")
-
- def download_bits(self):
- """
- downloads list of artifacts to self.dest_dir dir based on a given manifest
- """
- self.log('downloading and uploading artifacts to self_dest_dir...')
- dirs = self.query_abs_dirs()
-
- for locale in self.manifest['mapping']:
- for deliverable in self.manifest['mapping'][locale]:
- self.log("downloading '{}' deliverable for '{}' locale".format(deliverable, locale))
- source = self.manifest['mapping'][locale][deliverable]['artifact']
- self.retry(
- self.download_file,
- args=[source],
- kwargs={'parent_dir': dirs['abs_work_dir']},
- error_level=FATAL)
- self.log('Success!')
-
- def _strip_prefix(self, s3_key):
- """Return file name relative to prefix"""
- # "abc/def/hfg".split("abc/de")[-1] == "f/hfg"
- return s3_key.split(self._get_template_vars()["s3_prefix"])[-1]
-
- def upload_bits(self):
- """
- uploads list of artifacts to s3 candidates dir based on a given manifest
- """
- self.log('uploading artifacts to s3...')
- dirs = self.query_abs_dirs()
-
- # connect to s3
- boto = self.virtualenv_imports['boto']
- conn = boto.connect_s3(self.aws_key_id, self.aws_secret_key)
- bucket = conn.get_bucket(self.bucket)
-
- for locale in self.manifest['mapping']:
- for deliverable in self.manifest['mapping'][locale]:
- self.log("uploading '{}' deliverable for '{}' locale".format(deliverable, locale))
- # we have already downloaded the files locally so we can use that version
- source = self.manifest['mapping'][locale][deliverable]['artifact']
- s3_key = self.manifest['mapping'][locale][deliverable]['s3_key']
- downloaded_file = os.path.join(dirs['abs_work_dir'], self.get_filename_from_url(source))
- # generate checksums for every uploaded file
- beet_file_name = '{}.beet'.format(downloaded_file)
- # upload checksums to a separate subdirectory
- beet_dest = '{prefix}beetmover-checksums/{f}.beet'.format(
- prefix=self._get_template_vars()["s3_prefix"],
- f=self._strip_prefix(s3_key)
- )
- beet_contents = '\n'.join([
- '{hash} {fmt} {size} {name}'.format(
- hash=self.get_hash_for_file(downloaded_file, hash_type=fmt),
- fmt=fmt,
- size=os.path.getsize(downloaded_file),
- name=self._strip_prefix(s3_key)) for fmt in HASH_FORMATS
- ])
- self.write_to_file(beet_file_name, beet_contents)
- self.upload_bit(source=downloaded_file, s3_key=s3_key,
- bucket=bucket)
- self.upload_bit(source=beet_file_name, s3_key=beet_dest,
- bucket=bucket)
- self.log('Success!')
-
-
- def upload_bit(self, source, s3_key, bucket):
- boto = self.virtualenv_imports['boto']
- self.info('uploading to s3 with key: {}'.format(s3_key))
- key = boto.s3.key.Key(bucket) # create new key
- key.key = s3_key # set key name
-
- self.info("Checking if `{}` already exists".format(s3_key))
- key = bucket.get_key(s3_key)
- if not key:
- self.info("Uploading to `{}`".format(s3_key))
- key = bucket.new_key(s3_key)
- # set key value
- mime_type, _ = mimetypes.guess_type(source)
- self.retry(lambda: key.set_contents_from_filename(source, headers={'Content-Type': mime_type}),
- error_level=FATAL),
- else:
- if not get_hash(key.get_contents_as_string()) == get_hash(open(source).read()):
- # for now, let's halt. If necessary, we can revisit this and allow for overwrites
- # to the same buildnum release with different bits
- self.fatal("`{}` already exists with different checksum.".format(s3_key))
- self.log("`{}` has the same MD5 checksum, not uploading".format(s3_key))
-
- def scan_bits(self):
-
- dirs = self.query_abs_dirs()
-
- filenames = [f for f in listdir(dirs['abs_work_dir']) if isfile(join(dirs['abs_work_dir'], f))]
- self.mkdir_p(self.dest_dir)
- for file_name in filenames:
- if self._matches_exclude(file_name):
- self.info("Excluding {} from virus scan".format(file_name))
- else:
- self.info('Copying {} to {}'.format(file_name,self.dest_dir))
- self.copyfile(os.path.join(dirs['abs_work_dir'], file_name), os.path.join(self.dest_dir,file_name))
- self._scan_files()
- self.info('Emptying {}'.format(self.dest_dir))
- self.rmtree(self.dest_dir)
-
- def _scan_files(self):
- """Scan the files we've collected. We do the download and scan concurrently to make
- it easier to have a coherent log afterwards. Uses the venv python."""
- external_tools_path = os.path.join(
- os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))), 'external_tools')
- self.run_command([self.query_python_path(), os.path.join(external_tools_path,'extract_and_run_command.py'),
- '-j{}'.format(self.config['scan_parallelization']),
- 'clamscan', '--no-summary', '--', self.dest_dir])
-
- def _matches_exclude(self, keyname):
- return any(re.search(exclude, keyname) for exclude in self.excludes)
-
- def mime_fix(self):
- """ Add mimetypes for custom extensions """
- mimetypes.init()
- map(lambda (ext, mime_type,): mimetypes.add_type(mime_type, ext), MIME_MAP.items())
-
-if __name__ == '__main__':
- beet_mover = BeetMover(pop_aws_auth_from_env())
- beet_mover.run_and_exit()
diff --git a/testing/mozharness/scripts/release/generate-checksums.py b/testing/mozharness/scripts/release/generate-checksums.py
deleted file mode 100644
index 61a1c43d2..000000000
--- a/testing/mozharness/scripts/release/generate-checksums.py
+++ /dev/null
@@ -1,284 +0,0 @@
-from multiprocessing.pool import ThreadPool
-import os
-from os import path
-import re
-import sys
-import posixpath
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
-from mozharness.base.script import BaseScript
-from mozharness.base.vcs.vcsbase import VCSMixin
-from mozharness.mozilla.checksums import parse_checksums_file
-from mozharness.mozilla.signing import SigningMixin
-from mozharness.mozilla.buildbot import BuildbotMixin
-
-class ChecksumsGenerator(BaseScript, VirtualenvMixin, SigningMixin, VCSMixin, BuildbotMixin):
- config_options = [
- [["--stage-product"], {
- "dest": "stage_product",
- "help": "Name of product used in file server's directory structure, eg: firefox, mobile",
- }],
- [["--version"], {
- "dest": "version",
- "help": "Version of release, eg: 39.0b5",
- }],
- [["--build-number"], {
- "dest": "build_number",
- "help": "Build number of release, eg: 2",
- }],
- [["--bucket-name-prefix"], {
- "dest": "bucket_name_prefix",
- "help": "Prefix of bucket name, eg: net-mozaws-prod-delivery. This will be used to generate a full bucket name (such as net-mozaws-prod-delivery-{firefox,archive}.",
- }],
- [["--bucket-name-full"], {
- "dest": "bucket_name_full",
- "help": "Full bucket name, eg: net-mozaws-prod-delivery-firefox",
- }],
- [["-j", "--parallelization"], {
- "dest": "parallelization",
- "default": 20,
- "type": int,
- "help": "Number of checksums file to download concurrently",
- }],
- [["-f", "--format"], {
- "dest": "formats",
- "default": [],
- "action": "append",
- "help": "Format(s) to generate big checksums file for. Default: sha512",
- }],
- [["--include"], {
- "dest": "includes",
- "default": [],
- "action": "append",
- "help": "List of patterns to include in big checksums file. See script source for default.",
- }],
- [["--tools-repo"], {
- "dest": "tools_repo",
- "default": "https://hg.mozilla.org/build/tools",
- }],
- [["--credentials"], {
- "dest": "credentials",
- "help": "File containing access key and secret access key for S3",
- }],
- ] + virtualenv_config_options
-
- def __init__(self):
- BaseScript.__init__(self,
- config_options=self.config_options,
- require_config_file=False,
- config={
- "virtualenv_modules": [
- "pip==1.5.5",
- "boto",
- ],
- "virtualenv_path": "venv",
- 'buildbot_json_path': 'buildprops.json',
- },
- all_actions=[
- "create-virtualenv",
- "collect-individual-checksums",
- "create-big-checksums",
- "sign",
- "upload",
- "copy-info-files",
- ],
- default_actions=[
- "create-virtualenv",
- "collect-individual-checksums",
- "create-big-checksums",
- "sign",
- "upload",
- ],
- )
-
- self.checksums = {}
- self.bucket = None
- self.bucket_name = self._get_bucket_name()
- self.file_prefix = self._get_file_prefix()
- # set the env var for boto to read our special config file
- # rather than anything else we have at ~/.boto
- os.environ["BOTO_CONFIG"] = os.path.abspath(self.config["credentials"])
-
- def _pre_config_lock(self, rw_config):
- super(ChecksumsGenerator, self)._pre_config_lock(rw_config)
-
- # override properties from buildbot properties here as defined by
- # taskcluster properties
- self.read_buildbot_config()
- if not self.buildbot_config:
- self.warning("Skipping buildbot properties overrides")
- return
- # TODO: version should come from repo
- props = self.buildbot_config["properties"]
- for prop in ['version', 'build_number']:
- if props.get(prop):
- self.info("Overriding %s with %s" % (prop, props[prop]))
- self.config[prop] = props.get(prop)
-
- # These defaults are set here rather in the config because default
- # lists cannot be completely overidden, only appended to.
- if not self.config.get("formats"):
- self.config["formats"] = ["sha512", "sha256"]
-
- if not self.config.get("includes"):
- self.config["includes"] = [
- r"^.*\.tar\.bz2$",
- r"^.*\.tar\.xz$",
- r"^.*\.dmg$",
- r"^.*\.bundle$",
- r"^.*\.mar$",
- r"^.*Setup.*\.exe$",
- r"^.*\.xpi$",
- r"^.*fennec.*\.apk$",
- ]
-
- def _get_bucket_name(self):
- if self.config.get('bucket_name_full'):
- return self.config['bucket_name_full']
-
- suffix = "archive"
- # Firefox has a special bucket, per https://github.com/mozilla-services/product-delivery-tools/blob/master/bucketmap.go
- if self.config["stage_product"] == "firefox":
- suffix = "firefox"
-
- return "{}-{}".format(self.config["bucket_name_prefix"], suffix)
-
- def _get_file_prefix(self):
- return "pub/{}/candidates/{}-candidates/build{}/".format(
- self.config["stage_product"], self.config["version"], self.config["build_number"]
- )
-
- def _get_sums_filename(self, format_):
- return "{}SUMS".format(format_.upper())
-
- def _get_bucket(self):
- if not self.bucket:
- self.activate_virtualenv()
- from boto.s3.connection import S3Connection
-
- self.info("Connecting to S3")
- conn = S3Connection()
- self.debug("Successfully connected to S3")
- self.info("Connecting to bucket {}".format(self.bucket_name))
- self.bucket = conn.get_bucket(self.bucket_name)
- return self.bucket
-
- def collect_individual_checksums(self):
- """This step grabs all of the small checksums files for the release,
- filters out any unwanted files from within them, and adds the remainder
- to self.checksums for subsequent steps to use."""
- bucket = self._get_bucket()
- self.info("File prefix is: {}".format(self.file_prefix))
-
- # Temporary holding place for checksums
- raw_checksums = []
- def worker(item):
- self.debug("Downloading {}".format(item))
- # TODO: It would be nice to download the associated .asc file
- # and verify against it.
- sums = bucket.get_key(item).get_contents_as_string()
- raw_checksums.append(sums)
-
- def find_checksums_files():
- self.info("Getting key names from bucket")
- checksum_files = {"beets": [], "checksums": []}
- for key in bucket.list(prefix=self.file_prefix):
- if key.key.endswith(".checksums"):
- self.debug("Found checksums file: {}".format(key.key))
- checksum_files["checksums"].append(key.key)
- elif key.key.endswith(".beet"):
- self.debug("Found beet file: {}".format(key.key))
- checksum_files["beets"].append(key.key)
- else:
- self.debug("Ignoring non-checksums file: {}".format(key.key))
- if checksum_files["beets"]:
- self.log("Using beet format")
- return checksum_files["beets"]
- else:
- self.log("Using checksums format")
- return checksum_files["checksums"]
-
- pool = ThreadPool(self.config["parallelization"])
- pool.map(worker, find_checksums_files())
-
- for c in raw_checksums:
- for f, info in parse_checksums_file(c).iteritems():
- for pattern in self.config["includes"]:
- if re.search(pattern, f):
- if f in self.checksums:
- self.fatal("Found duplicate checksum entry for {}, don't know which one to pick.".format(f))
- if not set(self.config["formats"]) <= set(info["hashes"]):
- self.fatal("Missing necessary format for file {}".format(f))
- self.debug("Adding checksums for file: {}".format(f))
- self.checksums[f] = info
- break
- else:
- self.debug("Ignoring checksums for file: {}".format(f))
-
- def create_big_checksums(self):
- for fmt in self.config["formats"]:
- sums = self._get_sums_filename(fmt)
- self.info("Creating big checksums file: {}".format(sums))
- with open(sums, "w+") as output_file:
- for fn in sorted(self.checksums):
- output_file.write("{} {}\n".format(self.checksums[fn]["hashes"][fmt], fn))
-
- def sign(self):
- dirs = self.query_abs_dirs()
-
- tools_dir = path.join(dirs["abs_work_dir"], "tools")
- self.vcs_checkout(
- repo=self.config["tools_repo"],
- branch="default",
- vcs="hg",
- dest=tools_dir,
- )
-
- sign_cmd = self.query_moz_sign_cmd(formats=["gpg"])
-
- for fmt in self.config["formats"]:
- sums = self._get_sums_filename(fmt)
- self.info("Signing big checksums file: {}".format(sums))
- retval = self.run_command(sign_cmd + [sums])
- if retval != 0:
- self.fatal("Failed to sign {}".format(sums))
-
- def upload(self):
- # we need to provide the public side of the gpg key so that people can
- # verify the detached signatures
- dirs = self.query_abs_dirs()
- tools_dir = path.join(dirs["abs_work_dir"], "tools")
- self.copyfile(os.path.join(tools_dir, 'scripts', 'release', 'KEY'),
- 'KEY')
- files = ['KEY']
-
- for fmt in self.config["formats"]:
- files.append(self._get_sums_filename(fmt))
- files.append("{}.asc".format(self._get_sums_filename(fmt)))
-
- bucket = self._get_bucket()
- for f in files:
- dest = posixpath.join(self.file_prefix, f)
- self.info("Uploading {} to {}".format(f, dest))
- key = bucket.new_key(dest)
- key.set_contents_from_filename(f, headers={'Content-Type': 'text/plain'})
-
- def copy_info_files(self):
- bucket = self._get_bucket()
-
- for key in bucket.list(prefix=self.file_prefix):
- if re.search(r'/en-US/android.*_info\.txt$', key.name):
- self.info("Found {}".format(key.name))
- dest = posixpath.join(self.file_prefix, posixpath.basename(key.name))
- self.info("Copying to {}".format(dest))
- bucket.copy_key(new_key_name=dest,
- src_bucket_name=self.bucket_name,
- src_key_name=key.name,
- metadata={'Content-Type': 'text/plain'})
-
-
-if __name__ == "__main__":
- myScript = ChecksumsGenerator()
- myScript.run_and_exit()
diff --git a/testing/mozharness/scripts/release/postrelease_bouncer_aliases.py b/testing/mozharness/scripts/release/postrelease_bouncer_aliases.py
deleted file mode 100644
index 78a60b4bc..000000000
--- a/testing/mozharness/scripts/release/postrelease_bouncer_aliases.py
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/usr/bin/env python
-# lint_ignore=E501
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-""" postrelease_bouncer_aliases.py
-
-A script to replace the old-fashion way of updating the bouncer aliaes through
-tools script.
-"""
-
-import os
-import sys
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
-from mozharness.base.script import BaseScript
-from mozharness.mozilla.buildbot import BuildbotMixin
-
-
-# PostReleaseBouncerAliases {{{1
-class PostReleaseBouncerAliases(BaseScript, VirtualenvMixin, BuildbotMixin):
- config_options = virtualenv_config_options
-
- def __init__(self, require_config_file=True):
- super(PostReleaseBouncerAliases, self).__init__(
- config_options=self.config_options,
- require_config_file=require_config_file,
- config={
- "virtualenv_modules": [
- "redo",
- "requests",
- ],
- "virtualenv_path": "venv",
- 'credentials_file': 'oauth.txt',
- 'buildbot_json_path': 'buildprops.json',
- },
- all_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "update-bouncer-aliases",
- ],
- default_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "update-bouncer-aliases",
- ],
- )
-
- def _pre_config_lock(self, rw_config):
- super(PostReleaseBouncerAliases, self)._pre_config_lock(rw_config)
- # override properties from buildbot properties here as defined by
- # taskcluster properties
- self.read_buildbot_config()
- if not self.buildbot_config:
- self.warning("Skipping buildbot properties overrides")
- return
- props = self.buildbot_config["properties"]
- for prop in ['tuxedo_server_url', 'version']:
- if props.get(prop):
- self.info("Overriding %s with %s" % (prop, props[prop]))
- self.config[prop] = props.get(prop)
- else:
- self.warning("%s could not be found within buildprops" % prop)
- return
-
- def _update_bouncer_alias(self, tuxedo_server_url, auth,
- related_product, alias):
- from redo import retry
- import requests
-
- url = "%s/create_update_alias" % tuxedo_server_url
- data = {"alias": alias, "related_product": related_product}
- self.log("Updating {} to point to {} using {}".format(alias,
- related_product,
- url))
-
- # Wrap the real call to hide credentials from retry's logging
- def do_update_bouncer_alias():
- r = requests.post(url, data=data, auth=auth,
- verify=False, timeout=60)
- r.raise_for_status()
-
- retry(do_update_bouncer_alias)
-
- def update_bouncer_aliases(self):
- tuxedo_server_url = self.config['tuxedo_server_url']
- credentials_file = os.path.join(os.getcwd(),
- self.config['credentials_file'])
- credentials = {}
- execfile(credentials_file, credentials)
- auth = (credentials['tuxedoUsername'], credentials['tuxedoPassword'])
- version = self.config['version']
- for product, info in self.config["products"].iteritems():
- if "alias" in info:
- product_template = info["product-name"]
- related_product = product_template % {"version": version}
- self._update_bouncer_alias(tuxedo_server_url, auth,
- related_product, info["alias"])
-
-
-# __main__ {{{1
-if __name__ == '__main__':
- PostReleaseBouncerAliases().run_and_exit()
diff --git a/testing/mozharness/scripts/release/postrelease_mark_as_shipped.py b/testing/mozharness/scripts/release/postrelease_mark_as_shipped.py
deleted file mode 100644
index f84b5771c..000000000
--- a/testing/mozharness/scripts/release/postrelease_mark_as_shipped.py
+++ /dev/null
@@ -1,110 +0,0 @@
-#!/usr/bin/env python
-# lint_ignore=E501
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-""" postrelease_mark_as_shipped.py
-
-A script to automate the manual way of updating a release as shipped in Ship-it
-following its successful ship-to-the-door opertion.
-"""
-import os
-import sys
-from datetime import datetime
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
-from mozharness.base.script import BaseScript
-from mozharness.mozilla.buildbot import BuildbotMixin
-
-
-def build_release_name(product, version, buildno):
- """Function to reconstruct the name of the release based on product,
- version and buildnumber
- """
- return "{}-{}-build{}".format(product.capitalize(),
- str(version), str(buildno))
-
-
-class MarkReleaseAsShipped(BaseScript, VirtualenvMixin, BuildbotMixin):
- config_options = virtualenv_config_options
-
- def __init__(self, require_config_file=True):
- super(MarkReleaseAsShipped, self).__init__(
- config_options=self.config_options,
- require_config_file=require_config_file,
- config={
- "virtualenv_modules": [
- "shipitapi",
- ],
- "virtualenv_path": "venv",
- "credentials_file": "oauth.txt",
- "buildbot_json_path": "buildprops.json",
- "timeout": 60,
- },
- all_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "mark-as-shipped",
- ],
- default_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "mark-as-shipped",
- ],
- )
-
- def _pre_config_lock(self, rw_config):
- super(MarkReleaseAsShipped, self)._pre_config_lock(rw_config)
- # override properties from buildbot properties here as defined by
- # taskcluster properties
- self.read_buildbot_config()
- if not self.buildbot_config:
- self.warning("Skipping buildbot properties overrides")
- return
- props = self.buildbot_config['properties']
- mandatory_props = ['product', 'version', 'build_number']
- missing_props = []
- for prop in mandatory_props:
- if prop in props:
- self.info("Overriding %s with %s" % (prop, props[prop]))
- self.config[prop] = props.get(prop)
- else:
- self.warning("%s could not be found within buildprops" % prop)
- missing_props.append(prop)
-
- if missing_props:
- raise Exception("%s not found in configs" % missing_props)
-
- self.config['name'] = build_release_name(self.config['product'],
- self.config['version'],
- self.config['build_number'])
-
- def mark_as_shipped(self):
- """Method to make a simple call to Ship-it API to change a release
- status to 'shipped'
- """
- credentials_file = os.path.join(os.getcwd(),
- self.config["credentials_file"])
- credentials = {}
- execfile(credentials_file, credentials)
- ship_it_credentials = credentials["ship_it_credentials"]
- auth = (self.config["ship_it_username"],
- ship_it_credentials.get(self.config["ship_it_username"]))
- api_root = self.config['ship_it_root']
-
- from shipitapi import Release
- release_api = Release(auth, api_root=api_root,
- timeout=self.config['timeout'])
- shipped_at = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
-
- self.info("Mark the release as shipped with %s timestamp" % shipped_at)
- release_api.update(self.config['name'],
- status='shipped', shippedAt=shipped_at)
-
-
-if __name__ == '__main__':
- MarkReleaseAsShipped().run_and_exit()
diff --git a/testing/mozharness/scripts/release/postrelease_version_bump.py b/testing/mozharness/scripts/release/postrelease_version_bump.py
deleted file mode 100644
index dfffa699a..000000000
--- a/testing/mozharness/scripts/release/postrelease_version_bump.py
+++ /dev/null
@@ -1,184 +0,0 @@
-#!/usr/bin/env python
-# lint_ignore=E501
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-""" postrelease_version_bump.py
-
-A script to increase in-tree version number after shipping a release.
-"""
-
-import os
-import sys
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.buildbot import BuildbotMixin
-from mozharness.mozilla.repo_manupulation import MercurialRepoManipulationMixin
-
-
-# PostReleaseVersionBump {{{1
-class PostReleaseVersionBump(MercurialScript, BuildbotMixin,
- MercurialRepoManipulationMixin):
- config_options = [
- [['--hg-user', ], {
- "action": "store",
- "dest": "hg_user",
- "type": "string",
- "default": "ffxbld <release@mozilla.com>",
- "help": "Specify what user to use to commit to hg.",
- }],
- [['--next-version', ], {
- "action": "store",
- "dest": "next_version",
- "type": "string",
- "help": "Next version used in version bump",
- }],
- [['--ssh-user', ], {
- "action": "store",
- "dest": "ssh_user",
- "type": "string",
- "help": "SSH username with hg.mozilla.org permissions",
- }],
- [['--ssh-key', ], {
- "action": "store",
- "dest": "ssh_key",
- "type": "string",
- "help": "Path to SSH key.",
- }],
- [['--product', ], {
- "action": "store",
- "dest": "product",
- "type": "string",
- "help": "Product name",
- }],
- [['--version', ], {
- "action": "store",
- "dest": "version",
- "type": "string",
- "help": "Version",
- }],
- [['--build-number', ], {
- "action": "store",
- "dest": "build_number",
- "type": "string",
- "help": "Build number",
- }],
- [['--revision', ], {
- "action": "store",
- "dest": "revision",
- "type": "string",
- "help": "HG revision to tag",
- }],
- ]
-
- def __init__(self, require_config_file=True):
- super(PostReleaseVersionBump, self).__init__(
- config_options=self.config_options,
- all_actions=[
- 'clobber',
- 'clean-repos',
- 'pull',
- 'bump_postrelease',
- 'commit-changes',
- 'tag',
- 'push',
- ],
- default_actions=[
- 'clean-repos',
- 'pull',
- 'bump_postrelease',
- 'commit-changes',
- 'tag',
- 'push',
- ],
- config={
- 'buildbot_json_path': 'buildprops.json',
- },
- require_config_file=require_config_file
- )
-
- def _pre_config_lock(self, rw_config):
- super(PostReleaseVersionBump, self)._pre_config_lock(rw_config)
- # override properties from buildbot properties here as defined by
- # taskcluster properties
- self.read_buildbot_config()
- if not self.buildbot_config:
- self.warning("Skipping buildbot properties overrides")
- else:
- props = self.buildbot_config["properties"]
- for prop in ['next_version', 'product', 'version', 'build_number',
- 'revision']:
- if props.get(prop):
- self.info("Overriding %s with %s" % (prop, props[prop]))
- self.config[prop] = props.get(prop)
-
- if not self.config.get("next_version"):
- self.fatal("Next version has to be set. Use --next-version or "
- "pass `next_version' via buildbot properties.")
-
- def query_abs_dirs(self):
- """ Allow for abs_from_dir and abs_to_dir
- """
- if self.abs_dirs:
- return self.abs_dirs
- self.abs_dirs = super(PostReleaseVersionBump, self).query_abs_dirs()
- self.abs_dirs["abs_gecko_dir"] = os.path.join(
- self.abs_dirs['abs_work_dir'], self.config["repo"]["dest"])
- return self.abs_dirs
-
- def query_repos(self):
- """Build a list of repos to clone."""
- return [self.config["repo"]]
-
- def query_commit_dirs(self):
- return [self.query_abs_dirs()["abs_gecko_dir"]]
-
- def query_commit_message(self):
- return "Automatic version bump. CLOSED TREE NO BUG a=release"
-
- def query_push_dirs(self):
- return self.query_commit_dirs()
-
- def query_push_args(self, cwd):
- # cwd is not used here
- hg_ssh_opts = "ssh -l {user} -i {key}".format(
- user=self.config["ssh_user"],
- key=os.path.expanduser(self.config["ssh_key"])
- )
- return ["-e", hg_ssh_opts, "-r", "."]
-
- def pull(self):
- super(PostReleaseVersionBump, self).pull(
- repos=self.query_repos())
-
- def bump_postrelease(self, *args, **kwargs):
- """Bump version"""
- dirs = self.query_abs_dirs()
- for f in self.config["version_files"]:
- curr_version = ".".join(
- self.get_version(dirs['abs_gecko_dir'], f["file"]))
- self.replace(os.path.join(dirs['abs_gecko_dir'], f["file"]),
- curr_version, self.config["next_version"])
-
- def tag(self):
- dirs = self.query_abs_dirs()
- tags = ["{product}_{version}_BUILD{build_number}",
- "{product}_{version}_RELEASE"]
- tags = [t.format(product=self.config["product"].upper(),
- version=self.config["version"].replace(".", "_"),
- build_number=self.config["build_number"])
- for t in tags]
- message = "No bug - Tagging {revision} with {tags} a=release CLOSED TREE"
- message = message.format(
- revision=self.config["revision"],
- tags=', '.join(tags))
- self.hg_tag(cwd=dirs["abs_gecko_dir"], tags=tags,
- revision=self.config["revision"], message=message,
- user=self.config["hg_user"], force=True)
-
-# __main__ {{{1
-if __name__ == '__main__':
- PostReleaseVersionBump().run_and_exit()
diff --git a/testing/mozharness/scripts/release/publish_balrog.py b/testing/mozharness/scripts/release/publish_balrog.py
deleted file mode 100644
index edb381311..000000000
--- a/testing/mozharness/scripts/release/publish_balrog.py
+++ /dev/null
@@ -1,119 +0,0 @@
-#!/usr/bin/env python
-# lint_ignore=E501
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-""" updates.py
-
-A script publish a release to Balrog.
-
-"""
-
-import os
-import sys
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.buildbot import BuildbotMixin
-
-# PublishBalrog {{{1
-
-
-class PublishBalrog(MercurialScript, BuildbotMixin):
-
- def __init__(self, require_config_file=True):
- super(PublishBalrog, self).__init__(
- all_actions=[
- 'clobber',
- 'pull',
- 'submit-to-balrog',
- ],
- default_actions=[
- 'clobber',
- 'pull',
- 'submit-to-balrog',
- ],
- config={
- 'buildbot_json_path': 'buildprops.json',
- 'credentials_file': 'oauth.txt',
- },
- require_config_file=require_config_file
- )
-
- def _pre_config_lock(self, rw_config):
- super(PublishBalrog, self)._pre_config_lock(rw_config)
- # override properties from buildbot properties here as defined by
- # taskcluster properties
- self.read_buildbot_config()
- if not self.buildbot_config:
- self.warning("Skipping buildbot properties overrides")
- return
- # TODO: version and appVersion should come from repo
- props = self.buildbot_config["properties"]
- for prop in ['product', 'version', 'build_number', 'channels',
- 'balrog_api_root', 'schedule_at', 'background_rate']:
- if props.get(prop):
- self.info("Overriding %s with %s" % (prop, props[prop]))
- self.config[prop] = props.get(prop)
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- self.abs_dirs = super(PublishBalrog, self).query_abs_dirs()
- self.abs_dirs["abs_tools_dir"] = os.path.join(
- self.abs_dirs['abs_work_dir'], self.config["repo"]["dest"])
- return self.abs_dirs
-
- def query_channel_configs(self):
- """Return a list of channel configs.
- For RC builds it returns "release" and "beta" using
- "enabled_if_version_matches" to match RC.
-
- :return: list
- """
- return [(n, c) for n, c in self.config["update_channels"].items() if
- n in self.config["channels"]]
-
- def query_repos(self):
- """Build a list of repos to clone."""
- return [self.config["repo"]]
-
- def pull(self):
- super(PublishBalrog, self).pull(
- repos=self.query_repos())
-
-
- def submit_to_balrog(self):
- for _, channel_config in self.query_channel_configs():
- self._submit_to_balrog(channel_config)
-
- def _submit_to_balrog(self, channel_config):
- dirs = self.query_abs_dirs()
- auth = os.path.join(os.getcwd(), self.config['credentials_file'])
- cmd = [
- self.query_exe("python"),
- os.path.join(dirs["abs_tools_dir"],
- "scripts/build-promotion/balrog-release-shipper.py")]
- cmd.extend([
- "--api-root", self.config["balrog_api_root"],
- "--credentials-file", auth,
- "--username", self.config["balrog_username"],
- "--version", self.config["version"],
- "--product", self.config["product"],
- "--build-number", str(self.config["build_number"]),
- "--verbose",
- ])
- for r in channel_config["publish_rules"]:
- cmd.extend(["--rules", r])
- if self.config.get("schedule_at"):
- cmd.extend(["--schedule-at", self.config["schedule_at"]])
- if self.config.get("background_rate"):
- cmd.extend(["--background-rate", str(self.config["background_rate"])])
-
- self.retry(lambda: self.run_command(cmd, halt_on_failure=True))
-
-# __main__ {{{1
-if __name__ == '__main__':
- PublishBalrog().run_and_exit()
diff --git a/testing/mozharness/scripts/release/push-candidate-to-releases.py b/testing/mozharness/scripts/release/push-candidate-to-releases.py
deleted file mode 100644
index 977a26c48..000000000
--- a/testing/mozharness/scripts/release/push-candidate-to-releases.py
+++ /dev/null
@@ -1,199 +0,0 @@
-from multiprocessing.pool import ThreadPool
-import os
-import re
-import sys
-
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
-from mozharness.base.script import BaseScript
-from mozharness.mozilla.aws import pop_aws_auth_from_env
-
-
-class ReleasePusher(BaseScript, VirtualenvMixin):
- config_options = [
- [["--product"], {
- "dest": "product",
- "help": "Product being released, eg: firefox, thunderbird",
- }],
- [["--version"], {
- "dest": "version",
- "help": "Version of release, eg: 39.0b5",
- }],
- [["--build-number"], {
- "dest": "build_number",
- "help": "Build number of release, eg: 2",
- }],
- [["--bucket-name"], {
- "dest": "bucket_name",
- "help": "Bucket to copy files from candidates/ to releases/",
- }],
- [["--credentials"], {
- "dest": "credentials",
- "help": "File containing access key and secret access key",
- }],
- [["--exclude"], {
- "dest": "excludes",
- "default": [
- r"^.*tests.*$",
- r"^.*crashreporter.*$",
- r"^.*[^k]\.zip(\.asc)?$",
- r"^.*\.log$",
- r"^.*\.txt$",
- r"^.*/partner-repacks.*$",
- r"^.*.checksums(\.asc)?$",
- r"^.*/logs/.*$",
- r"^.*/jsshell.*$",
- r"^.*json$",
- r"^.*/host.*$",
- r"^.*/mar-tools/.*$",
- r"^.*bouncer.apk$",
- r"^.*contrib.*",
- r"^.*/beetmover-checksums/.*$",
- ],
- "action": "append",
- "help": "List of patterns to exclude from copy. The list can be "
- "extended by passing multiple --exclude arguments.",
- }],
- [["-j", "--parallelization"], {
- "dest": "parallelization",
- "default": 20,
- "type": "int",
- "help": "Number of copy requests to run concurrently",
- }],
- ] + virtualenv_config_options
-
- def __init__(self, aws_creds):
- BaseScript.__init__(self,
- config_options=self.config_options,
- require_config_file=False,
- config={
- "virtualenv_modules": [
- "boto",
- "redo",
- ],
- "virtualenv_path": "venv",
- },
- all_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "push-to-releases",
- ],
- default_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "push-to-releases",
- ],
- )
-
- # validate aws credentials
- if not (all(aws_creds) or self.config.get('credentials')):
- self.fatal("aws creds not defined. please add them to your config or env.")
- if any(aws_creds) and self.config.get('credentials'):
- self.fatal("aws creds found in env and self.config. please declare in one place only.")
-
- # set aws credentials
- if all(aws_creds):
- self.aws_key_id, self.aws_secret_key = aws_creds
- else: # use
- self.aws_key_id, self.aws_secret_key = None, None
- # set the env var for boto to read our special config file
- # rather than anything else we have at ~/.boto
- os.environ["BOTO_CONFIG"] = os.path.abspath(self.config["credentials"])
-
- def _get_candidates_prefix(self):
- return "pub/{}/candidates/{}-candidates/build{}/".format(
- self.config['product'],
- self.config["version"],
- self.config["build_number"]
- )
-
- def _get_releases_prefix(self):
- return "pub/{}/releases/{}/".format(
- self.config["product"],
- self.config["version"]
- )
-
- def _matches_exclude(self, keyname):
- for exclude in self.config["excludes"]:
- if re.search(exclude, keyname):
- return True
- return False
-
- def push_to_releases(self):
- """This step grabs the list of files in the candidates dir,
- filters out any unwanted files from within them, and copies
- the remainder."""
- from boto.s3.connection import S3Connection
- from boto.exception import S3CopyError, S3ResponseError
- from redo import retry
-
- # suppress boto debug logging, it's too verbose with --loglevel=debug
- import logging
- logging.getLogger('boto').setLevel(logging.INFO)
-
- self.info("Connecting to S3")
- conn = S3Connection(aws_access_key_id=self.aws_key_id,
- aws_secret_access_key=self.aws_secret_key)
- self.info("Getting bucket {}".format(self.config["bucket_name"]))
- bucket = conn.get_bucket(self.config["bucket_name"])
-
- # ensure the destination is empty
- self.info("Checking destination {} is empty".format(self._get_releases_prefix()))
- keys = [k for k in bucket.list(prefix=self._get_releases_prefix())]
- if keys:
- self.warning("Destination already exists with %s keys" % len(keys))
-
- def worker(item):
- source, destination = item
-
- def copy_key():
- source_key = bucket.get_key(source)
- dest_key = bucket.get_key(destination)
- # According to http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
- # S3 key MD5 is represented as ETag, except when objects are
- # uploaded using multipart method. In this case objects's ETag
- # is constructed using its MD5, minus symbol, and number of
- # part. See http://stackoverflow.com/questions/12186993/what-is-the-algorithm-to-compute-the-amazon-s3-etag-for-a-file-larger-than-5gb#answer-19896823
- source_md5 = source_key.etag.split("-")[0]
- if dest_key:
- dest_md5 = dest_key.etag.split("-")[0]
- else:
- dest_md5 = None
-
- if not dest_key:
- self.info("Copying {} to {}".format(source, destination))
- bucket.copy_key(destination, self.config["bucket_name"],
- source)
- elif source_md5 == dest_md5:
- self.warning(
- "{} already exists with the same content ({}), skipping copy".format(
- destination, dest_md5))
- else:
- self.fatal(
- "{} already exists with the different content (src ETag: {}, dest ETag: {}), aborting".format(
- destination, source_key.etag, dest_key.etag))
-
- return retry(copy_key, sleeptime=5, max_sleeptime=60,
- retry_exceptions=(S3CopyError, S3ResponseError))
-
- def find_release_files():
- candidates_prefix = self._get_candidates_prefix()
- release_prefix = self._get_releases_prefix()
- self.info("Getting key names from candidates")
- for key in bucket.list(prefix=candidates_prefix):
- keyname = key.name
- if self._matches_exclude(keyname):
- self.debug("Excluding {}".format(keyname))
- else:
- destination = keyname.replace(candidates_prefix,
- release_prefix)
- yield (keyname, destination)
-
- pool = ThreadPool(self.config["parallelization"])
- pool.map(worker, find_release_files())
-
-if __name__ == "__main__":
- myScript = ReleasePusher(pop_aws_auth_from_env())
- myScript.run_and_exit()
diff --git a/testing/mozharness/scripts/release/updates.py b/testing/mozharness/scripts/release/updates.py
deleted file mode 100644
index 4b660a67b..000000000
--- a/testing/mozharness/scripts/release/updates.py
+++ /dev/null
@@ -1,299 +0,0 @@
-#!/usr/bin/env python
-# lint_ignore=E501
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-""" updates.py
-
-A script to bump patcher configs, generate update verification configs, and
-publish top-level release blob information to Balrog.
-
-It clones the tools repo, modifies the existing patcher config to include
-current release build information, generates update verification configs,
-commits the changes and tags the repo using tags by Releng convention.
-After the changes are pushed to the repo, the script submits top-level release
-information to Balrog.
-"""
-
-import os
-import re
-import sys
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.buildbot import BuildbotMixin
-from mozharness.mozilla.repo_manupulation import MercurialRepoManipulationMixin
-from mozharness.mozilla.release import get_previous_version
-
-
-# UpdatesBumper {{{1
-class UpdatesBumper(MercurialScript, BuildbotMixin,
- MercurialRepoManipulationMixin):
- config_options = [
- [['--hg-user', ], {
- "action": "store",
- "dest": "hg_user",
- "type": "string",
- "default": "ffxbld <release@mozilla.com>",
- "help": "Specify what user to use to commit to hg.",
- }],
- [['--ssh-user', ], {
- "action": "store",
- "dest": "ssh_user",
- "type": "string",
- "help": "SSH username with hg.mozilla.org permissions",
- }],
- [['--ssh-key', ], {
- "action": "store",
- "dest": "ssh_key",
- "type": "string",
- "help": "Path to SSH key.",
- }],
- ]
-
- def __init__(self, require_config_file=True):
- super(UpdatesBumper, self).__init__(
- config_options=self.config_options,
- all_actions=[
- 'clobber',
- 'pull',
- 'download-shipped-locales',
- 'bump-configs',
- 'commit-changes',
- 'tag',
- 'push',
- 'submit-to-balrog',
- ],
- default_actions=[
- 'clobber',
- 'pull',
- 'download-shipped-locales',
- 'bump-configs',
- 'commit-changes',
- 'tag',
- 'push',
- 'submit-to-balrog',
- ],
- config={
- 'buildbot_json_path': 'buildprops.json',
- 'credentials_file': 'oauth.txt',
- },
- require_config_file=require_config_file
- )
-
- def _pre_config_lock(self, rw_config):
- super(UpdatesBumper, self)._pre_config_lock(rw_config)
- # override properties from buildbot properties here as defined by
- # taskcluster properties
- self.read_buildbot_config()
- if not self.buildbot_config:
- self.warning("Skipping buildbot properties overrides")
- return
- # TODO: version and appVersion should come from repo
- props = self.buildbot_config["properties"]
- for prop in ['product', 'version', 'build_number', 'revision',
- 'appVersion', 'balrog_api_root', "channels"]:
- if props.get(prop):
- self.info("Overriding %s with %s" % (prop, props[prop]))
- self.config[prop] = props.get(prop)
-
- partials = [v.strip() for v in props["partial_versions"].split(",")]
- self.config["partial_versions"] = [v.split("build") for v in partials]
- self.config["platforms"] = [p.strip() for p in
- props["platforms"].split(",")]
- self.config["channels"] = [c.strip() for c in
- props["channels"].split(",")]
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- self.abs_dirs = super(UpdatesBumper, self).query_abs_dirs()
- self.abs_dirs["abs_tools_dir"] = os.path.join(
- self.abs_dirs['abs_work_dir'], self.config["repo"]["dest"])
- return self.abs_dirs
-
- def query_repos(self):
- """Build a list of repos to clone."""
- return [self.config["repo"]]
-
- def query_commit_dirs(self):
- return [self.query_abs_dirs()["abs_tools_dir"]]
-
- def query_commit_message(self):
- return "Automated configuration bump"
-
- def query_push_dirs(self):
- return self.query_commit_dirs()
-
- def query_push_args(self, cwd):
- # cwd is not used here
- hg_ssh_opts = "ssh -l {user} -i {key}".format(
- user=self.config["ssh_user"],
- key=os.path.expanduser(self.config["ssh_key"])
- )
- return ["-e", hg_ssh_opts]
-
- def query_shipped_locales_path(self):
- dirs = self.query_abs_dirs()
- return os.path.join(dirs["abs_work_dir"], "shipped-locales")
-
- def query_channel_configs(self):
- """Return a list of channel configs.
- For RC builds it returns "release" and "beta" using
- "enabled_if_version_matches" to match RC.
-
- :return: list
- """
- return [(n, c) for n, c in self.config["update_channels"].items() if
- n in self.config["channels"]]
-
- def pull(self):
- super(UpdatesBumper, self).pull(
- repos=self.query_repos())
-
- def download_shipped_locales(self):
- dirs = self.query_abs_dirs()
- self.mkdir_p(dirs["abs_work_dir"])
- url = self.config["shipped-locales-url"].format(
- revision=self.config["revision"])
- if not self.download_file(url=url,
- file_name=self.query_shipped_locales_path()):
- self.fatal("Unable to fetch shipped-locales from %s" % url)
-
- def bump_configs(self):
- for channel, channel_config in self.query_channel_configs():
- self.bump_patcher_config(channel_config)
- self.bump_update_verify_configs(channel, channel_config)
-
- def query_matching_partials(self, channel_config):
- return [(v, b) for v, b in self.config["partial_versions"] if
- re.match(channel_config["version_regex"], v)]
-
- def query_patcher_config(self, channel_config):
- dirs = self.query_abs_dirs()
- patcher_config = os.path.join(
- dirs["abs_tools_dir"], "release/patcher-configs",
- channel_config["patcher_config"])
- return patcher_config
-
- def query_update_verify_config(self, channel, platform):
- dirs = self.query_abs_dirs()
- uvc = os.path.join(
- dirs["abs_tools_dir"], "release/updates",
- "{}-{}-{}.cfg".format(channel, self.config["product"], platform))
- return uvc
-
- def bump_patcher_config(self, channel_config):
- # TODO: to make it possible to run this before we have files copied to
- # the candidates directory, we need to add support to fetch build IDs
- # from tasks.
- dirs = self.query_abs_dirs()
- env = {"PERL5LIB": os.path.join(dirs["abs_tools_dir"], "lib/perl")}
- partial_versions = [v[0] for v in
- self.query_matching_partials(channel_config)]
- script = os.path.join(
- dirs["abs_tools_dir"], "release/patcher-config-bump.pl")
- patcher_config = self.query_patcher_config(channel_config)
- cmd = [self.query_exe("perl"), script]
- cmd.extend([
- "-p", self.config["product"],
- "-r", self.config["product"].capitalize(),
- "-v", self.config["version"],
- "-a", self.config["appVersion"],
- "-o", get_previous_version(
- self.config["version"], partial_versions),
- "-b", str(self.config["build_number"]),
- "-c", patcher_config,
- "-f", self.config["archive_domain"],
- "-d", self.config["download_domain"],
- "-l", self.query_shipped_locales_path(),
- ])
- for v in partial_versions:
- cmd.extend(["--partial-version", v])
- for p in self.config["platforms"]:
- cmd.extend(["--platform", p])
- for mar_channel_id in channel_config["mar_channel_ids"]:
- cmd.extend(["--mar-channel-id", mar_channel_id])
- self.run_command(cmd, halt_on_failure=True, env=env)
-
- def bump_update_verify_configs(self, channel, channel_config):
- dirs = self.query_abs_dirs()
- script = os.path.join(
- dirs["abs_tools_dir"],
- "scripts/build-promotion/create-update-verify-config.py")
- patcher_config = self.query_patcher_config(channel_config)
- for platform in self.config["platforms"]:
- cmd = [self.query_exe("python"), script]
- output = self.query_update_verify_config(channel, platform)
- cmd.extend([
- "--config", patcher_config,
- "--platform", platform,
- "--update-verify-channel",
- channel_config["update_verify_channel"],
- "--output", output,
- "--archive-prefix", self.config["archive_prefix"],
- "--previous-archive-prefix",
- self.config["previous_archive_prefix"],
- "--product", self.config["product"],
- "--balrog-url", self.config["balrog_url"],
- "--build-number", str(self.config["build_number"]),
- ])
-
- self.run_command(cmd, halt_on_failure=True)
-
- def tag(self):
- dirs = self.query_abs_dirs()
- tags = ["{product}_{version}_BUILD{build_number}_RUNTIME",
- "{product}_{version}_RELEASE_RUNTIME"]
- tags = [t.format(product=self.config["product"].upper(),
- version=self.config["version"].replace(".", "_"),
- build_number=self.config["build_number"])
- for t in tags]
- self.hg_tag(cwd=dirs["abs_tools_dir"], tags=tags,
- user=self.config["hg_user"], force=True)
-
- def submit_to_balrog(self):
- for _, channel_config in self.query_channel_configs():
- self._submit_to_balrog(channel_config)
-
- def _submit_to_balrog(self, channel_config):
- dirs = self.query_abs_dirs()
- auth = os.path.join(os.getcwd(), self.config['credentials_file'])
- cmd = [
- self.query_exe("python"),
- os.path.join(dirs["abs_tools_dir"],
- "scripts/build-promotion/balrog-release-pusher.py")]
- cmd.extend([
- "--api-root", self.config["balrog_api_root"],
- "--download-domain", self.config["download_domain"],
- "--archive-domain", self.config["archive_domain"],
- "--credentials-file", auth,
- "--product", self.config["product"],
- "--version", self.config["version"],
- "--build-number", str(self.config["build_number"]),
- "--app-version", self.config["appVersion"],
- "--username", self.config["balrog_username"],
- "--verbose",
- ])
- for c in channel_config["channel_names"]:
- cmd.extend(["--channel", c])
- for r in channel_config["rules_to_update"]:
- cmd.extend(["--rule-to-update", r])
- for p in self.config["platforms"]:
- cmd.extend(["--platform", p])
- for v, build_number in self.query_matching_partials(channel_config):
- partial = "{version}build{build_number}".format(
- version=v, build_number=build_number)
- cmd.extend(["--partial-update", partial])
- if channel_config["requires_mirrors"]:
- cmd.append("--requires-mirrors")
- if self.config["balrog_use_dummy_suffix"]:
- cmd.append("--dummy")
-
- self.retry(lambda: self.run_command(cmd, halt_on_failure=True))
-
-# __main__ {{{1
-if __name__ == '__main__':
- UpdatesBumper().run_and_exit()
diff --git a/testing/mozharness/scripts/release/uptake_monitoring.py b/testing/mozharness/scripts/release/uptake_monitoring.py
deleted file mode 100644
index 9ec24621f..000000000
--- a/testing/mozharness/scripts/release/uptake_monitoring.py
+++ /dev/null
@@ -1,188 +0,0 @@
-#!/usr/bin/env python
-# lint_ignore=E501
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-""" uptake_monitoring.py
-
-A script to replace the old-fashion way of computing the uptake monitoring
-from the scheduler within the slaves.
-"""
-
-import os
-import sys
-import datetime
-import time
-import xml.dom.minidom
-
-sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
-
-from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
-from mozharness.base.script import BaseScript
-from mozharness.mozilla.buildbot import BuildbotMixin
-
-
-def get_tuxedo_uptake_url(tuxedo_server_url, related_product, os):
- return '%s/uptake/?product=%s&os=%s' % (tuxedo_server_url,
- related_product, os)
-
-
-class UptakeMonitoring(BaseScript, VirtualenvMixin, BuildbotMixin):
- config_options = virtualenv_config_options
-
- def __init__(self, require_config_file=True):
- super(UptakeMonitoring, self).__init__(
- config_options=self.config_options,
- require_config_file=require_config_file,
- config={
- "virtualenv_modules": [
- "redo",
- "requests",
- ],
-
- "virtualenv_path": "venv",
- "credentials_file": "oauth.txt",
- "buildbot_json_path": "buildprops.json",
- "poll_interval": 60,
- "poll_timeout": 20*60,
- "min_uptake": 10000,
- },
- all_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "monitor-uptake",
- ],
- default_actions=[
- "create-virtualenv",
- "activate-virtualenv",
- "monitor-uptake",
- ],
- )
-
- def _pre_config_lock(self, rw_config):
- super(UptakeMonitoring, self)._pre_config_lock(rw_config)
- # override properties from buildbot properties here as defined by
- # taskcluster properties
- self.read_buildbot_config()
- if not self.buildbot_config:
- self.warning("Skipping buildbot properties overrides")
- return
- props = self.buildbot_config["properties"]
- for prop in ['tuxedo_server_url', 'version']:
- if props.get(prop):
- self.info("Overriding %s with %s" % (prop, props[prop]))
- self.config[prop] = props.get(prop)
- else:
- self.warning("%s could not be found within buildprops" % prop)
- return
- partials = [v.strip() for v in props["partial_versions"].split(",")]
- self.config["partial_versions"] = [v.split("build")[0] for v in partials]
- self.config["platforms"] = [p.strip() for p in
- props["platforms"].split(",")]
-
- def _get_product_uptake(self, tuxedo_server_url, auth,
- related_product, os):
- from redo import retry
- import requests
-
- url = get_tuxedo_uptake_url(tuxedo_server_url, related_product, os)
- self.info("Requesting {} from tuxedo".format(url))
-
- def get_tuxedo_page():
- r = requests.get(url, auth=auth,
- verify=False, timeout=60)
- r.raise_for_status()
- return r.content
-
- def calculateUptake(page):
- doc = xml.dom.minidom.parseString(page)
- uptake_values = []
-
- for element in doc.getElementsByTagName('available'):
- for node in element.childNodes:
- if node.nodeType == xml.dom.minidom.Node.TEXT_NODE and \
- node.data.isdigit():
- uptake_values.append(int(node.data))
- if not uptake_values:
- uptake_values = [0]
- return min(uptake_values)
-
- page = retry(get_tuxedo_page)
- uptake = calculateUptake(page)
- self.info("Current uptake for {} is {}".format(related_product, uptake))
- return uptake
-
- def _get_release_uptake(self, auth):
- assert isinstance(self.config["platforms"], (list, tuple))
-
- # handle the products first
- tuxedo_server_url = self.config["tuxedo_server_url"]
- version = self.config["version"]
- dl = []
-
- for product, info in self.config["products"].iteritems():
- if info.get("check_uptake"):
- product_template = info["product-name"]
- related_product = product_template % {"version": version}
-
- enUS_platforms = set(self.config["platforms"])
- paths_platforms = set(info["paths"].keys())
- platforms = enUS_platforms.intersection(paths_platforms)
-
- for platform in platforms:
- bouncer_platform = info["paths"].get(platform).get('bouncer-platform')
- dl.append(self._get_product_uptake(tuxedo_server_url, auth,
- related_product, bouncer_platform))
- # handle the partials as well
- prev_versions = self.config["partial_versions"]
- for product, info in self.config["partials"].iteritems():
- if info.get("check_uptake"):
- product_template = info["product-name"]
- for prev_version in prev_versions:
- subs = {
- "version": version,
- "prev_version": prev_version
- }
- related_product = product_template % subs
-
- enUS_platforms = set(self.config["platforms"])
- paths_platforms = set(info["paths"].keys())
- platforms = enUS_platforms.intersection(paths_platforms)
-
- for platform in platforms:
- bouncer_platform = info["paths"].get(platform).get('bouncer-platform')
- dl.append(self._get_product_uptake(tuxedo_server_url, auth,
- related_product, bouncer_platform))
- return min(dl)
-
- def monitor_uptake(self):
- credentials_file = os.path.join(os.getcwd(),
- self.config["credentials_file"])
- credentials = {}
- execfile(credentials_file, credentials)
- auth = (credentials['tuxedoUsername'], credentials['tuxedoPassword'])
- self.info("Starting the loop to determine the uptake monitoring ...")
-
- start_time = datetime.datetime.now()
- while True:
- delta = (datetime.datetime.now() - start_time).seconds
- if delta > self.config["poll_timeout"]:
- self.error("Uptake monitoring sadly timed-out")
- raise Exception("Time-out during uptake monitoring")
-
- uptake = self._get_release_uptake(auth)
- self.info("Current uptake value to check is {}".format(uptake))
-
- if uptake >= self.config["min_uptake"]:
- self.info("Uptake monitoring is complete!")
- break
- else:
- self.info("Mirrors not yet updated, sleeping for a bit ...")
- time.sleep(self.config["poll_interval"])
-
-
-if __name__ == '__main__':
- myScript = UptakeMonitoring()
- myScript.run_and_exit()
diff --git a/testing/mozharness/scripts/spidermonkey/build.b2g b/testing/mozharness/scripts/spidermonkey/build.b2g
deleted file mode 100755
index 958946230..000000000
--- a/testing/mozharness/scripts/spidermonkey/build.b2g
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash -e
-
-cd $SOURCE
-TOP=$(cd .. && echo $PWD)
-export MOZBUILD_STATE_PATH=$TOP/mozbuild-state
-[ -d $MOZBUILD_STATE_PATH ] || mkdir $MOZBUILD_STATE_PATH
-
-exec ./mach build -v -j8
diff --git a/testing/mozharness/scripts/spidermonkey/build.browser b/testing/mozharness/scripts/spidermonkey/build.browser
deleted file mode 100755
index 645d2ae86..000000000
--- a/testing/mozharness/scripts/spidermonkey/build.browser
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash -e
-
-cd $SOURCE
-TOP=$(cd ..; pwd)
-export MOZBUILD_STATE_PATH=$TOP/mozbuild-state
-[ -d $MOZBUILD_STATE_PATH ] || mkdir $MOZBUILD_STATE_PATH
-
-export MOZCONFIG=$SOURCE/browser/config/mozconfigs/linux64/hazards
-
-exec ./mach build -v -j8
diff --git a/testing/mozharness/scripts/spidermonkey/build.shell b/testing/mozharness/scripts/spidermonkey/build.shell
deleted file mode 100755
index 7aad477ea..000000000
--- a/testing/mozharness/scripts/spidermonkey/build.shell
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash -ex
-
-mkdir -p "$ANALYZED_OBJDIR"
-cd "$ANALYZED_OBJDIR"
-$SOURCE/js/src/configure --enable-debug --enable-optimize --enable-stdcxx-compat --enable-ctypes --enable-nspr-build
-make -j12 -s
diff --git a/testing/mozharness/scripts/spidermonkey_build.py b/testing/mozharness/scripts/spidermonkey_build.py
deleted file mode 100755
index 5522545da..000000000
--- a/testing/mozharness/scripts/spidermonkey_build.py
+++ /dev/null
@@ -1,482 +0,0 @@
-#!/usr/bin/env python
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import os
-import sys
-import copy
-from datetime import datetime
-from functools import wraps
-
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.errors import MakefileErrorList
-from mozharness.base.script import BaseScript
-from mozharness.base.transfer import TransferMixin
-from mozharness.base.vcs.vcsbase import VCSMixin
-from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
-from mozharness.mozilla.buildbot import BuildbotMixin
-from mozharness.mozilla.building.hazards import HazardError, HazardAnalysis
-from mozharness.mozilla.purge import PurgeMixin
-from mozharness.mozilla.mock import MockMixin
-from mozharness.mozilla.tooltool import TooltoolMixin
-
-SUCCESS, WARNINGS, FAILURE, EXCEPTION, RETRY = xrange(5)
-
-
-def requires(*queries):
- """Wrapper for detecting problems where some bit of information
- required by the wrapped step is unavailable. Use it put prepending
- @requires("foo"), which will check whether self.query_foo() returns
- something useful."""
- def make_wrapper(f):
- @wraps(f)
- def wrapper(self, *args, **kwargs):
- for query in queries:
- val = query(self)
- goodval = not (val is None or "None" in str(val))
- assert goodval, f.__name__ + " requires " + query.__name__ + " to return a value"
- return f(self, *args, **kwargs)
- return wrapper
- return make_wrapper
-
-
-nuisance_env_vars = ['TERMCAP', 'LS_COLORS', 'PWD', '_']
-
-
-class SpidermonkeyBuild(MockMixin,
- PurgeMixin, BaseScript,
- VCSMixin, BuildbotMixin, TooltoolMixin, TransferMixin, BlobUploadMixin):
- config_options = [
- [["--repo"], {
- "dest": "repo",
- "help": "which gecko repo to get spidermonkey from",
- }],
- [["--source"], {
- "dest": "source",
- "help": "directory containing gecko source tree (instead of --repo)",
- }],
- [["--revision"], {
- "dest": "revision",
- }],
- [["--branch"], {
- "dest": "branch",
- }],
- [["--vcs-share-base"], {
- "dest": "vcs_share_base",
- "help": "base directory for shared repositories",
- }],
- [["-j"], {
- "dest": "concurrency",
- "type": int,
- "default": 4,
- "help": "number of simultaneous jobs used while building the shell " +
- "(currently ignored for the analyzed build",
- }] + copy.deepcopy(blobupload_config_options)
- ]
-
- def __init__(self):
- super(SpidermonkeyBuild, self).__init__(
- config_options=self.config_options,
- # other stuff
- all_actions=[
- 'purge',
- 'checkout-tools',
-
- # First, build an optimized JS shell for running the analysis
- 'checkout-source',
- 'get-blobs',
- 'clobber-shell',
- 'configure-shell',
- 'build-shell',
-
- # Next, build a tree with the analysis plugin active. Note that
- # we are using the same checkout for the JS shell build and the
- # build of the source to be analyzed, which is a little
- # unnecessary (no need to rebuild the JS shell all the time).
- # (Different objdir, though.)
-
- 'clobber-analysis',
- 'setup-analysis',
- 'run-analysis',
- 'collect-analysis-output',
- 'upload-analysis',
- 'check-expectations',
- ],
- default_actions=[
- 'purge',
- 'checkout-tools',
- 'checkout-source',
- 'get-blobs',
- 'clobber-shell',
- 'configure-shell',
- 'build-shell',
- 'clobber-analysis',
- 'setup-analysis',
- 'run-analysis',
- 'collect-analysis-output',
- # Temporarily disabled, see bug 1211402
- # 'upload-analysis',
- 'check-expectations',
- ],
- config={
- 'default_vcs': 'hg',
- 'vcs_share_base': os.environ.get('HG_SHARE_BASE_DIR'),
- 'ccache': True,
- 'buildbot_json_path': os.environ.get('PROPERTIES_FILE'),
- 'tools_repo': 'https://hg.mozilla.org/build/tools',
-
- 'upload_ssh_server': None,
- 'upload_remote_basepath': None,
- 'enable_try_uploads': True,
- 'source': None,
- 'stage_product': 'firefox',
- },
- )
-
- self.buildid = None
- self.create_virtualenv()
- self.analysis = HazardAnalysis()
-
- def _pre_config_lock(self, rw_config):
- if self.config['source']:
- self.config['srcdir'] = self.config['source']
- super(SpidermonkeyBuild, self)._pre_config_lock(rw_config)
-
- if self.buildbot_config is None:
- self.info("Reading buildbot build properties...")
- self.read_buildbot_config()
-
- if self.buildbot_config:
- bb_props = [('mock_target', 'mock_target', None),
- ('hgurl', 'hgurl', None),
- ('clobberer_url', 'clobberer_url', 'https://api.pub.build.mozilla.org/clobberer/lastclobber'),
- ('force_clobber', 'force_clobber', None),
- ('branch', 'blob_upload_branch', None),
- ]
- buildbot_props = self.buildbot_config.get('properties', {})
- for bb_prop, cfg_prop, default in bb_props:
- if not self.config.get(cfg_prop) and buildbot_props.get(bb_prop, default):
- self.config[cfg_prop] = buildbot_props.get(bb_prop, default)
- self.config['is_automation'] = True
- else:
- self.config['is_automation'] = False
- self.config.setdefault('blob_upload_branch', 'devel')
-
- dirs = self.query_abs_dirs()
- replacements = self.config['env_replacements'].copy()
- for k,v in replacements.items():
- replacements[k] = v % dirs
-
- self.env = self.query_env(replace_dict=replacements,
- partial_env=self.config['partial_env'],
- purge_env=nuisance_env_vars)
- self.env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir']
- self.env['TOOLTOOL_DIR'] = dirs['abs_work_dir']
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = BaseScript.query_abs_dirs(self)
-
- abs_work_dir = abs_dirs['abs_work_dir']
- dirs = {
- 'shell_objdir':
- os.path.join(abs_work_dir, self.config['shell-objdir']),
- 'mozharness_scriptdir':
- os.path.abspath(os.path.dirname(__file__)),
- 'abs_analysis_dir':
- os.path.join(abs_work_dir, self.config['analysis-dir']),
- 'abs_analyzed_objdir':
- os.path.join(abs_work_dir, self.config['srcdir'], self.config['analysis-objdir']),
- 'analysis_scriptdir':
- os.path.join(self.config['srcdir'], self.config['analysis-scriptdir']),
- 'abs_tools_dir':
- os.path.join(abs_dirs['base_work_dir'], 'tools'),
- 'gecko_src':
- os.path.join(abs_work_dir, self.config['srcdir']),
- 'abs_blob_upload_dir':
- os.path.join(abs_work_dir, 'blobber_upload_dir'),
- }
-
- abs_dirs.update(dirs)
- self.abs_dirs = abs_dirs
-
- return self.abs_dirs
-
- def query_repo(self):
- if self.config.get('repo'):
- return self.config['repo']
- elif self.buildbot_config and 'properties' in self.buildbot_config:
- return self.config['hgurl'] + self.buildbot_config['properties']['repo_path']
- else:
- return None
-
- def query_revision(self):
- if 'revision' in self.buildbot_properties:
- revision = self.buildbot_properties['revision']
- elif self.buildbot_config and 'sourcestamp' in self.buildbot_config:
- revision = self.buildbot_config['sourcestamp']['revision']
- else:
- # Useful for local testing. In actual use, this would always be
- # None.
- revision = self.config.get('revision')
-
- return revision
-
- def query_branch(self):
- if self.buildbot_config and 'properties' in self.buildbot_config:
- return self.buildbot_config['properties']['branch']
- elif 'branch' in self.config:
- # Used for locally testing try vs non-try
- return self.config['branch']
- else:
- return os.path.basename(self.query_repo())
-
- def query_compiler_manifest(self):
- dirs = self.query_abs_dirs()
- manifest = os.path.join(dirs['abs_work_dir'], dirs['analysis_scriptdir'], self.config['compiler_manifest'])
- if os.path.exists(manifest):
- return manifest
- return os.path.join(dirs['abs_work_dir'], self.config['compiler_manifest'])
-
- def query_sixgill_manifest(self):
- dirs = self.query_abs_dirs()
- manifest = os.path.join(dirs['abs_work_dir'], dirs['analysis_scriptdir'], self.config['sixgill_manifest'])
- if os.path.exists(manifest):
- return manifest
- return os.path.join(dirs['abs_work_dir'], self.config['sixgill_manifest'])
-
- def query_buildid(self):
- if self.buildid:
- return self.buildid
- if self.buildbot_config and 'properties' in self.buildbot_config:
- self.buildid = self.buildbot_config['properties'].get('buildid')
- if not self.buildid:
- self.buildid = datetime.now().strftime("%Y%m%d%H%M%S")
- return self.buildid
-
- def query_upload_ssh_server(self):
- if self.buildbot_config and 'properties' in self.buildbot_config:
- return self.buildbot_config['properties']['upload_ssh_server']
- else:
- return self.config['upload_ssh_server']
-
- def query_upload_ssh_key(self):
- if self.buildbot_config and 'properties' in self.buildbot_config:
- key = self.buildbot_config['properties']['upload_ssh_key']
- else:
- key = self.config['upload_ssh_key']
- if self.mock_enabled and not key.startswith("/"):
- key = "/home/mock_mozilla/.ssh/" + key
- return key
-
- def query_upload_ssh_user(self):
- if self.buildbot_config and 'properties' in self.buildbot_config:
- return self.buildbot_config['properties']['upload_ssh_user']
- else:
- return self.config['upload_ssh_user']
-
- def query_product(self):
- if self.buildbot_config and 'properties' in self.buildbot_config:
- return self.buildbot_config['properties']['product']
- else:
- return self.config['product']
-
- def query_upload_remote_basepath(self):
- if self.config.get('upload_remote_basepath'):
- return self.config['upload_remote_basepath']
- else:
- return "/pub/mozilla.org/{product}".format(
- product=self.query_product(),
- )
-
- def query_upload_remote_baseuri(self):
- baseuri = self.config.get('upload_remote_baseuri')
- if self.buildbot_config and 'properties' in self.buildbot_config:
- buildprops = self.buildbot_config['properties']
- if 'upload_remote_baseuri' in buildprops:
- baseuri = buildprops['upload_remote_baseuri']
- return baseuri.strip("/") if baseuri else None
-
- def query_target(self):
- if self.buildbot_config and 'properties' in self.buildbot_config:
- return self.buildbot_config['properties']['platform']
- else:
- return self.config.get('target')
-
- def query_upload_path(self):
- branch = self.query_branch()
-
- common = {
- 'basepath': self.query_upload_remote_basepath(),
- 'branch': branch,
- 'target': self.query_target(),
- }
-
- if branch == 'try':
- if not self.config['enable_try_uploads']:
- return None
- try:
- user = self.buildbot_config['sourcestamp']['changes'][0]['who']
- except (KeyError, TypeError):
- user = "unknown"
- return "{basepath}/try-builds/{user}-{rev}/{branch}-{target}".format(
- user=user,
- rev=self.query_revision(),
- **common
- )
- else:
- return "{basepath}/tinderbox-builds/{branch}-{target}/{buildid}".format(
- buildid=self.query_buildid(),
- **common
- )
-
- def query_do_upload(self):
- if self.query_branch() == 'try':
- return self.config.get('enable_try_uploads')
- return True
-
- # Actions {{{2
- def purge(self):
- dirs = self.query_abs_dirs()
- self.info("purging, abs_upload_dir=" + dirs['abs_upload_dir'])
- PurgeMixin.clobber(
- self,
- always_clobber_dirs=[
- dirs['abs_upload_dir'],
- ],
- )
-
- def checkout_tools(self):
- dirs = self.query_abs_dirs()
-
- # If running from within a directory also passed as the --source dir,
- # this has the danger of clobbering <source>/tools/
- if self.config['source']:
- srcdir = self.config['source']
- if os.path.samefile(srcdir, os.path.dirname(dirs['abs_tools_dir'])):
- raise Exception("Cannot run from source checkout to avoid overwriting subdirs")
-
- rev = self.vcs_checkout(
- vcs='hg',
- branch="default",
- repo=self.config['tools_repo'],
- clean=False,
- dest=dirs['abs_tools_dir'],
- )
- self.set_buildbot_property("tools_revision", rev, write_to_file=True)
-
- def do_checkout_source(self):
- # --source option means to use an existing source directory instead of checking one out.
- if self.config['source']:
- return
-
- dirs = self.query_abs_dirs()
- dest = dirs['gecko_src']
-
- # Pre-create the directory to appease the share extension
- if not os.path.exists(dest):
- self.mkdir_p(dest)
-
- rev = self.vcs_checkout(
- repo=self.query_repo(),
- dest=dest,
- revision=self.query_revision(),
- branch=self.config.get('branch'),
- clean=True,
- )
- self.set_buildbot_property('source_revision', rev, write_to_file=True)
-
- def checkout_source(self):
- try:
- self.do_checkout_source()
- except Exception as e:
- self.fatal("checkout failed: " + str(e), exit_code=RETRY)
-
- def get_blobs(self):
- work_dir = self.query_abs_dirs()['abs_work_dir']
- if not os.path.exists(work_dir):
- self.mkdir_p(work_dir)
- self.tooltool_fetch(self.query_compiler_manifest(), output_dir=work_dir)
- self.tooltool_fetch(self.query_sixgill_manifest(), output_dir=work_dir)
-
- def clobber_shell(self):
- self.analysis.clobber_shell(self)
-
- def configure_shell(self):
- self.enable_mock()
-
- try:
- self.analysis.configure_shell(self)
- except HazardError as e:
- self.fatal(e, exit_code=FAILURE)
-
- self.disable_mock()
-
- def build_shell(self):
- self.enable_mock()
-
- try:
- self.analysis.build_shell(self)
- except HazardError as e:
- self.fatal(e, exit_code=FAILURE)
-
- self.disable_mock()
-
- def clobber_analysis(self):
- self.analysis.clobber(self)
-
- def setup_analysis(self):
- self.analysis.setup(self)
-
- def run_analysis(self):
- self.enable_mock()
-
- upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
- if not os.path.exists(upload_dir):
- self.mkdir_p(upload_dir)
-
- env = self.env.copy()
- env['MOZ_UPLOAD_DIR'] = upload_dir
-
- try:
- self.analysis.run(self, env=env, error_list=MakefileErrorList)
- except HazardError as e:
- self.fatal(e, exit_code=FAILURE)
-
- self.disable_mock()
-
- def collect_analysis_output(self):
- self.analysis.collect_output(self)
-
- def upload_analysis(self):
- if not self.config['is_automation']:
- return
-
- if not self.query_do_upload():
- self.info("Uploads disabled for this build. Skipping...")
- return
-
- self.enable_mock()
-
- try:
- self.analysis.upload_results(self)
- except HazardError as e:
- self.error(e)
- self.return_code = WARNINGS
-
- self.disable_mock()
-
- def check_expectations(self):
- try:
- self.analysis.check_expectations(self)
- except HazardError as e:
- self.fatal(e, exit_code=FAILURE)
-
-
-# main {{{1
-if __name__ == '__main__':
- myScript = SpidermonkeyBuild()
- myScript.run_and_exit()
diff --git a/testing/mozharness/scripts/talos_script.py b/testing/mozharness/scripts/talos_script.py
deleted file mode 100755
index dc4161193..000000000
--- a/testing/mozharness/scripts/talos_script.py
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-"""talos
-
-"""
-
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.mozilla.testing.talos import Talos
-
-if __name__ == '__main__':
- talos = Talos()
- talos.run_and_exit()
diff --git a/testing/mozharness/scripts/web_platform_tests.py b/testing/mozharness/scripts/web_platform_tests.py
deleted file mode 100755
index 7cd0e3842..000000000
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ /dev/null
@@ -1,258 +0,0 @@
-#!/usr/bin/env python
-# ***** BEGIN LICENSE BLOCK *****
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-# ***** END LICENSE BLOCK *****
-import copy
-import glob
-import json
-import os
-import sys
-
-# load modules from parent dir
-sys.path.insert(1, os.path.dirname(sys.path[0]))
-
-from mozharness.base.script import PreScriptAction
-from mozharness.base.vcs.vcsbase import MercurialScript
-from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
-from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options, TOOLTOOL_PLATFORM_DIR
-
-from mozharness.mozilla.structuredlog import StructuredOutputParser
-from mozharness.base.log import INFO
-
-class WebPlatformTest(TestingMixin, MercurialScript, BlobUploadMixin):
- config_options = [
- [['--test-type'], {
- "action": "extend",
- "dest": "test_type",
- "help": "Specify the test types to run."}
- ],
- [['--e10s'], {
- "action": "store_true",
- "dest": "e10s",
- "default": False,
- "help": "Run with e10s enabled"}
- ],
- [["--total-chunks"], {
- "action": "store",
- "dest": "total_chunks",
- "help": "Number of total chunks"}
- ],
- [["--this-chunk"], {
- "action": "store",
- "dest": "this_chunk",
- "help": "Number of this chunk"}
- ],
- [["--allow-software-gl-layers"], {
- "action": "store_true",
- "dest": "allow_software_gl_layers",
- "default": False,
- "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor."}]
- ] + copy.deepcopy(testing_config_options) + \
- copy.deepcopy(blobupload_config_options)
-
- def __init__(self, require_config_file=True):
- super(WebPlatformTest, self).__init__(
- config_options=self.config_options,
- all_actions=[
- 'clobber',
- 'read-buildbot-config',
- 'download-and-extract',
- 'fetch-geckodriver',
- 'create-virtualenv',
- 'pull',
- 'install',
- 'run-tests',
- ],
- require_config_file=require_config_file,
- config={'require_test_zip': True})
-
- # Surely this should be in the superclass
- c = self.config
- self.installer_url = c.get('installer_url')
- self.test_url = c.get('test_url')
- self.test_packages_url = c.get('test_packages_url')
- self.installer_path = c.get('installer_path')
- self.binary_path = c.get('binary_path')
- self.abs_app_dir = None
- self.geckodriver_path = None
-
- def query_abs_app_dir(self):
- """We can't set this in advance, because OSX install directories
- change depending on branding and opt/debug.
- """
- if self.abs_app_dir:
- return self.abs_app_dir
- if not self.binary_path:
- self.fatal("Can't determine abs_app_dir (binary_path not set!)")
- self.abs_app_dir = os.path.dirname(self.binary_path)
- return self.abs_app_dir
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(WebPlatformTest, self).query_abs_dirs()
-
- dirs = {}
- dirs['abs_app_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'application')
- dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
- dirs["abs_wpttest_dir"] = os.path.join(dirs['abs_test_install_dir'], "web-platform")
- dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
-
- abs_dirs.update(dirs)
- self.abs_dirs = abs_dirs
-
- return self.abs_dirs
-
- @PreScriptAction('create-virtualenv')
- def _pre_create_virtualenv(self, action):
- dirs = self.query_abs_dirs()
-
- requirements = os.path.join(dirs['abs_test_install_dir'],
- 'config',
- 'marionette_requirements.txt')
-
- self.register_virtualenv_module(requirements=[requirements],
- two_pass=True)
-
- def _query_cmd(self):
- if not self.binary_path:
- self.fatal("Binary path could not be determined")
- #And exit
-
- c = self.config
- dirs = self.query_abs_dirs()
- abs_app_dir = self.query_abs_app_dir()
- run_file_name = "runtests.py"
-
- cmd = [self.query_python_path('python'), '-u']
- cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name))
-
- # Make sure that the logging directory exists
- if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
- self.fatal("Could not create blobber upload directory")
- # Exit
-
- cmd += ["--log-raw=-",
- "--log-raw=%s" % os.path.join(dirs["abs_blob_upload_dir"],
- "wpt_raw.log"),
- "--log-errorsummary=%s" % os.path.join(dirs["abs_blob_upload_dir"],
- "wpt_errorsummary.log"),
- "--binary=%s" % self.binary_path,
- "--symbols-path=%s" % self.query_symbols_url(),
- "--stackwalk-binary=%s" % self.query_minidump_stackwalk(),
- "--stackfix-dir=%s" % os.path.join(dirs["abs_test_install_dir"], "bin")]
-
- for test_type in c.get("test_type", []):
- cmd.append("--test-type=%s" % test_type)
-
- if not c["e10s"]:
- cmd.append("--disable-e10s")
-
- for opt in ["total_chunks", "this_chunk"]:
- val = c.get(opt)
- if val:
- cmd.append("--%s=%s" % (opt.replace("_", "-"), val))
-
- if "wdspec" in c.get("test_type", []):
- assert self.geckodriver_path is not None
- cmd.append("--webdriver-binary=%s" % self.geckodriver_path)
-
- options = list(c.get("options", []))
-
- str_format_values = {
- 'binary_path': self.binary_path,
- 'test_path': dirs["abs_wpttest_dir"],
- 'test_install_path': dirs["abs_test_install_dir"],
- 'abs_app_dir': abs_app_dir,
- 'abs_work_dir': dirs["abs_work_dir"]
- }
-
- try_options, try_tests = self.try_args("web-platform-tests")
-
- cmd.extend(self.query_options(options,
- try_options,
- str_format_values=str_format_values))
- cmd.extend(self.query_tests_args(try_tests,
- str_format_values=str_format_values))
-
- return cmd
-
- def download_and_extract(self):
- super(WebPlatformTest, self).download_and_extract(
- extract_dirs=["bin/*",
- "config/*",
- "mozbase/*",
- "marionette/*",
- "tools/wptserve/*",
- "web-platform/*"],
- suite_categories=["web-platform"])
-
- def fetch_geckodriver(self):
- c = self.config
- dirs = self.query_abs_dirs()
-
- platform_name = self.platform_name()
-
- if "wdspec" not in c.get("test_type", []):
- return
-
- if platform_name != "linux64":
- self.fatal("Don't have a geckodriver for %s" % platform_name)
-
- tooltool_path = os.path.join(dirs["abs_test_install_dir"],
- "config",
- "tooltool-manifests",
- TOOLTOOL_PLATFORM_DIR[platform_name],
- "geckodriver.manifest")
-
- with open(tooltool_path) as f:
- manifest = json.load(f)
-
- assert len(manifest) == 1
- geckodriver_filename = manifest[0]["filename"]
- assert geckodriver_filename.endswith(".tar.gz")
-
- self.tooltool_fetch(
- manifest=tooltool_path,
- output_dir=dirs['abs_work_dir'],
- cache=c.get('tooltool_cache')
- )
-
- compressed_path = os.path.join(dirs['abs_work_dir'], geckodriver_filename)
- tar = self.query_exe('tar', return_type="list")
- self.run_command(tar + ["xf", compressed_path], cwd=dirs['abs_work_dir'],
- halt_on_failure=True, fatal_exit_code=3)
- self.geckodriver_path = os.path.join(dirs['abs_work_dir'], "geckodriver")
-
- def run_tests(self):
- dirs = self.query_abs_dirs()
- cmd = self._query_cmd()
-
- parser = StructuredOutputParser(config=self.config,
- log_obj=self.log_obj,
- log_compact=True)
-
- env = {'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']}
-
- if self.config['allow_software_gl_layers']:
- env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1'
-
- env = self.query_env(partial_env=env, log_level=INFO)
-
- return_code = self.run_command(cmd,
- cwd=dirs['abs_work_dir'],
- output_timeout=1000,
- output_parser=parser,
- env=env)
-
- tbpl_status, log_level = parser.evaluate_parser(return_code)
-
- self.buildbot_status(tbpl_status, level=log_level)
-
-
-# main {{{1
-if __name__ == '__main__':
- web_platform_tests = WebPlatformTest()
- web_platform_tests.run_and_exit()
diff --git a/testing/mozharness/setup.cfg b/testing/mozharness/setup.cfg
deleted file mode 100644
index d8057aec1..000000000
--- a/testing/mozharness/setup.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-[nosetests]
-exclude=TestingMixin
diff --git a/testing/mozharness/setup.py b/testing/mozharness/setup.py
deleted file mode 100644
index 5bcb36d63..000000000
--- a/testing/mozharness/setup.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import os
-from setuptools import setup, find_packages
-
-try:
- here = os.path.dirname(os.path.abspath(__file__))
- description = open(os.path.join(here, 'README.txt')).read()
-except IOError:
- description = ''
-
-import mozharness
-version = mozharness.version_string
-
-dependencies = ['virtualenv', 'mock', "coverage", "nose", "pylint", "pyflakes"]
-try:
- import json
-except ImportError:
- dependencies.append('simplejson')
-
-setup(name='mozharness',
- version=version,
- description="Mozharness is a configuration-driven script harness with full logging that allows production infrastructure and individual developers to use the same scripts. ",
- long_description=description,
- classifiers=[], # Get strings from http://www.python.org/pypi?%3Aaction=list_classifiers
- author='Aki Sasaki',
- author_email='aki@mozilla.com',
- url='https://hg.mozilla.org/build/mozharness/',
- license='MPL',
- packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
- include_package_data=True,
- zip_safe=False,
- install_requires=dependencies,
- entry_points="""
- # -*- Entry points: -*-
- """,
- )
diff --git a/testing/mozharness/test/README b/testing/mozharness/test/README
deleted file mode 100644
index 889c8a83d..000000000
--- a/testing/mozharness/test/README
+++ /dev/null
@@ -1,2 +0,0 @@
-test/ : non-network-dependent unit tests
-test/networked/ : network-dependent unit tests.
diff --git a/testing/mozharness/test/helper_files/.noserc b/testing/mozharness/test/helper_files/.noserc
deleted file mode 100644
index e6f21cf31..000000000
--- a/testing/mozharness/test/helper_files/.noserc
+++ /dev/null
@@ -1,2 +0,0 @@
-[nosetests]
-with-xunit=1
diff --git a/testing/mozharness/test/helper_files/archives/archive.tar b/testing/mozharness/test/helper_files/archives/archive.tar
deleted file mode 100644
index 1dc094198..000000000
--- a/testing/mozharness/test/helper_files/archives/archive.tar
+++ /dev/null
Binary files differ
diff --git a/testing/mozharness/test/helper_files/archives/archive.tar.bz2 b/testing/mozharness/test/helper_files/archives/archive.tar.bz2
deleted file mode 100644
index c393ea4b8..000000000
--- a/testing/mozharness/test/helper_files/archives/archive.tar.bz2
+++ /dev/null
Binary files differ
diff --git a/testing/mozharness/test/helper_files/archives/archive.tar.gz b/testing/mozharness/test/helper_files/archives/archive.tar.gz
deleted file mode 100644
index 0fbfa39b1..000000000
--- a/testing/mozharness/test/helper_files/archives/archive.tar.gz
+++ /dev/null
Binary files differ
diff --git a/testing/mozharness/test/helper_files/archives/archive.zip b/testing/mozharness/test/helper_files/archives/archive.zip
deleted file mode 100644
index aa2fb34c1..000000000
--- a/testing/mozharness/test/helper_files/archives/archive.zip
+++ /dev/null
Binary files differ
diff --git a/testing/mozharness/test/helper_files/archives/archive_invalid_filename.zip b/testing/mozharness/test/helper_files/archives/archive_invalid_filename.zip
deleted file mode 100644
index 20bdc5acd..000000000
--- a/testing/mozharness/test/helper_files/archives/archive_invalid_filename.zip
+++ /dev/null
Binary files differ
diff --git a/testing/mozharness/test/helper_files/archives/reference/bin/script.sh b/testing/mozharness/test/helper_files/archives/reference/bin/script.sh
deleted file mode 100755
index 134f2933c..000000000
--- a/testing/mozharness/test/helper_files/archives/reference/bin/script.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-echo Hello world!
diff --git a/testing/mozharness/test/helper_files/archives/reference/lorem.txt b/testing/mozharness/test/helper_files/archives/reference/lorem.txt
deleted file mode 100644
index d2cf010d3..000000000
--- a/testing/mozharness/test/helper_files/archives/reference/lorem.txt
+++ /dev/null
@@ -1 +0,0 @@
-Lorem ipsum dolor sit amet.
diff --git a/testing/mozharness/test/helper_files/create_archives.sh b/testing/mozharness/test/helper_files/create_archives.sh
deleted file mode 100755
index 314b55d27..000000000
--- a/testing/mozharness/test/helper_files/create_archives.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-# Script to auto-generate the different archive types under the archives directory.
-
-cd archives
-
-rm archive.*
-
-tar cf archive.tar -C reference .
-gzip -fk archive.tar >archive.tar.gz
-bzip2 -fk archive.tar >archive.tar.bz2
-cd reference && zip ../archive.zip -r * && cd ..
diff --git a/testing/mozharness/test/helper_files/init_hgrepo.sh b/testing/mozharness/test/helper_files/init_hgrepo.sh
deleted file mode 100755
index c978ebe73..000000000
--- a/testing/mozharness/test/helper_files/init_hgrepo.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-# Set up an hg repo for testing
-dest=$1
-if [ -z "$dest" ]; then
- echo You must specify a destination directory 1>&2
- exit 1
-fi
-
-rm -rf $dest
-mkdir $dest
-cd $dest
-hg init
-
-echo "Hello world $RANDOM" > hello.txt
-hg add hello.txt
-hg commit -m "Adding hello"
-
-hg branch branch2 > /dev/null
-echo "So long, farewell" >> hello.txt
-hg commit -m "Changing hello on branch"
-
-hg checkout default
-echo "Is this thing on?" >> hello.txt
-hg commit -m "Last change on default"
diff --git a/testing/mozharness/test/helper_files/locales.json b/testing/mozharness/test/helper_files/locales.json
deleted file mode 100644
index c9056b1d1..000000000
--- a/testing/mozharness/test/helper_files/locales.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "ar": {
- "revision": "default",
- "platforms": ["maemo"]
- },
- "be": {
- "revision": "default",
- "platforms": ["maemo"]
- },
- "de": {
- "revision": "default",
- "platforms": ["maemo", "maemo-multilocale", "android-multilocale"]
- },
- "es-ES": {
- "revision": "default",
- "platforms": ["maemo", "maemo-multilocale", "android-multilocale"]
- }
-}
diff --git a/testing/mozharness/test/helper_files/locales.txt b/testing/mozharness/test/helper_files/locales.txt
deleted file mode 100644
index 0b65ab76d..000000000
--- a/testing/mozharness/test/helper_files/locales.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-ar
-be
-de
-es-ES
diff --git a/testing/mozharness/test/hgrc b/testing/mozharness/test/hgrc
deleted file mode 100644
index 85e670518..000000000
--- a/testing/mozharness/test/hgrc
+++ /dev/null
@@ -1,9 +0,0 @@
-[extensions]
-mq =
-purge =
-rebase =
-share =
-transplant =
-
-[ui]
-username = tester <tester@example.com>
diff --git a/testing/mozharness/test/pip-freeze.example.txt b/testing/mozharness/test/pip-freeze.example.txt
deleted file mode 100644
index 56e06923f..000000000
--- a/testing/mozharness/test/pip-freeze.example.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-MakeItSo==0.2.6
-PyYAML==3.10
-Tempita==0.5.1
-WebOb==1.2b3
--e hg+http://k0s.org/mozilla/hg/configuration@35416ad140982c11eba0a2d6b96d683f53429e94#egg=configuration-dev
-coverage==3.5.1
--e hg+http://k0s.org/mozilla/hg/jetperf@4645ae34d2c41a353dcdbd856b486b6d3faabb99#egg=jetperf-dev
-logilab-astng==0.23.1
-logilab-common==0.57.1
-mozdevice==0.2
--e hg+https://hg.mozilla.org/build/mozharness@df6b7f1e14d8c472125ef7a77b8a3b40c96ae181#egg=mozharness-jetperf
-mozhttpd==0.3
-mozinfo==0.3.3
-nose==1.1.2
-pyflakes==0.5.0
-pylint==0.25.1
--e hg+https://hg.mozilla.org/build/talos@ee5c0b090d808e81a8fc5ba5f96b012797b3e785#egg=talos-dev
-virtualenv==1.7.1.2
-wsgiref==0.1.2
diff --git a/testing/mozharness/test/test_base_config.py b/testing/mozharness/test/test_base_config.py
deleted file mode 100644
index 42ec7a641..000000000
--- a/testing/mozharness/test/test_base_config.py
+++ /dev/null
@@ -1,308 +0,0 @@
-import os
-import unittest
-
-JSON_TYPE = None
-try:
- import simplejson as json
- assert json
-except ImportError:
- import json
- JSON_TYPE = 'json'
-else:
- JSON_TYPE = 'simplejson'
-
-import mozharness.base.config as config
-from copy import deepcopy
-
-MH_DIR = os.path.dirname(os.path.dirname(__file__))
-
-
-class TestParseConfigFile(unittest.TestCase):
- def _get_json_config(self, filename=os.path.join(MH_DIR, "configs", "test", "test.json"),
- output='dict'):
- fh = open(filename)
- contents = json.load(fh)
- fh.close()
- if 'output' == 'dict':
- return dict(contents)
- else:
- return contents
-
- def _get_python_config(self, filename=os.path.join(MH_DIR, "configs", "test", "test.py"),
- output='dict'):
- global_dict = {}
- local_dict = {}
- execfile(filename, global_dict, local_dict)
- return local_dict['config']
-
- def test_json_config(self):
- c = config.BaseConfig(initial_config_file='test/test.json')
- content_dict = self._get_json_config()
- for key in content_dict.keys():
- self.assertEqual(content_dict[key], c._config[key])
-
- def test_python_config(self):
- c = config.BaseConfig(initial_config_file='test/test.py')
- config_dict = self._get_python_config()
- for key in config_dict.keys():
- self.assertEqual(config_dict[key], c._config[key])
-
- def test_illegal_config(self):
- self.assertRaises(IOError, config.parse_config_file, "this_file_does_not_exist.py", search_path="yadda")
-
- def test_illegal_suffix(self):
- self.assertRaises(RuntimeError, config.parse_config_file, "test/test.illegal_suffix")
-
- def test_malformed_json(self):
- if JSON_TYPE == 'simplejson':
- self.assertRaises(json.decoder.JSONDecodeError, config.parse_config_file, "test/test_malformed.json")
- else:
- self.assertRaises(ValueError, config.parse_config_file, "test/test_malformed.json")
-
- def test_malformed_python(self):
- self.assertRaises(SyntaxError, config.parse_config_file, "test/test_malformed.py")
-
- def test_multiple_config_files_override_string(self):
- c = config.BaseConfig(initial_config_file='test/test.py')
- c.parse_args(['--cfg', 'test/test_override.py,test/test_override2.py'])
- self.assertEqual(c._config['override_string'], 'yay')
-
- def test_multiple_config_files_override_list(self):
- c = config.BaseConfig(initial_config_file='test/test.py')
- c.parse_args(['--cfg', 'test/test_override.py,test/test_override2.py'])
- self.assertEqual(c._config['override_list'], ['yay', 'worked'])
-
- def test_multiple_config_files_override_dict(self):
- c = config.BaseConfig(initial_config_file='test/test.py')
- c.parse_args(['--cfg', 'test/test_override.py,test/test_override2.py'])
- self.assertEqual(c._config['override_dict'], {'yay': 'worked'})
-
- def test_multiple_config_files_keep_string(self):
- c = config.BaseConfig(initial_config_file='test/test.py')
- c.parse_args(['--cfg', 'test/test_override.py,test/test_override2.py'])
- self.assertEqual(c._config['keep_string'], "don't change me")
-
- def test_optional_config_files_override_value(self):
- c = config.BaseConfig(initial_config_file='test/test.py')
- c.parse_args(['--cfg', 'test/test_override.py,test/test_override2.py',
- '--opt-cfg', 'test/test_optional.py'])
- self.assertEqual(c._config['opt_override'], "new stuff")
-
- def test_optional_config_files_missing_config(self):
- c = config.BaseConfig(initial_config_file='test/test.py')
- c.parse_args(['--cfg', 'test/test_override.py,test/test_override2.py',
- '--opt-cfg', 'test/test_optional.py,does_not_exist.py'])
- self.assertEqual(c._config['opt_override'], "new stuff")
-
- def test_optional_config_files_keep_string(self):
- c = config.BaseConfig(initial_config_file='test/test.py')
- c.parse_args(['--cfg', 'test/test_override.py,test/test_override2.py',
- '--opt-cfg', 'test/test_optional.py'])
- self.assertEqual(c._config['keep_string'], "don't change me")
-
-
-class TestReadOnlyDict(unittest.TestCase):
- control_dict = {
- 'b': '2',
- 'c': {'d': '4'},
- 'e': ['f', 'g'],
- 'e': ['f', 'g', {'turtles': ['turtle1']}],
- 'd': {
- 'turtles': ['turtle1']
- }
- }
-
- def get_unlocked_ROD(self):
- r = config.ReadOnlyDict(self.control_dict)
- return r
-
- def get_locked_ROD(self):
- r = config.ReadOnlyDict(self.control_dict)
- r.lock()
- return r
-
- def test_create_ROD(self):
- r = self.get_unlocked_ROD()
- self.assertEqual(r, self.control_dict,
- msg="can't transfer dict to ReadOnlyDict")
-
- def test_pop_item(self):
- r = self.get_unlocked_ROD()
- r.popitem()
- self.assertEqual(len(r), len(self.control_dict) - 1,
- msg="can't popitem() ReadOnlyDict when unlocked")
-
- def test_pop(self):
- r = self.get_unlocked_ROD()
- r.pop('e')
- self.assertEqual(len(r), len(self.control_dict) - 1,
- msg="can't pop() ReadOnlyDict when unlocked")
-
- def test_set(self):
- r = self.get_unlocked_ROD()
- r['e'] = 'yarrr'
- self.assertEqual(r['e'], 'yarrr',
- msg="can't set var in ReadOnlyDict when unlocked")
-
- def test_del(self):
- r = self.get_unlocked_ROD()
- del r['e']
- self.assertEqual(len(r), len(self.control_dict) - 1,
- msg="can't del in ReadOnlyDict when unlocked")
-
- def test_clear(self):
- r = self.get_unlocked_ROD()
- r.clear()
- self.assertEqual(r, {},
- msg="can't clear() ReadOnlyDict when unlocked")
-
- def test_set_default(self):
- r = self.get_unlocked_ROD()
- for key in self.control_dict.keys():
- r.setdefault(key, self.control_dict[key])
- self.assertEqual(r, self.control_dict,
- msg="can't setdefault() ReadOnlyDict when unlocked")
-
- def test_locked_set(self):
- r = self.get_locked_ROD()
- # TODO use |with self.assertRaises(AssertionError):| if/when we're
- # all on 2.7.
- try:
- r['e'] = 2
- except:
- pass
- else:
- self.assertEqual(0, 1, msg="can set r['e'] when locked")
-
- def test_locked_del(self):
- r = self.get_locked_ROD()
- try:
- del r['e']
- except:
- pass
- else:
- self.assertEqual(0, 1, "can del r['e'] when locked")
-
- def test_locked_popitem(self):
- r = self.get_locked_ROD()
- self.assertRaises(AssertionError, r.popitem)
-
- def test_locked_update(self):
- r = self.get_locked_ROD()
- self.assertRaises(AssertionError, r.update, {})
-
- def test_locked_set_default(self):
- r = self.get_locked_ROD()
- self.assertRaises(AssertionError, r.setdefault, {})
-
- def test_locked_pop(self):
- r = self.get_locked_ROD()
- self.assertRaises(AssertionError, r.pop)
-
- def test_locked_clear(self):
- r = self.get_locked_ROD()
- self.assertRaises(AssertionError, r.clear)
-
- def test_locked_second_level_dict_pop(self):
- r = self.get_locked_ROD()
- self.assertRaises(AssertionError, r['c'].update, {})
-
- def test_locked_second_level_list_pop(self):
- r = self.get_locked_ROD()
- with self.assertRaises(AttributeError):
- r['e'].pop()
-
- def test_locked_third_level_mutate(self):
- r = self.get_locked_ROD()
- with self.assertRaises(AttributeError):
- r['d']['turtles'].append('turtle2')
-
- def test_locked_object_in_tuple_mutate(self):
- r = self.get_locked_ROD()
- with self.assertRaises(AttributeError):
- r['e'][2]['turtles'].append('turtle2')
-
- def test_locked_second_level_dict_pop2(self):
- r = self.get_locked_ROD()
- self.assertRaises(AssertionError, r['c'].update, {})
-
- def test_locked_second_level_list_pop2(self):
- r = self.get_locked_ROD()
- with self.assertRaises(AttributeError):
- r['e'].pop()
-
- def test_locked_third_level_mutate2(self):
- r = self.get_locked_ROD()
- with self.assertRaises(AttributeError):
- r['d']['turtles'].append('turtle2')
-
- def test_locked_object_in_tuple_mutate2(self):
- r = self.get_locked_ROD()
- with self.assertRaises(AttributeError):
- r['e'][2]['turtles'].append('turtle2')
-
- def test_locked_deepcopy_set(self):
- r = self.get_locked_ROD()
- c = deepcopy(r)
- c['e'] = 'hey'
- self.assertEqual(c['e'], 'hey', "can't set var in ROD after deepcopy")
-
-
-class TestActions(unittest.TestCase):
- all_actions = ['a', 'b', 'c', 'd', 'e']
- default_actions = ['b', 'c', 'd']
-
- def test_verify_actions(self):
- c = config.BaseConfig(initial_config_file='test/test.json')
- try:
- c.verify_actions(['not_a_real_action'])
- except:
- pass
- else:
- self.assertEqual(0, 1, msg="verify_actions() didn't die on invalid action")
- c = config.BaseConfig(initial_config_file='test/test.json')
- returned_actions = c.verify_actions(c.all_actions)
- self.assertEqual(c.all_actions, returned_actions,
- msg="returned actions from verify_actions() changed")
-
- def test_default_actions(self):
- c = config.BaseConfig(default_actions=self.default_actions,
- all_actions=self.all_actions,
- initial_config_file='test/test.json')
- self.assertEqual(self.default_actions, c.get_actions(),
- msg="default_actions broken")
-
- def test_no_action1(self):
- c = config.BaseConfig(default_actions=self.default_actions,
- all_actions=self.all_actions,
- initial_config_file='test/test.json')
- c.parse_args(args=['foo', '--no-action', 'a'])
- self.assertEqual(self.default_actions, c.get_actions(),
- msg="--no-ACTION broken")
-
- def test_no_action2(self):
- c = config.BaseConfig(default_actions=self.default_actions,
- all_actions=self.all_actions,
- initial_config_file='test/test.json')
- c.parse_args(args=['foo', '--no-c'])
- self.assertEqual(['b', 'd'], c.get_actions(),
- msg="--no-ACTION broken")
-
- def test_add_action(self):
- c = config.BaseConfig(default_actions=self.default_actions,
- all_actions=self.all_actions,
- initial_config_file='test/test.json')
- c.parse_args(args=['foo', '--add-action', 'e'])
- self.assertEqual(['b', 'c', 'd', 'e'], c.get_actions(),
- msg="--add-action ACTION broken")
-
- def test_only_action(self):
- c = config.BaseConfig(default_actions=self.default_actions,
- all_actions=self.all_actions,
- initial_config_file='test/test.json')
- c.parse_args(args=['foo', '--a', '--e'])
- self.assertEqual(['a', 'e'], c.get_actions(),
- msg="--ACTION broken")
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/testing/mozharness/test/test_base_diskutils.py b/testing/mozharness/test/test_base_diskutils.py
deleted file mode 100644
index 79d36692f..000000000
--- a/testing/mozharness/test/test_base_diskutils.py
+++ /dev/null
@@ -1,84 +0,0 @@
-import mock
-import unittest
-from mozharness.base.diskutils import convert_to, DiskutilsError, DiskSize, DiskInfo
-
-
-class TestDiskutils(unittest.TestCase):
- def test_convert_to(self):
- # 0 is 0 regardless from_unit/to_unit
- self.assertTrue(convert_to(size=0, from_unit='GB', to_unit='MB') == 0)
- size = 524288 # 512 * 1024
- # converting from/to same unit
- self.assertTrue(convert_to(size=size, from_unit='MB', to_unit='MB') == size)
-
- self.assertTrue(convert_to(size=size, from_unit='MB', to_unit='GB') == 512)
-
- self.assertRaises(DiskutilsError,
- lambda: convert_to(size='a string', from_unit='MB', to_unit='MB'))
- self.assertRaises(DiskutilsError,
- lambda: convert_to(size=0, from_unit='foo', to_unit='MB'))
- self.assertRaises(DiskutilsError,
- lambda: convert_to(size=0, from_unit='MB', to_unit='foo'))
-
-
-class TestDiskInfo(unittest.TestCase):
-
- def testDiskinfo_to(self):
- di = DiskInfo()
- self.assertTrue(di.unit == 'bytes')
- self.assertTrue(di.free == 0)
- self.assertTrue(di.used == 0)
- self.assertTrue(di.total == 0)
- # convert to GB
- di._to('GB')
- self.assertTrue(di.unit == 'GB')
- self.assertTrue(di.free == 0)
- self.assertTrue(di.used == 0)
- self.assertTrue(di.total == 0)
-
-
-class MockStatvfs(object):
- def __init__(self):
- self.f_bsize = 0
- self.f_frsize = 0
- self.f_blocks = 0
- self.f_bfree = 0
- self.f_bavail = 0
- self.f_files = 0
- self.f_ffree = 0
- self.f_favail = 0
- self.f_flag = 0
- self.f_namemax = 0
-
-
-class TestDiskSpace(unittest.TestCase):
-
- @mock.patch('mozharness.base.diskutils.os')
- def testDiskSpacePosix(self, mock_os):
- ds = MockStatvfs()
- mock_os.statvfs.return_value = ds
- di = DiskSize()._posix_size('/')
- self.assertTrue(di.unit == 'bytes')
- self.assertTrue(di.free == 0)
- self.assertTrue(di.used == 0)
- self.assertTrue(di.total == 0)
-
- @mock.patch('mozharness.base.diskutils.ctypes')
- def testDiskSpaceWindows(self, mock_ctypes):
- mock_ctypes.windll.kernel32.GetDiskFreeSpaceExA.return_value = 0
- mock_ctypes.windll.kernel32.GetDiskFreeSpaceExW.return_value = 0
- di = DiskSize()._windows_size('/c/')
- self.assertTrue(di.unit == 'bytes')
- self.assertTrue(di.free == 0)
- self.assertTrue(di.used == 0)
- self.assertTrue(di.total == 0)
-
- @mock.patch('mozharness.base.diskutils.os')
- @mock.patch('mozharness.base.diskutils.ctypes')
- def testUnspportedPlafrom(self, mock_ctypes, mock_os):
- mock_os.statvfs.side_effect = AttributeError('')
- self.assertRaises(AttributeError, lambda: DiskSize()._posix_size('/'))
- mock_ctypes.windll.kernel32.GetDiskFreeSpaceExW.side_effect = AttributeError('')
- mock_ctypes.windll.kernel32.GetDiskFreeSpaceExA.side_effect = AttributeError('')
- self.assertRaises(AttributeError, lambda: DiskSize()._windows_size('/'))
- self.assertRaises(DiskutilsError, lambda: DiskSize().get_size(path='/', unit='GB'))
diff --git a/testing/mozharness/test/test_base_log.py b/testing/mozharness/test/test_base_log.py
deleted file mode 100644
index 0947834f7..000000000
--- a/testing/mozharness/test/test_base_log.py
+++ /dev/null
@@ -1,42 +0,0 @@
-import os
-import shutil
-import subprocess
-import unittest
-
-import mozharness.base.log as log
-
-tmp_dir = "test_log_dir"
-log_name = "test"
-
-
-def clean_log_dir():
- if os.path.exists(tmp_dir):
- shutil.rmtree(tmp_dir)
-
-
-def get_log_file_path(level=None):
- if level:
- return os.path.join(tmp_dir, "%s_%s.log" % (log_name, level))
- return os.path.join(tmp_dir, "%s.log" % log_name)
-
-
-class TestLog(unittest.TestCase):
- def setUp(self):
- clean_log_dir()
-
- def tearDown(self):
- clean_log_dir()
-
- def test_log_dir(self):
- fh = open(tmp_dir, 'w')
- fh.write("foo")
- fh.close()
- l = log.SimpleFileLogger(log_dir=tmp_dir, log_name=log_name,
- log_to_console=False)
- self.assertTrue(os.path.exists(tmp_dir))
- l.log_message('blah')
- self.assertTrue(os.path.exists(get_log_file_path()))
- del(l)
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/testing/mozharness/test/test_base_parallel.py b/testing/mozharness/test/test_base_parallel.py
deleted file mode 100644
index 8302be43a..000000000
--- a/testing/mozharness/test/test_base_parallel.py
+++ /dev/null
@@ -1,26 +0,0 @@
-import unittest
-
-from mozharness.base.parallel import ChunkingMixin
-
-
-class TestChunkingMixin(unittest.TestCase):
- def setUp(self):
- self.c = ChunkingMixin()
-
- def test_one_chunk(self):
- self.assertEquals(self.c.query_chunked_list([1, 3, 2], 1, 1), [1, 3, 2])
-
- def test_sorted(self):
- self.assertEquals(self.c.query_chunked_list([1, 3, 2], 1, 1, sort=True), [1, 2, 3])
-
- def test_first_chunk(self):
- self.assertEquals(self.c.query_chunked_list([4, 5, 4, 3], 1, 2), [4, 5])
-
- def test_last_chunk(self):
- self.assertEquals(self.c.query_chunked_list([1, 4, 5, 7, 5, 6], 3, 3), [5, 6])
-
- def test_not_evenly_divisble(self):
- thing = [1, 3, 6, 4, 3, 2, 6]
- self.assertEquals(self.c.query_chunked_list(thing, 1, 3), [1, 3, 6])
- self.assertEquals(self.c.query_chunked_list(thing, 2, 3), [4, 3])
- self.assertEquals(self.c.query_chunked_list(thing, 3, 3), [2, 6])
diff --git a/testing/mozharness/test/test_base_python.py b/testing/mozharness/test/test_base_python.py
deleted file mode 100644
index c013576f0..000000000
--- a/testing/mozharness/test/test_base_python.py
+++ /dev/null
@@ -1,37 +0,0 @@
-import os
-import unittest
-
-import mozharness.base.python as python
-
-here = os.path.dirname(os.path.abspath(__file__))
-
-
-class TestVirtualenvMixin(unittest.TestCase):
- def test_package_versions(self):
- example = os.path.join(here, 'pip-freeze.example.txt')
- output = file(example).read()
- mixin = python.VirtualenvMixin()
- packages = mixin.package_versions(output)
-
- # from the file
- expected = {'MakeItSo': '0.2.6',
- 'PyYAML': '3.10',
- 'Tempita': '0.5.1',
- 'WebOb': '1.2b3',
- 'coverage': '3.5.1',
- 'logilab-astng': '0.23.1',
- 'logilab-common': '0.57.1',
- 'mozdevice': '0.2',
- 'mozhttpd': '0.3',
- 'mozinfo': '0.3.3',
- 'nose': '1.1.2',
- 'pyflakes': '0.5.0',
- 'pylint': '0.25.1',
- 'virtualenv': '1.7.1.2',
- 'wsgiref': '0.1.2'}
-
- self.assertEqual(packages, expected)
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/testing/mozharness/test/test_base_script.py b/testing/mozharness/test/test_base_script.py
deleted file mode 100644
index c069a82f3..000000000
--- a/testing/mozharness/test/test_base_script.py
+++ /dev/null
@@ -1,898 +0,0 @@
-import gc
-import mock
-import os
-import re
-import shutil
-import tempfile
-import types
-import unittest
-PYWIN32 = False
-if os.name == 'nt':
- try:
- import win32file
- PYWIN32 = True
- except:
- pass
-
-
-import mozharness.base.errors as errors
-import mozharness.base.log as log
-from mozharness.base.log import DEBUG, INFO, WARNING, ERROR, CRITICAL, FATAL, IGNORE
-import mozharness.base.script as script
-from mozharness.base.config import parse_config_file
-
-
-here = os.path.dirname(os.path.abspath(__file__))
-
-test_string = '''foo
-bar
-baz'''
-
-
-class CleanupObj(script.ScriptMixin, log.LogMixin):
- def __init__(self):
- super(CleanupObj, self).__init__()
- self.log_obj = None
- self.config = {'log_level': ERROR}
-
-
-def cleanup(files=None):
- files = files or []
- files.extend(('test_logs', 'test_dir', 'tmpfile_stdout', 'tmpfile_stderr'))
- gc.collect()
- c = CleanupObj()
- for f in files:
- c.rmtree(f)
-
-
-def get_debug_script_obj():
- s = script.BaseScript(config={'log_type': 'multi',
- 'log_level': DEBUG},
- initial_config_file='test/test.json')
- return s
-
-
-def _post_fatal(self, **kwargs):
- fh = open('tmpfile_stdout', 'w')
- print >>fh, test_string
- fh.close()
-
-
-# TestScript {{{1
-class TestScript(unittest.TestCase):
- def setUp(self):
- cleanup()
- self.s = None
- self.tmpdir = tempfile.mkdtemp(suffix='.mozharness')
-
- def tearDown(self):
- # Close the logfile handles, or windows can't remove the logs
- if hasattr(self, 's') and isinstance(self.s, object):
- del(self.s)
- cleanup([self.tmpdir])
-
- # test _dump_config_hierarchy() when --dump-config-hierarchy is passed
- def test_dump_config_hierarchy_valid_files_len(self):
- try:
- self.s = script.BaseScript(
- initial_config_file='test/test.json',
- option_args=['--cfg', 'test/test_override.py,test/test_override2.py'],
- config={'dump_config_hierarchy': True}
- )
- except SystemExit:
- local_cfg_files = parse_config_file('test_logs/localconfigfiles.json')
- # first let's see if the correct number of config files were
- # realized
- self.assertEqual(
- len(local_cfg_files), 4,
- msg="--dump-config-hierarchy dumped wrong number of config files"
- )
-
- def test_dump_config_hierarchy_keys_unique_and_valid(self):
- try:
- self.s = script.BaseScript(
- initial_config_file='test/test.json',
- option_args=['--cfg', 'test/test_override.py,test/test_override2.py'],
- config={'dump_config_hierarchy': True}
- )
- except SystemExit:
- local_cfg_files = parse_config_file('test_logs/localconfigfiles.json')
- # now let's see if only unique items were added from each config
- t_override = local_cfg_files.get('test/test_override.py', {})
- self.assertTrue(
- t_override.get('keep_string') == "don't change me" and len(t_override.keys()) == 1,
- msg="--dump-config-hierarchy dumped wrong keys/value for "
- "`test/test_override.py`. There should only be one "
- "item and it should be unique to all the other "
- "items in test_log/localconfigfiles.json."
- )
-
- def test_dump_config_hierarchy_matches_self_config(self):
- try:
- ######
- # we need temp_cfg because self.s will be gcollected (NoneType) by
- # the time we get to SystemExit exception
- # temp_cfg will differ from self.s.config because of
- # 'dump_config_hierarchy'. we have to make a deepcopy because
- # config is a locked dict
- temp_s = script.BaseScript(
- initial_config_file='test/test.json',
- option_args=['--cfg', 'test/test_override.py,test/test_override2.py'],
- )
- from copy import deepcopy
- temp_cfg = deepcopy(temp_s.config)
- temp_cfg.update({'dump_config_hierarchy': True})
- ######
- self.s = script.BaseScript(
- initial_config_file='test/test.json',
- option_args=['--cfg', 'test/test_override.py,test/test_override2.py'],
- config={'dump_config_hierarchy': True}
- )
- except SystemExit:
- local_cfg_files = parse_config_file('test_logs/localconfigfiles.json')
- # finally let's just make sure that all the items added up, equals
- # what we started with: self.config
- target_cfg = {}
- for cfg_file in local_cfg_files:
- target_cfg.update(local_cfg_files[cfg_file])
- self.assertEqual(
- target_cfg, temp_cfg,
- msg="all of the items (combined) in each cfg file dumped via "
- "--dump-config-hierarchy does not equal self.config "
- )
-
- # test _dump_config() when --dump-config is passed
- def test_dump_config_equals_self_config(self):
- try:
- ######
- # we need temp_cfg because self.s will be gcollected (NoneType) by
- # the time we get to SystemExit exception
- # temp_cfg will differ from self.s.config because of
- # 'dump_config_hierarchy'. we have to make a deepcopy because
- # config is a locked dict
- temp_s = script.BaseScript(
- initial_config_file='test/test.json',
- option_args=['--cfg', 'test/test_override.py,test/test_override2.py'],
- )
- from copy import deepcopy
- temp_cfg = deepcopy(temp_s.config)
- temp_cfg.update({'dump_config': True})
- ######
- self.s = script.BaseScript(
- initial_config_file='test/test.json',
- option_args=['--cfg', 'test/test_override.py,test/test_override2.py'],
- config={'dump_config': True}
- )
- except SystemExit:
- target_cfg = parse_config_file('test_logs/localconfig.json')
- self.assertEqual(
- target_cfg, temp_cfg,
- msg="all of the items (combined) in each cfg file dumped via "
- "--dump-config does not equal self.config "
- )
-
- def test_nonexistent_mkdir_p(self):
- self.s = script.BaseScript(initial_config_file='test/test.json')
- self.s.mkdir_p('test_dir/foo/bar/baz')
- self.assertTrue(os.path.isdir('test_dir/foo/bar/baz'),
- msg="mkdir_p error")
-
- def test_existing_mkdir_p(self):
- self.s = script.BaseScript(initial_config_file='test/test.json')
- os.makedirs('test_dir/foo/bar/baz')
- self.s.mkdir_p('test_dir/foo/bar/baz')
- self.assertTrue(os.path.isdir('test_dir/foo/bar/baz'),
- msg="mkdir_p error when dir exists")
-
- def test_chdir(self):
- self.s = script.BaseScript(initial_config_file='test/test.json')
- cwd = os.getcwd()
- self.s.chdir('test_logs')
- self.assertEqual(os.path.join(cwd, "test_logs"), os.getcwd(),
- msg="chdir error")
- self.s.chdir(cwd)
-
- def _test_log_helper(self, obj):
- obj.debug("Testing DEBUG")
- obj.warning("Testing WARNING")
- obj.error("Testing ERROR")
- obj.critical("Testing CRITICAL")
- try:
- obj.fatal("Testing FATAL")
- except SystemExit:
- pass
- else:
- self.assertTrue(False, msg="fatal() didn't SystemExit!")
-
- def test_log(self):
- self.s = get_debug_script_obj()
- self.s.log_obj = None
- self._test_log_helper(self.s)
- del(self.s)
- self.s = script.BaseScript(initial_config_file='test/test.json')
- self._test_log_helper(self.s)
-
- def test_run_nonexistent_command(self):
- self.s = get_debug_script_obj()
- self.s.run_command(command="this_cmd_should_not_exist --help",
- env={'GARBLE': 'FARG'},
- error_list=errors.PythonErrorList)
- error_logsize = os.path.getsize("test_logs/test_error.log")
- self.assertTrue(error_logsize > 0,
- msg="command not found error not hit")
-
- def test_run_command_in_bad_dir(self):
- self.s = get_debug_script_obj()
- self.s.run_command(command="ls",
- cwd='/this_dir_should_not_exist',
- error_list=errors.PythonErrorList)
- error_logsize = os.path.getsize("test_logs/test_error.log")
- self.assertTrue(error_logsize > 0,
- msg="bad dir error not hit")
-
- def test_get_output_from_command_in_bad_dir(self):
- self.s = get_debug_script_obj()
- self.s.get_output_from_command(command="ls", cwd='/this_dir_should_not_exist')
- error_logsize = os.path.getsize("test_logs/test_error.log")
- self.assertTrue(error_logsize > 0,
- msg="bad dir error not hit")
-
- def test_get_output_from_command_with_missing_file(self):
- self.s = get_debug_script_obj()
- self.s.get_output_from_command(command="ls /this_file_should_not_exist")
- error_logsize = os.path.getsize("test_logs/test_error.log")
- self.assertTrue(error_logsize > 0,
- msg="bad file error not hit")
-
- def test_get_output_from_command_with_missing_file2(self):
- self.s = get_debug_script_obj()
- self.s.run_command(
- command="cat mozharness/base/errors.py",
- error_list=[{
- 'substr': "error", 'level': ERROR
- }, {
- 'regex': re.compile(',$'), 'level': IGNORE,
- }, {
- 'substr': ']$', 'level': WARNING,
- }])
- error_logsize = os.path.getsize("test_logs/test_error.log")
- self.assertTrue(error_logsize > 0,
- msg="error list not working properly")
-
- def test_download_unpack(self):
- # NOTE: The action is called *download*, however, it can work for files in disk
- self.s = get_debug_script_obj()
-
- archives_path = os.path.join(here, 'helper_files', 'archives')
-
- # Test basic decompression
- for archive in ('archive.tar', 'archive.tar.bz2', 'archive.tar.gz', 'archive.zip'):
- self.s.download_unpack(
- url=os.path.join(archives_path, archive),
- extract_to=self.tmpdir
- )
- self.assertIn('script.sh', os.listdir(os.path.join(self.tmpdir, 'bin')))
- self.assertIn('lorem.txt', os.listdir(self.tmpdir))
- shutil.rmtree(self.tmpdir)
-
- # Test permissions for extracted entries from zip archive
- self.s.download_unpack(
- url=os.path.join(archives_path, 'archive.zip'),
- extract_to=self.tmpdir,
- )
- file_stats = os.stat(os.path.join(self.tmpdir, 'bin', 'script.sh'))
- orig_fstats = os.stat(os.path.join(archives_path, 'reference', 'bin', 'script.sh'))
- self.assertEqual(file_stats.st_mode, orig_fstats.st_mode)
- shutil.rmtree(self.tmpdir)
-
- # Test unzip specific dirs only
- self.s.download_unpack(
- url=os.path.join(archives_path, 'archive.zip'),
- extract_to=self.tmpdir,
- extract_dirs=['bin/*']
- )
- self.assertIn('bin', os.listdir(self.tmpdir))
- self.assertNotIn('lorem.txt', os.listdir(self.tmpdir))
- shutil.rmtree(self.tmpdir)
-
- # Test for invalid filenames (Windows only)
- if PYWIN32:
- with self.assertRaises(IOError):
- self.s.download_unpack(
- url=os.path.join(archives_path, 'archive_invalid_filename.zip'),
- extract_to=self.tmpdir
- )
-
- def test_unpack(self):
- self.s = get_debug_script_obj()
-
- archives_path = os.path.join(here, 'helper_files', 'archives')
-
- # Test basic decompression
- for archive in ('archive.tar', 'archive.tar.bz2', 'archive.tar.gz', 'archive.zip'):
- self.s.unpack(os.path.join(archives_path, archive), self.tmpdir)
- self.assertIn('script.sh', os.listdir(os.path.join(self.tmpdir, 'bin')))
- self.assertIn('lorem.txt', os.listdir(self.tmpdir))
- shutil.rmtree(self.tmpdir)
-
- # Test permissions for extracted entries from zip archive
- self.s.unpack(os.path.join(archives_path, 'archive.zip'), self.tmpdir)
- file_stats = os.stat(os.path.join(self.tmpdir, 'bin', 'script.sh'))
- orig_fstats = os.stat(os.path.join(archives_path, 'reference', 'bin', 'script.sh'))
- self.assertEqual(file_stats.st_mode, orig_fstats.st_mode)
- shutil.rmtree(self.tmpdir)
-
- # Test extract specific dirs only
- self.s.unpack(os.path.join(archives_path, 'archive.zip'), self.tmpdir,
- extract_dirs=['bin/*'])
- self.assertIn('bin', os.listdir(self.tmpdir))
- self.assertNotIn('lorem.txt', os.listdir(self.tmpdir))
- shutil.rmtree(self.tmpdir)
-
- # Test for invalid filenames (Windows only)
- if PYWIN32:
- with self.assertRaises(IOError):
- self.s.unpack(os.path.join(archives_path, 'archive_invalid_filename.zip'),
- self.tmpdir)
-
-
-# TestHelperFunctions {{{1
-class TestHelperFunctions(unittest.TestCase):
- temp_file = "test_dir/mozilla"
-
- def setUp(self):
- cleanup()
- self.s = None
-
- def tearDown(self):
- # Close the logfile handles, or windows can't remove the logs
- if hasattr(self, 's') and isinstance(self.s, object):
- del(self.s)
- cleanup()
-
- def _create_temp_file(self, contents=test_string):
- os.mkdir('test_dir')
- fh = open(self.temp_file, "w+")
- fh.write(contents)
- fh.close
-
- def test_mkdir_p(self):
- self.s = script.BaseScript(initial_config_file='test/test.json')
- self.s.mkdir_p('test_dir')
- self.assertTrue(os.path.isdir('test_dir'),
- msg="mkdir_p error")
-
- def test_get_output_from_command(self):
- self._create_temp_file()
- self.s = script.BaseScript(initial_config_file='test/test.json')
- contents = self.s.get_output_from_command(["bash", "-c", "cat %s" % self.temp_file])
- self.assertEqual(test_string, contents,
- msg="get_output_from_command('cat file') differs from fh.write")
-
- def test_run_command(self):
- self._create_temp_file()
- self.s = script.BaseScript(initial_config_file='test/test.json')
- temp_file_name = os.path.basename(self.temp_file)
- self.assertEqual(self.s.run_command("cat %s" % temp_file_name,
- cwd="test_dir"), 0,
- msg="run_command('cat file') did not exit 0")
-
- def test_move1(self):
- self._create_temp_file()
- self.s = script.BaseScript(initial_config_file='test/test.json')
- temp_file2 = '%s2' % self.temp_file
- self.s.move(self.temp_file, temp_file2)
- self.assertFalse(os.path.exists(self.temp_file),
- msg="%s still exists after move()" % self.temp_file)
-
- def test_move2(self):
- self._create_temp_file()
- self.s = script.BaseScript(initial_config_file='test/test.json')
- temp_file2 = '%s2' % self.temp_file
- self.s.move(self.temp_file, temp_file2)
- self.assertTrue(os.path.exists(temp_file2),
- msg="%s doesn't exist after move()" % temp_file2)
-
- def test_copyfile(self):
- self._create_temp_file()
- self.s = script.BaseScript(initial_config_file='test/test.json')
- temp_file2 = '%s2' % self.temp_file
- self.s.copyfile(self.temp_file, temp_file2)
- self.assertEqual(os.path.getsize(self.temp_file),
- os.path.getsize(temp_file2),
- msg="%s and %s are different sizes after copyfile()" %
- (self.temp_file, temp_file2))
-
- def test_existing_rmtree(self):
- self._create_temp_file()
- self.s = script.BaseScript(initial_config_file='test/test.json')
- self.s.mkdir_p('test_dir/foo/bar/baz')
- self.s.rmtree('test_dir')
- self.assertFalse(os.path.exists('test_dir'),
- msg="rmtree unsuccessful")
-
- def test_nonexistent_rmtree(self):
- self.s = script.BaseScript(initial_config_file='test/test.json')
- status = self.s.rmtree('test_dir')
- self.assertFalse(status, msg="nonexistent rmtree error")
-
- @unittest.skipUnless(PYWIN32, "PyWin32 specific")
- def test_long_dir_rmtree(self):
- self.s = script.BaseScript(initial_config_file='test/test.json')
- # create a very long path that the command-prompt cannot delete
- # by using unicode format (max path length 32000)
- path = u'\\\\?\\%s\\test_dir' % os.getcwd()
- win32file.CreateDirectoryExW(u'.', path)
-
- for x in range(0, 20):
- print("path=%s" % path)
- path = path + u'\\%sxxxxxxxxxxxxxxxxxxxx' % x
- win32file.CreateDirectoryExW(u'.', path)
- self.s.rmtree('test_dir')
- self.assertFalse(os.path.exists('test_dir'),
- msg="rmtree unsuccessful")
-
- @unittest.skipUnless(PYWIN32, "PyWin32 specific")
- def test_chmod_rmtree(self):
- self._create_temp_file()
- win32file.SetFileAttributesW(self.temp_file, win32file.FILE_ATTRIBUTE_READONLY)
- self.s = script.BaseScript(initial_config_file='test/test.json')
- self.s.rmtree('test_dir')
- self.assertFalse(os.path.exists('test_dir'),
- msg="rmtree unsuccessful")
-
- @unittest.skipIf(os.name == "nt", "Not for Windows")
- def test_chmod(self):
- self._create_temp_file()
- self.s = script.BaseScript(initial_config_file='test/test.json')
- self.s.chmod(self.temp_file, 0100700)
- self.assertEqual(os.stat(self.temp_file)[0], 33216,
- msg="chmod unsuccessful")
-
- def test_env_normal(self):
- self.s = script.BaseScript(initial_config_file='test/test.json')
- script_env = self.s.query_env()
- self.assertEqual(script_env, os.environ,
- msg="query_env() != env\n%s\n%s" % (script_env, os.environ))
-
- def test_env_normal2(self):
- self.s = script.BaseScript(initial_config_file='test/test.json')
- self.s.query_env()
- script_env = self.s.query_env()
- self.assertEqual(script_env, os.environ,
- msg="Second query_env() != env\n%s\n%s" % (script_env, os.environ))
-
- def test_env_partial(self):
- self.s = script.BaseScript(initial_config_file='test/test.json')
- script_env = self.s.query_env(partial_env={'foo': 'bar'})
- self.assertTrue('foo' in script_env and script_env['foo'] == 'bar')
-
- def test_env_path(self):
- self.s = script.BaseScript(initial_config_file='test/test.json')
- partial_path = "yaddayadda:%(PATH)s"
- full_path = partial_path % {'PATH': os.environ['PATH']}
- script_env = self.s.query_env(partial_env={'PATH': partial_path})
- self.assertEqual(script_env['PATH'], full_path)
-
- def test_query_exe(self):
- self.s = script.BaseScript(
- initial_config_file='test/test.json',
- config={'exes': {'foo': 'bar'}},
- )
- path = self.s.query_exe('foo')
- self.assertEqual(path, 'bar')
-
- def test_query_exe_string_replacement(self):
- self.s = script.BaseScript(
- initial_config_file='test/test.json',
- config={
- 'base_work_dir': 'foo',
- 'work_dir': 'bar',
- 'exes': {'foo': os.path.join('%(abs_work_dir)s', 'baz')},
- },
- )
- path = self.s.query_exe('foo')
- self.assertEqual(path, os.path.join('foo', 'bar', 'baz'))
-
- def test_read_from_file(self):
- self._create_temp_file()
- self.s = script.BaseScript(initial_config_file='test/test.json')
- contents = self.s.read_from_file(self.temp_file)
- self.assertEqual(contents, test_string)
-
- def test_read_from_nonexistent_file(self):
- self.s = script.BaseScript(initial_config_file='test/test.json')
- contents = self.s.read_from_file("nonexistent_file!!!")
- self.assertEqual(contents, None)
-
-
-# TestScriptLogging {{{1
-class TestScriptLogging(unittest.TestCase):
- # I need a log watcher helper function, here and in test_log.
- def setUp(self):
- cleanup()
- self.s = None
-
- def tearDown(self):
- # Close the logfile handles, or windows can't remove the logs
- if hasattr(self, 's') and isinstance(self.s, object):
- del(self.s)
- cleanup()
-
- def test_info_logsize(self):
- self.s = script.BaseScript(config={'log_type': 'multi'},
- initial_config_file='test/test.json')
- info_logsize = os.path.getsize("test_logs/test_info.log")
- self.assertTrue(info_logsize > 0,
- msg="initial info logfile missing/size 0")
-
- def test_add_summary_info(self):
- self.s = script.BaseScript(config={'log_type': 'multi'},
- initial_config_file='test/test.json')
- info_logsize = os.path.getsize("test_logs/test_info.log")
- self.s.add_summary('one')
- info_logsize2 = os.path.getsize("test_logs/test_info.log")
- self.assertTrue(info_logsize < info_logsize2,
- msg="add_summary() info not logged")
-
- def test_add_summary_warning(self):
- self.s = script.BaseScript(config={'log_type': 'multi'},
- initial_config_file='test/test.json')
- warning_logsize = os.path.getsize("test_logs/test_warning.log")
- self.s.add_summary('two', level=WARNING)
- warning_logsize2 = os.path.getsize("test_logs/test_warning.log")
- self.assertTrue(warning_logsize < warning_logsize2,
- msg="add_summary(level=%s) not logged in warning log" % WARNING)
-
- def test_summary(self):
- self.s = script.BaseScript(config={'log_type': 'multi'},
- initial_config_file='test/test.json')
- self.s.add_summary('one')
- self.s.add_summary('two', level=WARNING)
- info_logsize = os.path.getsize("test_logs/test_info.log")
- warning_logsize = os.path.getsize("test_logs/test_warning.log")
- self.s.summary()
- info_logsize2 = os.path.getsize("test_logs/test_info.log")
- warning_logsize2 = os.path.getsize("test_logs/test_warning.log")
- msg = ""
- if info_logsize >= info_logsize2:
- msg += "summary() didn't log to info!\n"
- if warning_logsize >= warning_logsize2:
- msg += "summary() didn't log to warning!\n"
- self.assertEqual(msg, "", msg=msg)
-
- def _test_log_level(self, log_level, log_level_file_list):
- self.s = script.BaseScript(config={'log_type': 'multi'},
- initial_config_file='test/test.json')
- if log_level != FATAL:
- self.s.log('testing', level=log_level)
- else:
- self.s._post_fatal = types.MethodType(_post_fatal, self.s)
- try:
- self.s.fatal('testing')
- except SystemExit:
- contents = None
- if os.path.exists('tmpfile_stdout'):
- fh = open('tmpfile_stdout')
- contents = fh.read()
- fh.close()
- self.assertEqual(contents.rstrip(), test_string, "_post_fatal failed!")
- del(self.s)
- msg = ""
- for level in log_level_file_list:
- log_path = "test_logs/test_%s.log" % level
- if not os.path.exists(log_path):
- msg += "%s doesn't exist!\n" % log_path
- else:
- filesize = os.path.getsize(log_path)
- if not filesize > 0:
- msg += "%s is size 0!\n" % log_path
- self.assertEqual(msg, "", msg=msg)
-
- def test_debug(self):
- self._test_log_level(DEBUG, [])
-
- def test_ignore(self):
- self._test_log_level(IGNORE, [])
-
- def test_info(self):
- self._test_log_level(INFO, [INFO])
-
- def test_warning(self):
- self._test_log_level(WARNING, [INFO, WARNING])
-
- def test_error(self):
- self._test_log_level(ERROR, [INFO, WARNING, ERROR])
-
- def test_critical(self):
- self._test_log_level(CRITICAL, [INFO, WARNING, ERROR, CRITICAL])
-
- def test_fatal(self):
- self._test_log_level(FATAL, [INFO, WARNING, ERROR, CRITICAL, FATAL])
-
-
-# TestRetry {{{1
-class NewError(Exception):
- pass
-
-
-class OtherError(Exception):
- pass
-
-
-class TestRetry(unittest.TestCase):
- def setUp(self):
- self.ATTEMPT_N = 1
- self.s = script.BaseScript(initial_config_file='test/test.json')
-
- def tearDown(self):
- # Close the logfile handles, or windows can't remove the logs
- if hasattr(self, 's') and isinstance(self.s, object):
- del(self.s)
- cleanup()
-
- def _succeedOnSecondAttempt(self, foo=None, exception=Exception):
- if self.ATTEMPT_N == 2:
- self.ATTEMPT_N += 1
- return
- self.ATTEMPT_N += 1
- raise exception("Fail")
-
- def _raiseCustomException(self):
- return self._succeedOnSecondAttempt(exception=NewError)
-
- def _alwaysPass(self):
- self.ATTEMPT_N += 1
- return True
-
- def _mirrorArgs(self, *args, **kwargs):
- return args, kwargs
-
- def _alwaysFail(self):
- raise Exception("Fail")
-
- def testRetrySucceed(self):
- # Will raise if anything goes wrong
- self.s.retry(self._succeedOnSecondAttempt, attempts=2, sleeptime=0)
-
- def testRetryFailWithoutCatching(self):
- self.assertRaises(Exception, self.s.retry, self._alwaysFail, sleeptime=0,
- exceptions=())
-
- def testRetryFailEnsureRaisesLastException(self):
- self.assertRaises(SystemExit, self.s.retry, self._alwaysFail, sleeptime=0,
- error_level=FATAL)
-
- def testRetrySelectiveExceptionSucceed(self):
- self.s.retry(self._raiseCustomException, attempts=2, sleeptime=0,
- retry_exceptions=(NewError,))
-
- def testRetrySelectiveExceptionFail(self):
- self.assertRaises(NewError, self.s.retry, self._raiseCustomException, attempts=2,
- sleeptime=0, retry_exceptions=(OtherError,))
-
- # TODO: figure out a way to test that the sleep actually happened
- def testRetryWithSleep(self):
- self.s.retry(self._succeedOnSecondAttempt, attempts=2, sleeptime=1)
-
- def testRetryOnlyRunOnce(self):
- """Tests that retry() doesn't call the action again after success"""
- self.s.retry(self._alwaysPass, attempts=3, sleeptime=0)
- # self.ATTEMPT_N gets increased regardless of pass/fail
- self.assertEquals(2, self.ATTEMPT_N)
-
- def testRetryReturns(self):
- ret = self.s.retry(self._alwaysPass, sleeptime=0)
- self.assertEquals(ret, True)
-
- def testRetryCleanupIsCalled(self):
- cleanup = mock.Mock()
- self.s.retry(self._succeedOnSecondAttempt, cleanup=cleanup, sleeptime=0)
- self.assertEquals(cleanup.call_count, 1)
-
- def testRetryArgsPassed(self):
- args = (1, 'two', 3)
- kwargs = dict(foo='a', bar=7)
- ret = self.s.retry(self._mirrorArgs, args=args, kwargs=kwargs.copy(), sleeptime=0)
- print ret
- self.assertEqual(ret[0], args)
- self.assertEqual(ret[1], kwargs)
-
-
-class BaseScriptWithDecorators(script.BaseScript):
- def __init__(self, *args, **kwargs):
- super(BaseScriptWithDecorators, self).__init__(*args, **kwargs)
-
- self.pre_run_1_args = []
- self.raise_during_pre_run_1 = False
- self.pre_action_1_args = []
- self.raise_during_pre_action_1 = False
- self.pre_action_2_args = []
- self.pre_action_3_args = []
- self.post_action_1_args = []
- self.raise_during_post_action_1 = False
- self.post_action_2_args = []
- self.post_action_3_args = []
- self.post_run_1_args = []
- self.raise_during_post_run_1 = False
- self.post_run_2_args = []
- self.raise_during_build = False
-
- @script.PreScriptRun
- def pre_run_1(self, *args, **kwargs):
- self.pre_run_1_args.append((args, kwargs))
-
- if self.raise_during_pre_run_1:
- raise Exception(self.raise_during_pre_run_1)
-
- @script.PreScriptAction
- def pre_action_1(self, *args, **kwargs):
- self.pre_action_1_args.append((args, kwargs))
-
- if self.raise_during_pre_action_1:
- raise Exception(self.raise_during_pre_action_1)
-
- @script.PreScriptAction
- def pre_action_2(self, *args, **kwargs):
- self.pre_action_2_args.append((args, kwargs))
-
- @script.PreScriptAction('clobber')
- def pre_action_3(self, *args, **kwargs):
- self.pre_action_3_args.append((args, kwargs))
-
- @script.PostScriptAction
- def post_action_1(self, *args, **kwargs):
- self.post_action_1_args.append((args, kwargs))
-
- if self.raise_during_post_action_1:
- raise Exception(self.raise_during_post_action_1)
-
- @script.PostScriptAction
- def post_action_2(self, *args, **kwargs):
- self.post_action_2_args.append((args, kwargs))
-
- @script.PostScriptAction('build')
- def post_action_3(self, *args, **kwargs):
- self.post_action_3_args.append((args, kwargs))
-
- @script.PostScriptRun
- def post_run_1(self, *args, **kwargs):
- self.post_run_1_args.append((args, kwargs))
-
- if self.raise_during_post_run_1:
- raise Exception(self.raise_during_post_run_1)
-
- @script.PostScriptRun
- def post_run_2(self, *args, **kwargs):
- self.post_run_2_args.append((args, kwargs))
-
- def build(self):
- if self.raise_during_build:
- raise Exception(self.raise_during_build)
-
-
-class TestScriptDecorators(unittest.TestCase):
- def setUp(self):
- cleanup()
- self.s = None
-
- def tearDown(self):
- if hasattr(self, 's') and isinstance(self.s, object):
- del self.s
-
- cleanup()
-
- def test_decorators_registered(self):
- self.s = BaseScriptWithDecorators(initial_config_file='test/test.json')
-
- self.assertEqual(len(self.s._listeners['pre_run']), 1)
- self.assertEqual(len(self.s._listeners['pre_action']), 3)
- self.assertEqual(len(self.s._listeners['post_action']), 3)
- self.assertEqual(len(self.s._listeners['post_run']), 3)
-
- def test_pre_post_fired(self):
- self.s = BaseScriptWithDecorators(initial_config_file='test/test.json')
- self.s.run()
-
- self.assertEqual(len(self.s.pre_run_1_args), 1)
- self.assertEqual(len(self.s.pre_action_1_args), 2)
- self.assertEqual(len(self.s.pre_action_2_args), 2)
- self.assertEqual(len(self.s.pre_action_3_args), 1)
- self.assertEqual(len(self.s.post_action_1_args), 2)
- self.assertEqual(len(self.s.post_action_2_args), 2)
- self.assertEqual(len(self.s.post_action_3_args), 1)
- self.assertEqual(len(self.s.post_run_1_args), 1)
-
- self.assertEqual(self.s.pre_run_1_args[0], ((), {}))
-
- self.assertEqual(self.s.pre_action_1_args[0], (('clobber',), {}))
- self.assertEqual(self.s.pre_action_1_args[1], (('build',), {}))
-
- # pre_action_3 should only get called for the action it is registered
- # with.
- self.assertEqual(self.s.pre_action_3_args[0], (('clobber',), {}))
-
- self.assertEqual(self.s.post_action_1_args[0][0], ('clobber',))
- self.assertEqual(self.s.post_action_1_args[0][1], dict(success=True))
- self.assertEqual(self.s.post_action_1_args[1][0], ('build',))
- self.assertEqual(self.s.post_action_1_args[1][1], dict(success=True))
-
- # post_action_3 should only get called for the action it is registered
- # with.
- self.assertEqual(self.s.post_action_3_args[0], (('build',),
- dict(success=True)))
-
- self.assertEqual(self.s.post_run_1_args[0], ((), {}))
-
- def test_post_always_fired(self):
- self.s = BaseScriptWithDecorators(initial_config_file='test/test.json')
- self.s.raise_during_build = 'Testing post always fired.'
-
- with self.assertRaises(SystemExit):
- self.s.run()
-
- self.assertEqual(len(self.s.pre_run_1_args), 1)
- self.assertEqual(len(self.s.pre_action_1_args), 2)
- self.assertEqual(len(self.s.post_action_1_args), 2)
- self.assertEqual(len(self.s.post_action_2_args), 2)
- self.assertEqual(len(self.s.post_run_1_args), 1)
- self.assertEqual(len(self.s.post_run_2_args), 1)
-
- self.assertEqual(self.s.post_action_1_args[0][1], dict(success=True))
- self.assertEqual(self.s.post_action_1_args[1][1], dict(success=False))
- self.assertEqual(self.s.post_action_2_args[1][1], dict(success=False))
-
- def test_pre_run_exception(self):
- self.s = BaseScriptWithDecorators(initial_config_file='test/test.json')
- self.s.raise_during_pre_run_1 = 'Error during pre run 1'
-
- with self.assertRaises(SystemExit):
- self.s.run()
-
- self.assertEqual(len(self.s.pre_run_1_args), 1)
- self.assertEqual(len(self.s.pre_action_1_args), 0)
- self.assertEqual(len(self.s.post_run_1_args), 1)
- self.assertEqual(len(self.s.post_run_2_args), 1)
-
- def test_pre_action_exception(self):
- self.s = BaseScriptWithDecorators(initial_config_file='test/test.json')
- self.s.raise_during_pre_action_1 = 'Error during pre 1'
-
- with self.assertRaises(SystemExit):
- self.s.run()
-
- self.assertEqual(len(self.s.pre_run_1_args), 1)
- self.assertEqual(len(self.s.pre_action_1_args), 1)
- self.assertEqual(len(self.s.pre_action_2_args), 0)
- self.assertEqual(len(self.s.post_action_1_args), 1)
- self.assertEqual(len(self.s.post_action_2_args), 1)
- self.assertEqual(len(self.s.post_run_1_args), 1)
- self.assertEqual(len(self.s.post_run_2_args), 1)
-
- def test_post_action_exception(self):
- self.s = BaseScriptWithDecorators(initial_config_file='test/test.json')
- self.s.raise_during_post_action_1 = 'Error during post 1'
-
- with self.assertRaises(SystemExit):
- self.s.run()
-
- self.assertEqual(len(self.s.pre_run_1_args), 1)
- self.assertEqual(len(self.s.post_action_1_args), 1)
- self.assertEqual(len(self.s.post_action_2_args), 1)
- self.assertEqual(len(self.s.post_run_1_args), 1)
- self.assertEqual(len(self.s.post_run_2_args), 1)
-
- def test_post_run_exception(self):
- self.s = BaseScriptWithDecorators(initial_config_file='test/test.json')
- self.s.raise_during_post_run_1 = 'Error during post run 1'
-
- with self.assertRaises(SystemExit):
- self.s.run()
-
- self.assertEqual(len(self.s.post_run_1_args), 1)
- self.assertEqual(len(self.s.post_run_2_args), 1)
-
-
-# main {{{1
-if __name__ == '__main__':
- unittest.main()
diff --git a/testing/mozharness/test/test_base_transfer.py b/testing/mozharness/test/test_base_transfer.py
deleted file mode 100644
index f3f907254..000000000
--- a/testing/mozharness/test/test_base_transfer.py
+++ /dev/null
@@ -1,127 +0,0 @@
-import unittest
-import mock
-
-from mozharness.base.transfer import TransferMixin
-
-
-class GoodMockMixin(object):
- def query_abs_dirs(self):
- return {'abs_work_dir': ''}
-
- def query_exe(self, exe):
- return exe
-
- def info(self, msg):
- pass
-
- def log(self, msg, level):
- pass
-
- def run_command(*args, **kwargs):
- return 0
-
-
-class BadMockMixin(GoodMockMixin):
- def run_command(*args, **kwargs):
- return 1
-
-
-class GoodTransferMixin(TransferMixin, GoodMockMixin):
- pass
-
-
-class BadTransferMixin(TransferMixin, BadMockMixin):
- pass
-
-
-class TestTranferMixin(unittest.TestCase):
- @mock.patch('mozharness.base.transfer.os')
- def test_rsync_upload_dir_not_a_dir(self, os_mock):
- # simulates upload dir but dir is a file
- os_mock.path.isdir.return_value = False
- tm = GoodTransferMixin()
- self.assertEqual(tm.rsync_upload_directory(
- local_path='',
- ssh_key='my ssh key',
- ssh_user='my ssh user',
- remote_host='remote host',
- remote_path='remote path',), -1)
-
- @mock.patch('mozharness.base.transfer.os')
- def test_rsync_upload_fails_create_remote_dir(self, os_mock):
- # we cannot create the remote directory
- os_mock.path.isdir.return_value = True
- tm = BadTransferMixin()
- self.assertEqual(tm.rsync_upload_directory(
- local_path='',
- ssh_key='my ssh key',
- ssh_user='my ssh user',
- remote_host='remote host',
- remote_path='remote path',
- create_remote_directory=True), -2)
-
- @mock.patch('mozharness.base.transfer.os')
- def test_rsync_upload_fails_do_not_create_remote_dir(self, os_mock):
- # upload fails, remote directory is not created
- os_mock.path.isdir.return_value = True
- tm = BadTransferMixin()
- self.assertEqual(tm.rsync_upload_directory(
- local_path='',
- ssh_key='my ssh key',
- ssh_user='my ssh user',
- remote_host='remote host',
- remote_path='remote path',
- create_remote_directory=False), -3)
-
- @mock.patch('mozharness.base.transfer.os')
- def test_rsync_upload(self, os_mock):
- # simulates an upload with no errors
- os_mock.path.isdir.return_value = True
- tm = GoodTransferMixin()
- self.assertEqual(tm.rsync_upload_directory(
- local_path='',
- ssh_key='my ssh key',
- ssh_user='my ssh user',
- remote_host='remote host',
- remote_path='remote path',
- create_remote_directory=False), None)
-
- @mock.patch('mozharness.base.transfer.os')
- def test_rsync_download_in_not_a_dir(self, os_mock):
- # local path is not a directory
- os_mock.path.isdir.return_value = False
- tm = GoodTransferMixin()
- self.assertEqual(tm.rsync_download_directory(
- local_path='',
- ssh_key='my ssh key',
- ssh_user='my ssh user',
- remote_host='remote host',
- remote_path='remote path',), -1)
-
- @mock.patch('mozharness.base.transfer.os')
- def test_rsync_download(self, os_mock):
- # successful rsync
- os_mock.path.isdir.return_value = True
- tm = GoodTransferMixin()
- self.assertEqual(tm.rsync_download_directory(
- local_path='',
- ssh_key='my ssh key',
- ssh_user='my ssh user',
- remote_host='remote host',
- remote_path='remote path',), None)
-
- @mock.patch('mozharness.base.transfer.os')
- def test_rsync_download_fail(self, os_mock):
- # ops download has failed
- os_mock.path.isdir.return_value = True
- tm = BadTransferMixin()
- self.assertEqual(tm.rsync_download_directory(
- local_path='',
- ssh_key='my ssh key',
- ssh_user='my ssh user',
- remote_host='remote host',
- remote_path='remote path',), -3)
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/testing/mozharness/test/test_base_vcs_mercurial.py b/testing/mozharness/test/test_base_vcs_mercurial.py
deleted file mode 100644
index 1463d8963..000000000
--- a/testing/mozharness/test/test_base_vcs_mercurial.py
+++ /dev/null
@@ -1,440 +0,0 @@
-import os
-import platform
-import shutil
-import tempfile
-import unittest
-
-import mozharness.base.errors as errors
-import mozharness.base.vcs.mercurial as mercurial
-
-test_string = '''foo
-bar
-baz'''
-
-HG = ['hg'] + mercurial.HG_OPTIONS
-
-# Known default .hgrc
-os.environ['HGRCPATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'helper_files', '.hgrc'))
-
-
-def cleanup():
- if os.path.exists('test_logs'):
- shutil.rmtree('test_logs')
- if os.path.exists('test_dir'):
- if os.path.isdir('test_dir'):
- shutil.rmtree('test_dir')
- else:
- os.remove('test_dir')
- for filename in ('localconfig.json', 'localconfig.json.bak'):
- if os.path.exists(filename):
- os.remove(filename)
-
-
-def get_mercurial_vcs_obj():
- m = mercurial.MercurialVCS()
- m.config = {}
- return m
-
-
-def get_revisions(dest):
- m = get_mercurial_vcs_obj()
- retval = []
- for rev in m.get_output_from_command(HG + ['log', '-R', dest, '--template', '{node}\n']).split('\n'):
- rev = rev.strip()
- if not rev:
- continue
- retval.append(rev)
- return retval
-
-
-class TestMakeAbsolute(unittest.TestCase):
- # _make_absolute() doesn't play nicely with windows/msys paths.
- # TODO: fix _make_absolute, write it out of the picture, or determine
- # that it's not needed on windows.
- if platform.system() not in ("Windows",):
- def test_absolute_path(self):
- m = get_mercurial_vcs_obj()
- self.assertEquals(m._make_absolute("/foo/bar"), "/foo/bar")
-
- def test_relative_path(self):
- m = get_mercurial_vcs_obj()
- self.assertEquals(m._make_absolute("foo/bar"), os.path.abspath("foo/bar"))
-
- def test_HTTP_paths(self):
- m = get_mercurial_vcs_obj()
- self.assertEquals(m._make_absolute("http://foo/bar"), "http://foo/bar")
-
- def test_absolute_file_path(self):
- m = get_mercurial_vcs_obj()
- self.assertEquals(m._make_absolute("file:///foo/bar"), "file:///foo/bar")
-
- def test_relative_file_path(self):
- m = get_mercurial_vcs_obj()
- self.assertEquals(m._make_absolute("file://foo/bar"), "file://%s/foo/bar" % os.getcwd())
-
-
-class TestHg(unittest.TestCase):
- def _init_hg_repo(self, hg_obj, repodir):
- hg_obj.run_command(["bash",
- os.path.join(os.path.dirname(__file__),
- "helper_files", "init_hgrepo.sh"),
- repodir])
-
- def setUp(self):
- self.tmpdir = tempfile.mkdtemp()
- self.repodir = os.path.join(self.tmpdir, 'repo')
- m = get_mercurial_vcs_obj()
- self._init_hg_repo(m, self.repodir)
- self.revisions = get_revisions(self.repodir)
- self.wc = os.path.join(self.tmpdir, 'wc')
- self.pwd = os.getcwd()
-
- def tearDown(self):
- shutil.rmtree(self.tmpdir)
- os.chdir(self.pwd)
-
- def test_get_branch(self):
- m = get_mercurial_vcs_obj()
- m.clone(self.repodir, self.wc)
- b = m.get_branch_from_path(self.wc)
- self.assertEquals(b, 'default')
-
- def test_get_branches(self):
- m = get_mercurial_vcs_obj()
- m.clone(self.repodir, self.wc)
- branches = m.get_branches_from_path(self.wc)
- self.assertEquals(sorted(branches), sorted(["branch2", "default"]))
-
- def test_clone(self):
- m = get_mercurial_vcs_obj()
- rev = m.clone(self.repodir, self.wc, update_dest=False)
- self.assertEquals(rev, None)
- self.assertEquals(self.revisions, get_revisions(self.wc))
- self.assertEquals(sorted(os.listdir(self.wc)), ['.hg'])
-
- def test_clone_into_non_empty_dir(self):
- m = get_mercurial_vcs_obj()
- m.mkdir_p(self.wc)
- open(os.path.join(self.wc, 'test.txt'), 'w').write('hello')
- m.clone(self.repodir, self.wc, update_dest=False)
- self.failUnless(not os.path.exists(os.path.join(self.wc, 'test.txt')))
-
- def test_clone_update(self):
- m = get_mercurial_vcs_obj()
- rev = m.clone(self.repodir, self.wc, update_dest=True)
- self.assertEquals(rev, self.revisions[0])
-
- def test_clone_branch(self):
- m = get_mercurial_vcs_obj()
- m.clone(self.repodir, self.wc, branch='branch2',
- update_dest=False)
- # On hg 1.6, we should only have a subset of the revisions
- if m.hg_ver() >= (1, 6, 0):
- self.assertEquals(self.revisions[1:],
- get_revisions(self.wc))
- else:
- self.assertEquals(self.revisions,
- get_revisions(self.wc))
-
- def test_clone_update_branch(self):
- m = get_mercurial_vcs_obj()
- rev = m.clone(self.repodir, os.path.join(self.tmpdir, 'wc'),
- branch="branch2", update_dest=True)
- self.assertEquals(rev, self.revisions[1], self.revisions)
-
- def test_clone_revision(self):
- m = get_mercurial_vcs_obj()
- m.clone(self.repodir, self.wc,
- revision=self.revisions[0], update_dest=False)
- # We'll only get a subset of the revisions
- self.assertEquals(self.revisions[:1] + self.revisions[2:],
- get_revisions(self.wc))
-
- def test_update_revision(self):
- m = get_mercurial_vcs_obj()
- rev = m.clone(self.repodir, self.wc, update_dest=False)
- self.assertEquals(rev, None)
-
- rev = m.update(self.wc, revision=self.revisions[1])
- self.assertEquals(rev, self.revisions[1])
-
- def test_pull(self):
- m = get_mercurial_vcs_obj()
- # Clone just the first rev
- m.clone(self.repodir, self.wc, revision=self.revisions[-1], update_dest=False)
- self.assertEquals(get_revisions(self.wc), self.revisions[-1:])
-
- # Now pull in new changes
- rev = m.pull(self.repodir, self.wc, update_dest=False)
- self.assertEquals(rev, None)
- self.assertEquals(get_revisions(self.wc), self.revisions)
-
- def test_pull_revision(self):
- m = get_mercurial_vcs_obj()
- # Clone just the first rev
- m.clone(self.repodir, self.wc, revision=self.revisions[-1], update_dest=False)
- self.assertEquals(get_revisions(self.wc), self.revisions[-1:])
-
- # Now pull in just the last revision
- rev = m.pull(self.repodir, self.wc, revision=self.revisions[0], update_dest=False)
- self.assertEquals(rev, None)
-
- # We'll be missing the middle revision (on another branch)
- self.assertEquals(get_revisions(self.wc), self.revisions[:1] + self.revisions[2:])
-
- def test_pull_branch(self):
- m = get_mercurial_vcs_obj()
- # Clone just the first rev
- m.clone(self.repodir, self.wc, revision=self.revisions[-1], update_dest=False)
- self.assertEquals(get_revisions(self.wc), self.revisions[-1:])
-
- # Now pull in the other branch
- rev = m.pull(self.repodir, self.wc, branch="branch2", update_dest=False)
- self.assertEquals(rev, None)
-
- # On hg 1.6, we'll be missing the last revision (on another branch)
- if m.hg_ver() >= (1, 6, 0):
- self.assertEquals(get_revisions(self.wc), self.revisions[1:])
- else:
- self.assertEquals(get_revisions(self.wc), self.revisions)
-
- def test_pull_unrelated(self):
- m = get_mercurial_vcs_obj()
- # Create a new repo
- repo2 = os.path.join(self.tmpdir, 'repo2')
- self._init_hg_repo(m, repo2)
-
- self.assertNotEqual(self.revisions, get_revisions(repo2))
-
- # Clone the original repo
- m.clone(self.repodir, self.wc, update_dest=False)
- # Hide the wanted error
- m.config = {'log_to_console': False}
- # Try and pull in changes from the new repo
- self.assertRaises(mercurial.VCSException, m.pull, repo2, self.wc, update_dest=False)
-
- def test_push(self):
- m = get_mercurial_vcs_obj()
- m.clone(self.repodir, self.wc, revision=self.revisions[-2])
- m.push(src=self.repodir, remote=self.wc)
- self.assertEquals(get_revisions(self.wc), self.revisions)
-
- def test_push_with_branch(self):
- m = get_mercurial_vcs_obj()
- if m.hg_ver() >= (1, 6, 0):
- m.clone(self.repodir, self.wc, revision=self.revisions[-1])
- m.push(src=self.repodir, remote=self.wc, branch='branch2')
- m.push(src=self.repodir, remote=self.wc, branch='default')
- self.assertEquals(get_revisions(self.wc), self.revisions)
-
- def test_push_with_revision(self):
- m = get_mercurial_vcs_obj()
- m.clone(self.repodir, self.wc, revision=self.revisions[-2])
- m.push(src=self.repodir, remote=self.wc, revision=self.revisions[-1])
- self.assertEquals(get_revisions(self.wc), self.revisions[-2:])
-
- def test_mercurial(self):
- m = get_mercurial_vcs_obj()
- m.vcs_config = {
- 'repo': self.repodir,
- 'dest': self.wc,
- 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
- }
- m.ensure_repo_and_revision()
- rev = m.ensure_repo_and_revision()
- self.assertEquals(rev, self.revisions[0])
-
- def test_push_new_branches_not_allowed(self):
- m = get_mercurial_vcs_obj()
- m.clone(self.repodir, self.wc, revision=self.revisions[0])
- # Hide the wanted error
- m.config = {'log_to_console': False}
- self.assertRaises(Exception, m.push, self.repodir, self.wc, push_new_branches=False)
-
- def test_mercurial_relative_dir(self):
- m = get_mercurial_vcs_obj()
- repo = os.path.basename(self.repodir)
- wc = os.path.basename(self.wc)
- m.vcs_config = {
- 'repo': repo,
- 'dest': wc,
- 'revision': self.revisions[-1],
- 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
- }
- m.chdir(os.path.dirname(self.repodir))
- try:
- rev = m.ensure_repo_and_revision()
- self.assertEquals(rev, self.revisions[-1])
- m.info("Creating test.txt")
- open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
-
- m = get_mercurial_vcs_obj()
- m.vcs_config = {
- 'repo': repo,
- 'dest': wc,
- 'revision': self.revisions[0],
- 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
- }
- rev = m.ensure_repo_and_revision()
- self.assertEquals(rev, self.revisions[0])
- # Make sure our local file didn't go away
- self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
- finally:
- m.chdir(self.pwd)
-
- def test_mercurial_update_tip(self):
- m = get_mercurial_vcs_obj()
- m.vcs_config = {
- 'repo': self.repodir,
- 'dest': self.wc,
- 'revision': self.revisions[-1],
- 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
- }
- rev = m.ensure_repo_and_revision()
- self.assertEquals(rev, self.revisions[-1])
- open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
-
- m = get_mercurial_vcs_obj()
- m.vcs_config = {
- 'repo': self.repodir,
- 'dest': self.wc,
- 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
- }
- rev = m.ensure_repo_and_revision()
- self.assertEquals(rev, self.revisions[0])
- # Make sure our local file didn't go away
- self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
-
- def test_mercurial_update_rev(self):
- m = get_mercurial_vcs_obj()
- m.vcs_config = {
- 'repo': self.repodir,
- 'dest': self.wc,
- 'revision': self.revisions[-1],
- 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
- }
- rev = m.ensure_repo_and_revision()
- self.assertEquals(rev, self.revisions[-1])
- open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
-
- m = get_mercurial_vcs_obj()
- m.vcs_config = {
- 'repo': self.repodir,
- 'dest': self.wc,
- 'revision': self.revisions[0],
- 'vcs_share_base': os.path.join(self.tmpdir, 'share'),
- }
- rev = m.ensure_repo_and_revision()
- self.assertEquals(rev, self.revisions[0])
- # Make sure our local file didn't go away
- self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
-
- def test_make_hg_url(self):
- #construct an hg url specific to revision, branch and filename and try to pull it down
- file_url = mercurial.make_hg_url(
- "hg.mozilla.org",
- '//build/tools/',
- revision='FIREFOX_3_6_12_RELEASE',
- filename="/lib/python/util/hg.py",
- protocol='https',
- )
- expected_url = "https://hg.mozilla.org/build/tools/raw-file/FIREFOX_3_6_12_RELEASE/lib/python/util/hg.py"
- self.assertEquals(file_url, expected_url)
-
- def test_make_hg_url_no_filename(self):
- file_url = mercurial.make_hg_url(
- "hg.mozilla.org",
- "/build/tools",
- revision="default",
- protocol='https',
- )
- expected_url = "https://hg.mozilla.org/build/tools/rev/default"
- self.assertEquals(file_url, expected_url)
-
- def test_make_hg_url_no_revision_no_filename(self):
- repo_url = mercurial.make_hg_url(
- "hg.mozilla.org",
- "/build/tools",
- protocol='https',
- )
- expected_url = "https://hg.mozilla.org/build/tools"
- self.assertEquals(repo_url, expected_url)
-
- def test_make_hg_url_different_protocol(self):
- repo_url = mercurial.make_hg_url(
- "hg.mozilla.org",
- "/build/tools",
- protocol='ssh',
- )
- expected_url = "ssh://hg.mozilla.org/build/tools"
- self.assertEquals(repo_url, expected_url)
-
- def test_apply_and_push(self):
- m = get_mercurial_vcs_obj()
- m.clone(self.repodir, self.wc)
-
- def c(repo, attempt):
- m.run_command(HG + ['tag', '-f', 'TEST'], cwd=repo)
- m.apply_and_push(self.wc, self.repodir, c)
- self.assertEquals(get_revisions(self.wc), get_revisions(self.repodir))
-
- def test_apply_and_push_fail(self):
- m = get_mercurial_vcs_obj()
- m.clone(self.repodir, self.wc)
-
- def c(repo, attempt, remote):
- m.run_command(HG + ['tag', '-f', 'TEST'], cwd=repo)
- m.run_command(HG + ['tag', '-f', 'CONFLICTING_TAG'], cwd=remote)
- m.config = {'log_to_console': False}
- self.assertRaises(errors.VCSException, m.apply_and_push, self.wc,
- self.repodir, lambda r, a: c(r, a, self.repodir),
- max_attempts=2)
-
- def test_apply_and_push_with_rebase(self):
- m = get_mercurial_vcs_obj()
- m.clone(self.repodir, self.wc)
- m.config = {'log_to_console': False}
-
- def c(repo, attempt, remote):
- m.run_command(HG + ['tag', '-f', 'TEST'], cwd=repo)
- if attempt == 1:
- m.run_command(HG + ['rm', 'hello.txt'], cwd=remote)
- m.run_command(HG + ['commit', '-m', 'test'], cwd=remote)
- m.apply_and_push(self.wc, self.repodir,
- lambda r, a: c(r, a, self.repodir), max_attempts=2)
- self.assertEquals(get_revisions(self.wc), get_revisions(self.repodir))
-
- def test_apply_and_push_rebase_fails(self):
- m = get_mercurial_vcs_obj()
- m.clone(self.repodir, self.wc)
- m.config = {'log_to_console': False}
-
- def c(repo, attempt, remote):
- m.run_command(HG + ['tag', '-f', 'TEST'], cwd=repo)
- if attempt in (1, 2):
- m.run_command(HG + ['tag', '-f', 'CONFLICTING_TAG'], cwd=remote)
- m.apply_and_push(self.wc, self.repodir,
- lambda r, a: c(r, a, self.repodir), max_attempts=4)
- self.assertEquals(get_revisions(self.wc), get_revisions(self.repodir))
-
- def test_apply_and_push_on_branch(self):
- m = get_mercurial_vcs_obj()
- if m.hg_ver() >= (1, 6, 0):
- m.clone(self.repodir, self.wc)
-
- def c(repo, attempt):
- m.run_command(HG + ['branch', 'branch3'], cwd=repo)
- m.run_command(HG + ['tag', '-f', 'TEST'], cwd=repo)
- m.apply_and_push(self.wc, self.repodir, c)
- self.assertEquals(get_revisions(self.wc), get_revisions(self.repodir))
-
- def test_apply_and_push_with_no_change(self):
- m = get_mercurial_vcs_obj()
- m.clone(self.repodir, self.wc)
-
- def c(r, a):
- pass
- self.assertRaises(errors.VCSException, m.apply_and_push, self.wc, self.repodir, c)
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/testing/mozharness/test/test_l10n_locales.py b/testing/mozharness/test/test_l10n_locales.py
deleted file mode 100644
index e8372a9fb..000000000
--- a/testing/mozharness/test/test_l10n_locales.py
+++ /dev/null
@@ -1,132 +0,0 @@
-import os
-import shutil
-import subprocess
-import sys
-import unittest
-
-import mozharness.base.log as log
-import mozharness.base.script as script
-import mozharness.mozilla.l10n.locales as locales
-
-ALL_LOCALES = ['ar', 'be', 'de', 'es-ES']
-
-MH_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-def cleanup():
- if os.path.exists('test_logs'):
- shutil.rmtree('test_logs')
-
-class LocalesTest(locales.LocalesMixin, script.BaseScript):
- def __init__(self, **kwargs):
- if 'config' not in kwargs:
- kwargs['config'] = {'log_type': 'simple',
- 'log_level': 'error'}
- if 'initial_config_file' not in kwargs:
- kwargs['initial_config_file'] = 'test/test.json'
- super(LocalesTest, self).__init__(**kwargs)
- self.config = {}
- self.log_obj = None
-
-class TestLocalesMixin(unittest.TestCase):
- BASE_ABS_DIRS = ['abs_compare_locales_dir', 'abs_log_dir',
- 'abs_upload_dir', 'abs_work_dir', 'base_work_dir']
- def setUp(self):
- cleanup()
-
- def tearDown(self):
- cleanup()
-
- def test_query_locales_locales(self):
- l = LocalesTest()
- l.locales = ['a', 'b', 'c']
- self.assertEqual(l.locales, l.query_locales())
-
- def test_query_locales_ignore_locales(self):
- l = LocalesTest()
- l.config['locales'] = ['a', 'b', 'c']
- l.config['ignore_locales'] = ['a', 'c']
- self.assertEqual(['b'], l.query_locales())
-
- def test_query_locales_config(self):
- l = LocalesTest()
- l.config['locales'] = ['a', 'b', 'c']
- self.assertEqual(l.config['locales'], l.query_locales())
-
- def test_query_locales_json(self):
- l = LocalesTest()
- l.config['locales_file'] = os.path.join(MH_DIR, "test/helper_files/locales.json")
- l.config['base_work_dir'] = '.'
- l.config['work_dir'] = '.'
- locales = l.query_locales()
- locales.sort()
- self.assertEqual(ALL_LOCALES, locales)
-
-# Commenting out til we can hide the FATAL ?
-# def test_query_locales_no_file(self):
-# l = LocalesTest()
-# l.config['base_work_dir'] = '.'
-# l.config['work_dir'] = '.'
-# try:
-# l.query_locales()
-# except SystemExit:
-# pass # Good
-# else:
-# self.assertTrue(False, "query_locales with no file doesn't fatal()!")
-
- def test_parse_locales_file(self):
- l = LocalesTest()
- self.assertEqual(ALL_LOCALES, l.parse_locales_file(os.path.join(MH_DIR, 'test/helper_files/locales.txt')))
-
- def _get_query_abs_dirs_obj(self):
- l = LocalesTest()
- l.config['base_work_dir'] = "base_work_dir"
- l.config['work_dir'] = "work_dir"
- return l
-
- def test_query_abs_dirs_base(self):
- l = self._get_query_abs_dirs_obj()
- dirs = l.query_abs_dirs().keys()
- dirs.sort()
- self.assertEqual(dirs, self.BASE_ABS_DIRS)
-
- def test_query_abs_dirs_base2(self):
- l = self._get_query_abs_dirs_obj()
- l.query_abs_dirs().keys()
- dirs = l.query_abs_dirs().keys()
- dirs.sort()
- self.assertEqual(dirs, self.BASE_ABS_DIRS)
-
- def test_query_abs_dirs_l10n(self):
- l = self._get_query_abs_dirs_obj()
- l.config['l10n_dir'] = "l10n_dir"
- dirs = l.query_abs_dirs().keys()
- dirs.sort()
- expected_dirs = self.BASE_ABS_DIRS + ['abs_l10n_dir']
- expected_dirs.sort()
- self.assertEqual(dirs, expected_dirs)
-
- def test_query_abs_dirs_mozilla(self):
- l = self._get_query_abs_dirs_obj()
- l.config['l10n_dir'] = "l10n_dir"
- l.config['mozilla_dir'] = "mozilla_dir"
- l.config['locales_dir'] = "locales_dir"
- dirs = l.query_abs_dirs().keys()
- dirs.sort()
- expected_dirs = self.BASE_ABS_DIRS + ['abs_mozilla_dir', 'abs_locales_src_dir', 'abs_l10n_dir']
- expected_dirs.sort()
- self.assertEqual(dirs, expected_dirs)
-
- def test_query_abs_dirs_objdir(self):
- l = self._get_query_abs_dirs_obj()
- l.config['l10n_dir'] = "l10n_dir"
- l.config['mozilla_dir'] = "mozilla_dir"
- l.config['locales_dir'] = "locales_dir"
- l.config['objdir'] = "objdir"
- dirs = l.query_abs_dirs().keys()
- dirs.sort()
- expected_dirs = self.BASE_ABS_DIRS + ['abs_mozilla_dir', 'abs_locales_src_dir', 'abs_l10n_dir', 'abs_objdir', 'abs_merge_dir', 'abs_locales_dir']
- expected_dirs.sort()
- self.assertEqual(dirs, expected_dirs)
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/testing/mozharness/test/test_mozilla_blob_upload.py b/testing/mozharness/test/test_mozilla_blob_upload.py
deleted file mode 100644
index 4918d6c73..000000000
--- a/testing/mozharness/test/test_mozilla_blob_upload.py
+++ /dev/null
@@ -1,103 +0,0 @@
-import os
-import gc
-import unittest
-import copy
-import mock
-
-import mozharness.base.log as log
-from mozharness.base.log import ERROR
-import mozharness.base.script as script
-from mozharness.mozilla.blob_upload import BlobUploadMixin, \
- blobupload_config_options
-
-class CleanupObj(script.ScriptMixin, log.LogMixin):
- def __init__(self):
- super(CleanupObj, self).__init__()
- self.log_obj = None
- self.config = {'log_level': ERROR}
-
-
-def cleanup():
- gc.collect()
- c = CleanupObj()
- for f in ('test_logs', 'test_dir', 'tmpfile_stdout', 'tmpfile_stderr'):
- c.rmtree(f)
-
-
-class BlobUploadScript(BlobUploadMixin, script.BaseScript):
- config_options = copy.deepcopy(blobupload_config_options)
- def __init__(self, **kwargs):
- self.abs_dirs = None
- self.set_buildbot_property = mock.Mock()
- super(BlobUploadScript, self).__init__(
- config_options=self.config_options,
- **kwargs
- )
-
- def query_python_path(self, binary="python"):
- if binary == "blobberc.py":
- return mock.Mock(return_value='/path/to/blobberc').return_value
- elif binary == "python":
- return mock.Mock(return_value='/path/to/python').return_value
-
- def query_abs_dirs(self):
- if self.abs_dirs:
- return self.abs_dirs
- abs_dirs = super(BlobUploadScript, self).query_abs_dirs()
- dirs = {}
- dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'],
- 'blobber_upload_dir')
- abs_dirs.update(dirs)
- self.abs_dirs = abs_dirs
-
- return self.abs_dirs
-
- def run_command(self, command):
- self.command = command
-
-# TestBlobUploadMechanism {{{1
-class TestBlobUploadMechanism(unittest.TestCase):
- # I need a log watcher helper function, here and in test_log.
- def setUp(self):
- cleanup()
- self.s = None
-
- def tearDown(self):
- # Close the logfile handles, or windows can't remove the logs
- if hasattr(self, 's') and isinstance(self.s, object):
- del(self.s)
- cleanup()
-
- def test_blob_upload_mechanism(self):
- self.s = BlobUploadScript(config={'log_type': 'multi',
- 'blob_upload_branch': 'test-branch',
- 'default_blob_upload_servers':
- ['http://blob_server.me'],
- 'blob_uploader_auth_file':
- os.path.abspath(__file__)},
- initial_config_file='test/test.json')
-
- content = "Hello world!"
- parent_dir = self.s.query_abs_dirs()['abs_blob_upload_dir']
- if not os.path.isdir(parent_dir):
- self.s.mkdir_p(parent_dir)
-
- file_name = os.path.join(parent_dir, 'test_mock_blob_file')
- self.s.write_to_file(file_name, content)
- self.s.upload_blobber_files()
- self.assertTrue(self.s.set_buildbot_property.called)
-
- expected_result = ['/path/to/python', '/path/to/blobberc', '-u',
- 'http://blob_server.me', '-a',
- os.path.abspath(__file__), '-b', 'test-branch', '-d']
- expected_result.append(self.s.query_abs_dirs()['abs_blob_upload_dir'])
- expected_result += [
- '--output-manifest',
- os.path.join(self.s.query_abs_dirs()['abs_work_dir'], "uploaded_files.json")
- ]
- self.assertEqual(expected_result, self.s.command)
-
-
-# main {{{1
-if __name__ == '__main__':
- unittest.main()
diff --git a/testing/mozharness/test/test_mozilla_buildbot.py b/testing/mozharness/test/test_mozilla_buildbot.py
deleted file mode 100644
index afc715026..000000000
--- a/testing/mozharness/test/test_mozilla_buildbot.py
+++ /dev/null
@@ -1,62 +0,0 @@
-import gc
-import unittest
-
-
-import mozharness.base.log as log
-from mozharness.base.log import ERROR
-import mozharness.base.script as script
-from mozharness.mozilla.buildbot import BuildbotMixin, TBPL_SUCCESS, \
- TBPL_FAILURE, EXIT_STATUS_DICT
-
-
-class CleanupObj(script.ScriptMixin, log.LogMixin):
- def __init__(self):
- super(CleanupObj, self).__init__()
- self.log_obj = None
- self.config = {'log_level': ERROR}
-
-
-def cleanup():
- gc.collect()
- c = CleanupObj()
- for f in ('test_logs', 'test_dir', 'tmpfile_stdout', 'tmpfile_stderr'):
- c.rmtree(f)
-
-
-class BuildbotScript(BuildbotMixin, script.BaseScript):
- def __init__(self, **kwargs):
- super(BuildbotScript, self).__init__(**kwargs)
-
-
-# TestBuildbotStatus {{{1
-class TestBuildbotStatus(unittest.TestCase):
- # I need a log watcher helper function, here and in test_log.
- def setUp(self):
- cleanup()
- self.s = None
-
- def tearDown(self):
- # Close the logfile handles, or windows can't remove the logs
- if hasattr(self, 's') and isinstance(self.s, object):
- del(self.s)
- cleanup()
-
- def test_over_max_log_size(self):
- self.s = BuildbotScript(config={'log_type': 'multi',
- 'buildbot_max_log_size': 200},
- initial_config_file='test/test.json')
- self.s.info("foo!")
- self.s.buildbot_status(TBPL_SUCCESS)
- self.assertEqual(self.s.return_code, EXIT_STATUS_DICT[TBPL_FAILURE])
-
- def test_under_max_log_size(self):
- self.s = BuildbotScript(config={'log_type': 'multi',
- 'buildbot_max_log_size': 20000},
- initial_config_file='test/test.json')
- self.s.info("foo!")
- self.s.buildbot_status(TBPL_SUCCESS)
- self.assertEqual(self.s.return_code, EXIT_STATUS_DICT[TBPL_SUCCESS])
-
-# main {{{1
-if __name__ == '__main__':
- unittest.main()
diff --git a/testing/mozharness/test/test_mozilla_release.py b/testing/mozharness/test/test_mozilla_release.py
deleted file mode 100644
index adbe322c4..000000000
--- a/testing/mozharness/test/test_mozilla_release.py
+++ /dev/null
@@ -1,42 +0,0 @@
-import unittest
-from mozharness.mozilla.release import get_previous_version
-
-
-class TestGetPreviousVersion(unittest.TestCase):
- def testESR(self):
- self.assertEquals(
- '31.5.3esr',
- get_previous_version('31.6.0esr',
- ['31.5.3esr', '31.5.2esr', '31.4.0esr']))
-
- def testReleaseBuild1(self):
- self.assertEquals(
- '36.0.4',
- get_previous_version('37.0', ['36.0.4', '36.0.1', '35.0.1']))
-
- def testReleaseBuild2(self):
- self.assertEquals(
- '36.0.4',
- get_previous_version('37.0',
- ['37.0', '36.0.4', '36.0.1', '35.0.1']))
-
- def testBetaMidCycle(self):
- self.assertEquals(
- '37.0b4',
- get_previous_version('37.0b5', ['37.0b4', '37.0b3']))
-
- def testBetaEarlyCycle(self):
- # 37.0 is the RC build
- self.assertEquals(
- '38.0b1',
- get_previous_version('38.0b2', ['38.0b1', '37.0']))
-
- def testBetaFirstInCycle(self):
- self.assertEquals(
- '37.0',
- get_previous_version('38.0b1', ['37.0', '37.0b7']))
-
- def testTwoDots(self):
- self.assertEquals(
- '37.1.0',
- get_previous_version('38.0b1', ['37.1.0', '36.0']))
diff --git a/testing/mozharness/tox.ini b/testing/mozharness/tox.ini
deleted file mode 100644
index e2e1c3009..000000000
--- a/testing/mozharness/tox.ini
+++ /dev/null
@@ -1,27 +0,0 @@
-[tox]
-envlist = py27-hg3.7
-
-[base]
-deps =
- coverage
- nose
- rednose
-
-[testenv]
-basepython = python2.7
-setenv =
- HGRCPATH = {toxinidir}/test/hgrc
-
-commands =
- coverage run --source configs,mozharness,scripts --branch {envbindir}/nosetests -v --with-xunit --rednose --force-color {posargs}
-
-[testenv:py27-hg3.7]
-deps =
- {[base]deps}
- mercurial==3.7.3
-
-[testenv:py27-coveralls]
-deps=
- python-coveralls==2.4.3
-commands=
- coveralls
diff --git a/testing/mozharness/unit.sh b/testing/mozharness/unit.sh
deleted file mode 100755
index a4a27a837..000000000
--- a/testing/mozharness/unit.sh
+++ /dev/null
@@ -1,85 +0,0 @@
-#!/bin/bash
-###########################################################################
-# This requires coverage and nosetests:
-#
-# pip install -r requirements.txt
-#
-# test_base_vcs_mercurial.py requires hg >= 1.6.0 with mq, rebase, share
-# extensions to fully test.
-###########################################################################
-
-COVERAGE_ARGS="--omit='/usr/*,/opt/*'"
-OS_TYPE='linux'
-uname -v | grep -q Darwin
-if [ $? -eq 0 ] ; then
- OS_TYPE='osx'
- COVERAGE_ARGS="--omit='/Library/*,/usr/*,/opt/*'"
-fi
-uname -s | egrep -q MINGW32 # Cygwin will be linux in this case?
-if [ $? -eq 0 ] ; then
- OS_TYPE='windows'
-fi
-NOSETESTS=`env which nosetests`
-
-echo "### Finding mozharness/ .py files..."
-files=`find mozharness -name [a-z]\*.py`
-if [ $OS_TYPE == 'windows' ] ; then
- MOZHARNESS_PY_FILES=""
- for f in $files; do
- file $f | grep -q "Assembler source"
- if [ $? -ne 0 ] ; then
- MOZHARNESS_PY_FILES="$MOZHARNESS_PY_FILES $f"
- fi
- done
-else
- MOZHARNESS_PY_FILES=$files
-fi
-echo "### Finding scripts/ .py files..."
-files=`find scripts -name [a-z]\*.py`
-if [ $OS_TYPE == 'windows' ] ; then
- SCRIPTS_PY_FILES=""
- for f in $files; do
- file $f | grep -q "Assembler source"
- if [ $? -ne 0 ] ; then
- SCRIPTS_PY_FILES="$SCRIPTS_PY_FILES $f"
- fi
- done
-else
- SCRIPTS_PY_FILES=$files
-fi
-export PYTHONPATH=`env pwd`:$PYTHONPATH
-
-echo "### Running pyflakes"
-pyflakes $MOZHARNESS_PY_FILES $SCRIPTS_PY_FILES | grep -v "local variable 'url' is assigned to" | grep -v "redefinition of unused 'json'" | egrep -v "mozharness/mozilla/testing/mozpool\.py.*undefined name 'requests'"
-
-echo "### Running pylint"
-pylint -E -e F -f parseable $MOZHARNESS_PY_FILES $SCRIPTS_PY_FILES 2>&1 | egrep -v '(No config file found, using default configuration|Instance of .* has no .* member|Unable to import .devicemanager|Undefined variable .DMError|Module .hashlib. has no .sha512. member)'
-
-rm -rf build logs
-if [ $OS_TYPE != 'windows' ] ; then
- echo "### Testing non-networked unit tests"
- coverage run -a --branch $COVERAGE_ARGS $NOSETESTS test/test_*.py
- echo "### Running *.py [--list-actions]"
- for filename in $MOZHARNESS_PY_FILES; do
- coverage run -a --branch $COVERAGE_ARGS $filename
- done
- for filename in $SCRIPTS_PY_FILES ; do
- coverage run -a --branch $COVERAGE_ARGS $filename --list-actions > /dev/null
- done
- echo "### Running scripts/configtest.py --log-level warning"
- coverage run -a --branch $COVERAGE_ARGS scripts/configtest.py --log-level warning
-
- echo "### Creating coverage html"
- coverage html $COVERAGE_ARGS -d coverage.new
- if [ -e coverage ] ; then
- mv coverage coverage.old
- mv coverage.new coverage
- rm -rf coverage.old
- else
- mv coverage.new coverage
- fi
-else
- echo "### Running nosetests..."
- nosetests test/
-fi
-rm -rf build logs