diff options
Diffstat (limited to 'security/nss/automation/taskcluster')
28 files changed, 51 insertions, 587 deletions
diff --git a/security/nss/automation/taskcluster/docker-clang-3.9/setup.sh b/security/nss/automation/taskcluster/docker-clang-3.9/setup.sh index 3076667a6e..7b7d534e66 100644 --- a/security/nss/automation/taskcluster/docker-clang-3.9/setup.sh +++ b/security/nss/automation/taskcluster/docker-clang-3.9/setup.sh @@ -25,8 +25,8 @@ apt-get -y update apt-get install -y --no-install-recommends ${apt_packages[@]} # Download clang. -curl -LO https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -curl -LO https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig +curl -LO http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz +curl -LO http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig # Verify the signature. gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D gpg --verify *.tar.xz.sig diff --git a/security/nss/automation/taskcluster/docker-decision/Dockerfile b/security/nss/automation/taskcluster/docker-decision/Dockerfile index 473ce64ba3..35777c0b7c 100644 --- a/security/nss/automation/taskcluster/docker-decision/Dockerfile +++ b/security/nss/automation/taskcluster/docker-decision/Dockerfile @@ -12,9 +12,6 @@ RUN chmod +x /home/worker/bin/* ADD setup.sh /tmp/setup.sh RUN bash /tmp/setup.sh -# Change user. -USER worker - # Env variables. ENV HOME /home/worker ENV SHELL /bin/bash diff --git a/security/nss/automation/taskcluster/docker-decision/bin/checkout.sh b/security/nss/automation/taskcluster/docker-decision/bin/checkout.sh index 0cdd2ac405..9167f6bda6 100644 --- a/security/nss/automation/taskcluster/docker-decision/bin/checkout.sh +++ b/security/nss/automation/taskcluster/docker-decision/bin/checkout.sh @@ -2,6 +2,11 @@ set -v -e -x +if [ $(id -u) = 0 ]; then + # Drop privileges by re-running this script. + exec su worker $0 +fi + # Default values for testing. REVISION=${NSS_HEAD_REVISION:-default} REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss} diff --git a/security/nss/automation/taskcluster/docker-gcc-4.4/Dockerfile b/security/nss/automation/taskcluster/docker-gcc-4.4/Dockerfile deleted file mode 100644 index 3330c007fe..0000000000 --- a/security/nss/automation/taskcluster/docker-gcc-4.4/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM ubuntu:14.04 -MAINTAINER Tim Taubert <ttaubert@mozilla.com> - -RUN useradd -d /home/worker -s /bin/bash -m worker -WORKDIR /home/worker - -# Add build and test scripts. -ADD bin /home/worker/bin -RUN chmod +x /home/worker/bin/* - -# Install dependencies. -ADD setup.sh /tmp/setup.sh -RUN bash /tmp/setup.sh - -# Change user. -USER worker - -# Env variables. -ENV HOME /home/worker -ENV SHELL /bin/bash -ENV USER worker -ENV LOGNAME worker -ENV HOSTNAME taskcluster-worker -ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 -ENV HOST localhost -ENV DOMSUF localdomain - -# Set a default command for debugging. -CMD ["/bin/bash", "--login"] diff --git a/security/nss/automation/taskcluster/docker-gcc-4.4/bin/checkout.sh b/security/nss/automation/taskcluster/docker-gcc-4.4/bin/checkout.sh deleted file mode 100644 index 9167f6bda6..0000000000 --- a/security/nss/automation/taskcluster/docker-gcc-4.4/bin/checkout.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -if [ $(id -u) = 0 ]; then - # Drop privileges by re-running this script. - exec su worker $0 -fi - -# Default values for testing. -REVISION=${NSS_HEAD_REVISION:-default} -REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss} - -# Clone NSS. -for i in 0 2 5; do - sleep $i - hg clone -r $REVISION $REPOSITORY nss && exit 0 - rm -rf nss -done -exit 1 diff --git a/security/nss/automation/taskcluster/docker-gcc-4.4/setup.sh b/security/nss/automation/taskcluster/docker-gcc-4.4/setup.sh deleted file mode 100644 index f6325d966c..0000000000 --- a/security/nss/automation/taskcluster/docker-gcc-4.4/setup.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -# Update packages. -export DEBIAN_FRONTEND=noninteractive -apt-get -y update && apt-get -y upgrade - -apt_packages=() -apt_packages+=('ca-certificates') -apt_packages+=('g++-4.4') -apt_packages+=('gcc-4.4') -apt_packages+=('locales') -apt_packages+=('make') -apt_packages+=('mercurial') -apt_packages+=('zlib1g-dev') - -# Install packages. -apt-get -y update -apt-get install -y --no-install-recommends ${apt_packages[@]} - -locale-gen en_US.UTF-8 -dpkg-reconfigure locales - -# Cleanup. -rm -rf ~/.ccache ~/.cache -apt-get autoremove -y -apt-get clean -apt-get autoclean -rm $0 diff --git a/security/nss/automation/taskcluster/docker-hacl/Dockerfile b/security/nss/automation/taskcluster/docker-hacl/Dockerfile deleted file mode 100644 index e8a88f06c7..0000000000 --- a/security/nss/automation/taskcluster/docker-hacl/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM ubuntu:xenial - -MAINTAINER Franziskus Kiefer <franziskuskiefer@gmail.com> -# Based on the HACL* image from Benjamin Beurdouche and -# the original F* formula with Daniel Fabian - -# Pinned versions of HACL* (F* and KreMLin are pinned as submodules) -ENV haclrepo https://github.com/mitls/hacl-star.git - -# Define versions of dependencies -ENV opamv 4.04.2 -ENV haclversion dcd48329d535727dbde93877b124c5ec4a7a2b20 - -# Install required packages and set versions -ADD setup.sh /tmp/setup.sh -RUN bash /tmp/setup.sh - -# Create user, add scripts. -RUN useradd -ms /bin/bash worker -WORKDIR /home/worker -ADD bin /home/worker/bin -RUN chmod +x /home/worker/bin/* -USER worker - -# Build F*, HACL*, verify. Install a few more dependencies. -ENV OPAMYES true -ENV PATH "/home/worker/hacl-star/dependencies/z3/bin:$PATH" -ADD setup-user.sh /tmp/setup-user.sh -ADD license.txt /tmp/license.txt -RUN bash /tmp/setup-user.sh diff --git a/security/nss/automation/taskcluster/docker-hacl/bin/checkout.sh b/security/nss/automation/taskcluster/docker-hacl/bin/checkout.sh deleted file mode 100644 index 9167f6bda6..0000000000 --- a/security/nss/automation/taskcluster/docker-hacl/bin/checkout.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -if [ $(id -u) = 0 ]; then - # Drop privileges by re-running this script. - exec su worker $0 -fi - -# Default values for testing. -REVISION=${NSS_HEAD_REVISION:-default} -REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss} - -# Clone NSS. -for i in 0 2 5; do - sleep $i - hg clone -r $REVISION $REPOSITORY nss && exit 0 - rm -rf nss -done -exit 1 diff --git a/security/nss/automation/taskcluster/docker-hacl/license.txt b/security/nss/automation/taskcluster/docker-hacl/license.txt deleted file mode 100644 index 03d25c4d31..0000000000 --- a/security/nss/automation/taskcluster/docker-hacl/license.txt +++ /dev/null @@ -1,15 +0,0 @@ -/* Copyright 2016-2017 INRIA and Microsoft Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - diff --git a/security/nss/automation/taskcluster/docker-hacl/setup-user.sh b/security/nss/automation/taskcluster/docker-hacl/setup-user.sh deleted file mode 100644 index b8accaf584..0000000000 --- a/security/nss/automation/taskcluster/docker-hacl/setup-user.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -# Prepare build (OCaml packages) -opam init -echo ". /home/worker/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true" >> .bashrc -opam switch -v ${opamv} -opam install ocamlfind batteries sqlite3 fileutils yojson ppx_deriving_yojson zarith pprint menhir ulex process fix wasm stdint - -# Get the HACL* code -git clone ${haclrepo} hacl-star -git -C hacl-star checkout ${haclversion} - -# Prepare submodules, and build, verify, test, and extract c code -# This caches the extracted c code (pins the HACL* version). All we need to do -# on CI now is comparing the code in this docker image with the one in NSS. -opam config exec -- make -C hacl-star prepare -j$(nproc) -make -C hacl-star verify-nss -j$(nproc) -make -C hacl-star -f Makefile.build snapshots/nss -j$(nproc) -KOPTS="-funroll-loops 5" make -C hacl-star/code/curve25519 test -j$(nproc) -make -C hacl-star/code/salsa-family test -j$(nproc) -make -C hacl-star/code/poly1305 test -j$(nproc) - -# Cleanup. -rm -rf ~/.ccache ~/.cache diff --git a/security/nss/automation/taskcluster/docker-hacl/setup.sh b/security/nss/automation/taskcluster/docker-hacl/setup.sh deleted file mode 100644 index f5f8bd7d5e..0000000000 --- a/security/nss/automation/taskcluster/docker-hacl/setup.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -# Update packages. -export DEBIAN_FRONTEND=noninteractive -apt-get -qq update -apt-get install --yes libssl-dev libsqlite3-dev g++-5 gcc-5 m4 make opam pkg-config python libgmp3-dev cmake curl libtool-bin autoconf wget locales -update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 200 -update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 200 - -# Get clang-format-3.9 -curl -LO https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -curl -LO https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig -# Verify the signature. -gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D -gpg --verify *.tar.xz.sig -# Install into /usr/local/. -tar xJvf *.tar.xz -C /usr/local --strip-components=1 -# Cleanup. -rm *.tar.xz* - -locale-gen en_US.UTF-8 -dpkg-reconfigure locales - -# Cleanup. -rm -rf ~/.ccache ~/.cache -apt-get autoremove -y -apt-get clean -apt-get autoclean diff --git a/security/nss/automation/taskcluster/docker/setup.sh b/security/nss/automation/taskcluster/docker/setup.sh index 01f9c413a5..3ba4e854ef 100644 --- a/security/nss/automation/taskcluster/docker/setup.sh +++ b/security/nss/automation/taskcluster/docker/setup.sh @@ -48,8 +48,8 @@ apt-get -y update apt-get install -y --no-install-recommends ${apt_packages[@]} # Download clang. -curl -LO https://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz -curl -LO https://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig +curl -LO http://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz +curl -LO http://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig # Verify the signature. gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D gpg --verify *.tar.xz.sig diff --git a/security/nss/automation/taskcluster/graph/src/context_hash.js b/security/nss/automation/taskcluster/graph/src/context_hash.js index 0699a0590e..f0a2e9a88c 100644 --- a/security/nss/automation/taskcluster/graph/src/context_hash.js +++ b/security/nss/automation/taskcluster/graph/src/context_hash.js @@ -27,24 +27,14 @@ function collectFilesInDirectory(dir) { }); } -// A list of hashes for each file in the given path. -function collectFileHashes(context_path) { +// Compute a context hash for the given context path. +export default function (context_path) { let root = path.join(__dirname, "../../../.."); let dir = path.join(root, context_path); let files = collectFilesInDirectory(dir).sort(); - - return files.map(file => { + let hashes = files.map(file => { return sha256(file + "|" + fs.readFileSync(file, "utf-8")); }); -} - -// Compute a context hash for the given context path. -export default function (context_path) { - // Regenerate all images when the image_builder changes. - let hashes = collectFileHashes("automation/taskcluster/image_builder"); - - // Regenerate images when the image itself changes. - hashes = hashes.concat(collectFileHashes(context_path)); // Generate a new prefix every month to ensure the image stays buildable. let now = new Date(); diff --git a/security/nss/automation/taskcluster/graph/src/extend.js b/security/nss/automation/taskcluster/graph/src/extend.js index 90e23ae601..d541a1a3b6 100644 --- a/security/nss/automation/taskcluster/graph/src/extend.js +++ b/security/nss/automation/taskcluster/graph/src/extend.js @@ -15,29 +15,15 @@ const LINUX_CLANG39_IMAGE = { path: "automation/taskcluster/docker-clang-3.9" }; -const LINUX_GCC44_IMAGE = { - name: "linux-gcc-4.4", - path: "automation/taskcluster/docker-gcc-4.4" -}; - const FUZZ_IMAGE = { name: "fuzz", path: "automation/taskcluster/docker-fuzz" }; -const HACL_GEN_IMAGE = { - name: "hacl", - path: "automation/taskcluster/docker-hacl" -}; - const WINDOWS_CHECKOUT_CMD = "bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " + "(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " + "(sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)\""; -const MAC_CHECKOUT_CMD = ["bash", "-c", - "hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " + - "(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " + - "(sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)"]; /*****************************************************************************/ @@ -65,15 +51,6 @@ queue.filter(task => { if (task.platform == "aarch64") { return false; } - - // No mac - if (task.platform == "mac") { - return false; - } - } - - if (task.tests == "fips" && task.platform == "mac") { - return false; } // Only old make builds have -Ddisable_libpkix=0 and can run chain tests. @@ -82,8 +59,8 @@ queue.filter(task => { } if (task.group == "Test") { - // Don't run test builds on old make platforms, and not for fips gyp. - if (task.collection == "make" || task.collection == "fips") { + // Don't run test builds on old make platforms + if (task.collection == "make") { return false; } } @@ -101,19 +78,11 @@ queue.filter(task => { queue.map(task => { if (task.collection == "asan") { // CRMF and FIPS tests still leak, unfortunately. - if (task.tests == "crmf") { + if (task.tests == "crmf" || task.tests == "fips") { task.env.ASAN_OPTIONS = "detect_leaks=0"; } } - // We don't run FIPS SSL tests - if (task.tests == "ssl") { - if (!task.env) { - task.env = {}; - } - task.env.NSS_SSL_TESTS = "crl iopr policy"; - } - // Windows is slow. if (task.platform == "windows2012-64" && task.tests == "chains") { task.maxRunTime = 7200; @@ -159,18 +128,6 @@ export default async function main() { ], }); - await scheduleLinux("Linux 64 (opt, make)", { - env: {USE_64: "1", BUILD_OPT: "1"}, - platform: "linux64", - image: LINUX_IMAGE, - collection: "make", - command: [ - "/bin/bash", - "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" - ], - }); - await scheduleLinux("Linux 32 (debug, make)", { platform: "linux32", image: LINUX_IMAGE, @@ -196,12 +153,6 @@ export default async function main() { features: ["allowPtrace"], }, "--ubsan --asan"); - await scheduleLinux("Linux 64 (FIPS opt)", { - platform: "linux64", - collection: "fips", - image: LINUX_IMAGE, - }, "--enable-fips --opt"); - await scheduleWindows("Windows 2012 64 (debug, make)", { platform: "windows2012-64", collection: "make", @@ -265,70 +216,6 @@ export default async function main() { collection: "opt", }, aarch64_base) ); - - await scheduleMac("Mac (opt)", {collection: "opt"}, "--opt"); - await scheduleMac("Mac (debug)", {collection: "debug"}); -} - - -async function scheduleMac(name, base, args = "") { - let mac_base = merge(base, { - env: { - PATH: "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", - NSS_TASKCLUSTER_MAC: "1", - DOMSUF: "localdomain", - HOST: "localhost", - }, - provisioner: "localprovisioner", - workerType: "nss-macos-10-12", - platform: "mac" - }); - - // Build base definition. - let build_base = merge({ - command: [ - MAC_CHECKOUT_CMD, - ["bash", "-c", - "nss/automation/taskcluster/scripts/build_gyp.sh", args] - ], - provisioner: "localprovisioner", - workerType: "nss-macos-10-12", - platform: "mac", - maxRunTime: 7200, - artifacts: [{ - expires: 24 * 7, - type: "directory", - path: "public" - }], - kind: "build", - symbol: "B" - }, mac_base); - - // The task that builds NSPR+NSS. - let task_build = queue.scheduleTask(merge(build_base, {name})); - - // The task that generates certificates. - let task_cert = queue.scheduleTask(merge(build_base, { - name: "Certificates", - command: [ - MAC_CHECKOUT_CMD, - ["bash", "-c", - "nss/automation/taskcluster/scripts/gen_certs.sh"] - ], - parent: task_build, - symbol: "Certs" - })); - - // Schedule tests. - scheduleTests(task_build, task_cert, merge(mac_base, { - command: [ - MAC_CHECKOUT_CMD, - ["bash", "-c", - "nss/automation/taskcluster/scripts/run_tests.sh"] - ] - })); - - return queue.submit(); } /*****************************************************************************/ @@ -355,45 +242,6 @@ async function scheduleLinux(name, base, args = "") { // The task that builds NSPR+NSS. let task_build = queue.scheduleTask(merge(build_base, {name})); - // Make builds run FIPS tests, which need an extra FIPS build. - if (base.collection == "make") { - let extra_build = queue.scheduleTask(merge(build_base, { - env: { NSS_FORCE_FIPS: "1" }, - group: "FIPS", - name: `${name} w/ NSS_FORCE_FIPS` - })); - - // The task that generates certificates. - let task_cert = queue.scheduleTask(merge(build_base, { - name: "Certificates", - command: [ - "/bin/bash", - "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/gen_certs.sh" - ], - parent: extra_build, - symbol: "Certs-F", - group: "FIPS", - })); - - // Schedule FIPS tests. - queue.scheduleTask(merge(base, { - parent: task_cert, - name: "FIPS", - command: [ - "/bin/bash", - "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" - ], - cycle: "standard", - kind: "test", - name: "FIPS tests", - symbol: "Tests-F", - tests: "fips", - group: "FIPS" - })); - } - // The task that generates certificates. let task_cert = queue.scheduleTask(merge(build_base, { name: "Certificates", @@ -427,26 +275,6 @@ async function scheduleLinux(name, base, args = "") { })); queue.scheduleTask(merge(extra_base, { - name: `${name} w/ gcc-4.4`, - image: LINUX_GCC44_IMAGE, - env: { - USE_64: "1", - CC: "gcc-4.4", - CCC: "g++-4.4", - // gcc-4.6 introduced nullptr. - NSS_DISABLE_GTESTS: "1", - }, - // Use the old Makefile-based build system, GYP doesn't have a proper GCC - // version check for __int128 support. It's mainly meant to cover RHEL6. - command: [ - "/bin/bash", - "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh", - ], - symbol: "gcc-4.4" - })); - - queue.scheduleTask(merge(extra_base, { name: `${name} w/ gcc-4.8`, env: { CC: "gcc-4.8", @@ -575,13 +403,12 @@ async function scheduleFuzzing() { // Schedule MPI fuzzing runs. let mpi_base = merge(run_base, {group: "MPI"}); - let mpi_names = ["add", "addmod", "div", "mod", "mulmod", "sqr", + let mpi_names = ["add", "addmod", "div", "expmod", "mod", "mulmod", "sqr", "sqrmod", "sub", "submod"]; for (let name of mpi_names) { scheduleFuzzingRun(mpi_base, `MPI (${name})`, `mpi-${name}`, 4096, name); } scheduleFuzzingRun(mpi_base, `MPI (invmod)`, `mpi-invmod`, 256, "invmod"); - scheduleFuzzingRun(mpi_base, `MPI (expmod)`, `mpi-expmod`, 2048, "expmod"); // Schedule TLS fuzzing runs (non-fuzzing mode). let tls_base = merge(run_base, {group: "TLS"}); @@ -798,43 +625,6 @@ async function scheduleWindows(name, base, build_script) { symbol: "B" }); - // Make builds run FIPS tests, which need an extra FIPS build. - if (base.collection == "make") { - let extra_build = queue.scheduleTask(merge(build_base, { - env: { NSS_FORCE_FIPS: "1" }, - group: "FIPS", - name: `${name} w/ NSS_FORCE_FIPS` - })); - - // The task that generates certificates. - let task_cert = queue.scheduleTask(merge(build_base, { - name: "Certificates", - command: [ - WINDOWS_CHECKOUT_CMD, - "bash -c nss/automation/taskcluster/windows/gen_certs.sh" - ], - parent: extra_build, - symbol: "Certs-F", - group: "FIPS", - })); - - // Schedule FIPS tests. - queue.scheduleTask(merge(base, { - parent: task_cert, - name: "FIPS", - command: [ - WINDOWS_CHECKOUT_CMD, - "bash -c nss/automation/taskcluster/windows/run_tests.sh" - ], - cycle: "standard", - kind: "test", - name: "FIPS tests", - symbol: "Tests-F", - tests: "fips", - group: "FIPS" - })); - } - // The task that builds NSPR+NSS. let task_build = queue.scheduleTask(merge(build_base, {name})); @@ -913,6 +703,9 @@ function scheduleTests(task_build, task_cert, test_base) { name: "DB tests", symbol: "DB", tests: "dbtests" })); queue.scheduleTask(merge(cert_base, { + name: "FIPS tests", symbol: "FIPS", tests: "fips" + })); + queue.scheduleTask(merge(cert_base, { name: "Merge tests", symbol: "Merge", tests: "merge" })); queue.scheduleTask(merge(cert_base, { @@ -980,16 +773,5 @@ async function scheduleTools() { ] })); - queue.scheduleTask(merge(base, { - symbol: "hacl", - name: "hacl", - image: HACL_GEN_IMAGE, - command: [ - "/bin/bash", - "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_hacl.sh" - ] - })); - return queue.submit(); } diff --git a/security/nss/automation/taskcluster/graph/src/image_builder.js b/security/nss/automation/taskcluster/graph/src/image_builder.js index b89b6980c1..bc90e0242f 100644 --- a/security/nss/automation/taskcluster/graph/src/image_builder.js +++ b/security/nss/automation/taskcluster/graph/src/image_builder.js @@ -31,11 +31,13 @@ export async function buildTask({name, path}) { return { name: "Image Builder", - image: "nssdev/image_builder:0.1.5", + image: "taskcluster/image_builder:0.1.5", routes: ["index." + ns], env: { - NSS_HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, - NSS_HEAD_REVISION: process.env.NSS_HEAD_REVISION, + HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, + BASE_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, + HEAD_REV: process.env.NSS_HEAD_REVISION, + HEAD_REF: process.env.NSS_HEAD_REVISION, PROJECT: process.env.TC_PROJECT, CONTEXT_PATH: path, HASH: hash @@ -50,11 +52,10 @@ export async function buildTask({name, path}) { command: [ "/bin/bash", "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/build_image.sh" + "/home/worker/bin/build_image.sh" ], platform: "nss-decision", features: ["dind"], - maxRunTime: 7200, kind: "build", symbol: "I" }; diff --git a/security/nss/automation/taskcluster/graph/src/try_syntax.js b/security/nss/automation/taskcluster/graph/src/try_syntax.js index 1f4e12eeee..7748e068a0 100644 --- a/security/nss/automation/taskcluster/graph/src/try_syntax.js +++ b/security/nss/automation/taskcluster/graph/src/try_syntax.js @@ -22,10 +22,10 @@ function parseOptions(opts) { } // Parse platforms. - let allPlatforms = ["linux", "linux64", "linux64-asan", "linux64-fips", + let allPlatforms = ["linux", "linux64", "linux64-asan", "win", "win64", "win-make", "win64-make", "linux64-make", "linux-make", "linux-fuzz", - "linux64-fuzz", "aarch64", "mac"]; + "linux64-fuzz", "aarch64"]; let platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms); // If the given value is nonsense or "none" default to all platforms. @@ -51,7 +51,7 @@ function parseOptions(opts) { } // Parse tools. - let allTools = ["clang-format", "scan-build", "hacl"]; + let allTools = ["clang-format", "scan-build"]; let tools = intersect(opts.tools.split(/\s*,\s*/), allTools); // If the given value is "all" run all tools. @@ -111,7 +111,6 @@ function filter(opts) { "linux": "linux32", "linux-fuzz": "linux32", "linux64-asan": "linux64", - "linux64-fips": "linux64", "linux64-fuzz": "linux64", "linux64-make": "linux64", "linux-make": "linux32", @@ -127,8 +126,6 @@ function filter(opts) { // Additional checks. if (platform == "linux64-asan") { keep &= coll("asan"); - } else if (platform == "linux64-fips") { - keep &= coll("fips"); } else if (platform == "linux64-make" || platform == "linux-make" || platform == "win64-make" || platform == "win-make") { keep &= coll("make"); diff --git a/security/nss/automation/taskcluster/image_builder/Dockerfile b/security/nss/automation/taskcluster/image_builder/Dockerfile deleted file mode 100644 index f8b4edcc53..0000000000 --- a/security/nss/automation/taskcluster/image_builder/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM ubuntu:16.04 -MAINTAINER Tim Taubert <ttaubert@mozilla.com> - -WORKDIR /home/worker - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update && apt-get install -y apt-transport-https apt-utils -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 && \ - sh -c "echo deb https://get.docker.io/ubuntu docker main \ - > /etc/apt/sources.list.d/docker.list" -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 41BD8711B1F0EC2B0D85B91CF59CE3A8323293EE && \ - sh -c "echo deb http://ppa.launchpad.net/mercurial-ppa/releases/ubuntu xenial main \ - > /etc/apt/sources.list.d/mercurial.list" -RUN apt-get update && apt-get install -y \ - lxc-docker-1.6.1 \ - mercurial - -ADD bin /home/worker/bin -RUN chmod +x /home/worker/bin/* - -# Set a default command useful for debugging -CMD ["/bin/bash", "--login"] diff --git a/security/nss/automation/taskcluster/image_builder/VERSION b/security/nss/automation/taskcluster/image_builder/VERSION deleted file mode 100644 index 9faa1b7a73..0000000000 --- a/security/nss/automation/taskcluster/image_builder/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.1.5 diff --git a/security/nss/automation/taskcluster/image_builder/bin/checkout.sh b/security/nss/automation/taskcluster/image_builder/bin/checkout.sh deleted file mode 100644 index 0cdd2ac405..0000000000 --- a/security/nss/automation/taskcluster/image_builder/bin/checkout.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -# Default values for testing. -REVISION=${NSS_HEAD_REVISION:-default} -REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss} - -# Clone NSS. -for i in 0 2 5; do - sleep $i - hg clone -r $REVISION $REPOSITORY nss && exit 0 - rm -rf nss -done -exit 1 diff --git a/security/nss/automation/taskcluster/scripts/build_gyp.sh b/security/nss/automation/taskcluster/scripts/build_gyp.sh index fb3a33a52b..7190bd5c49 100755 --- a/security/nss/automation/taskcluster/scripts/build_gyp.sh +++ b/security/nss/automation/taskcluster/scripts/build_gyp.sh @@ -9,10 +9,5 @@ hg_clone https://hg.mozilla.org/projects/nspr ./nspr default nss/build.sh -g -v "$@" # Package. -if [[ $(uname) = "Darwin" ]]; then - mkdir -p public - tar cvfjh public/dist.tar.bz2 dist -else - mkdir artifacts - tar cvfjh artifacts/dist.tar.bz2 dist -fi +mkdir artifacts +tar cvfjh artifacts/dist.tar.bz2 dist diff --git a/security/nss/automation/taskcluster/scripts/build_image.sh b/security/nss/automation/taskcluster/scripts/build_image.sh deleted file mode 100644 index b422214e71..0000000000 --- a/security/nss/automation/taskcluster/scripts/build_image.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -vex - -set -x -e -v - -# Prefix errors with taskcluster error prefix so that they are parsed by Treeherder -raise_error() { - echo - echo "[taskcluster-image-build:error] $1" - exit 1 -} - -# Ensure that the PROJECT is specified so the image can be indexed -test -n "$PROJECT" || raise_error "Project must be provided." -test -n "$HASH" || raise_error "Context Hash must be provided." - -CONTEXT_PATH=/home/worker/nss/$CONTEXT_PATH - -test -d $CONTEXT_PATH || raise_error "Context Path $CONTEXT_PATH does not exist." -test -f "$CONTEXT_PATH/Dockerfile" || raise_error "Dockerfile must be present in $CONTEXT_PATH." - -docker build -t $PROJECT:$HASH $CONTEXT_PATH - -mkdir /artifacts -docker save $PROJECT:$HASH > /artifacts/image.tar diff --git a/security/nss/automation/taskcluster/scripts/gen_certs.sh b/security/nss/automation/taskcluster/scripts/gen_certs.sh index c03db7e9c2..b8d4f60bae 100755 --- a/security/nss/automation/taskcluster/scripts/gen_certs.sh +++ b/security/nss/automation/taskcluster/scripts/gen_certs.sh @@ -12,10 +12,5 @@ NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" $(dirname $0)/run_tests.sh echo 1 > tests_results/security/localhost # Package. -if [[ $(uname) = "Darwin" ]]; then - mkdir -p public - tar cvfjh public/dist.tar.bz2 dist tests_results -else - mkdir artifacts - tar cvfjh artifacts/dist.tar.bz2 dist tests_results -fi +mkdir artifacts +tar cvfjh artifacts/dist.tar.bz2 dist tests_results diff --git a/security/nss/automation/taskcluster/scripts/run_hacl.sh b/security/nss/automation/taskcluster/scripts/run_hacl.sh deleted file mode 100644 index 281075eef4..0000000000 --- a/security/nss/automation/taskcluster/scripts/run_hacl.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -if [[ $(id -u) -eq 0 ]]; then - # Drop privileges by re-running this script. - # Note: this mangles arguments, better to avoid running scripts as root. - exec su worker -c "$0 $*" -fi - -set -e -x -v - -# The docker image this is running in has the HACL* and NSS sources. -# The extracted C code from HACL* is already generated and the HACL* tests were -# successfully executed. - -# Verify Poly1305 (doesn't work in docker image build) -make verify -C ~/hacl-star/code/poly1305 -j$(nproc) - -# Add license header to specs -spec_files=($(find ~/hacl-star/specs -type f -name '*.fst')) -for f in "${spec_files[@]}"; do - cat /tmp/license.txt "$f" > /tmp/tmpfile && mv /tmp/tmpfile "$f" -done - -# Format the extracted C code. -cd ~/hacl-star/snapshots/nss -cp ~/nss/.clang-format . -find . -type f -name '*.[ch]' -exec clang-format -i {} \+ - -# These diff commands will return 1 if there are differences and stop the script. -files=($(find ~/nss/lib/freebl/verified/ -type f -name '*.[ch]')) -for f in "${files[@]}"; do - diff $f $(basename "$f") -done - -# Check that the specs didn't change either. -cd ~/hacl-star/specs -files=($(find ~/nss/lib/freebl/verified/specs -type f)) -for f in "${files[@]}"; do - diff $f $(basename "$f") -done diff --git a/security/nss/automation/taskcluster/scripts/split.sh b/security/nss/automation/taskcluster/scripts/split.sh index fded64e1b2..4d18385eca 100644 --- a/security/nss/automation/taskcluster/scripts/split.sh +++ b/security/nss/automation/taskcluster/scripts/split.sh @@ -23,10 +23,16 @@ split_util() { # Copy everything. cp -R $nssdir $dstdir + # Skip gtests when building. + sed '/^DIRS = /s/ cpputil gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn + # Remove subdirectories that we don't want. rm -rf $dstdir/cmd + rm -rf $dstdir/tests rm -rf $dstdir/lib rm -rf $dstdir/automation + rm -rf $dstdir/gtests + rm -rf $dstdir/cpputil rm -rf $dstdir/doc # Start with an empty cmd lib directories to be filled selectively. diff --git a/security/nss/automation/taskcluster/windows/releng.manifest b/security/nss/automation/taskcluster/windows/releng.manifest index d571c544d6..68d2c1d9e6 100644 --- a/security/nss/automation/taskcluster/windows/releng.manifest +++ b/security/nss/automation/taskcluster/windows/releng.manifest @@ -1,10 +1,10 @@ [ { - "version": "Visual Studio 2017 15.4.2 / SDK 10.0.15063.0", - "size": 303146863, - "digest": "18700889e6b5e81613b9cf57ce4e0d46a6ee45bb4c5c33bae2604a5275326128775b8a032a1eb178c5db973746d565340c4e36d98375789e1d5bd836ab16ba58", + "version": "Visual Studio 2015 Update 3 14.0.25425.01 / SDK 10.0.14393.0", + "size": 326656969, + "digest": "babc414ffc0457d27f5a1ed24a8e4873afbe2f1c1a4075469a27c005e1babc3b2a788f643f825efedff95b79686664c67ec4340ed535487168a3482e68559bc7", "algorithm": "sha512", - "filename": "vs2017_15.4.2.zip", + "filename": "vs2015u3.zip", "unpack": true }, { diff --git a/security/nss/automation/taskcluster/windows/setup.sh b/security/nss/automation/taskcluster/windows/setup.sh index 36a040ba1c..7def50db4f 100644 --- a/security/nss/automation/taskcluster/windows/setup.sh +++ b/security/nss/automation/taskcluster/windows/setup.sh @@ -2,12 +2,12 @@ set -v -e -x -export VSPATH="$(pwd)/vs2017_15.4.2" +export VSPATH="$(pwd)/vs2015u3" export NINJA_PATH="$(pwd)/ninja/bin" export WINDOWSSDKDIR="${VSPATH}/SDK" export VS90COMNTOOLS="${VSPATH}/VC" -export INCLUDE="${VSPATH}/VC/include:${VSPATH}/SDK/Include/10.0.15063.0/ucrt:${VSPATH}/SDK/Include/10.0.15063.0/shared:${VSPATH}/SDK/Include/10.0.15063.0/um" +export INCLUDE="${VSPATH}/VC/include:${VSPATH}/SDK/Include/10.0.14393.0/ucrt:${VSPATH}/SDK/Include/10.0.14393.0/shared:${VSPATH}/SDK/Include/10.0.14393.0/um" # Usage: hg_clone repo dir [revision=@] hg_clone() { @@ -23,4 +23,4 @@ hg_clone() { } hg_clone https://hg.mozilla.org/build/tools tools default -tools/scripts/tooltool/tooltool_wrapper.sh $(dirname $0)/releng.manifest https://tooltool.mozilla-releng.net/ non-existant-file.sh /c/mozilla-build/python/python.exe /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok -c /c/builds/tooltool_cache +tools/scripts/tooltool/tooltool_wrapper.sh $(dirname $0)/releng.manifest https://api.pub.build.mozilla.org/tooltool/ non-existant-file.sh /c/mozilla-build/python/python.exe /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok -c /c/builds/tooltool_cache diff --git a/security/nss/automation/taskcluster/windows/setup32.sh b/security/nss/automation/taskcluster/windows/setup32.sh index 19bed284d1..bcddabfa39 100644 --- a/security/nss/automation/taskcluster/windows/setup32.sh +++ b/security/nss/automation/taskcluster/windows/setup32.sh @@ -4,7 +4,7 @@ set -v -e -x source $(dirname $0)/setup.sh -export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x86/Microsoft.VC141.CRT" +export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x86/Microsoft.VC140.CRT" export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x86" -export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/Hostx64/x86:${VSPATH}/VC/bin/Hostx64/x64:${VSPATH}/VC/Hostx86/x86:${VSPATH}/SDK/bin/10.0.15063.0/x64:${VSPATH}/VC/redist/x86/Microsoft.VC141.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x86:${PATH}" -export LIB="${VSPATH}/VC/lib/x86:${VSPATH}/SDK/lib/10.0.15063.0/ucrt/x86:${VSPATH}/SDK/lib/10.0.15063.0/um/x86" +export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/amd64_x86:${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x86:${VSPATH}/SDK/bin/x64:${VSPATH}/VC/redist/x86/Microsoft.VC140.CRT:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x86:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}" +export LIB="${VSPATH}/VC/lib:${VSPATH}/SDK/lib/10.0.14393.0/ucrt/x86:${VSPATH}/SDK/lib/10.0.14393.0/um/x86" diff --git a/security/nss/automation/taskcluster/windows/setup64.sh b/security/nss/automation/taskcluster/windows/setup64.sh index d16cb0ec9d..f308298c18 100644 --- a/security/nss/automation/taskcluster/windows/setup64.sh +++ b/security/nss/automation/taskcluster/windows/setup64.sh @@ -4,7 +4,7 @@ set -v -e -x source $(dirname $0)/setup.sh -export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x64/Microsoft.VC141.CRT" +export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT" export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x64" -export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/Hostx64/x64:${VSPATH}/VC/bin/Hostx86/x86:${VSPATH}/SDK/bin/10.0.15063.0/x64:${VSPATH}/VC/redist/x64/Microsoft.VC141.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}" -export LIB="${VSPATH}/VC/lib/x64:${VSPATH}/SDK/lib/10.0.15063.0/ucrt/x64:${VSPATH}/SDK/lib/10.0.15063.0/um/x64" +export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x64:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}" +export LIB="${VSPATH}/VC/lib/amd64:${VSPATH}/SDK/lib/10.0.14393.0/ucrt/x64:${VSPATH}/SDK/lib/10.0.14393.0/um/x64" |