diff options
author | Moonchild <moonchild@palemoon.org> | 2022-02-12 17:47:03 +0000 |
---|---|---|
committer | Matt A. Tobin <email@mattatobin.com> | 2022-02-12 14:23:18 -0600 |
commit | f66babd8b8368ada3e5aa29cdef1c77291ee4ddd (patch) | |
tree | e3842e2a6bf19090185f9c475b3846e1bb79ac97 /tools | |
download | GRE-f66babd8b8368ada3e5aa29cdef1c77291ee4ddd.tar.gz |
Create the Goanna Runtime Environment
Diffstat (limited to 'tools')
135 files changed, 19697 insertions, 0 deletions
diff --git a/tools/bloatview/bloatdiff.pl b/tools/bloatview/bloatdiff.pl new file mode 100755 index 000000000..8c93ad2b0 --- /dev/null +++ b/tools/bloatview/bloatdiff.pl @@ -0,0 +1,372 @@ +#!/usr/bin/perl -w +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +################################################################################ + +sub usage() { + print <<EOUSAGE; +# bloatdiff.pl - munges the output from +# XPCOM_MEM_BLOAT_LOG=1 +# firefox-bin -P default resource:///res/bloatcycle.html +# so that it does some summary and stats stuff. +# +# To show leak test results for a set of changes, do something like this: +# +# XPCOM_MEM_BLOAT_LOG=1 +# firefox-bin -P default resource:///res/bloatcycle.html > a.out +# **make change** +# firefox-bin -P default resource:///res/bloatcycle.html > b.out +# bloatdiff.pl a.out b.out + +EOUSAGE +} + +$OLDFILE = $ARGV[0]; +$NEWFILE = $ARGV[1]; +#$LABEL = $ARGV[2]; + +if (!$OLDFILE or + ! -e $OLDFILE or + -z $OLDFILE) { + print "\nError: Previous log file not specified, does not exist, or is empty.\n\n"; + &usage(); + exit 1; +} + +if (!$NEWFILE or + ! -e $NEWFILE or + -z $NEWFILE) { + print "\nError: Current log file not specified, does not exist, or is empty.\n\n"; + &usage(); + exit 1; +} + +sub processFile { + my ($filename, $map, $prevMap) = @_; + open(FH, $filename); + while (<FH>) { + if (m{ + ^\s*(\d+)\s # Line number + ([\w:]+)\s+ # Name + (-?\d+)\s+ # Size + (-?\d+)\s+ # Leaked + (-?\d+)\s+ # Objects Total + (-?\d+)\s+ # Objects Rem + \(\s*(-?[\d.]+)\s+ # Objects Mean + \+/-\s+ + ([\w.]+)\)\s+ # Objects StdDev + (-?\d+)\s+ # Reference Total + (-?\d+)\s+ # Reference Rem + \(\s*(-?[\d.]+)\s+ # Reference Mean + \+/-\s+ + ([\w\.]+)\) # Reference StdDev + }x) { + $$map{$2} = { name => $2, + size => $3, + leaked => $4, + objTotal => $5, + objRem => $6, + objMean => $7, + objStdDev => $8, + refTotal => $9, + refRem => $10, + refMean => $11, + refStdDev => $12, + bloat => $3 * $5 # size * objTotal + }; + } else { +# print "failed to parse: $_\n"; + } + } + close(FH); +} + +%oldMap = (); +processFile($OLDFILE, \%oldMap); + +%newMap = (); +processFile($NEWFILE, \%newMap); + +################################################################################ + +$inf = 9999999.99; + +sub getLeaksDelta { + my ($key) = @_; + my $oldLeaks = $oldMap{$key}{leaked} || 0; + my $newLeaks = $newMap{$key}{leaked}; + my $percentLeaks = 0; + if ($oldLeaks == 0) { + if ($newLeaks != 0) { + # there weren't any leaks before, but now there are! + $percentLeaks = $inf; + } + } + else { + $percentLeaks = ($newLeaks - $oldLeaks) / $oldLeaks * 100; + } + # else we had no record of this class before + return ($newLeaks - $oldLeaks, $percentLeaks); +} + +################################################################################ + +sub getBloatDelta { + my ($key) = @_; + my $newBloat = $newMap{$key}{bloat}; + my $percentBloat = 0; + my $oldSize = $oldMap{$key}{size} || 0; + my $oldTotal = $oldMap{$key}{objTotal} || 0; + my $oldBloat = $oldTotal * $oldSize; + if ($oldBloat == 0) { + if ($newBloat != 0) { + # this class wasn't used before, but now it is + $percentBloat = $inf; + } + } + else { + $percentBloat = ($newBloat - $oldBloat) / $oldBloat * 100; + } + # else we had no record of this class before + return ($newBloat - $oldBloat, $percentBloat); +} + +################################################################################ + +foreach $key (keys %newMap) { + my ($newLeaks, $percentLeaks) = getLeaksDelta($key); + my ($newBloat, $percentBloat) = getBloatDelta($key); + $newMap{$key}{leakDelta} = $newLeaks; + $newMap{$key}{leakPercent} = $percentLeaks; + $newMap{$key}{bloatDelta} = $newBloat; + $newMap{$key}{bloatPercent} = $percentBloat; +} + +################################################################################ + +# Print a value of bytes out in a reasonable +# KB, MB, or GB form. Copied from build-seamonkey-util.pl, sorry. -mcafee +sub PrintSize($) { + + # print a number with 3 significant figures + sub PrintNum($) { + my ($num) = @_; + my $rv; + if ($num < 1) { + $rv = sprintf "%.3f", ($num); + } elsif ($num < 10) { + $rv = sprintf "%.2f", ($num); + } elsif ($num < 100) { + $rv = sprintf "%.1f", ($num); + } else { + $rv = sprintf "%d", ($num); + } + } + + my ($size) = @_; + my $rv; + if ($size > 1000000000) { + $rv = PrintNum($size / 1000000000.0) . "G"; + } elsif ($size > 1000000) { + $rv = PrintNum($size / 1000000.0) . "M"; + } elsif ($size > 1000) { + $rv = PrintNum($size / 1000.0) . "K"; + } else { + $rv = PrintNum($size); + } +} + + +print "Bloat/Leak Delta Report\n"; +print "--------------------------------------------------------------------------------------\n"; +print "Current file: $NEWFILE\n"; +print "Previous file: $OLDFILE\n"; +print "----------------------------------------------leaks------leaks%------bloat------bloat%\n"; + + if (! $newMap{"TOTAL"} or + ! $newMap{"TOTAL"}{bloat}) { + # It's OK if leaked or leakPercent are 0 (in fact, that would be good). + # If bloatPercent is zero, it is also OK, because we may have just had + # two runs exactly the same or with no new bloat. + print "\nError: unable to calculate bloat/leak data.\n"; + print "There is no data present.\n\n"; + print "HINT - Did your test run complete successfully?\n"; + print "HINT - Are you pointing at the right log files?\n\n"; + &usage(); + exit 1; + } + +printf "%-40s %10s %10.2f%% %10s %10.2f%%\n", + ("TOTAL", + $newMap{"TOTAL"}{leaked}, $newMap{"TOTAL"}{leakPercent}, + $newMap{"TOTAL"}{bloat}, $newMap{"TOTAL"}{bloatPercent}); + +################################################################################ + +sub percentStr { + my ($p) = @_; + if ($p == $inf) { + return "-"; + } + else { + return sprintf "%10.2f%%", $p; + } +} + +# NEW LEAKS +@keys = sort { $newMap{$b}{leakPercent} <=> $newMap{$a}{leakPercent} } keys %newMap; +my $needsHeading = 1; +my $total = 0; +foreach $key (@keys) { + my $percentLeaks = $newMap{$key}{leakPercent}; + my $leaks = $newMap{$key}{leaked}; + if ($percentLeaks > 0 && $key !~ /TOTAL/) { + if ($needsHeading) { + printf "--NEW-LEAKS-----------------------------------leaks------leaks%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, percentStr($percentLeaks)); + $total += $leaks; + } +} +if (!$needsHeading) { + printf "%-40s %10s\n", ("TOTAL", $total); +} + +# FIXED LEAKS +@keys = sort { $newMap{$b}{leakPercent} <=> $newMap{$a}{leakPercent} } keys %newMap; +$needsHeading = 1; +$total = 0; +foreach $key (@keys) { + my $percentLeaks = $newMap{$key}{leakPercent}; + my $leaks = $newMap{$key}{leaked}; + if ($percentLeaks < 0 && $key !~ /TOTAL/) { + if ($needsHeading) { + printf "--FIXED-LEAKS---------------------------------leaks------leaks%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, percentStr($percentLeaks)); + $total += $leaks; + } +} +if (!$needsHeading) { + printf "%-40s %10s\n", ("TOTAL", $total); +} + +# NEW BLOAT +@keys = sort { $newMap{$b}{bloatPercent} <=> $newMap{$a}{bloatPercent} } keys %newMap; +$needsHeading = 1; +$total = 0; +foreach $key (@keys) { + my $percentBloat = $newMap{$key}{bloatPercent}; + my $bloat = $newMap{$key}{bloat}; + if ($percentBloat > 0 && $key !~ /TOTAL/) { + if ($needsHeading) { + printf "--NEW-BLOAT-----------------------------------bloat------bloat%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $bloat, percentStr($percentBloat)); + $total += $bloat; + } +} +if (!$needsHeading) { + printf "%-40s %10s\n", ("TOTAL", $total); +} + +# ALL LEAKS +@keys = sort { $newMap{$b}{leaked} <=> $newMap{$a}{leaked} } keys %newMap; +$needsHeading = 1; +$total = 0; +foreach $key (@keys) { + my $leaks = $newMap{$key}{leaked}; + my $percentLeaks = $newMap{$key}{leakPercent}; + if ($leaks > 0) { + if ($needsHeading) { + printf "--ALL-LEAKS-----------------------------------leaks------leaks%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, percentStr($percentLeaks)); + if ($key !~ /TOTAL/) { + $total += $leaks; + } + } +} +if (!$needsHeading) { +# printf "%-40s %10s\n", ("TOTAL", $total); +} + +# ALL BLOAT +@keys = sort { $newMap{$b}{bloat} <=> $newMap{$a}{bloat} } keys %newMap; +$needsHeading = 1; +$total = 0; +foreach $key (@keys) { + my $bloat = $newMap{$key}{bloat}; + my $percentBloat = $newMap{$key}{bloatPercent}; + if ($bloat > 0) { + if ($needsHeading) { + printf "--ALL-BLOAT-----------------------------------bloat------bloat%%-----------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $bloat, percentStr($percentBloat)); + if ($key !~ /TOTAL/) { + $total += $bloat; + } + } +} +if (!$needsHeading) { +# printf "%-40s %10s\n", ("TOTAL", $total); +} + +# NEW CLASSES +@keys = sort { $newMap{$b}{bloatDelta} <=> $newMap{$a}{bloatDelta} } keys %newMap; +$needsHeading = 1; +my $ltotal = 0; +my $btotal = 0; +foreach $key (@keys) { + my $leaks = $newMap{$key}{leaked}; + my $bloat = $newMap{$key}{bloat}; + my $percentBloat = $newMap{$key}{bloatPercent}; + if ($percentBloat == $inf && $key !~ /TOTAL/) { + if ($needsHeading) { + printf "--CLASSES-NOT-REPORTED-LAST-TIME--------------leaks------bloat------------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, $bloat); + if ($key !~ /TOTAL/) { + $ltotal += $leaks; + $btotal += $bloat; + } + } +} +if (!$needsHeading) { + printf "%-40s %10s %10s\n", ("TOTAL", $ltotal, $btotal); +} + +# OLD CLASSES +@keys = sort { ($oldMap{$b}{bloat} || 0) <=> ($oldMap{$a}{bloat} || 0) } keys %oldMap; +$needsHeading = 1; +$ltotal = 0; +$btotal = 0; +foreach $key (@keys) { + if (!defined($newMap{$key})) { + my $leaks = $oldMap{$key}{leaked}; + my $bloat = $oldMap{$key}{bloat}; + if ($needsHeading) { + printf "--CLASSES-THAT-WENT-AWAY----------------------leaks------bloat------------------------\n"; + $needsHeading = 0; + } + printf "%-40s %10s %10s\n", ($key, $leaks, $bloat); + if ($key !~ /TOTAL/) { + $ltotal += $leaks; + $btotal += $bloat; + } + } +} +if (!$needsHeading) { + printf "%-40s %10s %10s\n", ("TOTAL", $ltotal, $btotal); +} + +print "--------------------------------------------------------------------------------------\n"; diff --git a/tools/bloatview/bloattable.pl b/tools/bloatview/bloattable.pl new file mode 100755 index 000000000..e8acfabed --- /dev/null +++ b/tools/bloatview/bloattable.pl @@ -0,0 +1,590 @@ +#!/usr/bin/perl -w +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# bloattable [-debug] [-source] [-byte n|-obj n|-ref n] <file1> <file2> ... <filen> > <html-file> +# +# file1, file2, ... filen should be successive BloatView files generated from the same run. +# Summarize them in an HTML table. Output the HTML to the standard output. +# +# If -debug is set, create a slightly larger html file which is more suitable for debugging this script. +# If -source is set, create an html file that prints the html source as the output +# If -byte n, -obj n, or -ref n is given, make the page default to showing byte, object, or reference statistics, +# respectively, and sort by the nth column (n is zero-based, so the first column has n==0). +# +# See http://lxr.mozilla.org/mozilla/source/xpcom/doc/MemoryTools.html + +use 5.004; +use strict; +use diagnostics; +use File::Basename; +use Getopt::Long; + +# The generated HTML is almost entirely generated by a script. Only the <HTML>, <HEAD>, and <BODY> elements are explicit +# because a <SCRIPT> element cannot officially be a direct descendant of an <HTML> element. +# The script itself is almost all generated by an eval of a large string. This allows the script to reproduce itself +# when making a new page using document.write's. Re-sorting the page causes it to regenerate itself in this way. + + + +# Return the file's modification date. +sub fileModDate($) { + my ($pathName) = @_; + my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks) = + stat $pathName or die "Can't stat '$pathName'"; + return $mtime; +} + + +sub fileCoreName($) { + my ($pathName) = @_; + my $fileName = basename($pathName, ""); + $fileName =~ s/\..*//; + return $fileName; +} + + +# Convert a raw string into a single-quoted JavaScript string. +sub singleQuoteString($) { + local ($_) = @_; + s/\\/\\\\/g; + s/'/\\'/g; + s/\n/\\n/g; + s/<\//<\\\//g; + return "'$_'"; +} + + +# Convert a raw string into a double-quoted JavaScript string. +sub doubleQuoteString($) { + local ($_) = @_; + s/\\/\\\\/g; + s/"/\\"/g; + s/\n/\\n/g; + s/<\//<\\\//g; + return "\"$_\""; +} + + +# Quote special HTML characters in the string. +sub quoteHTML($) { + local ($_) = @_; + s/\&/&/g; + s/</</g; + s/>/>/g; + s/ / /g; + s/\n/<BR>\n/g; + return $_; +} + + +# Write the generated page to the standard output. +# The script source code is read from this file past the __END__ marker +# @$scriptData is the JavaScript source for the tables passed to JavaScript. Each entry is one line of JavaScript. +# @$persistentScriptData is the same as @scriptData, but persists when the page reloads itself. +# If $debug is true, generate the script directly instead of having it eval itself. +# If $source is true, generate a script that displays the page's source instead of the page itself. +sub generate(\@\@$$$$) { + my ($scriptData, $persistentScriptData, $debug, $source, $showMode, $sortColumn) = @_; + + my @scriptSource = <DATA>; + chomp @scriptSource; + print <<'EOS'; +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd"> +<HTML> +<HEAD> +<SCRIPT type="text/javascript"> +EOS + + foreach (@$scriptData) {print "$_\n";} + print "\n"; + + print "var srcArray = [\n"; + my @quotedScriptSource = map { + my $line = $_; + $line =~ s/^\s+//g; + # $line =~ s/^\/\/SOURCE\s+//g if $source; + $line =~ s/^\/\/.*//g; + $line =~ s/\s+$//g; + $line eq "" ? () : $line + } @$persistentScriptData, @scriptSource; + my $lastQuotedLine = pop @quotedScriptSource; + foreach (@quotedScriptSource) {print doubleQuoteString($_), ",\n";} + print doubleQuoteString($lastQuotedLine), "];\n\n"; + + if ($debug) { + push @quotedScriptSource, $lastQuotedLine; + foreach (@quotedScriptSource) { + s/<\//<\\\//g; # This fails if a regexp ends with a '<'. Oh well.... + print "$_\n"; + } + print "\n"; + } else { + print "eval(srcArray.join(\"\\n\"));\n\n"; + } + print "showMode = $showMode;\n"; + print "sortColumn = $sortColumn;\n"; + if ($source) { + print <<'EOS'; +function writeQuotedHTML(s) { + document.write(quoteHTML(s.toString()).replace(/\n/g, '<BR>\n')); +} + +var quotingDocument = { + write: function () { + for (var i = 0; i < arguments.length; i++) + writeQuotedHTML(arguments[i]); + }, + writeln: function () { + for (var i = 0; i < arguments.length; i++) + writeQuotedHTML(arguments[i]); + document.writeln('<BR>'); + } +}; +EOS + } else { + print "showHead(document);\n"; + } + print "</SCRIPT>\n"; + print "</HEAD>\n\n"; + print "<BODY>\n"; + if ($source) { + print "<P><TT>"; + print quoteHTML "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 Transitional//EN\" \"http://www.w3.org/TR/REC-html40/loose.dtd\">\n"; + print quoteHTML "<HTML>\n"; + print quoteHTML "<HEAD>\n"; + print "<SCRIPT type=\"text/javascript\">showHead(quotingDocument);</SCRIPT>\n"; + print quoteHTML "</HEAD>\n\n"; + print quoteHTML "<BODY>\n"; + print "<SCRIPT type=\"text/javascript\">showBody(quotingDocument);</SCRIPT>\n"; + print quoteHTML "</BODY>\n"; + print quoteHTML "</HTML>\n"; + print "</TT></P>\n"; + } else { + print "<SCRIPT type=\"text/javascript\">showBody(document);</SCRIPT>\n"; + } + print "</BODY>\n"; + print "</HTML>\n"; +} + + + +# Read the bloat file into hash table $h. The hash table is indexed by class names; +# each entry is a list with the following elements: +# bytesAlloc Total number of bytes allocated +# bytesNet Total number of bytes allocated but not deallocated +# objectsAlloc Total number of objects allocated +# objectsNet Total number of objects allocated but not deallocated +# refsAlloc Total number of references AddRef'd +# refsNet Total number of references AddRef'd but not Released +# Except for TOTAL, all hash table entries refer to mutually exclusive data. +# $sizes is a hash table indexed by class names. Each entry of that table contains the class's instance size. +sub readBloatFile($\%\%) { + my ($file, $h, $sizes) = @_; + local $_; # Needed for 'while (<FILE>)' below. + + my $readSomething = 0; + open FILE, $file; + while (<FILE>) { + if (my ($name, $size, $bytesNet, $objectsAlloc, $objectsNet, $refsAlloc, $refsNet) = + /^\s*(?:\d+)\s+([\w:]+)\s+(\d+)\s+(-?\d+)\s+(\d+)\s+(-?\d+)\s*\([^()]*\)\s*(\d+)\s+(-?\d+)\s*\([^()]*\)\s*$/) { + my $bytesAlloc; + if ($name eq "TOTAL") { + $size = "undefined"; + $bytesAlloc = "undefined"; + } else { + $bytesAlloc = $objectsAlloc * $size; + if ($bytesNet != $objectsNet * $size) { + print STDERR "In '$file', class $name bytesNet != objectsNet * size: $bytesNet != $objectsNet * $size\n"; + } + } + print STDERR "Duplicate entry $name in '$file'\n" if $$h{$name}; + $$h{$name} = [$bytesAlloc, $bytesNet, $objectsAlloc, $objectsNet, $refsAlloc, $refsNet]; + + my $oldSize = $$sizes{$name}; + print STDERR "Mismatch of sizes of class $name: $oldSize and $size\n" if defined($oldSize) && $size ne $oldSize; + $$sizes{$name} = $size; + $readSomething = 1; + } elsif (/^\s*(?:\d+)\s+([\w:]+)\s/) { + print STDERR "Unable to parse '$file' line: $_"; + } + } + close FILE; + print STDERR "No data in '$file'\n" unless $readSomething; + return $h; +} + + +my %sizes; # <class-name> => <instance-size> +my %tables; # <file-name> => <bloat-table>; see readBloatFile for format of <bloat-table> + +# Generate the JavaScript source code for the row named $c. $l can contain the initial entries of the row. +sub genTableRowSource($$) { + my ($l, $c) = @_; + my $lastE; + foreach (@ARGV) { + my $e = $tables{$_}{$c}; + if (defined($lastE) && !defined($e)) { + $e = [0,0,0,0,0,0]; + print STDERR "Class $c is defined in an earlier file but not in '$_'\n"; + } + if (defined $e) { + if (defined $lastE) { + for (my $i = 0; $i <= $#$e; $i++) { + my $n = $$e[$i]; + $l .= ($n eq "undefined" ? "undefined" : $n - $$lastE[$i]) . ","; + } + $l .= " "; + } else { + $l .= join(",", @$e) . ", "; + } + $lastE = $e; + } else { + $l .= "0,0,0,0,0,0, "; + } + } + $l .= join(",", @$lastE); + return "[$l]"; +} + + + +my $debug; +my $source; +my $showMode; +my $sortColumn; +my @modeOptions; + +GetOptions("debug" => \$debug, "source" => \$source, "byte=i" => \$modeOptions[0], "obj=i" => \$modeOptions[1], "ref=i" => \$modeOptions[2]); +for (my $i = 0; $i != 3; $i++) { + my $modeOption = $modeOptions[$i]; + if ($modeOption) { + die "Only one of -byte, -obj, or -ref may be given" if defined $showMode; + my $nFileColumns = scalar(@ARGV) + 1; + die "-byte, -obj, or -ref column number out of range" if $modeOption < 0 || $modeOption >= 2 + 2*$nFileColumns; + $showMode = $i; + if ($modeOption >= 2) { + $modeOption -= 2; + $sortColumn = 2 + $showMode*2; + if ($modeOption >= $nFileColumns) { + $sortColumn++; + $modeOption -= $nFileColumns; + } + $sortColumn += $modeOption*6; + } else { + $sortColumn = $modeOption; + } + } +} +unless (defined $showMode) { + $showMode = 0; + $sortColumn = 0; +} + +# Read all of the bloat files. +foreach (@ARGV) { + unless ($tables{$_}) { + my $f = $_; + my %table; + + readBloatFile $_, %table, %sizes; + $tables{$_} = \%table; + } +} +die "No input" unless %sizes; + +my @scriptData; # JavaScript source for the tables passed to JavaScript. Each entry is one line of JavaScript. +my @persistentScriptData; # Same as @scriptData, but persists the page reloads itself. + +# Print a list of bloat file names. +push @persistentScriptData, "var nFiles = " . scalar(@ARGV) . ";"; +push @persistentScriptData, "var fileTags = [" . join(", ", map {singleQuoteString substr(fileCoreName($_), -10)} @ARGV) . "];"; +push @persistentScriptData, "var fileNames = [" . join(", ", map {singleQuoteString $_} @ARGV) . "];"; +push @persistentScriptData, "var fileDates = [" . join(", ", map {singleQuoteString localtime fileModDate $_} @ARGV) . "];"; + +# Print the bloat tables. +push @persistentScriptData, "var totals = " . genTableRowSource('"TOTAL", undefined, ', "TOTAL") . ";"; +push @scriptData, "var classTables = ["; +delete $sizes{"TOTAL"}; +my @classes = sort(keys %sizes); +for (my $i = 0; $i <= $#classes; $i++) { + my $c = $classes[$i]; + push @scriptData, genTableRowSource(doubleQuoteString($c).", ".$sizes{$c}.", ", $c) . ($i == $#classes ? "];" : ","); +} + +generate(@scriptData, @persistentScriptData, $debug, $source, $showMode, $sortColumn); +1; + + +# The source of the eval'd JavaScript follows. +# Comments starting with // that are alone on a line are stripped by the Perl script. +__END__ + +// showMode: 0=bytes, 1=objects, 2=references +var showMode; +var modeName; +var modeNameUpper; + +var sortColumn; + +// Sort according to the sortColumn. Column 0 is sorted alphabetically in ascending order. +// All other columns are sorted numerically in descending order, with column 0 used for a secondary sort. +// Undefined is always listed last. +function sortCompare(x, y) { + if (sortColumn) { + var xc = x[sortColumn]; + var yc = y[sortColumn]; + if (xc < yc || xc === undefined && yc !== undefined) return 1; + if (yc < xc || yc === undefined && xc !== undefined) return -1; + } + + var x0 = x[0]; + var y0 = y[0]; + if (x0 > y0 || x0 === undefined && y0 !== undefined) return 1; + if (y0 > x0 || y0 === undefined && x0 !== undefined) return -1; + return 0; +} + + +// Quote special HTML characters in the string. +function quoteHTML(s) { + s = s.replace(/&/g, '&'); + // Can't use /</g because HTML interprets '</g' as ending the script! + s = s.replace(/\x3C/g, '<'); + s = s.replace(/>/g, '>'); + s = s.replace(/ /g, ' '); + return s; +} + + +function writeFileTable(d) { + d.writeln('<TABLE border=1 cellspacing=1 cellpadding=0>'); + d.writeln('<TR>\n<TH>Name</TH>\n<TH>File</TH>\n<TH>Date</TH>\n</TR>'); + for (var i = 0; i < nFiles; i++) + d.writeln('<TR>\n<TD>'+quoteHTML(fileTags[i])+'</TD>\n<TD><TT>'+quoteHTML(fileNames[i])+'</TT></TD>\n<TD>'+quoteHTML(fileDates[i])+'</TD>\n</TR>'); + d.writeln('</TABLE>'); +} + + +function writeReloadLink(d, column, s, rowspan) { + d.write(rowspan == 1 ? '<TH>' : '<TH rowspan='+rowspan+'>'); + if (column != sortColumn) + d.write('<A href="javascript:reloadSelf('+column+','+showMode+')">'); + d.write(s); + if (column != sortColumn) + d.write('</A>'); + d.writeln('</TH>'); +} + +function writeClassTableRow(d, row, base, modeName) { + if (modeName) { + d.writeln('<TR>\n<TH>'+modeName+'</TH>'); + } else { + d.writeln('<TR>\n<TD><A href="javascript:showRowDetail(\''+row[0]+'\')">'+quoteHTML(row[0])+'</A></TD>'); + var v = row[1]; + d.writeln('<TD class=num>'+(v === undefined ? '' : v)+'</TD>'); + } + for (var i = 0; i != 2; i++) { + var c = base + i; + for (var j = 0; j <= nFiles; j++) { + v = row[c]; + var style = 'num'; + if (j != nFiles) + if (v > 0) { + style = 'pos'; + v = '+'+v; + } else + style = 'neg'; + d.writeln('<TD class='+style+'>'+(v === undefined ? '' : v)+'</TD>'); + c += 6; + } + } + d.writeln('</TR>'); +} + +function writeClassTable(d) { + var base = 2 + showMode*2; + + // Make a copy because a sort is destructive. + var table = classTables.concat(); + table.sort(sortCompare); + + d.writeln('<TABLE border=1 cellspacing=1 cellpadding=0>'); + + d.writeln('<TR>'); + writeReloadLink(d, 0, 'Class Name', 2); + writeReloadLink(d, 1, 'Instance<BR>Size', 2); + d.writeln('<TH colspan='+(nFiles+1)+'>'+modeNameUpper+'s allocated</TH>'); + d.writeln('<TH colspan='+(nFiles+1)+'>'+modeNameUpper+'s allocated but not freed</TH>\n</TR>'); + d.writeln('<TR>'); + for (var i = 0; i != 2; i++) { + var c = base + i; + for (var j = 0; j <= nFiles; j++) { + writeReloadLink(d, c, j == nFiles ? 'Total' : quoteHTML(fileTags[j]), 1); + c += 6; + } + } + d.writeln('</TR>'); + + writeClassTableRow(d, totals, base, 0); + for (var r = 0; r < table.length; r++) + writeClassTableRow(d, table[r], base, 0); + + d.writeln('</TABLE>'); +} + + +var modeNames = ["byte", "object", "reference"]; +var modeNamesUpper = ["Byte", "Object", "Reference"]; +var styleSheet = '<STYLE type="TEXT/CSS">\n'+ + 'BODY {background-color: #FFFFFF; color: #000000}\n'+ + '.num {text-align: right}\n'+ + '.pos {text-align: right; color: #CC0000}\n'+ + '.neg {text-align: right; color: #009900}\n'+ + '</STYLE>'; + + +function showHead(d) { + modeName = modeNames[showMode]; + modeNameUpper = modeNamesUpper[showMode]; + d.writeln('<TITLE>'+modeNameUpper+' Bloats</TITLE>'); + d.writeln(styleSheet); +} + +function showBody(d) { + d.writeln('<H1>'+modeNameUpper+' Bloats</H1>'); + writeFileTable(d); + d.write('<FORM>'); + for (var i = 0; i != 3; i++) + if (i != showMode) { + var newSortColumn = sortColumn; + if (sortColumn >= 2) + newSortColumn = sortColumn + (i-showMode)*2; + d.write('<INPUT type="button" value="Show '+modeNamesUpper[i]+'s" onClick="reloadSelf('+newSortColumn+','+i+')">'); + } + d.writeln('</FORM>'); + d.writeln('<P>The numbers do not include <CODE>malloc</CODE>\'d data such as string contents.</P>'); + d.writeln('<P>Click on a column heading to sort by that column. Click on a class name to see details for that class.</P>'); + writeClassTable(d); +} + + +function showRowDetail(rowName) { + var row; + var i; + + if (rowName == "TOTAL") + row = totals; + else { + for (i = 0; i < classTables.length; i++) + if (rowName == classTables[i][0]) { + row = classTables[i]; + break; + } + } + if (row) { + var w = window.open("", "ClassTableRowDetails"); + var d = w.document; + d.open(); + d.writeln('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">'); + d.writeln('<HTML>\n<HEAD>\n<TITLE>'+quoteHTML(rowName)+' bloat details</TITLE>'); + d.writeln(styleSheet); + d.writeln('</HEAD>\n\n<BODY>'); + d.writeln('<H2>'+quoteHTML(rowName)+'</H2>'); + if (row[1] !== undefined) + d.writeln('<P>Each instance has '+row[1]+' bytes.</P>'); + + d.writeln('<TABLE border=1 cellspacing=1 cellpadding=0>'); + d.writeln('<TR>\n<TH></TH>\n<TH colspan='+(nFiles+1)+'>Allocated</TH>'); + d.writeln('<TH colspan='+(nFiles+1)+'>Allocated but not freed</TH>\n</TR>'); + d.writeln('<TR>\n<TH></TH>'); + for (i = 0; i != 2; i++) + for (var j = 0; j <= nFiles; j++) + d.writeln('<TH>'+(j == nFiles ? 'Total' : quoteHTML(fileTags[j]))+'</TH>'); + d.writeln('</TR>'); + + for (i = 0; i != 3; i++) + writeClassTableRow(d, row, 2+i*2, modeNamesUpper[i]+'s'); + + d.writeln('</TABLE>\n</BODY>\n</HTML>'); + d.close(); + } + return undefined; +} + + +function stringSource(s) { + s = s.replace(/\\/g, '\\\\'); + s = s.replace(/"/g, '\\"'); + s = s.replace(/<\//g, '<\\/'); + return '"'+s+'"'; +} + +function reloadSelf(n,m) { + // Need to cache these because globals go away on document.open(). + var sa = srcArray; + var ss = stringSource; + var ct = classTables; + var i; + + document.open(); + // Uncomment this and comment the document.open() line above to see the reloaded page's source. + //var w = window.open("", "NewDoc"); + //var d = w.document; + //var document = new Object; + //document.write = function () { + // for (var i = 0; i < arguments.length; i++) { + // var s = arguments[i].toString(); + // s = s.replace(/&/g, '&'); + // s = s.replace(/\x3C/g, '<'); + // s = s.replace(/>/g, '>'); + // s = s.replace(/ /g, ' '); + // d.write(s); + // } + //}; + //document.writeln = function () { + // for (var i = 0; i < arguments.length; i++) { + // var s = arguments[i].toString(); + // s = s.replace(/&/g, '&'); + // s = s.replace(/\x3C/g, '<'); + // s = s.replace(/>/g, '>'); + // s = s.replace(/ /g, ' '); + // d.write(s); + // } + // d.writeln('<BR>'); + //}; + + document.writeln('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">'); + document.writeln('<HTML>\n<HEAD>\n<SCRIPT type="text/javascript">'); + + // Manually copy non-persistent script data + if (!ct.length) + document.writeln('var classTables = [];'); + else { + document.writeln('var classTables = ['); + for (i = 0; i < ct.length; i++) { + var row = ct[i]; + document.write('[' + ss(row[0])); + for (var j = 1; j < row.length; j++) + document.write(',' + row[j]); + document.writeln(']' + (i == ct.length-1 ? '];' : ',')); + } + } + + document.writeln('var srcArray = ['); + for (i = 0; i < sa.length; i++) { + document.write(ss(sa[i])); + if (i != sa.length-1) + document.writeln(','); + } + document.writeln('];'); + document.writeln('eval(srcArray.join("\\n"));'); + document.writeln('showMode = '+m+';'); + document.writeln('sortColumn = '+n+';'); + document.writeln('showHead(document);'); + document.writeln('</SCRIPT>\n</HEAD>\n\n<BODY>\n<SCRIPT type="text/javascript">showBody(document);</SCRIPT>\n</BODY>\n</HTML>'); + document.close(); + return undefined; +} diff --git a/tools/check-moz-style/checkmozstyle.py b/tools/check-moz-style/checkmozstyle.py new file mode 100755 index 000000000..d8261aec5 --- /dev/null +++ b/tools/check-moz-style/checkmozstyle.py @@ -0,0 +1,172 @@ +#!/usr/bin/python +# +# Copyright (C) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Script to run the linter for source code of WebKit.""" + +import os +import os.path +import re +import sys + +import modules.cpplint as cpplint +from modules.diff_parser import DiffParser +from modules.scm import detect_scm_system + + +# Override the usage of the lint tool. +cpplint._USAGE = """ +Syntax: %(program_name)s [--verbose=#] [--git-commit=<COMMITISH>] [--output=vs7] [--filter=-x,+y,...] + + The style guidelines this tries to follow are those in + http://webkit.org/coding/coding-style.html + + Every problem is given a confidence score from 1-5, with 5 meaning we are + certain of the problem, and 1 meaning it could be a legitimate construct. + This will miss some errors, and is not a substitute for a code review. + + To prevent specific lines from being linted, add a '// NOLINT' comment to the + end of the line. + + Linted extensions are .cpp, .c and .h. Other file types will be ignored. + + Flags: + + verbose=# + Specify a number 0-5 to restrict errors to certain verbosity levels. + + git-commit=<COMMITISH> + Check style for a specified git commit. + Note that the program checks style based on current local file + instead of actual diff of the git commit. So, if the files are + updated after the specified git commit, the information of line + number may be wrong. + + output=vs7 + By default, the output is formatted to ease emacs parsing. Visual Studio + compatible output (vs7) may also be used. Other formats are unsupported. + + filter=-x,+y,... + Specify a comma-separated list of category-filters to apply: only + error messages whose category names pass the filters will be printed. + (Category names are printed with the message and look like + "[whitespace/indent]".) Filters are evaluated left to right. + "-FOO" and "FOO" means "do not print categories that start with FOO". + "+FOO" means "do print categories that start with FOO". + + Examples: --filter=-whitespace,+whitespace/braces + --filter=whitespace,runtime/printf,+runtime/printf_format + --filter=-,+build/include_what_you_use + + To see a list of all the categories used in %(program_name)s, pass no arg: + --filter= +""" % {'program_name': sys.argv[0]} + +def process_patch(patch_string, root, cwd, scm): + """Does lint on a single patch. + + Args: + patch_string: A string of a patch. + """ + patch = DiffParser(patch_string.splitlines()) + + if not len(patch.files): + cpplint.error("patch", 0, "patch/notempty", 3, + "Patch does not appear to diff against any file.") + return + + if not patch.status_line: + cpplint.error("patch", 0, "patch/nosummary", 3, + "Patch does not have a summary.") + else: + proper_format = re.match(r"^Bug [0-9]+ - ", patch.status_line) + if not proper_format: + proper_format = re.match(r"^No bug - ", patch.status_line) + cpplint.error("patch", 0, "patch/bugnumber", 3, + "Patch summary should begin with 'Bug XXXXX - ' " + + "or 'No bug -'.") + + if not patch.patch_description: + cpplint.error("patch", 0, "patch/nodescription", 3, + "Patch does not have a description.") + + for filename, diff in patch.files.iteritems(): + file_extension = os.path.splitext(filename)[1] + + if file_extension in ['.cpp', '.c', '.h']: + line_numbers = set() + orig_filename = filename + + def error_for_patch(filename, line_number, category, confidence, + message): + """Wrapper function of cpplint.error for patches. + + This function outputs errors only if the line number + corresponds to lines which are modified or added. + """ + if not line_numbers: + for line in diff.lines: + # When deleted line is not set, it means that + # the line is newly added. + if not line[0]: + line_numbers.add(line[1]) + + if line_number in line_numbers: + cpplint.error(orig_filename, line_number, + category, confidence, message) + + cpplint.process_file(os.path.join(root, filename), + relative_name=orig_filename, + error=error_for_patch) + + +def main(): + cpplint.use_mozilla_styles() + + (args, flags) = cpplint.parse_arguments(sys.argv[1:], ["git-commit="]) + if args: + sys.stderr.write("ERROR: We don't support files as arguments for " + + "now.\n" + cpplint._USAGE) + sys.exit(1) + + cwd = os.path.abspath('.') + scm = detect_scm_system(cwd) + root = scm.find_checkout_root(cwd) + + if "--git-commit" in flags: + process_patch(scm.create_patch_from_local_commit(flags["--git-commit"]), root, cwd, scm) + else: + process_patch(scm.create_patch(), root, cwd, scm) + + sys.stderr.write('Total errors found: %d\n' % cpplint.error_count()) + sys.exit(cpplint.error_count() > 0) + + +if __name__ == "__main__": + main() diff --git a/tools/check-moz-style/diff_parser.py b/tools/check-moz-style/diff_parser.py new file mode 100644 index 000000000..91898af31 --- /dev/null +++ b/tools/check-moz-style/diff_parser.py @@ -0,0 +1,162 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""WebKit's Python module for interacting with patches.""" + +import logging +import re + + +_regexp_compile_cache = {} + + +def match(pattern, string): + """Matches the string with the pattern, caching the compiled regexp.""" + if not pattern in _regexp_compile_cache: + _regexp_compile_cache[pattern] = re.compile(pattern) + return _regexp_compile_cache[pattern].match(string) + + +def git_diff_to_svn_diff(line): + """Converts a git formatted diff line to a svn formatted line. + + Args: + line: A string representing a line of the diff. + """ + conversion_patterns = (("^diff --git a/(.+) b/(?P<FilePath>.+)", lambda matched: "Index: " + matched.group('FilePath') + "\n"), + ("^new file.*", lambda matched: "\n"), + ("^index [0-9a-f]{7}\.\.[0-9a-f]{7} [0-9]{6}", lambda matched: "===================================================================\n"), + ("^--- a/(?P<FilePath>.+)", lambda matched: "--- " + matched.group('FilePath') + "\n"), + ("^\+\+\+ b/(?P<FilePath>.+)", lambda matched: "+++ " + matched.group('FilePath') + "\n")) + + for pattern, conversion in conversion_patterns: + matched = match(pattern, line) + if matched: + return conversion(matched) + return line + + +def get_diff_converter(first_diff_line): + """Gets a converter function of diff lines. + + Args: + first_diff_line: The first filename line of a diff file. + If this line is git formatted, we'll return a + converter from git to SVN. + """ + if match(r"^diff --git a/", first_diff_line): + return git_diff_to_svn_diff + return lambda input: input + + +_INITIAL_STATE = 1 +_DECLARED_FILE_PATH = 2 +_PROCESSING_CHUNK = 3 + + +class DiffFile: + """Contains the information for one file in a patch. + + The field "lines" is a list which contains tuples in this format: + (deleted_line_number, new_line_number, line_string) + If deleted_line_number is zero, it means this line is newly added. + If new_line_number is zero, it means this line is deleted. + """ + + def __init__(self, filename): + self.filename = filename + self.lines = [] + + def add_new_line(self, line_number, line): + self.lines.append((0, line_number, line)) + + def add_deleted_line(self, line_number, line): + self.lines.append((line_number, 0, line)) + + def add_unchanged_line(self, deleted_line_number, new_line_number, line): + self.lines.append((deleted_line_number, new_line_number, line)) + + +class DiffParser: + """A parser for a patch file. + + The field "files" is a dict whose key is the filename and value is + a DiffFile object. + """ + + def __init__(self, diff_input): + """Parses a diff. + + Args: + diff_input: An iterable object. + """ + state = _INITIAL_STATE + + self.files = {} + current_file = None + old_diff_line = None + new_diff_line = None + for line in diff_input: + line = line.rstrip("\n") + if state == _INITIAL_STATE: + transform_line = get_diff_converter(line) + line = transform_line(line) + + file_declaration = match(r"^Index: (?P<FilePath>.+)", line) + if file_declaration: + filename = file_declaration.group('FilePath') + current_file = DiffFile(filename) + self.files[filename] = current_file + state = _DECLARED_FILE_PATH + continue + + lines_changed = match(r"^@@ -(?P<OldStartLine>\d+)(,\d+)? \+(?P<NewStartLine>\d+)(,\d+)? @@", line) + if lines_changed: + if state != _DECLARED_FILE_PATH and state != _PROCESSING_CHUNK: + logging.error('Unexpected line change without file path declaration: %r' % line) + old_diff_line = int(lines_changed.group('OldStartLine')) + new_diff_line = int(lines_changed.group('NewStartLine')) + state = _PROCESSING_CHUNK + continue + + if state == _PROCESSING_CHUNK: + if line.startswith('+'): + current_file.add_new_line(new_diff_line, line[1:]) + new_diff_line += 1 + elif line.startswith('-'): + current_file.add_deleted_line(old_diff_line, line[1:]) + old_diff_line += 1 + elif line.startswith(' '): + current_file.add_unchanged_line(old_diff_line, new_diff_line, line[1:]) + old_diff_line += 1 + new_diff_line += 1 + elif line == '\\ No newline at end of file': + # Nothing to do. We may still have some added lines. + pass + else: + logging.error('Unexpected diff format when parsing a chunk: %r' % line) diff --git a/tools/check-moz-style/modules/__init__.py b/tools/check-moz-style/modules/__init__.py new file mode 100644 index 000000000..ef65bee5b --- /dev/null +++ b/tools/check-moz-style/modules/__init__.py @@ -0,0 +1 @@ +# Required for Python to search this directory for module files diff --git a/tools/check-moz-style/modules/cpplint.py b/tools/check-moz-style/modules/cpplint.py new file mode 100644 index 000000000..c01e82d45 --- /dev/null +++ b/tools/check-moz-style/modules/cpplint.py @@ -0,0 +1,3150 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Copyright (C) 2009 Google Inc. All rights reserved. +# Copyright (C) 2009 Torch Mobile Inc. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This is the modified version of Google's cpplint. The original code is +# http://google-styleguide.googlecode.com/svn/trunk/cpplint/cpplint.py + +"""Does WebKit-lint on c++ files. + +The goal of this script is to identify places in the code that *may* +be in non-compliance with WebKit style. It does not attempt to fix +up these problems -- the point is to educate. It does also not +attempt to find all problems, or to ensure that everything it does +find is legitimately a problem. + +In particular, we can get very confused by /* and // inside strings! +We do a small hack, which is to ignore //'s with "'s after them on the +same line, but it is far from perfect (in either direction). +""" + +import codecs +import getopt +import math # for log +import os +import os.path +import re +import sre_compile +import string +import sys +import unicodedata + + +_USAGE = """ +Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] + <file> [file] ... + + The style guidelines this tries to follow are those in + http://webkit.org/coding/coding-style.html + + Every problem is given a confidence score from 1-5, with 5 meaning we are + certain of the problem, and 1 meaning it could be a legitimate construct. + This will miss some errors, and is not a substitute for a code review. + + To prevent specific lines from being linted, add a '// NOLINT' comment to the + end of the line. + + The files passed in will be linted; at least one file must be provided. + Linted extensions are .cpp, .c and .h. Other file types will be ignored. + + Flags: + + output=vs7 + By default, the output is formatted to ease emacs parsing. Visual Studio + compatible output (vs7) may also be used. Other formats are unsupported. + + verbose=# + Specify a number 0-5 to restrict errors to certain verbosity levels. + + filter=-x,+y,... + Specify a comma-separated list of category-filters to apply: only + error messages whose category names pass the filters will be printed. + (Category names are printed with the message and look like + "[whitespace/indent]".) Filters are evaluated left to right. + "-FOO" and "FOO" means "do not print categories that start with FOO". + "+FOO" means "do print categories that start with FOO". + + Examples: --filter=-whitespace,+whitespace/braces + --filter=whitespace,runtime/printf,+runtime/printf_format + --filter=-,+build/include_what_you_use + + To see a list of all the categories used in cpplint, pass no arg: + --filter= +""" + +# We categorize each error message we print. Here are the categories. +# We want an explicit list so we can list them all in cpplint --filter=. +# If you add a new error message with a new category, add it to the list +# here! cpplint_unittest.py should tell you if you forget to do this. +# \ used for clearer layout -- pylint: disable-msg=C6013 +_ERROR_CATEGORIES = '''\ + build/class + build/deprecated + build/endif_comment + build/forward_decl + build/header_guard + build/include + build/include_order + build/include_what_you_use + build/namespaces + build/printf_format + build/storage_class + legal/copyright + readability/braces + readability/casting + readability/check + readability/comparison_to_zero + readability/constructors + readability/control_flow + readability/fn_size + readability/function + readability/multiline_comment + readability/multiline_string + readability/null + readability/streams + readability/todo + readability/utf8 + runtime/arrays + runtime/casting + runtime/explicit + runtime/int + runtime/init + runtime/invalid_increment + runtime/memset + runtime/printf + runtime/printf_format + runtime/references + runtime/rtti + runtime/sizeof + runtime/string + runtime/threadsafe_fn + runtime/virtual + whitespace/blank_line + whitespace/braces + whitespace/comma + whitespace/comments + whitespace/comments-doublespace + whitespace/end_of_line + whitespace/ending_newline + whitespace/indent + whitespace/labels + whitespace/line_length + whitespace/newline + whitespace/operators + whitespace/parens + whitespace/semicolon + whitespace/tab + whitespace/todo +''' + +# The default state of the category filter. This is overrided by the --filter= +# flag. By default all errors are on, so only add here categories that should be +# off by default (i.e., categories that must be enabled by the --filter= flags). +# All entries here should start with a '-' or '+', as in the --filter= flag. +_DEFAULT_FILTERS = [] + +# Headers that we consider STL headers. +_STL_HEADERS = frozenset([ + 'algobase.h', 'algorithm', 'alloc.h', 'bitset', 'deque', 'exception', + 'function.h', 'functional', 'hash_map', 'hash_map.h', 'hash_set', + 'hash_set.h', 'iterator', 'list', 'list.h', 'map', 'memory', 'pair.h', + 'pthread_alloc', 'queue', 'set', 'set.h', 'sstream', 'stack', + 'stl_alloc.h', 'stl_relops.h', 'type_traits.h', + 'utility', 'vector', 'vector.h', + ]) + + +# Non-STL C++ system headers. +_CPP_HEADERS = frozenset([ + 'algo.h', 'builtinbuf.h', 'bvector.h', 'cassert', 'cctype', + 'cerrno', 'cfloat', 'ciso646', 'climits', 'clocale', 'cmath', + 'complex', 'complex.h', 'csetjmp', 'csignal', 'cstdarg', 'cstddef', + 'cstdio', 'cstdlib', 'cstring', 'ctime', 'cwchar', 'cwctype', + 'defalloc.h', 'deque.h', 'editbuf.h', 'exception', 'fstream', + 'fstream.h', 'hashtable.h', 'heap.h', 'indstream.h', 'iomanip', + 'iomanip.h', 'ios', 'iosfwd', 'iostream', 'iostream.h', 'istream.h', + 'iterator.h', 'limits', 'map.h', 'multimap.h', 'multiset.h', + 'numeric', 'ostream.h', 'parsestream.h', 'pfstream.h', 'PlotFile.h', + 'procbuf.h', 'pthread_alloc.h', 'rope', 'rope.h', 'ropeimpl.h', + 'SFile.h', 'slist', 'slist.h', 'stack.h', 'stdexcept', + 'stdiostream.h', 'streambuf.h', 'stream.h', 'strfile.h', 'string', + 'strstream', 'strstream.h', 'tempbuf.h', 'tree.h', 'typeinfo', 'valarray', + ]) + + +# Assertion macros. These are defined in base/logging.h and +# testing/base/gunit.h. Note that the _M versions need to come first +# for substring matching to work. +_CHECK_MACROS = [ + 'DCHECK', 'CHECK', + 'EXPECT_TRUE_M', 'EXPECT_TRUE', + 'ASSERT_TRUE_M', 'ASSERT_TRUE', + 'EXPECT_FALSE_M', 'EXPECT_FALSE', + 'ASSERT_FALSE_M', 'ASSERT_FALSE', + ] + +# Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE +_CHECK_REPLACEMENT = dict([(m, {}) for m in _CHECK_MACROS]) + +for op, replacement in [('==', 'EQ'), ('!=', 'NE'), + ('>=', 'GE'), ('>', 'GT'), + ('<=', 'LE'), ('<', 'LT')]: + _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement + _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE_M'][op] = 'EXPECT_%s_M' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE_M'][op] = 'ASSERT_%s_M' % replacement + +for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), + ('>=', 'LT'), ('>', 'LE'), + ('<=', 'GT'), ('<', 'GE')]: + _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement + _CHECK_REPLACEMENT['EXPECT_FALSE_M'][op] = 'EXPECT_%s_M' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE_M'][op] = 'ASSERT_%s_M' % inv_replacement + + +# These constants define types of headers for use with +# _IncludeState.check_next_include_order(). +_CONFIG_HEADER = 0 +_PRIMARY_HEADER = 1 +_OTHER_HEADER = 2 + + +_regexp_compile_cache = {} + + +def match(pattern, s): + """Matches the string with the pattern, caching the compiled regexp.""" + # The regexp compilation caching is inlined in both match and search for + # performance reasons; factoring it out into a separate function turns out + # to be noticeably expensive. + if not pattern in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].match(s) + + +def search(pattern, s): + """Searches the string for the pattern, caching the compiled regexp.""" + if not pattern in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].search(s) + + +class _IncludeState(dict): + """Tracks line numbers for includes, and the order in which includes appear. + + As a dict, an _IncludeState object serves as a mapping between include + filename and line number on which that file was included. + + Call check_next_include_order() once for each header in the file, passing + in the type constants defined above. Calls in an illegal order will + raise an _IncludeError with an appropriate error message. + + """ + # self._section will move monotonically through this set. If it ever + # needs to move backwards, check_next_include_order will raise an error. + _INITIAL_SECTION = 0 + _CONFIG_SECTION = 1 + _PRIMARY_SECTION = 2 + _OTHER_SECTION = 3 + + _TYPE_NAMES = { + _CONFIG_HEADER: 'WebCore config.h', + _PRIMARY_HEADER: 'header this file implements', + _OTHER_HEADER: 'other header', + } + _SECTION_NAMES = { + _INITIAL_SECTION: "... nothing.", + _CONFIG_SECTION: "WebCore config.h.", + _PRIMARY_SECTION: 'a header this file implements.', + _OTHER_SECTION: 'other header.', + } + + def __init__(self): + dict.__init__(self) + self._section = self._INITIAL_SECTION + self._visited_primary_section = False + self.header_types = dict(); + + def visited_primary_section(self): + return self._visited_primary_section + + def check_next_include_order(self, header_type, file_is_header): + """Returns a non-empty error message if the next header is out of order. + + This function also updates the internal state to be ready to check + the next include. + + Args: + header_type: One of the _XXX_HEADER constants defined above. + file_is_header: Whether the file that owns this _IncludeState is itself a header + + Returns: + The empty string if the header is in the right order, or an + error message describing what's wrong. + + """ + if header_type == _CONFIG_HEADER and file_is_header: + return 'Header file should not contain WebCore config.h.' + if header_type == _PRIMARY_HEADER and file_is_header: + return 'Header file should not contain itself.' + + error_message = '' + if self._section != self._OTHER_SECTION: + before_error_message = ('Found %s before %s' % + (self._TYPE_NAMES[header_type], + self._SECTION_NAMES[self._section + 1])) + after_error_message = ('Found %s after %s' % + (self._TYPE_NAMES[header_type], + self._SECTION_NAMES[self._section])) + + if header_type == _CONFIG_HEADER: + if self._section >= self._CONFIG_SECTION: + error_message = after_error_message + self._section = self._CONFIG_SECTION + elif header_type == _PRIMARY_HEADER: + if self._section >= self._PRIMARY_SECTION: + error_message = after_error_message + elif self._section < self._CONFIG_SECTION: + error_message = before_error_message + self._section = self._PRIMARY_SECTION + self._visited_primary_section = True + else: + assert header_type == _OTHER_HEADER + if not file_is_header and self._section < self._PRIMARY_SECTION: + error_message = before_error_message + self._section = self._OTHER_SECTION + + return error_message + + +class _CppLintState(object): + """Maintains module-wide state..""" + + def __init__(self): + self.verbose_level = 1 # global setting. + self.error_count = 0 # global count of reported errors + # filters to apply when emitting error messages + self.filters = _DEFAULT_FILTERS[:] + + # output format: + # "emacs" - format that emacs can parse (default) + # "vs7" - format that Microsoft Visual Studio 7 can parse + self.output_format = 'emacs' + + self.output_stream = sys.stderr + + def set_output_format(self, output_format): + """Sets the output format for errors.""" + self.output_format = output_format + + def set_verbose_level(self, level): + """Sets the module's verbosity, and returns the previous setting.""" + last_verbose_level = self.verbose_level + self.verbose_level = level + return last_verbose_level + + def set_filters(self, filters): + """Sets the error-message filters. + + These filters are applied when deciding whether to emit a given + error message. + + Args: + filters: A string of comma-separated filters (eg "+whitespace/indent"). + Each filter should start with + or -; else we die. + + Raises: + ValueError: The comma-separated filters did not all start with '+' or '-'. + E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter" + """ + # Default filters always have less priority than the flag ones. + self.filters = _DEFAULT_FILTERS[:] + for filter in filters.split(','): + clean_filter = filter.strip() + if clean_filter: + self.filters.append(clean_filter) + for filter in self.filters: + if not (filter.startswith('+') or filter.startswith('-')): + raise ValueError('Every filter in --filter must start with ' + '+ or - (%s does not)' % filter) + + def reset_error_count(self): + """Sets the module's error statistic back to zero.""" + self.error_count = 0 + + def increment_error_count(self): + """Bumps the module's error statistic.""" + self.error_count += 1 + + def set_stream(self, stream): + self.output_stream = stream + + def write_error(self, error): + self.output_stream.write(error) + + +_cpplint_state = _CppLintState() + + +def _output_format(): + """Gets the module's output format.""" + return _cpplint_state.output_format + + +def _set_output_format(output_format): + """Sets the module's output format.""" + _cpplint_state.set_output_format(output_format) + + +def _verbose_level(): + """Returns the module's verbosity setting.""" + return _cpplint_state.verbose_level + + +def _set_verbose_level(level): + """Sets the module's verbosity, and returns the previous setting.""" + return _cpplint_state.set_verbose_level(level) + + +def _filters(): + """Returns the module's list of output filters, as a list.""" + return _cpplint_state.filters + + +def _set_filters(filters): + """Sets the module's error-message filters. + + These filters are applied when deciding whether to emit a given + error message. + + Args: + filters: A string of comma-separated filters (eg "whitespace/indent"). + Each filter should start with + or -; else we die. + """ + _cpplint_state.set_filters(filters) + + +def error_count(): + """Returns the global count of reported errors.""" + return _cpplint_state.error_count + + +class _FunctionState(object): + """Tracks current function name and the number of lines in its body.""" + + _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. + _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. + + def __init__(self): + self.in_a_function = False + self.lines_in_function = 0 + self.current_function = '' + + def begin(self, function_name): + """Start analyzing function body. + + Args: + function_name: The name of the function being tracked. + """ + self.in_a_function = True + self.lines_in_function = 0 + self.current_function = function_name + + def count(self): + """Count line in current function body.""" + if self.in_a_function: + self.lines_in_function += 1 + + def check(self, error, filename, line_number): + """Report if too many lines in function body. + + Args: + error: The function to call with any errors found. + filename: The name of the current file. + line_number: The number of the line to check. + """ + if match(r'T(EST|est)', self.current_function): + base_trigger = self._TEST_TRIGGER + else: + base_trigger = self._NORMAL_TRIGGER + trigger = base_trigger * 2 ** _verbose_level() + + if self.lines_in_function > trigger: + error_level = int(math.log(self.lines_in_function / base_trigger, 2)) + # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... + if error_level > 5: + error_level = 5 + error(filename, line_number, 'readability/fn_size', error_level, + 'Small and focused functions are preferred:' + ' %s has %d non-comment lines' + ' (error triggered by exceeding %d lines).' % ( + self.current_function, self.lines_in_function, trigger)) + + def end(self): + """Stop analizing function body.""" + self.in_a_function = False + + +class _IncludeError(Exception): + """Indicates a problem with the include order in a file.""" + pass + + +class FileInfo: + """Provides utility functions for filenames. + + FileInfo provides easy access to the components of a file's path + relative to the project root. + """ + + def __init__(self, filename): + self._filename = filename + + def full_name(self): + """Make Windows paths like Unix.""" + return os.path.abspath(self._filename).replace('\\', '/') + + def repository_name(self): + """Full name after removing the local path to the repository. + + If we have a real absolute path name here we can try to do something smart: + detecting the root of the checkout and truncating /path/to/checkout from + the name so that we get header guards that don't include things like + "C:\Documents and Settings\..." or "/home/username/..." in them and thus + people on different computers who have checked the source out to different + locations won't see bogus errors. + """ + fullname = self.full_name() + + if os.path.exists(fullname): + project_dir = os.path.dirname(fullname) + + if os.path.exists(os.path.join(project_dir, ".svn")): + # If there's a .svn file in the current directory, we + # recursively look up the directory tree for the top + # of the SVN checkout + root_dir = project_dir + one_up_dir = os.path.dirname(root_dir) + while os.path.exists(os.path.join(one_up_dir, ".svn")): + root_dir = os.path.dirname(root_dir) + one_up_dir = os.path.dirname(one_up_dir) + + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Not SVN? Try to find a git top level directory by + # searching up from the current path. + root_dir = os.path.dirname(fullname) + while (root_dir != os.path.dirname(root_dir) + and not os.path.exists(os.path.join(root_dir, ".git"))): + root_dir = os.path.dirname(root_dir) + if os.path.exists(os.path.join(root_dir, ".git")): + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Don't know what to do; header guard warnings may be wrong... + return fullname + + def split(self): + """Splits the file into the directory, basename, and extension. + + For 'chrome/browser/browser.cpp', Split() would + return ('chrome/browser', 'browser', '.cpp') + + Returns: + A tuple of (directory, basename, extension). + """ + + googlename = self.repository_name() + project, rest = os.path.split(googlename) + return (project,) + os.path.splitext(rest) + + def base_name(self): + """File base name - text after the final slash, before the final period.""" + return self.split()[1] + + def extension(self): + """File extension - text following the final period.""" + return self.split()[2] + + def no_extension(self): + """File has no source file extension.""" + return '/'.join(self.split()[0:2]) + + def is_source(self): + """File has a source file extension.""" + return self.extension()[1:] in ('c', 'cc', 'cpp', 'cxx') + + +def _should_print_error(category, confidence): + """Returns true iff confidence >= verbose, and category passes filter.""" + # There are two ways we might decide not to print an error message: + # the verbosity level isn't high enough, or the filters filter it out. + if confidence < _cpplint_state.verbose_level: + return False + + is_filtered = False + for one_filter in _filters(): + if one_filter.startswith('-'): + if category.startswith(one_filter[1:]): + is_filtered = True + elif one_filter.startswith('+'): + if category.startswith(one_filter[1:]): + is_filtered = False + else: + assert False # should have been checked for in set_filter. + if is_filtered: + return False + + return True + + +def error(filename, line_number, category, confidence, message): + """Logs the fact we've found a lint error. + + We log where the error was found, and also our confidence in the error, + that is, how certain we are this is a legitimate style regression, and + not a misidentification or a use that's sometimes justified. + + Args: + filename: The name of the file containing the error. + line_number: The number of the line containing the error. + category: A string used to describe the "category" this bug + falls under: "whitespace", say, or "runtime". Categories + may have a hierarchy separated by slashes: "whitespace/indent". + confidence: A number from 1-5 representing a confidence score for + the error, with 5 meaning that we are certain of the problem, + and 1 meaning that it could be a legitimate construct. + message: The error message. + """ + # There are two ways we might decide not to print an error message: + # the verbosity level isn't high enough, or the filters filter it out. + if _should_print_error(category, confidence): + _cpplint_state.increment_error_count() + if _cpplint_state.output_format == 'vs7': + write_error('%s(%s): %s [%s] [%d]\n' % ( + filename, line_number, message, category, confidence)) + else: + write_error('%s:%s: %s [%s] [%d]\n' % ( + filename, line_number, message, category, confidence)) + + +# Matches standard C++ escape esequences per 2.13.2.3 of the C++ standard. +_RE_PATTERN_CLEANSE_LINE_ESCAPES = re.compile( + r'\\([abfnrtv?"\\\']|\d+|x[0-9a-fA-F]+)') +# Matches strings. Escape codes should already be removed by ESCAPES. +_RE_PATTERN_CLEANSE_LINE_DOUBLE_QUOTES = re.compile(r'"[^"]*"') +# Matches characters. Escape codes should already be removed by ESCAPES. +_RE_PATTERN_CLEANSE_LINE_SINGLE_QUOTES = re.compile(r"'.'") +# Matches multi-line C++ comments. +# This RE is a little bit more complicated than one might expect, because we +# have to take care of space removals tools so we can handle comments inside +# statements better. +# The current rule is: We only clear spaces from both sides when we're at the +# end of the line. Otherwise, we try to remove spaces from the right side, +# if this doesn't work we try on left side but only if there's a non-character +# on the right. +_RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile( + r"""(\s*/\*.*\*/\s*$| + /\*.*\*/\s+| + \s+/\*.*\*/(?=\W)| + /\*.*\*/)""", re.VERBOSE) + + +def is_cpp_string(line): + """Does line terminate so, that the next symbol is in string constant. + + This function does not consider single-line nor multi-line comments. + + Args: + line: is a partial line of code starting from the 0..n. + + Returns: + True, if next character appended to 'line' is inside a + string constant. + """ + + line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" + return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 + + +def find_next_multi_line_comment_start(lines, line_index): + """Find the beginning marker for a multiline comment.""" + while line_index < len(lines): + if lines[line_index].strip().startswith('/*'): + # Only return this marker if the comment goes beyond this line + if lines[line_index].strip().find('*/', 2) < 0: + return line_index + line_index += 1 + return len(lines) + + +def find_next_multi_line_comment_end(lines, line_index): + """We are inside a comment, find the end marker.""" + while line_index < len(lines): + if lines[line_index].strip().endswith('*/'): + return line_index + line_index += 1 + return len(lines) + + +def remove_multi_line_comments_from_range(lines, begin, end): + """Clears a range of lines for multi-line comments.""" + # Having // dummy comments makes the lines non-empty, so we will not get + # unnecessary blank line warnings later in the code. + for i in range(begin, end): + lines[i] = '// dummy' + + +def remove_multi_line_comments(filename, lines, error): + """Removes multiline (c-style) comments from lines.""" + line_index = 0 + while line_index < len(lines): + line_index_begin = find_next_multi_line_comment_start(lines, line_index) + if line_index_begin >= len(lines): + return + line_index_end = find_next_multi_line_comment_end(lines, line_index_begin) + if line_index_end >= len(lines): + error(filename, line_index_begin + 1, 'readability/multiline_comment', 5, + 'Could not find end of multi-line comment') + return + remove_multi_line_comments_from_range(lines, line_index_begin, line_index_end + 1) + line_index = line_index_end + 1 + + +def cleanse_comments(line): + """Removes //-comments and single-line C-style /* */ comments. + + Args: + line: A line of C++ source. + + Returns: + The line with single-line comments removed. + """ + comment_position = line.find('//') + if comment_position != -1 and not is_cpp_string(line[:comment_position]): + line = line[:comment_position] + # get rid of /* ... */ + return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) + + +class CleansedLines(object): + """Holds 3 copies of all lines with different preprocessing applied to them. + + 1) elided member contains lines without strings and comments, + 2) lines member contains lines without comments, and + 3) raw member contains all the lines without processing. + All these three members are of <type 'list'>, and of the same length. + """ + + def __init__(self, lines): + self.elided = [] + self.lines = [] + self.raw_lines = lines + self._num_lines = len(lines) + for line_number in range(len(lines)): + self.lines.append(cleanse_comments(lines[line_number])) + elided = self.collapse_strings(lines[line_number]) + self.elided.append(cleanse_comments(elided)) + + def num_lines(self): + """Returns the number of lines represented.""" + return self._num_lines + + @staticmethod + def collapse_strings(elided): + """Collapses strings and chars on a line to simple "" or '' blocks. + + We nix strings first so we're not fooled by text like '"http://"' + + Args: + elided: The line being processed. + + Returns: + The line with collapsed strings. + """ + if not _RE_PATTERN_INCLUDE.match(elided): + # Remove escaped characters first to make quote/single quote collapsing + # basic. Things that look like escaped characters shouldn't occur + # outside of strings and chars. + elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) + elided = _RE_PATTERN_CLEANSE_LINE_SINGLE_QUOTES.sub("''", elided) + elided = _RE_PATTERN_CLEANSE_LINE_DOUBLE_QUOTES.sub('""', elided) + return elided + + +def close_expression(clean_lines, line_number, pos): + """If input points to ( or { or [, finds the position that closes it. + + If lines[line_number][pos] points to a '(' or '{' or '[', finds the the + line_number/pos that correspond to the closing of the expression. + + Args: + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + pos: A position on the line. + + Returns: + A tuple (line, line_number, pos) pointer *past* the closing brace, or + (line, len(lines), -1) if we never find a close. Note we ignore + strings and comments when matching; and the line we return is the + 'cleansed' line at line_number. + """ + + line = clean_lines.elided[line_number] + start_character = line[pos] + if start_character not in '({[': + return (line, clean_lines.num_lines(), -1) + if start_character == '(': + end_character = ')' + if start_character == '[': + end_character = ']' + if start_character == '{': + end_character = '}' + + num_open = line.count(start_character) - line.count(end_character) + while line_number < clean_lines.num_lines() and num_open > 0: + line_number += 1 + line = clean_lines.elided[line_number] + num_open += line.count(start_character) - line.count(end_character) + # OK, now find the end_character that actually got us back to even + endpos = len(line) + while num_open >= 0: + endpos = line.rfind(')', 0, endpos) + num_open -= 1 # chopped off another ) + return (line, line_number, endpos + 1) + + +def check_for_copyright(filename, lines, error): + """Logs an error if no Copyright message appears at the top of the file.""" + + # We'll say it should occur by line 10. Don't forget there's a + # dummy line at the front. + for line in xrange(1, min(len(lines), 11)): + if re.search(r'Copyright|License', lines[line], re.I): + break + else: # means no copyright line was found + error(filename, 1, 'legal/copyright', 3, + 'No copyright message found.') + + +def get_header_guard_cpp_variable(filename): + """Returns the CPP variable that should be used as a header guard. + + Args: + filename: The name of a C++ header file. + + Returns: + The CPP variable that should be used as a header guard in the + named file. + + """ + + fileinfo = FileInfo(filename) + return re.sub(r'[-./\s]', '_', fileinfo.repository_name()).upper() + '_' + + +def check_for_header_guard(filename, lines, error): + """Checks that the file contains a header guard. + + Logs an error if no #ifndef header guard is present. For other + headers, checks that the full pathname is used. + + Args: + filename: The name of the C++ header file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + + cppvar = get_header_guard_cpp_variable(filename) + + ifndef = None + ifndef_line_number = 0 + define = None + endif = None + endif_line_number = 0 + for line_number, line in enumerate(lines): + line_split = line.split() + if len(line_split) >= 2: + # find the first occurrence of #ifndef and #define, save arg + if not ifndef and line_split[0] == '#ifndef': + # set ifndef to the header guard presented on the #ifndef line. + ifndef = line_split[1] + ifndef_line_number = line_number + if not define and line_split[0] == '#define': + define = line_split[1] + # find the last occurrence of #endif, save entire line + if line.startswith('#endif'): + endif = line + endif_line_number = line_number + + if not ifndef or not define or ifndef != define: + error(filename, 1, 'build/header_guard', 5, + 'No #ifndef header guard found, suggested CPP variable is: %s' % + cppvar) + return + + # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ + # for backward compatibility. + if ifndef != cppvar: + error_level = 0 + if ifndef != cppvar + '_': + error_level = 5 + + error(filename, ifndef_line_number, 'build/header_guard', error_level, + '#ifndef header guard has wrong style, please use: %s' % cppvar) + + if endif != ('#endif // %s' % cppvar): + error_level = 0 + if endif != ('#endif // %s' % (cppvar + '_')): + error_level = 5 + + error(filename, endif_line_number, 'build/header_guard', error_level, + '#endif line should be "#endif // %s"' % cppvar) + + +def check_for_unicode_replacement_characters(filename, lines, error): + """Logs an error for each line containing Unicode replacement characters. + + These indicate that either the file contained invalid UTF-8 (likely) + or Unicode replacement characters (which it shouldn't). Note that + it's possible for this to throw off line numbering if the invalid + UTF-8 occurred adjacent to a newline. + + Args: + filename: The name of the current file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + for line_number, line in enumerate(lines): + if u'\ufffd' in line: + error(filename, line_number, 'readability/utf8', 5, + 'Line contains invalid UTF-8 (or Unicode replacement character).') + + +def check_for_new_line_at_eof(filename, lines, error): + """Logs an error if there is no newline char at the end of the file. + + Args: + filename: The name of the current file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + + # The array lines() was created by adding two newlines to the + # original file (go figure), then splitting on \n. + # To verify that the file ends in \n, we just have to make sure the + # last-but-two element of lines() exists and is empty. + if len(lines) < 3 or lines[-2]: + error(filename, len(lines) - 2, 'whitespace/ending_newline', 5, + 'Could not find a newline character at the end of the file.') + + +def check_for_multiline_comments_and_strings(filename, clean_lines, line_number, error): + """Logs an error if we see /* ... */ or "..." that extend past one line. + + /* ... */ comments are legit inside macros, for one line. + Otherwise, we prefer // comments, so it's ok to warn about the + other. Likewise, it's ok for strings to extend across multiple + lines, as long as a line continuation character (backslash) + terminates each line. Although not currently prohibited by the C++ + style guide, it's ugly and unnecessary. We don't do well with either + in this lint program, so we warn about both. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[line_number] + + # Remove all \\ (escaped backslashes) from the line. They are OK, and the + # second (escaped) slash may trigger later \" detection erroneously. + line = line.replace('\\\\', '') + + if line.count('/*') > line.count('*/'): + error(filename, line_number, 'readability/multiline_comment', 5, + 'Complex multi-line /*...*/-style comment found. ' + 'Lint may give bogus warnings. ' + 'Consider replacing these with //-style comments, ' + 'with #if 0...#endif, ' + 'or with more clearly structured multi-line comments.') + + if (line.count('"') - line.count('\\"')) % 2: + error(filename, line_number, 'readability/multiline_string', 5, + 'Multi-line string ("...") found. This lint script doesn\'t ' + 'do well with such strings, and may give bogus warnings. They\'re ' + 'ugly and unnecessary, and you should use concatenation instead".') + + +_THREADING_LIST = ( + ('asctime(', 'asctime_r('), + ('ctime(', 'ctime_r('), + ('getgrgid(', 'getgrgid_r('), + ('getgrnam(', 'getgrnam_r('), + ('getlogin(', 'getlogin_r('), + ('getpwnam(', 'getpwnam_r('), + ('getpwuid(', 'getpwuid_r('), + ('gmtime(', 'gmtime_r('), + ('localtime(', 'localtime_r('), + ('rand(', 'rand_r('), + ('readdir(', 'readdir_r('), + ('strtok(', 'strtok_r('), + ('ttyname(', 'ttyname_r('), + ) + + +def check_posix_threading(filename, clean_lines, line_number, error): + """Checks for calls to thread-unsafe functions. + + Much code has been originally written without consideration of + multi-threading. Also, engineers are relying on their old experience; + they have learned posix before threading extensions were added. These + tests guide the engineers to use thread-safe functions (when using + posix directly). + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[line_number] + for single_thread_function, multithread_safe_function in _THREADING_LIST: + index = line.find(single_thread_function) + # Comparisons made explicit for clarity -- pylint: disable-msg=C6403 + if index >= 0 and (index == 0 or (not line[index - 1].isalnum() + and line[index - 1] not in ('_', '.', '>'))): + error(filename, line_number, 'runtime/threadsafe_fn', 2, + 'Consider using ' + multithread_safe_function + + '...) instead of ' + single_thread_function + + '...) for improved thread safety.') + + +# Matches invalid increment: *count++, which moves pointer instead of +# incrementing a value. +_RE_PATTERN_INVALID_INCREMENT = re.compile( + r'^\s*\*\w+(\+\+|--);') + + +def check_invalid_increment(filename, clean_lines, line_number, error): + """Checks for invalid increment *count++. + + For example following function: + void increment_counter(int* count) { + *count++; + } + is invalid, because it effectively does count++, moving pointer, and should + be replaced with ++*count, (*count)++ or *count += 1. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[line_number] + if _RE_PATTERN_INVALID_INCREMENT.match(line): + error(filename, line_number, 'runtime/invalid_increment', 5, + 'Changing pointer instead of value (or unused value of operator*).') + + +class _ClassInfo(object): + """Stores information about a class.""" + + def __init__(self, name, line_number): + self.name = name + self.line_number = line_number + self.seen_open_brace = False + self.is_derived = False + self.virtual_method_line_number = None + self.has_virtual_destructor = False + self.brace_depth = 0 + + +class _ClassState(object): + """Holds the current state of the parse relating to class declarations. + + It maintains a stack of _ClassInfos representing the parser's guess + as to the current nesting of class declarations. The innermost class + is at the top (back) of the stack. Typically, the stack will either + be empty or have exactly one entry. + """ + + def __init__(self): + self.classinfo_stack = [] + + def check_finished(self, filename, error): + """Checks that all classes have been completely parsed. + + Call this when all lines in a file have been processed. + Args: + filename: The name of the current file. + error: The function to call with any errors found. + """ + if self.classinfo_stack: + # Note: This test can result in false positives if #ifdef constructs + # get in the way of brace matching. See the testBuildClass test in + # cpplint_unittest.py for an example of this. + error(filename, self.classinfo_stack[0].line_number, 'build/class', 5, + 'Failed to find complete declaration of class %s' % + self.classinfo_stack[0].name) + + +def check_for_non_standard_constructs(filename, clean_lines, line_number, + class_state, error): + """Logs an error if we see certain non-ANSI constructs ignored by gcc-2. + + Complain about several constructs which gcc-2 accepts, but which are + not standard C++. Warning about these in lint is one way to ease the + transition to new compilers. + - put storage class first (e.g. "static const" instead of "const static"). + - "%lld" instead of %qd" in printf-type functions. + - "%1$d" is non-standard in printf-type functions. + - "\%" is an undefined character escape sequence. + - text after #endif is not allowed. + - invalid inner-style forward declaration. + - >? and <? operators, and their >?= and <?= cousins. + - classes with virtual methods need virtual destructors (compiler warning + available, but not turned on yet.) + + Additionally, check for constructor/destructor style violations as it + is very convenient to do so while checking for gcc-2 compliance. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + class_state: A _ClassState instance which maintains information about + the current stack of nested class declarations being parsed. + error: A callable to which errors are reported, which takes 4 arguments: + filename, line number, error level, and message + """ + + # Remove comments from the line, but leave in strings for now. + line = clean_lines.lines[line_number] + + if search(r'printf\s*\(.*".*%[-+ ]?\d*q', line): + error(filename, line_number, 'runtime/printf_format', 3, + '%q in format strings is deprecated. Use %ll instead.') + + if search(r'printf\s*\(.*".*%\d+\$', line): + error(filename, line_number, 'runtime/printf_format', 2, + '%N$ formats are unconventional. Try rewriting to avoid them.') + + # Remove escaped backslashes before looking for undefined escapes. + line = line.replace('\\\\', '') + + if search(r'("|\').*\\(%|\[|\(|{)', line): + error(filename, line_number, 'build/printf_format', 3, + '%, [, (, and { are undefined character escapes. Unescape them.') + + # For the rest, work with both comments and strings removed. + line = clean_lines.elided[line_number] + + if search(r'\b(const|volatile|void|char|short|int|long' + r'|float|double|signed|unsigned' + r'|schar|u?int8|u?int16|u?int32|u?int64)' + r'\s+(auto|register|static|extern|typedef)\b', + line): + error(filename, line_number, 'build/storage_class', 5, + 'Storage class (static, extern, typedef, etc) should be first.') + + if match(r'\s*#\s*endif\s*[^/\s]+', line): + error(filename, line_number, 'build/endif_comment', 5, + 'Uncommented text after #endif is non-standard. Use a comment.') + + if match(r'\s*class\s+(\w+\s*::\s*)+\w+\s*;', line): + error(filename, line_number, 'build/forward_decl', 5, + 'Inner-style forward declarations are invalid. Remove this line.') + + if search(r'(\w+|[+-]?\d+(\.\d*)?)\s*(<|>)\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', line): + error(filename, line_number, 'build/deprecated', 3, + '>? and <? (max and min) operators are non-standard and deprecated.') + + # Track class entry and exit, and attempt to find cases within the + # class declaration that don't meet the C++ style + # guidelines. Tracking is very dependent on the code matching Google + # style guidelines, but it seems to perform well enough in testing + # to be a worthwhile addition to the checks. + classinfo_stack = class_state.classinfo_stack + # Look for a class declaration + class_decl_match = match( + r'\s*(template\s*<[\w\s<>,:]*>\s*)?(class|struct)\s+(\w+(::\w+)*)', line) + if class_decl_match: + classinfo_stack.append(_ClassInfo(class_decl_match.group(3), line_number)) + + # Everything else in this function uses the top of the stack if it's + # not empty. + if not classinfo_stack: + return + + classinfo = classinfo_stack[-1] + + # If the opening brace hasn't been seen look for it and also + # parent class declarations. + if not classinfo.seen_open_brace: + # If the line has a ';' in it, assume it's a forward declaration or + # a single-line class declaration, which we won't process. + if line.find(';') != -1: + classinfo_stack.pop() + return + classinfo.seen_open_brace = (line.find('{') != -1) + # Look for a bare ':' + if search('(^|[^:]):($|[^:])', line): + classinfo.is_derived = True + if not classinfo.seen_open_brace: + return # Everything else in this function is for after open brace + + # The class may have been declared with namespace or classname qualifiers. + # The constructor and destructor will not have those qualifiers. + base_classname = classinfo.name.split('::')[-1] + + # Look for single-argument constructors that aren't marked explicit. + # Technically a valid construct, but against style. + args = match(r'(?<!explicit)\s+%s\s*\(([^,()]+)\)' + % re.escape(base_classname), + line) + if (args + and args.group(1) != 'void' + and not match(r'(const\s+)?%s\s*&' % re.escape(base_classname), + args.group(1).strip())): + error(filename, line_number, 'runtime/explicit', 5, + 'Single-argument constructors should be marked explicit.') + + # Look for methods declared virtual. + if search(r'\bvirtual\b', line): + classinfo.virtual_method_line_number = line_number + # Only look for a destructor declaration on the same line. It would + # be extremely unlikely for the destructor declaration to occupy + # more than one line. + if search(r'~%s\s*\(' % base_classname, line): + classinfo.has_virtual_destructor = True + + # Look for class end. + brace_depth = classinfo.brace_depth + brace_depth = brace_depth + line.count('{') - line.count('}') + if brace_depth <= 0: + classinfo = classinfo_stack.pop() + # Try to detect missing virtual destructor declarations. + # For now, only warn if a non-derived class with virtual methods lacks + # a virtual destructor. This is to make it less likely that people will + # declare derived virtual destructors without declaring the base + # destructor virtual. + if ((classinfo.virtual_method_line_number is not None) + and (not classinfo.has_virtual_destructor) + and (not classinfo.is_derived)): # Only warn for base classes + error(filename, classinfo.line_number, 'runtime/virtual', 4, + 'The class %s probably needs a virtual destructor due to ' + 'having virtual method(s), one declared at line %d.' + % (classinfo.name, classinfo.virtual_method_line_number)) + else: + classinfo.brace_depth = brace_depth + + +def check_spacing_for_function_call(filename, line, line_number, error): + """Checks for the correctness of various spacing around function calls. + + Args: + filename: The name of the current file. + line: The text of the line to check. + line_number: The number of the line to check. + error: The function to call with any errors found. + """ + + # Since function calls often occur inside if/for/foreach/while/switch + # expressions - which have their own, more liberal conventions - we + # first see if we should be looking inside such an expression for a + # function call, to which we can apply more strict standards. + function_call = line # if there's no control flow construct, look at whole line + for pattern in (r'\bif\s*\((.*)\)\s*{', + r'\bfor\s*\((.*)\)\s*{', + r'\bforeach\s*\((.*)\)\s*{', + r'\bwhile\s*\((.*)\)\s*[{;]', + r'\bswitch\s*\((.*)\)\s*{'): + matched = search(pattern, line) + if matched: + function_call = matched.group(1) # look inside the parens for function calls + break + + # Except in if/for/foreach/while/switch, there should never be space + # immediately inside parens (eg "f( 3, 4 )"). We make an exception + # for nested parens ( (a+b) + c ). Likewise, there should never be + # a space before a ( when it's a function argument. I assume it's a + # function argument when the char before the whitespace is legal in + # a function name (alnum + _) and we're not starting a macro. Also ignore + # pointers and references to arrays and functions coz they're too tricky: + # we use a very simple way to recognize these: + # " (something)(maybe-something)" or + # " (something)(maybe-something," or + # " (something)[something]" + # Note that we assume the contents of [] to be short enough that + # they'll never need to wrap. + if ( # Ignore control structures. + not search(r'\b(if|for|foreach|while|switch|return|new|delete)\b', function_call) + # Ignore pointers/references to functions. + and not search(r' \([^)]+\)\([^)]*(\)|,$)', function_call) + # Ignore pointers/references to arrays. + and not search(r' \([^)]+\)\[[^\]]+\]', function_call)): + if search(r'\w\s*\([ \t](?!\s*\\$)', function_call): # a ( used for a fn call + error(filename, line_number, 'whitespace/parens', 4, + 'Extra space after ( in function call') + elif search(r'\([ \t]+(?!(\s*\\)|\()', function_call): + error(filename, line_number, 'whitespace/parens', 2, + 'Extra space after (') + if (search(r'\w\s+\(', function_call) + and not search(r'#\s*define|typedef', function_call)): + error(filename, line_number, 'whitespace/parens', 4, + 'Extra space before ( in function call') + # If the ) is followed only by a newline or a { + newline, assume it's + # part of a control statement (if/while/etc), and don't complain + if search(r'[^)\s]\s+\)(?!\s*$|{\s*$)', function_call): + error(filename, line_number, 'whitespace/parens', 2, + 'Extra space before )') + + +def is_blank_line(line): + """Returns true if the given line is blank. + + We consider a line to be blank if the line is empty or consists of + only white spaces. + + Args: + line: A line of a string. + + Returns: + True, if the given line is blank. + """ + return not line or line.isspace() + + +def check_for_function_lengths(filename, clean_lines, line_number, + function_state, error): + """Reports for long function bodies. + + For an overview why this is done, see: + http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Write_Short_Functions + + Uses a simplistic algorithm assuming other style guidelines + (especially spacing) are followed. + Only checks unindented functions, so class members are unchecked. + Trivial bodies are unchecked, so constructors with huge initializer lists + may be missed. + Blank/comment lines are not counted so as to avoid encouraging the removal + of vertical space and commments just to get through a lint check. + NOLINT *on the last line of a function* disables this check. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + function_state: Current function name and lines in body so far. + error: The function to call with any errors found. + """ + lines = clean_lines.lines + line = lines[line_number] + raw = clean_lines.raw_lines + raw_line = raw[line_number] + joined_line = '' + + starting_func = False + regexp = r'(\w(\w|::|\*|\&|\s)*)\(' # decls * & space::name( ... + match_result = match(regexp, line) + if match_result: + # If the name is all caps and underscores, figure it's a macro and + # ignore it, unless it's TEST or TEST_F. + function_name = match_result.group(1).split()[-1] + if function_name == 'TEST' or function_name == 'TEST_F' or (not match(r'[A-Z_]+$', function_name)): + starting_func = True + + if starting_func: + body_found = False + for start_line_number in xrange(line_number, clean_lines.num_lines()): + start_line = lines[start_line_number] + joined_line += ' ' + start_line.lstrip() + if search(r'(;|})', start_line): # Declarations and trivial functions + body_found = True + break # ... ignore + if search(r'{', start_line): + body_found = True + function = search(r'((\w|:)*)\(', line).group(1) + if match(r'TEST', function): # Handle TEST... macros + parameter_regexp = search(r'(\(.*\))', joined_line) + if parameter_regexp: # Ignore bad syntax + function += parameter_regexp.group(1) + else: + function += '()' + function_state.begin(function) + break + if not body_found: + # No body for the function (or evidence of a non-function) was found. + error(filename, line_number, 'readability/fn_size', 5, + 'Lint failed to find start of function body.') + elif match(r'^\}\s*$', line): # function end + if not search(r'\bNOLINT\b', raw_line): + function_state.check(error, filename, line_number) + function_state.end() + elif not match(r'^\s*$', line): + function_state.count() # Count non-blank/non-comment lines. + + +def check_spacing(filename, clean_lines, line_number, error): + """Checks for the correctness of various spacing issues in the code. + + Things we check for: spaces around operators, spaces after + if/for/while/switch, no spaces around parens in function calls, two + spaces between code and comment, don't start a block with a blank + line, don't end a function with a blank line, don't have too many + blank lines in a row. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + error: The function to call with any errors found. + """ + + raw = clean_lines.raw_lines + line = raw[line_number] + + # Before nixing comments, check if the line is blank for no good + # reason. This includes the first line after a block is opened, and + # blank lines at the end of a function (ie, right before a line like '}'). + if is_blank_line(line): + elided = clean_lines.elided + previous_line = elided[line_number - 1] + previous_brace = previous_line.rfind('{') + # FIXME: Don't complain if line before blank line, and line after, + # both start with alnums and are indented the same amount. + # This ignores whitespace at the start of a namespace block + # because those are not usually indented. + if (previous_brace != -1 and previous_line[previous_brace:].find('}') == -1 + and previous_line[:previous_brace].find('namespace') == -1): + # OK, we have a blank line at the start of a code block. Before we + # complain, we check if it is an exception to the rule: The previous + # non-empty line has the parameters of a function header that are indented + # 4 spaces (because they did not fit in a 80 column line when placed on + # the same line as the function name). We also check for the case where + # the previous line is indented 6 spaces, which may happen when the + # initializers of a constructor do not fit into a 80 column line. + exception = False + if match(r' {6}\w', previous_line): # Initializer list? + # We are looking for the opening column of initializer list, which + # should be indented 4 spaces to cause 6 space indentation afterwards. + search_position = line_number - 2 + while (search_position >= 0 + and match(r' {6}\w', elided[search_position])): + search_position -= 1 + exception = (search_position >= 0 + and elided[search_position][:5] == ' :') + else: + # Search for the function arguments or an initializer list. We use a + # simple heuristic here: If the line is indented 4 spaces; and we have a + # closing paren, without the opening paren, followed by an opening brace + # or colon (for initializer lists) we assume that it is the last line of + # a function header. If we have a colon indented 4 spaces, it is an + # initializer list. + exception = (match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', + previous_line) + or match(r' {4}:', previous_line)) + + if not exception: + error(filename, line_number, 'whitespace/blank_line', 2, + 'Blank line at the start of a code block. Is this needed?') + # This doesn't ignore whitespace at the end of a namespace block + # because that is too hard without pairing open/close braces; + # however, a special exception is made for namespace closing + # brackets which have a comment containing "namespace". + # + # Also, ignore blank lines at the end of a block in a long if-else + # chain, like this: + # if (condition1) { + # // Something followed by a blank line + # + # } else if (condition2) { + # // Something else + # } + if line_number + 1 < clean_lines.num_lines(): + next_line = raw[line_number + 1] + if (next_line + and match(r'\s*}', next_line) + and next_line.find('namespace') == -1 + and next_line.find('} else ') == -1): + error(filename, line_number, 'whitespace/blank_line', 3, + 'Blank line at the end of a code block. Is this needed?') + + # Next, we complain if there's a comment too near the text + comment_position = line.find('//') + if comment_position != -1: + # Check if the // may be in quotes. If so, ignore it + # Comparisons made explicit for clarity -- pylint: disable-msg=C6403 + if (line.count('"', 0, comment_position) - line.count('\\"', 0, comment_position)) % 2 == 0: # not in quotes + # Allow one space for new scopes, two spaces otherwise: + if (not match(r'^\s*{ //', line) + and ((comment_position >= 1 + and line[comment_position-1] not in string.whitespace) + or (comment_position >= 2 + and line[comment_position-2] not in string.whitespace))): + error(filename, line_number, 'whitespace/comments-doublespace', 2, + 'At least two spaces is best between code and comments') + # There should always be a space between the // and the comment + commentend = comment_position + 2 + if commentend < len(line) and not line[commentend] == ' ': + # but some lines are exceptions -- e.g. if they're big + # comment delimiters like: + # //---------------------------------------------------------- + # or they begin with multiple slashes followed by a space: + # //////// Header comment + matched = (search(r'[=/-]{4,}\s*$', line[commentend:]) + or search(r'^/+ ', line[commentend:])) + if not matched: + error(filename, line_number, 'whitespace/comments', 4, + 'Should have a space between // and comment') + + line = clean_lines.elided[line_number] # get rid of comments and strings + + # Don't try to do spacing checks for operator methods + line = re.sub(r'operator(==|!=|<|<<|<=|>=|>>|>)\(', 'operator\(', line) + + # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". + # Otherwise not. Note we only check for non-spaces on *both* sides; + # sometimes people put non-spaces on one side when aligning ='s among + # many lines (not that this is behavior that I approve of...) + if search(r'[\w.]=[\w.]', line) and not search(r'\b(if|while) ', line): + error(filename, line_number, 'whitespace/operators', 4, + 'Missing spaces around =') + + # FIXME: It's not ok to have spaces around binary operators like + - * / . + + # You should always have whitespace around binary operators. + # Alas, we can't test < or > because they're legitimately used sans spaces + # (a->b, vector<int> a). The only time we can tell is a < with no >, and + # only if it's not template params list spilling into the next line. + matched = search(r'[^<>=!\s](==|!=|<=|>=)[^<>=!\s]', line) + if not matched: + # Note that while it seems that the '<[^<]*' term in the following + # regexp could be simplified to '<.*', which would indeed match + # the same class of strings, the [^<] means that searching for the + # regexp takes linear rather than quadratic time. + if not search(r'<[^<]*,\s*$', line): # template params spill + matched = search(r'[^<>=!\s](<)[^<>=!\s]([^>]|->)*$', line) + if matched: + error(filename, line_number, 'whitespace/operators', 3, + 'Missing spaces around %s' % matched.group(1)) + # We allow no-spaces around << and >> when used like this: 10<<20, but + # not otherwise (particularly, not when used as streams) + matched = search(r'[^0-9\s](<<|>>)[^0-9\s]', line) + if matched: + error(filename, line_number, 'whitespace/operators', 3, + 'Missing spaces around %s' % matched.group(1)) + + # There shouldn't be space around unary operators + matched = search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) + if matched: + error(filename, line_number, 'whitespace/operators', 4, + 'Extra space for operator %s' % matched.group(1)) + + # A pet peeve of mine: no spaces after an if, while, switch, or for + matched = search(r' (if\(|for\(|foreach\(|while\(|switch\()', line) + if matched: + error(filename, line_number, 'whitespace/parens', 5, + 'Missing space before ( in %s' % matched.group(1)) + + # For if/for/foreach/while/switch, the left and right parens should be + # consistent about how many spaces are inside the parens, and + # there should either be zero or one spaces inside the parens. + # We don't want: "if ( foo)" or "if ( foo )". + # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. + matched = search(r'\b(if|for|foreach|while|switch)\s*\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', + line) + if matched: + if len(matched.group(2)) != len(matched.group(4)): + if not (matched.group(3) == ';' + and len(matched.group(2)) == 1 + len(matched.group(4)) + or not matched.group(2) and search(r'\bfor\s*\(.*; \)', line)): + error(filename, line_number, 'whitespace/parens', 5, + 'Mismatching spaces inside () in %s' % matched.group(1)) + if not len(matched.group(2)) in [0, 1]: + error(filename, line_number, 'whitespace/parens', 5, + 'Should have zero or one spaces inside ( and ) in %s' % + matched.group(1)) + + # You should always have a space after a comma (either as fn arg or operator) + if search(r',[^\s]', line): + error(filename, line_number, 'whitespace/comma', 3, + 'Missing space after ,') + + # Next we will look for issues with function calls. + check_spacing_for_function_call(filename, line, line_number, error) + + # Except after an opening paren, you should have spaces before your braces. + # And since you should never have braces at the beginning of a line, this is + # an easy test. + if search(r'[^ ({]{', line): + error(filename, line_number, 'whitespace/braces', 5, + 'Missing space before {') + + # Make sure '} else {' has spaces. + if search(r'}else', line): + error(filename, line_number, 'whitespace/braces', 5, + 'Missing space before else') + + # You shouldn't have spaces before your brackets, except maybe after + # 'delete []' or 'new char * []'. + if search(r'\w\s+\[', line) and not search(r'delete\s+\[', line): + error(filename, line_number, 'whitespace/braces', 5, + 'Extra space before [') + + # You shouldn't have a space before a semicolon at the end of the line. + # There's a special case for "for" since the style guide allows space before + # the semicolon there. + if search(r':\s*;\s*$', line): + error(filename, line_number, 'whitespace/semicolon', 5, + 'Semicolon defining empty statement. Use { } instead.') + elif search(r'^\s*;\s*$', line): + error(filename, line_number, 'whitespace/semicolon', 5, + 'Line contains only semicolon. If this should be an empty statement, ' + 'use { } instead.') + elif (search(r'\s+;\s*$', line) and not search(r'\bfor\b', line)): + error(filename, line_number, 'whitespace/semicolon', 5, + 'Extra space before last semicolon. If this should be an empty ' + 'statement, use { } instead.') + elif (search(r'\b(for|while)\s*\(.*\)\s*;\s*$', line) + and line.count('(') == line.count(')') + # Allow do {} while(); + and not search(r'}\s*while', line)): + error(filename, line_number, 'whitespace/semicolon', 5, + 'Semicolon defining empty statement for this loop. Use { } instead.') + + +def get_previous_non_blank_line(clean_lines, line_number): + """Return the most recent non-blank line and its line number. + + Args: + clean_lines: A CleansedLines instance containing the file contents. + line_number: The number of the line to check. + + Returns: + A tuple with two elements. The first element is the contents of the last + non-blank line before the current line, or the empty string if this is the + first non-blank line. The second is the line number of that line, or -1 + if this is the first non-blank line. + """ + + previous_line_number = line_number - 1 + while previous_line_number >= 0: + previous_line = clean_lines.elided[previous_line_number] + if not is_blank_line(previous_line): # if not a blank line... + return (previous_line, previous_line_number) + previous_line_number -= 1 + return ('', -1) + + +def check_namespace_indentation(filename, clean_lines, line_number, file_extension, error): + """Looks for indentation errors inside of namespaces. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + file_extension: The extension (dot not included) of the file. + error: The function to call with any errors found. + """ + + line = clean_lines.elided[line_number] # Get rid of comments and strings. + + namespace_match = match(r'(?P<namespace_indentation>\s*)namespace\s+\S+\s*{\s*$', line) + if not namespace_match: + return + + namespace_indentation = namespace_match.group('namespace_indentation') + + is_header_file = file_extension == 'h' + is_implementation_file = not is_header_file + line_offset = 0 + + if is_header_file: + inner_indentation = namespace_indentation + ' ' * 4 + + for current_line in clean_lines.raw_lines[line_number + 1:]: + line_offset += 1 + + # Skip not only empty lines but also those with preprocessor directives. + # Goto labels don't occur in header files, so no need to check for those. + if current_line.strip() == '' or current_line.startswith('#'): + continue + + if not current_line.startswith(inner_indentation): + # If something unindented was discovered, make sure it's a closing brace. + if not current_line.startswith(namespace_indentation + '}'): + error(filename, line_number + line_offset, 'whitespace/indent', 4, + 'In a header, code inside a namespace should be indented.') + break + + if is_implementation_file: + for current_line in clean_lines.raw_lines[line_number + 1:]: + line_offset += 1 + + # Skip not only empty lines but also those with (goto) labels. + # The goto label regexp accepts spaces or the beginning of a + # comment (if anything) after the initial colon. + if current_line.strip() == '' or match(r'\w+\s*:([\s\/].*)?$', current_line): + continue + + remaining_line = current_line[len(namespace_indentation):] + if not match(r'\S', remaining_line): + error(filename, line_number + line_offset, 'whitespace/indent', 4, + 'In an implementation file, code inside a namespace should not be indented.') + + # Just check the first non-empty line in any case, because + # otherwise we would need to count opened and closed braces, + # which is obviously a lot more complicated. + break + + +def check_switch_indentation(filename, clean_lines, line_number, error): + """Looks for indentation errors inside of switch statements. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + error: The function to call with any errors found. + """ + + line = clean_lines.elided[line_number] # Get rid of comments and strings. + + switch_match = match(r'(?P<switch_indentation>\s*)switch\s*\(.+\)\s*{\s*$', line) + if not switch_match: + return + + switch_indentation = switch_match.group('switch_indentation') + inner_indentation = switch_indentation + ' ' * 4 + line_offset = 0 + encountered_nested_switch = False + + for current_line in clean_lines.elided[line_number + 1:]: + line_offset += 1 + + # Skip not only empty lines but also those with preprocessor directives. + if current_line.strip() == '' or current_line.startswith('#'): + continue + + if match(r'\s*switch\s*\(.+\)\s*{\s*$', current_line): + # Complexity alarm - another switch statement nested inside the one + # that we're currently testing. We'll need to track the extent of + # that inner switch if the upcoming label tests are still supposed + # to work correctly. Let's not do that; instead, we'll finish + # checking this line, and then leave it like that. Assuming the + # indentation is done consistently (even if incorrectly), this will + # still catch all indentation issues in practice. + encountered_nested_switch = True + + current_indentation_match = match(r'(?P<indentation>\s*)(?P<remaining_line>.*)$', current_line); + current_indentation = current_indentation_match.group('indentation') + remaining_line = current_indentation_match.group('remaining_line') + + # End the check at the end of the switch statement. + if remaining_line.startswith('}') and current_indentation == switch_indentation: + break + # Case and default branches should not be indented. The regexp also + # catches single-line cases like "default: break;" but does not trigger + # on stuff like "Document::Foo();". + elif match(r'(default|case\s+.*)\s*:([^:].*)?$', remaining_line): + if current_indentation != switch_indentation: + error(filename, line_number + line_offset, 'whitespace/indent', 4, + 'A case label should not be indented, but line up with its switch statement.') + # Don't throw an error for multiple badly indented labels, + # one should be enough to figure out the problem. + break + # We ignore goto labels at the very beginning of a line. + elif match(r'\w+\s*:\s*$', remaining_line): + continue + # It's not a goto label, so check if it's indented at least as far as + # the switch statement plus one more level of indentation. + elif not current_indentation.startswith(inner_indentation): + error(filename, line_number + line_offset, 'whitespace/indent', 4, + 'Non-label code inside switch statements should be indented.') + # Don't throw an error for multiple badly indented statements, + # one should be enough to figure out the problem. + break + + if encountered_nested_switch: + break + + +def check_braces(filename, clean_lines, line_number, error): + """Looks for misplaced braces (e.g. at the end of line). + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + error: The function to call with any errors found. + """ + + line = clean_lines.elided[line_number] # Get rid of comments and strings. + + """ + These don't match our style guideline: + https://developer.mozilla.org/en-US/docs/Developer_Guide/Coding_Style#Control_Structures + + TODO: Spin this off in a different rule and disable that rule for mozilla + rather then commenting this out + + + if match(r'\s*{\s*$', line): + # We allow an open brace to start a line in the case where someone + # is using braces for function definition or in a block to + # explicitly create a new scope, which is commonly used to control + # the lifetime of stack-allocated variables. We don't detect this + # perfectly: we just don't complain if the last non-whitespace + # character on the previous non-blank line is ';', ':', '{', '}', + # ')', or ') const' and doesn't begin with 'if|for|while|switch|else'. + # We also allow '#' for #endif and '=' for array initialization. + previous_line = get_previous_non_blank_line(clean_lines, line_number)[0] + if ((not search(r'[;:}{)=]\s*$|\)\s*const\s*$', previous_line) + or search(r'\b(if|for|foreach|while|switch|else)\b', previous_line)) + and previous_line.find('#') < 0): + error(filename, line_number, 'whitespace/braces', 4, + 'This { should be at the end of the previous line') + elif (search(r'\)\s*(const\s*)?{\s*$', line) + and line.count('(') == line.count(')') + and not search(r'\b(if|for|foreach|while|switch)\b', line)): + error(filename, line_number, 'whitespace/braces', 4, + 'Place brace on its own line for function definitions.') + + if (match(r'\s*}\s*$', line) and line_number > 1): + # We check if a closed brace has started a line to see if a + # one line control statement was previous. + previous_line = clean_lines.elided[line_number - 2] + if (previous_line.find('{') > 0 + and search(r'\b(if|for|foreach|while|else)\b', previous_line)): + error(filename, line_number, 'whitespace/braces', 4, + 'One line control clauses should not use braces.') + """ + + # An else clause should be on the same line as the preceding closing brace. + if match(r'\s*else\s*', line): + previous_line = get_previous_non_blank_line(clean_lines, line_number)[0] + if match(r'\s*}\s*$', previous_line): + error(filename, line_number, 'whitespace/newline', 4, + 'An else should appear on the same line as the preceding }') + + # Likewise, an else should never have the else clause on the same line + if search(r'\belse [^\s{]', line) and not search(r'\belse if\b', line): + error(filename, line_number, 'whitespace/newline', 4, + 'Else clause should never be on same line as else (use 2 lines)') + + # In the same way, a do/while should never be on one line + if match(r'\s*do [^\s{]', line): + error(filename, line_number, 'whitespace/newline', 4, + 'do/while clauses should not be on a single line') + + # Braces shouldn't be followed by a ; unless they're defining a struct + # or initializing an array. + # We can't tell in general, but we can for some common cases. + previous_line_number = line_number + while True: + (previous_line, previous_line_number) = get_previous_non_blank_line(clean_lines, previous_line_number) + if match(r'\s+{.*}\s*;', line) and not previous_line.count(';'): + line = previous_line + line + else: + break + if (search(r'{.*}\s*;', line) + and line.count('{') == line.count('}') + and not search(r'struct|class|enum|\s*=\s*{', line)): + error(filename, line_number, 'readability/braces', 4, + "You don't need a ; after a }") + + +def check_exit_statement_simplifications(filename, clean_lines, line_number, error): + """Looks for else or else-if statements that should be written as an + if statement when the prior if concludes with a return, break, continue or + goto statement. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + error: The function to call with any errors found. + """ + + line = clean_lines.elided[line_number] # Get rid of comments and strings. + + else_match = match(r'(?P<else_indentation>\s*)(\}\s*)?else(\s+if\s*\(|(?P<else>\s*(\{\s*)?\Z))', line) + if not else_match: + return + + else_indentation = else_match.group('else_indentation') + inner_indentation = else_indentation + ' ' * 4 + + previous_lines = clean_lines.elided[:line_number] + previous_lines.reverse() + line_offset = 0 + encountered_exit_statement = False + + for current_line in previous_lines: + line_offset -= 1 + + # Skip not only empty lines but also those with preprocessor directives + # and goto labels. + if current_line.strip() == '' or current_line.startswith('#') or match(r'\w+\s*:\s*$', current_line): + continue + + # Skip lines with closing braces on the original indentation level. + # Even though the styleguide says they should be on the same line as + # the "else if" statement, we also want to check for instances where + # the current code does not comply with the coding style. Thus, ignore + # these lines and proceed to the line before that. + if current_line == else_indentation + '}': + continue + + current_indentation_match = match(r'(?P<indentation>\s*)(?P<remaining_line>.*)$', current_line); + current_indentation = current_indentation_match.group('indentation') + remaining_line = current_indentation_match.group('remaining_line') + + # As we're going up the lines, the first real statement to encounter + # has to be an exit statement (return, break, continue or goto) - + # otherwise, this check doesn't apply. + if not encountered_exit_statement: + # We only want to find exit statements if they are on exactly + # the same level of indentation as expected from the code inside + # the block. If the indentation doesn't strictly match then we + # might have a nested if or something, which must be ignored. + if current_indentation != inner_indentation: + break + if match(r'(return(\W+.*)|(break|continue)\s*;|goto\s*\w+;)$', remaining_line): + encountered_exit_statement = True + continue + break + + # When code execution reaches this point, we've found an exit statement + # as last statement of the previous block. Now we only need to make + # sure that the block belongs to an "if", then we can throw an error. + + # Skip lines with opening braces on the original indentation level, + # similar to the closing braces check above. ("if (condition)\n{") + if current_line == else_indentation + '{': + continue + + # Skip everything that's further indented than our "else" or "else if". + if current_indentation.startswith(else_indentation) and current_indentation != else_indentation: + continue + + # So we've got a line with same (or less) indentation. Is it an "if"? + # If yes: throw an error. If no: don't throw an error. + # Whatever the outcome, this is the end of our loop. + if match(r'if\s*\(', remaining_line): + if else_match.start('else') != -1: + error(filename, line_number + line_offset, 'readability/control_flow', 4, + 'An else statement can be removed when the prior "if" ' + 'concludes with a return, break, continue or goto statement.') + else: + error(filename, line_number + line_offset, 'readability/control_flow', 4, + 'An else if statement should be written as an if statement ' + 'when the prior "if" concludes with a return, break, ' + 'continue or goto statement.') + break + + +def replaceable_check(operator, macro, line): + """Determine whether a basic CHECK can be replaced with a more specific one. + + For example suggest using CHECK_EQ instead of CHECK(a == b) and + similarly for CHECK_GE, CHECK_GT, CHECK_LE, CHECK_LT, CHECK_NE. + + Args: + operator: The C++ operator used in the CHECK. + macro: The CHECK or EXPECT macro being called. + line: The current source line. + + Returns: + True if the CHECK can be replaced with a more specific one. + """ + + # This matches decimal and hex integers, strings, and chars (in that order). + match_constant = r'([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')' + + # Expression to match two sides of the operator with something that + # looks like a literal, since CHECK(x == iterator) won't compile. + # This means we can't catch all the cases where a more specific + # CHECK is possible, but it's less annoying than dealing with + # extraneous warnings. + match_this = (r'\s*' + macro + r'\((\s*' + + match_constant + r'\s*' + operator + r'[^<>].*|' + r'.*[^<>]' + operator + r'\s*' + match_constant + + r'\s*\))') + + # Don't complain about CHECK(x == NULL) or similar because + # CHECK_EQ(x, NULL) won't compile (requires a cast). + # Also, don't complain about more complex boolean expressions + # involving && or || such as CHECK(a == b || c == d). + return match(match_this, line) and not search(r'NULL|&&|\|\|', line) + + +def check_check(filename, clean_lines, line_number, error): + """Checks the use of CHECK and EXPECT macros. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + error: The function to call with any errors found. + """ + + # Decide the set of replacement macros that should be suggested + raw_lines = clean_lines.raw_lines + current_macro = '' + for macro in _CHECK_MACROS: + if raw_lines[line_number].find(macro) >= 0: + current_macro = macro + break + if not current_macro: + # Don't waste time here if line doesn't contain 'CHECK' or 'EXPECT' + return + + line = clean_lines.elided[line_number] # get rid of comments and strings + + # Encourage replacing plain CHECKs with CHECK_EQ/CHECK_NE/etc. + for operator in ['==', '!=', '>=', '>', '<=', '<']: + if replaceable_check(operator, current_macro, line): + error(filename, line_number, 'readability/check', 2, + 'Consider using %s instead of %s(a %s b)' % ( + _CHECK_REPLACEMENT[current_macro][operator], + current_macro, operator)) + break + + +def check_for_comparisons_to_zero(filename, clean_lines, line_number, error): + # Get the line without comments and strings. + line = clean_lines.elided[line_number] + + # Include NULL here so that users don't have to convert NULL to 0 first and then get this error. + if search(r'[=!]=\s*(NULL|0|true|false)\W', line) or search(r'\W(NULL|0|true|false)\s*[=!]=', line): + error(filename, line_number, 'readability/comparison_to_zero', 5, + 'Tests for true/false, null/non-null, and zero/non-zero should all be done without equality comparisons.') + + +def check_for_null(filename, clean_lines, line_number, error): + # This check doesn't apply to C or Objective-C implementation files. + if filename.endswith('.c') or filename.endswith('.m'): + return + + line = clean_lines.elided[line_number] + if search(r'\bNULL\b', line): + error(filename, line_number, 'readability/null', 5, 'Use 0 instead of NULL.') + return + + line = clean_lines.raw_lines[line_number] + # See if NULL occurs in any comments in the line. If the search for NULL using the raw line + # matches, then do the check with strings collapsed to avoid giving errors for + # NULLs occurring in strings. + if search(r'\bNULL\b', line) and search(r'\bNULL\b', CleansedLines.collapse_strings(line)): + error(filename, line_number, 'readability/null', 4, 'Use 0 instead of NULL.') + +def get_line_width(line): + """Determines the width of the line in column positions. + + Args: + line: A string, which may be a Unicode string. + + Returns: + The width of the line in column positions, accounting for Unicode + combining characters and wide characters. + """ + if isinstance(line, unicode): + width = 0 + for c in unicodedata.normalize('NFC', line): + if unicodedata.east_asian_width(c) in ('W', 'F'): + width += 2 + elif not unicodedata.combining(c): + width += 1 + return width + return len(line) + + +def check_style(filename, clean_lines, line_number, file_extension, error): + """Checks rules from the 'C++ style rules' section of cppguide.html. + + Most of these rules are hard to test (naming, comment style), but we + do what we can. In particular we check for 4-space indents, line lengths, + tab usage, spaces inside code, etc. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + file_extension: The extension (without the dot) of the filename. + error: The function to call with any errors found. + """ + + raw_lines = clean_lines.raw_lines + line = raw_lines[line_number] + + if line.find('\t') != -1: + error(filename, line_number, 'whitespace/tab', 1, + 'Tab found; better to use spaces') + + # One or three blank spaces at the beginning of the line is weird; it's + # hard to reconcile that with 4-space indents. + # NOTE: here are the conditions rob pike used for his tests. Mine aren't + # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces + # if(RLENGTH > 20) complain = 0; + # if(match($0, " +(error|private|public|protected):")) complain = 0; + # if(match(prev, "&& *$")) complain = 0; + # if(match(prev, "\\|\\| *$")) complain = 0; + # if(match(prev, "[\",=><] *$")) complain = 0; + # if(match($0, " <<")) complain = 0; + # if(match(prev, " +for \\(")) complain = 0; + # if(prevodd && match(prevprev, " +for \\(")) complain = 0; + initial_spaces = 0 + cleansed_line = clean_lines.elided[line_number] + while initial_spaces < len(line) and line[initial_spaces] == ' ': + initial_spaces += 1 + if line and line[-1].isspace(): + error(filename, line_number, 'whitespace/end_of_line', 4, + 'Line ends in whitespace. Consider deleting these extra spaces.') + # There are certain situations we allow one space, notably for labels + elif ((initial_spaces == 1 or initial_spaces == 3) + and not match(r'\s*\w+\s*:\s*$', cleansed_line)): + error(filename, line_number, 'whitespace/indent', 3, + 'Weird number of spaces at line-start. ' + 'Are you using at least 2-space indent?') + # Labels should always be indented at least one space. + elif not initial_spaces and line[:2] != '//': + label_match = match(r'(?P<label>[^:]+):\s*$', line) + + if label_match: + label = label_match.group('label') + # Only throw errors for stuff that is definitely not a goto label, + # because goto labels can in fact occur at the start of the line. + if label in ['public', 'private', 'protected'] or label.find(' ') != -1: + error(filename, line_number, 'whitespace/labels', 4, + 'Labels should always be indented at least one space. ' + 'If this is a member-initializer list in a constructor, ' + 'the colon should be on the line after the definition header.') + + if (cleansed_line.count(';') > 1 + # for loops are allowed two ;'s (and may run over two lines). + and cleansed_line.find('for') == -1 + and (get_previous_non_blank_line(clean_lines, line_number)[0].find('for') == -1 + or get_previous_non_blank_line(clean_lines, line_number)[0].find(';') != -1) + # It's ok to have many commands in a switch case that fits in 1 line + and not ((cleansed_line.find('case ') != -1 + or cleansed_line.find('default:') != -1) + and cleansed_line.find('break;') != -1)): + error(filename, line_number, 'whitespace/newline', 4, + 'More than one command on the same line') + + if cleansed_line.strip().endswith('||') or cleansed_line.strip().endswith('&&'): + error(filename, line_number, 'whitespace/operators', 4, + 'Boolean expressions that span multiple lines should have their ' + 'operators on the left side of the line instead of the right side.') + + # Some more style checks + check_namespace_indentation(filename, clean_lines, line_number, file_extension, error) + check_switch_indentation(filename, clean_lines, line_number, error) + check_braces(filename, clean_lines, line_number, error) + check_exit_statement_simplifications(filename, clean_lines, line_number, error) + check_spacing(filename, clean_lines, line_number, error) + check_check(filename, clean_lines, line_number, error) + check_for_comparisons_to_zero(filename, clean_lines, line_number, error) + check_for_null(filename, clean_lines, line_number, error) + + +_RE_PATTERN_INCLUDE_NEW_STYLE = re.compile(r'#include +"[^/]+\.h"') +_RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$') +# Matches the first component of a filename delimited by -s and _s. That is: +# _RE_FIRST_COMPONENT.match('foo').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo.cpp').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo-bar_baz.cpp').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo_bar-baz.cpp').group(0) == 'foo' +_RE_FIRST_COMPONENT = re.compile(r'^[^-_.]+') + + +def _drop_common_suffixes(filename): + """Drops common suffixes like _test.cpp or -inl.h from filename. + + For example: + >>> _drop_common_suffixes('foo/foo-inl.h') + 'foo/foo' + >>> _drop_common_suffixes('foo/bar/foo.cpp') + 'foo/bar/foo' + >>> _drop_common_suffixes('foo/foo_internal.h') + 'foo/foo' + >>> _drop_common_suffixes('foo/foo_unusualinternal.h') + 'foo/foo_unusualinternal' + + Args: + filename: The input filename. + + Returns: + The filename with the common suffix removed. + """ + for suffix in ('test.cpp', 'regtest.cpp', 'unittest.cpp', + 'inl.h', 'impl.h', 'internal.h'): + if (filename.endswith(suffix) and len(filename) > len(suffix) + and filename[-len(suffix) - 1] in ('-', '_')): + return filename[:-len(suffix) - 1] + return os.path.splitext(filename)[0] + + +def _is_test_filename(filename): + """Determines if the given filename has a suffix that identifies it as a test. + + Args: + filename: The input filename. + + Returns: + True if 'filename' looks like a test, False otherwise. + """ + if (filename.endswith('_test.cpp') + or filename.endswith('_unittest.cpp') + or filename.endswith('_regtest.cpp')): + return True + return False + + +def _classify_include(filename, include, is_system, include_state): + """Figures out what kind of header 'include' is. + + Args: + filename: The current file cpplint is running over. + include: The path to a #included file. + is_system: True if the #include used <> rather than "". + include_state: An _IncludeState instance in which the headers are inserted. + + Returns: + One of the _XXX_HEADER constants. + + For example: + >>> _classify_include('foo.cpp', 'config.h', False) + _CONFIG_HEADER + >>> _classify_include('foo.cpp', 'foo.h', False) + _PRIMARY_HEADER + >>> _classify_include('foo.cpp', 'bar.h', False) + _OTHER_HEADER + """ + + # If it is a system header we know it is classified as _OTHER_HEADER. + if is_system: + return _OTHER_HEADER + + # If the include is named config.h then this is WebCore/config.h. + if include == "config.h": + return _CONFIG_HEADER + + # There cannot be primary includes in header files themselves. Only an + # include exactly matches the header filename will be is flagged as + # primary, so that it triggers the "don't include yourself" check. + if filename.endswith('.h') and filename != include: + return _OTHER_HEADER; + + # If the target file basename starts with the include we're checking + # then we consider it the primary header. + target_base = FileInfo(filename).base_name() + include_base = FileInfo(include).base_name() + + # If we haven't encountered a primary header, then be lenient in checking. + if not include_state.visited_primary_section() and target_base.startswith(include_base): + return _PRIMARY_HEADER + # If we already encountered a primary header, perform a strict comparison. + # In case the two filename bases are the same then the above lenient check + # probably was a false positive. + elif include_state.visited_primary_section() and target_base == include_base: + return _PRIMARY_HEADER + + return _OTHER_HEADER + + + +def check_include_line(filename, clean_lines, line_number, include_state, error): + """Check rules that are applicable to #include lines. + + Strings on #include lines are NOT removed from elided line, to make + certain tasks easier. However, to prevent false positives, checks + applicable to #include lines in CheckLanguage must be put here. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + include_state: An _IncludeState instance in which the headers are inserted. + error: The function to call with any errors found. + """ + + line = clean_lines.lines[line_number] + + # we shouldn't include a file more than once. actually, there are a + # handful of instances where doing so is okay, but in general it's + # not. + matched = _RE_PATTERN_INCLUDE.search(line) + if matched: + include = matched.group(2) + is_system = (matched.group(1) == '<') + if include in include_state: + error(filename, line_number, 'build/include', 4, + '"%s" already included at %s:%s' % + (include, filename, include_state[include])) + else: + include_state[include] = line_number + + # We want to ensure that headers appear in the right order: + # 1) for implementation files: config.h, primary header, blank line, alphabetically sorted + # 2) for header files: alphabetically sorted + # + # We classify each include statement as one of 4 types + # using a number of techniques. The include_state object keeps + # track of the highest type seen, and complains if we see a + # lower type after that. + header_type = _classify_include(filename, include, is_system, include_state) + error_message = include_state.check_next_include_order(header_type, filename.endswith('.h')) + include_state.header_types[line_number] = header_type + + # Check to make sure we have a blank line after primary header. + if not error_message and header_type == _PRIMARY_HEADER: + next_line = clean_lines.raw_lines[line_number + 1] + if not is_blank_line(next_line): + error(filename, line_number, 'build/include_order', 4, + 'You should add a blank line after implementation file\'s own header.') + + # Check to make sure all headers besides config.h and the primary header are + # alphabetically sorted. + if not error_message and header_type == _OTHER_HEADER: + previous_line_number = line_number - 1; + previous_line = clean_lines.lines[previous_line_number] + previous_match = _RE_PATTERN_INCLUDE.search(previous_line) + while (not previous_match and previous_line_number > 0 + and not search(r'\A(#if|#ifdef|#ifndef|#else|#elif|#endif)', previous_line)): + previous_line_number -= 1; + previous_line = clean_lines.lines[previous_line_number] + previous_match = _RE_PATTERN_INCLUDE.search(previous_line) + if previous_match: + previous_header_type = include_state.header_types[previous_line_number] + if previous_header_type == _OTHER_HEADER and previous_line.strip() > line.strip(): + error(filename, line_number, 'build/include_order', 4, + 'Alphabetical sorting problem.') + + if error_message: + if filename.endswith('.h'): + error(filename, line_number, 'build/include_order', 4, + '%s Should be: alphabetically sorted.' % + error_message) + else: + error(filename, line_number, 'build/include_order', 4, + '%s Should be: config.h, primary header, blank line, and then alphabetically sorted.' % + error_message) + + # Look for any of the stream classes that are part of standard C++. + if match(r'(f|ind|io|i|o|parse|pf|stdio|str|)?stream$', include): + # Many unit tests use cout, so we exempt them. + if not _is_test_filename(filename): + error(filename, line_number, 'readability/streams', 3, + 'Streams are highly discouraged.') + + # Look for specific includes to fix. + if include.startswith('wtf/') and not is_system: + error(filename, line_number, 'build/include', 4, + 'wtf includes should be <wtf/file.h> instead of "wtf/file.h".') + + +def check_language(filename, clean_lines, line_number, file_extension, include_state, + error): + """Checks rules from the 'C++ language rules' section of cppguide.html. + + Some of these rules are hard to test (function overloading, using + uint32 inappropriately), but we do the best we can. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + line_number: The number of the line to check. + file_extension: The extension (without the dot) of the filename. + include_state: An _IncludeState instance in which the headers are inserted. + error: The function to call with any errors found. + """ + # If the line is empty or consists of entirely a comment, no need to + # check it. + line = clean_lines.elided[line_number] + if not line: + return + + matched = _RE_PATTERN_INCLUDE.search(line) + if matched: + check_include_line(filename, clean_lines, line_number, include_state, error) + return + + # FIXME: figure out if they're using default arguments in fn proto. + + # Check to see if they're using an conversion function cast. + # I just try to capture the most common basic types, though there are more. + # Parameterless conversion functions, such as bool(), are allowed as they are + # probably a member operator declaration or default constructor. + matched = search( + r'\b(int|float|double|bool|char|int32|uint32|int64|uint64)\([^)]', line) + if matched: + # gMock methods are defined using some variant of MOCK_METHODx(name, type) + # where type may be float(), int(string), etc. Without context they are + # virtually indistinguishable from int(x) casts. + if not match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line): + error(filename, line_number, 'readability/casting', 4, + 'Using deprecated casting style. ' + 'Use static_cast<%s>(...) instead' % + matched.group(1)) + + check_c_style_cast(filename, line_number, line, clean_lines.raw_lines[line_number], + 'static_cast', + r'\((int|float|double|bool|char|u?int(16|32|64))\)', + error) + # This doesn't catch all cases. Consider (const char * const)"hello". + check_c_style_cast(filename, line_number, line, clean_lines.raw_lines[line_number], + 'reinterpret_cast', r'\((\w+\s?\*+\s?)\)', error) + + # In addition, we look for people taking the address of a cast. This + # is dangerous -- casts can assign to temporaries, so the pointer doesn't + # point where you think. + """ + if search( + r'(&\([^)]+\)[\w(])|(&(static|dynamic|reinterpret)_cast\b)', line): + error(filename, line_number, 'runtime/casting', 4, + ('Are you taking an address of a cast? ' + 'This is dangerous: could be a temp var. ' + 'Take the address before doing the cast, rather than after')) + """ + + # Check for people declaring static/global STL strings at the top level. + # This is dangerous because the C++ language does not guarantee that + # globals with constructors are initialized before the first access. + matched = match( + r'((?:|static +)(?:|const +))string +([a-zA-Z0-9_:]+)\b(.*)', + line) + # Make sure it's not a function. + # Function template specialization looks like: "string foo<Type>(...". + # Class template definitions look like: "string Foo<Type>::Method(...". + if matched and not match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)?\s*\(([^"]|$)', + matched.group(3)): + error(filename, line_number, 'runtime/string', 4, + 'For a static/global string constant, use a C style string instead: ' + '"%schar %s[]".' % + (matched.group(1), matched.group(2))) + + # Check that we're not using RTTI outside of testing code. + if search(r'\bdynamic_cast<', line) and not _is_test_filename(filename): + error(filename, line_number, 'runtime/rtti', 5, + 'Do not use dynamic_cast<>. If you need to cast within a class ' + "hierarchy, use static_cast<> to upcast. Mozilla doesn't support " + 'RTTI.') + + if search(r'\b([A-Za-z0-9_]*_)\(\1\)', line): + error(filename, line_number, 'runtime/init', 4, + 'You seem to be initializing a member variable with itself.') + + if file_extension == 'h': + # FIXME: check that 1-arg constructors are explicit. + # How to tell it's a constructor? + # (handled in check_for_non_standard_constructs for now) + pass + + # Check if people are using the verboten C basic types. The only exception + # we regularly allow is "unsigned short port" for port. + if search(r'\bshort port\b', line): + if not search(r'\bunsigned short port\b', line): + error(filename, line_number, 'runtime/int', 4, + 'Use "unsigned short" for ports, not "short"') + + # When snprintf is used, the second argument shouldn't be a literal. + matched = search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) + if matched: + error(filename, line_number, 'runtime/printf', 3, + 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' + 'to snprintf.' % (matched.group(1), matched.group(2))) + + # Check if some verboten C functions are being used. + if search(r'\bsprintf\b', line): + error(filename, line_number, 'runtime/printf', 5, + 'Never use sprintf. Use snprintf instead.') + matched = search(r'\b(strcpy|strcat)\b', line) + if matched: + error(filename, line_number, 'runtime/printf', 4, + 'Almost always, snprintf is better than %s' % matched.group(1)) + + if search(r'\bsscanf\b', line): + error(filename, line_number, 'runtime/printf', 1, + 'sscanf can be ok, but is slow and can overflow buffers.') + + # Check for suspicious usage of "if" like + # } if (a == b) { + if search(r'\}\s*if\s*\(', line): + error(filename, line_number, 'readability/braces', 4, + 'Did you mean "else if"? If not, start a new line for "if".') + + # Check for potential format string bugs like printf(foo). + # We constrain the pattern not to pick things like DocidForPrintf(foo). + # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) + matched = re.search(r'\b((?:string)?printf)\s*\(([\w.\->()]+)\)', line, re.I) + if matched: + error(filename, line_number, 'runtime/printf', 4, + 'Potential format string bug. Do %s("%%s", %s) instead.' + % (matched.group(1), matched.group(2))) + + # Check for potential memset bugs like memset(buf, sizeof(buf), 0). + matched = search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) + if matched and not match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", matched.group(2)): + error(filename, line_number, 'runtime/memset', 4, + 'Did you mean "memset(%s, 0, %s)"?' + % (matched.group(1), matched.group(2))) + + # Detect variable-length arrays. + matched = match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) + if (matched and matched.group(2) != 'return' and matched.group(2) != 'delete' and + matched.group(3).find(']') == -1): + # Split the size using space and arithmetic operators as delimiters. + # If any of the resulting tokens are not compile time constants then + # report the error. + tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', matched.group(3)) + is_const = True + skip_next = False + for tok in tokens: + if skip_next: + skip_next = False + continue + + if search(r'sizeof\(.+\)', tok): + continue + if search(r'arraysize\(\w+\)', tok): + continue + + tok = tok.lstrip('(') + tok = tok.rstrip(')') + if not tok: + continue + if match(r'\d+', tok): + continue + if match(r'0[xX][0-9a-fA-F]+', tok): + continue + if match(r'k[A-Z0-9]\w*', tok): + continue + if match(r'(.+::)?k[A-Z0-9]\w*', tok): + continue + if match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): + continue + # A catch all for tricky sizeof cases, including 'sizeof expression', + # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' + # requires skipping the next token becasue we split on ' ' and '*'. + if tok.startswith('sizeof'): + skip_next = True + continue + is_const = False + break + if not is_const: + error(filename, line_number, 'runtime/arrays', 1, + 'Do not use variable-length arrays. Use an appropriately named ' + "('k' followed by CamelCase) compile-time constant for the size.") + + # Check for use of unnamed namespaces in header files. Registration + # macros are typically OK, so we allow use of "namespace {" on lines + # that end with backslashes. + if (file_extension == 'h' + and search(r'\bnamespace\s*{', line) + and line[-1] != '\\'): + error(filename, line_number, 'build/namespaces', 4, + 'Do not use unnamed namespaces in header files. See ' + 'http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces' + ' for more information.') + + +def check_c_style_cast(filename, line_number, line, raw_line, cast_type, pattern, + error): + """Checks for a C-style cast by looking for the pattern. + + This also handles sizeof(type) warnings, due to similarity of content. + + Args: + filename: The name of the current file. + line_number: The number of the line to check. + line: The line of code to check. + raw_line: The raw line of code to check, with comments. + cast_type: The string for the C++ cast to recommend. This is either + reinterpret_cast or static_cast, depending. + pattern: The regular expression used to find C-style casts. + error: The function to call with any errors found. + """ + matched = search(pattern, line) + if not matched: + return + + # e.g., sizeof(int) + sizeof_match = match(r'.*sizeof\s*$', line[0:matched.start(1) - 1]) + if sizeof_match: + error(filename, line_number, 'runtime/sizeof', 1, + 'Using sizeof(type). Use sizeof(varname) instead if possible') + return + + remainder = line[matched.end(0):] + + # The close paren is for function pointers as arguments to a function. + # eg, void foo(void (*bar)(int)); + # The semicolon check is a more basic function check; also possibly a + # function pointer typedef. + # eg, void foo(int); or void foo(int) const; + # The equals check is for function pointer assignment. + # eg, void *(*foo)(int) = ... + # + # Right now, this will only catch cases where there's a single argument, and + # it's unnamed. It should probably be expanded to check for multiple + # arguments with some unnamed. + function_match = match(r'\s*(\)|=|(const)?\s*(;|\{|throw\(\)))', remainder) + if function_match: + if (not function_match.group(3) + or function_match.group(3) == ';' + or raw_line.find('/*') < 0): + error(filename, line_number, 'readability/function', 3, + 'All parameters should be named in a function') + return + + # At this point, all that should be left is actual casts. + error(filename, line_number, 'readability/casting', 4, + 'Using C-style cast. Use %s<%s>(...) instead' % + (cast_type, matched.group(1))) + + +_HEADERS_CONTAINING_TEMPLATES = ( + ('<deque>', ('deque',)), + ('<functional>', ('unary_function', 'binary_function', + 'plus', 'minus', 'multiplies', 'divides', 'modulus', + 'negate', + 'equal_to', 'not_equal_to', 'greater', 'less', + 'greater_equal', 'less_equal', + 'logical_and', 'logical_or', 'logical_not', + 'unary_negate', 'not1', 'binary_negate', 'not2', + 'bind1st', 'bind2nd', + 'pointer_to_unary_function', + 'pointer_to_binary_function', + 'ptr_fun', + 'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t', + 'mem_fun_ref_t', + 'const_mem_fun_t', 'const_mem_fun1_t', + 'const_mem_fun_ref_t', 'const_mem_fun1_ref_t', + 'mem_fun_ref', + )), + ('<limits>', ('numeric_limits',)), + ('<list>', ('list',)), + ('<map>', ('map', 'multimap',)), + ('<memory>', ('allocator',)), + ('<queue>', ('queue', 'priority_queue',)), + ('<set>', ('set', 'multiset',)), + ('<stack>', ('stack',)), + ('<string>', ('char_traits', 'basic_string',)), + ('<utility>', ('pair',)), + ('<vector>', ('vector',)), + + # gcc extensions. + # Note: std::hash is their hash, ::hash is our hash + ('<hash_map>', ('hash_map', 'hash_multimap',)), + ('<hash_set>', ('hash_set', 'hash_multiset',)), + ('<slist>', ('slist',)), + ) + +_HEADERS_ACCEPTED_BUT_NOT_PROMOTED = { + # We can trust with reasonable confidence that map gives us pair<>, too. + 'pair<>': ('map', 'multimap', 'hash_map', 'hash_multimap') +} + +_RE_PATTERN_STRING = re.compile(r'\bstring\b') + +_re_pattern_algorithm_header = [] +for _template in ('copy', 'max', 'min', 'min_element', 'sort', 'swap', + 'transform'): + # Match max<type>(..., ...), max(..., ...), but not foo->max, foo.max or + # type::max(). + _re_pattern_algorithm_header.append( + (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'), + _template, + '<algorithm>')) + +_re_pattern_templates = [] +for _header, _templates in _HEADERS_CONTAINING_TEMPLATES: + for _template in _templates: + _re_pattern_templates.append( + (re.compile(r'(\<|\b)' + _template + r'\s*\<'), + _template + '<>', + _header)) + + +def files_belong_to_same_module(filename_cpp, filename_h): + """Check if these two filenames belong to the same module. + + The concept of a 'module' here is a as follows: + foo.h, foo-inl.h, foo.cpp, foo_test.cpp and foo_unittest.cpp belong to the + same 'module' if they are in the same directory. + some/path/public/xyzzy and some/path/internal/xyzzy are also considered + to belong to the same module here. + + If the filename_cpp contains a longer path than the filename_h, for example, + '/absolute/path/to/base/sysinfo.cpp', and this file would include + 'base/sysinfo.h', this function also produces the prefix needed to open the + header. This is used by the caller of this function to more robustly open the + header file. We don't have access to the real include paths in this context, + so we need this guesswork here. + + Known bugs: tools/base/bar.cpp and base/bar.h belong to the same module + according to this implementation. Because of this, this function gives + some false positives. This should be sufficiently rare in practice. + + Args: + filename_cpp: is the path for the .cpp file + filename_h: is the path for the header path + + Returns: + Tuple with a bool and a string: + bool: True if filename_cpp and filename_h belong to the same module. + string: the additional prefix needed to open the header file. + """ + + if not filename_cpp.endswith('.cpp'): + return (False, '') + filename_cpp = filename_cpp[:-len('.cpp')] + if filename_cpp.endswith('_unittest'): + filename_cpp = filename_cpp[:-len('_unittest')] + elif filename_cpp.endswith('_test'): + filename_cpp = filename_cpp[:-len('_test')] + filename_cpp = filename_cpp.replace('/public/', '/') + filename_cpp = filename_cpp.replace('/internal/', '/') + + if not filename_h.endswith('.h'): + return (False, '') + filename_h = filename_h[:-len('.h')] + if filename_h.endswith('-inl'): + filename_h = filename_h[:-len('-inl')] + filename_h = filename_h.replace('/public/', '/') + filename_h = filename_h.replace('/internal/', '/') + + files_belong_to_same_module = filename_cpp.endswith(filename_h) + common_path = '' + if files_belong_to_same_module: + common_path = filename_cpp[:-len(filename_h)] + return files_belong_to_same_module, common_path + + +def update_include_state(filename, include_state, io=codecs): + """Fill up the include_state with new includes found from the file. + + Args: + filename: the name of the header to read. + include_state: an _IncludeState instance in which the headers are inserted. + io: The io factory to use to read the file. Provided for testability. + + Returns: + True if a header was succesfully added. False otherwise. + """ + header_file = None + try: + header_file = io.open(filename, 'r', 'utf8', 'replace') + except IOError: + return False + line_number = 0 + for line in header_file: + line_number += 1 + clean_line = cleanse_comments(line) + matched = _RE_PATTERN_INCLUDE.search(clean_line) + if matched: + include = matched.group(2) + # The value formatting is cute, but not really used right now. + # What matters here is that the key is in include_state. + include_state.setdefault(include, '%s:%d' % (filename, line_number)) + return True + + +def check_for_include_what_you_use(filename, clean_lines, include_state, error, + io=codecs): + """Reports for missing stl includes. + + This function will output warnings to make sure you are including the headers + necessary for the stl containers and functions that you use. We only give one + reason to include a header. For example, if you use both equal_to<> and + less<> in a .h file, only one (the latter in the file) of these will be + reported as a reason to include the <functional>. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + include_state: An _IncludeState instance. + error: The function to call with any errors found. + io: The IO factory to use to read the header file. Provided for unittest + injection. + """ + required = {} # A map of header name to line_number and the template entity. + # Example of required: { '<functional>': (1219, 'less<>') } + + for line_number in xrange(clean_lines.num_lines()): + line = clean_lines.elided[line_number] + if not line or line[0] == '#': + continue + + # String is special -- it is a non-templatized type in STL. + if _RE_PATTERN_STRING.search(line): + required['<string>'] = (line_number, 'string') + + for pattern, template, header in _re_pattern_algorithm_header: + if pattern.search(line): + required[header] = (line_number, template) + + # The following function is just a speed up, no semantics are changed. + if not '<' in line: # Reduces the cpu time usage by skipping lines. + continue + + for pattern, template, header in _re_pattern_templates: + if pattern.search(line): + required[header] = (line_number, template) + + # The policy is that if you #include something in foo.h you don't need to + # include it again in foo.cpp. Here, we will look at possible includes. + # Let's copy the include_state so it is only messed up within this function. + include_state = include_state.copy() + + # Did we find the header for this file (if any) and succesfully load it? + header_found = False + + # Use the absolute path so that matching works properly. + abs_filename = os.path.abspath(filename) + + # For Emacs's flymake. + # If cpplint is invoked from Emacs's flymake, a temporary file is generated + # by flymake and that file name might end with '_flymake.cpp'. In that case, + # restore original file name here so that the corresponding header file can be + # found. + # e.g. If the file name is 'foo_flymake.cpp', we should search for 'foo.h' + # instead of 'foo_flymake.h' + emacs_flymake_suffix = '_flymake.cpp' + if abs_filename.endswith(emacs_flymake_suffix): + abs_filename = abs_filename[:-len(emacs_flymake_suffix)] + '.cpp' + + # include_state is modified during iteration, so we iterate over a copy of + # the keys. + for header in include_state.keys(): #NOLINT + (same_module, common_path) = files_belong_to_same_module(abs_filename, header) + fullpath = common_path + header + if same_module and update_include_state(fullpath, include_state, io): + header_found = True + + # If we can't find the header file for a .cpp, assume it's because we don't + # know where to look. In that case we'll give up as we're not sure they + # didn't include it in the .h file. + # FIXME: Do a better job of finding .h files so we are confident that + # not having the .h file means there isn't one. + if filename.endswith('.cpp') and not header_found: + return + + # All the lines have been processed, report the errors found. + for required_header_unstripped in required: + template = required[required_header_unstripped][1] + if template in _HEADERS_ACCEPTED_BUT_NOT_PROMOTED: + headers = _HEADERS_ACCEPTED_BUT_NOT_PROMOTED[template] + if [True for header in headers if header in include_state]: + continue + if required_header_unstripped.strip('<>"') not in include_state: + error(filename, required[required_header_unstripped][0], + 'build/include_what_you_use', 4, + 'Add #include ' + required_header_unstripped + ' for ' + template) + + +def process_line(filename, file_extension, + clean_lines, line, include_state, function_state, + class_state, error): + """Processes a single line in the file. + + Args: + filename: Filename of the file that is being processed. + file_extension: The extension (dot not included) of the file. + clean_lines: An array of strings, each representing a line of the file, + with comments stripped. + line: Number of line being processed. + include_state: An _IncludeState instance in which the headers are inserted. + function_state: A _FunctionState instance which counts function lines, etc. + class_state: A _ClassState instance which maintains information about + the current stack of nested class declarations being parsed. + error: A callable to which errors are reported, which takes 4 arguments: + filename, line number, error level, and message + + """ + raw_lines = clean_lines.raw_lines + check_for_function_lengths(filename, clean_lines, line, function_state, error) + if search(r'\bNOLINT\b', raw_lines[line]): # ignore nolint lines + return + check_for_multiline_comments_and_strings(filename, clean_lines, line, error) + check_style(filename, clean_lines, line, file_extension, error) + check_language(filename, clean_lines, line, file_extension, include_state, + error) + check_for_non_standard_constructs(filename, clean_lines, line, + class_state, error) + check_posix_threading(filename, clean_lines, line, error) + check_invalid_increment(filename, clean_lines, line, error) + + +def process_file_data(filename, file_extension, lines, error): + """Performs lint checks and reports any errors to the given error function. + + Args: + filename: Filename of the file that is being processed. + file_extension: The extension (dot not included) of the file. + lines: An array of strings, each representing a line of the file, with the + last element being empty if the file is termined with a newline. + error: A callable to which errors are reported, which takes 4 arguments: + """ + lines = (['// marker so line numbers and indices both start at 1'] + lines + + ['// marker so line numbers end in a known way']) + + include_state = _IncludeState() + function_state = _FunctionState() + class_state = _ClassState() + + check_for_copyright(filename, lines, error) + + if file_extension == 'h': + check_for_header_guard(filename, lines, error) + + remove_multi_line_comments(filename, lines, error) + clean_lines = CleansedLines(lines) + for line in xrange(clean_lines.num_lines()): + process_line(filename, file_extension, clean_lines, line, + include_state, function_state, class_state, error) + class_state.check_finished(filename, error) + + check_for_include_what_you_use(filename, clean_lines, include_state, error) + + # We check here rather than inside process_line so that we see raw + # lines rather than "cleaned" lines. + check_for_unicode_replacement_characters(filename, lines, error) + + check_for_new_line_at_eof(filename, lines, error) + + +def process_file(filename, relative_name=None, error=error): + """Performs cpplint on a single file. + + Args: + filename: The name of the file to parse. + error: The function to call with any errors found. + """ + + if not relative_name: + relative_name = filename + + try: + # Support the UNIX convention of using "-" for stdin. Note that + # we are not opening the file with universal newline support + # (which codecs doesn't support anyway), so the resulting lines do + # contain trailing '\r' characters if we are reading a file that + # has CRLF endings. + # If after the split a trailing '\r' is present, it is removed + # below. If it is not expected to be present (i.e. os.linesep != + # '\r\n' as in Windows), a warning is issued below if this file + # is processed. + + if filename == '-': + lines = codecs.StreamReaderWriter(sys.stdin, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace').read().split('\n') + else: + lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n') + + carriage_return_found = False + # Remove trailing '\r'. + for line_number in range(len(lines)): + if lines[line_number].endswith('\r'): + lines[line_number] = lines[line_number].rstrip('\r') + carriage_return_found = True + + except IOError: + write_error( + "Skipping input '%s': Can't open for reading\n" % relative_name) + return + + # Note, if no dot is found, this will give the entire filename as the ext. + file_extension = filename[filename.rfind('.') + 1:] + + # When reading from stdin, the extension is unknown, so no cpplint tests + # should rely on the extension. + if (filename != '-' and file_extension != 'h' and file_extension != 'cpp' + and file_extension != 'c'): + write_error('Ignoring %s; not a .cpp, .c or .h file\n' % filename) + else: + process_file_data(relative_name, file_extension, lines, error) + if carriage_return_found and os.linesep != '\r\n': + # Use 0 for line_number since outputing only one error for potentially + # several lines. + error(relative_name, 1, 'whitespace/newline', 1, + 'One or more unexpected \\r (^M) found;' + 'better to use only a \\n') + + write_error('Done processing %s\n' % relative_name) + + +def print_usage(message): + """Prints a brief usage string and exits, optionally with an error message. + + Args: + message: The optional error message. + """ + write_error(_USAGE) + if message: + sys.exit('\nFATAL ERROR: ' + message) + else: + sys.exit(1) + + +def print_categories(): + """Prints a list of all the error-categories used by error messages. + + These are the categories used to filter messages via --filter. + """ + write_error(_ERROR_CATEGORIES) + sys.exit(0) + + +def parse_arguments(args, additional_flags=[]): + """Parses the command line arguments. + + This may set the output format and verbosity level as side-effects. + + Args: + args: The command line arguments: + additional_flags: A list of strings which specifies flags we allow. + + Returns: + A tuple of (filenames, flags) + + filenames: The list of filenames to lint. + flags: The dict of the flag names and the flag values. + """ + flags = ['help', 'output=', 'verbose=', 'filter='] + additional_flags + additional_flag_values = {} + try: + (opts, filenames) = getopt.getopt(args, '', flags) + except getopt.GetoptError: + print_usage('Invalid arguments.') + + verbosity = _verbose_level() + output_format = _output_format() + filters = '' + + for (opt, val) in opts: + if opt == '--help': + print_usage(None) + elif opt == '--output': + if not val in ('emacs', 'vs7'): + print_usage('The only allowed output formats are emacs and vs7.') + output_format = val + elif opt == '--verbose': + verbosity = int(val) + elif opt == '--filter': + filters = val + if not filters: + print_categories() + else: + additional_flag_values[opt] = val + + _set_output_format(output_format) + _set_verbose_level(verbosity) + _set_filters(filters) + + return (filenames, additional_flag_values) + + +def set_stream(stream): + _cpplint_state.set_stream(stream) + +def write_error(error): + _cpplint_state.write_error(error) + +def use_mozilla_styles(): + """Disables some features which are not suitable for WebKit.""" + # FIXME: For filters we will never want to have, remove them. + # For filters we want to have similar functionalities, + # modify the implementation and enable them. + global _DEFAULT_FILTERS + _DEFAULT_FILTERS = [ + '-whitespace/comments-doublespace', + '-whitespace/blank_line', + '-build/include', # Webkit specific + '-build/include_what_you_use', # <string> for std::string + '-readability/braces', # int foo() {}; + '-readability/null', + '-readability/fn_size', + '-build/storage_class', # const static + '-build/endif_comment', + '-whitespace/labels', + '-runtime/arrays', # variable length array + '-build/header_guard', # TODO Write a mozilla header_guard variant + '-runtime/casting', + ] + + +def main(): + write_error( + '''********************* WARNING WARNING WARNING ********************* + +This tool is in the process of development and may give inaccurate +results at present. Please file bugs (and/or patches) for things +that you notice that it flags incorrectly. + +********************* WARNING WARNING WARNING ********************* + +''') + + use_webkit_styles() + + (filenames, flags) = parse_arguments(sys.argv[1:]) + if not filenames: + print_usage('No files were specified.') + + # Change stderr to write with replacement characters so we don't die + # if we try to print something containing non-ASCII characters. + sys.stderr = codecs.StreamReaderWriter(sys.stderr, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace') + + _cpplint_state.reset_error_count() + for filename in filenames: + process_file(filename) + write_error('Total errors found: %d\n' % _cpplint_state.error_count) + sys.exit(_cpplint_state.error_count > 0) + + +if __name__ == '__main__': + main() diff --git a/tools/check-moz-style/modules/diff_parser.py b/tools/check-moz-style/modules/diff_parser.py new file mode 100644 index 000000000..e232f5504 --- /dev/null +++ b/tools/check-moz-style/modules/diff_parser.py @@ -0,0 +1,180 @@ +# Copyright (C) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""WebKit's Python module for interacting with patches.""" + +import logging +import re + + +_regexp_compile_cache = {} + + +def match(pattern, string): + """Matches the string with the pattern, caching the compiled regexp.""" + if not pattern in _regexp_compile_cache: + _regexp_compile_cache[pattern] = re.compile(pattern) + return _regexp_compile_cache[pattern].match(string) + + +def git_diff_to_svn_diff(line): + """Converts a git formatted diff line to a svn formatted line. + + Args: + line: A string representing a line of the diff. + """ + conversion_patterns = (("^diff --git a/(.+) b/(?P<FilePath>.+)", lambda matched: "Index: " + matched.group('FilePath') + "\n"), + ("^new file.*", lambda matched: "\n"), + ("^index [0-9a-f]{7}\.\.[0-9a-f]{7} [0-9]{6}", lambda matched: "===================================================================\n"), + ("^--- a/(?P<FilePath>.+)", lambda matched: "--- " + matched.group('FilePath') + "\n"), + ("^\+\+\+ b/(?P<FilePath>.+)", lambda matched: "+++ " + matched.group('FilePath') + "\n")) + + for pattern, conversion in conversion_patterns: + matched = match(pattern, line) + if matched: + return conversion(matched) + return line + + +def get_diff_converter(first_diff_line): + """Gets a converter function of diff lines. + + Args: + first_diff_line: The first filename line of a diff file. + If this line is git formatted, we'll return a + converter from git to SVN. + """ + if match(r"^diff --git a/", first_diff_line): + return git_diff_to_svn_diff + return lambda input: input + + +_INITIAL_STATE = 1 +_DECLARED_FILE_PATH = 2 +_PROCESSING_CHUNK = 3 + + +class DiffFile: + """Contains the information for one file in a patch. + + The field "lines" is a list which contains tuples in this format: + (deleted_line_number, new_line_number, line_string) + If deleted_line_number is zero, it means this line is newly added. + If new_line_number is zero, it means this line is deleted. + """ + + def __init__(self, filename): + self.filename = filename + self.lines = [] + + def add_new_line(self, line_number, line): + self.lines.append((0, line_number, line)) + + def add_deleted_line(self, line_number, line): + self.lines.append((line_number, 0, line)) + + def add_unchanged_line(self, deleted_line_number, new_line_number, line): + self.lines.append((deleted_line_number, new_line_number, line)) + + +class DiffParser: + """A parser for a patch file. + + The field "files" is a dict whose key is the filename and value is + a DiffFile object. + """ + + def __init__(self, diff_input): + """Parses a diff. + + Args: + diff_input: An iterable object. + """ + state = _INITIAL_STATE + + self.files = {} + self.status_line = None + self.patch_description = None + current_file = None + old_diff_line = None + new_diff_line = None + for line in diff_input: + line = line.rstrip("\n") + if state == _INITIAL_STATE: + transform_line = get_diff_converter(line) + line = transform_line(line) + + comment_line = match(r"^\#", line) + if comment_line: + continue + + file_declaration = match(r"^Index: (?P<FilePath>.+)", line) + if file_declaration: + filename = file_declaration.group('FilePath') + current_file = DiffFile(filename) + self.files[filename] = current_file + state = _DECLARED_FILE_PATH + continue + + lines_changed = match(r"^@@ -(?P<OldStartLine>\d+)(,\d+)? \+(?P<NewStartLine>\d+)(,\d+)? @@", line) + if lines_changed: + if state != _DECLARED_FILE_PATH and state != _PROCESSING_CHUNK: + logging.error('Unexpected line change without file path declaration: %r' % line) + old_diff_line = int(lines_changed.group('OldStartLine')) + new_diff_line = int(lines_changed.group('NewStartLine')) + state = _PROCESSING_CHUNK + continue + + if state == _PROCESSING_CHUNK: + if line.startswith('+'): + current_file.add_new_line(new_diff_line, line[1:]) + new_diff_line += 1 + elif line.startswith('-'): + current_file.add_deleted_line(old_diff_line, line[1:]) + old_diff_line += 1 + elif line.startswith(' '): + current_file.add_unchanged_line(old_diff_line, new_diff_line, line[1:]) + old_diff_line += 1 + new_diff_line += 1 + elif line == '\\ No newline at end of file': + # Nothing to do. We may still have some added lines. + pass + else: + logging.error('Unexpected diff format when parsing a chunk: %r' % line) + + # Patch description + if state == _INITIAL_STATE: + if not self.status_line: + self.status_line = line + else: + if not self.patch_description: + # Skip the first blank line after the patch description + if line != "": + self.patch_description = line + else: + self.patch_description = self.patch_description + "\n" + line diff --git a/tools/check-moz-style/modules/logging.py b/tools/check-moz-style/modules/logging.py new file mode 100644 index 000000000..ea03a489c --- /dev/null +++ b/tools/check-moz-style/modules/logging.py @@ -0,0 +1,39 @@ +# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# WebKit's Python module for logging + +import sys + +def log(string): + print >> sys.stderr, string + +def error(string): + log("ERROR: " + string) + exit(1) diff --git a/tools/check-moz-style/modules/scm.py b/tools/check-moz-style/modules/scm.py new file mode 100644 index 000000000..e7cb9ffc6 --- /dev/null +++ b/tools/check-moz-style/modules/scm.py @@ -0,0 +1,420 @@ +# Copyright (c) 2009, Google Inc. All rights reserved. +# Copyright (c) 2009 Apple Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Python module for interacting with an SCM system (like SVN or Git) + +import os +import re +import subprocess +import sys + +# Import WebKit-specific modules. +from modules.logging import error, log + +def detect_scm_system(path): + if HG.in_working_directory(path): + return HG(cwd=path) + + if SVN.in_working_directory(path): + return SVN(cwd=path) + + if Git.in_working_directory(path): + return Git(cwd=path) + + raise ScriptError("working directory is not a HG/SVN/Git repo") + +def first_non_empty_line_after_index(lines, index=0): + first_non_empty_line = index + for line in lines[index:]: + if re.match("^\s*$", line): + first_non_empty_line += 1 + else: + break + return first_non_empty_line + + +class CommitMessage: + def __init__(self, message): + self.message_lines = message[first_non_empty_line_after_index(message, 0):] + + def body(self, lstrip=False): + lines = self.message_lines[first_non_empty_line_after_index(self.message_lines, 1):] + if lstrip: + lines = [line.lstrip() for line in lines] + return "\n".join(lines) + "\n" + + def description(self, lstrip=False, strip_url=False): + line = self.message_lines[0] + if lstrip: + line = line.lstrip() + if strip_url: + line = re.sub("^(\s*)<.+> ", "\1", line) + return line + + def message(self): + return "\n".join(self.message_lines) + "\n" + + def parse_bug_id(self): + for line in self.message_lines: + match = re.search("http\://webkit\.org/b/(?P<bug_id>\d+)", line) + if match: + return match.group('bug_id') + match = re.search(Bugzilla.bug_server_regex + "show_bug\.cgi\?id=(?P<bug_id>\d+)", line) + if match: + return match.group('bug_id') + return None + + +class ScriptError(Exception): + pass + + +class SCM: + def __init__(self, cwd, dryrun=False): + self.cwd = cwd + self.checkout_root = self.find_checkout_root(self.cwd) + self.dryrun = dryrun + + @staticmethod + def run_command(args, cwd=None, input=None, raise_on_failure=True, return_exit_code=False): + stdin = subprocess.PIPE if input else None + process = subprocess.Popen(args, stdout=subprocess.PIPE, stdin=stdin, cwd=cwd) + output = process.communicate(input)[0].rstrip() + exit_code = process.wait() + if raise_on_failure and exit_code: + raise ScriptError('Failed to run "%s" exit_code: %d cwd: %s' % (args, exit_code, cwd)) + if return_exit_code: + return exit_code + return output + + def script_path(self, script_name): + return os.path.join(self.checkout_root, "WebKitTools", "Scripts", script_name) + + def ensure_clean_working_directory(self, force): + if not force and not self.working_directory_is_clean(): + print self.run_command(self.status_command(), raise_on_failure=False) + error("Working directory has modifications, pass --force-clean or --no-clean to continue.") + + log("Cleaning working directory") + self.clean_working_directory() + + def ensure_no_local_commits(self, force): + if not self.supports_local_commits(): + return + commits = self.local_commits() + if not len(commits): + return + if not force: + error("Working directory has local commits, pass --force-clean to continue.") + self.discard_local_commits() + + def apply_patch(self, patch): + # It's possible that the patch was not made from the root directory. + # We should detect and handle that case. + curl_process = subprocess.Popen(['curl', patch['url']], stdout=subprocess.PIPE) + patch_apply_process = subprocess.Popen([self.script_path('svn-apply'), '--reviewer', patch['reviewer']], stdin=curl_process.stdout) + + return_code = patch_apply_process.wait() + if return_code: + raise ScriptError("Patch %s from bug %s failed to download and apply." % (patch['url'], patch['bug_id'])) + + def run_status_and_extract_filenames(self, status_command, status_regexp): + filenames = [] + for line in self.run_command(status_command).splitlines(): + match = re.search(status_regexp, line) + if not match: + continue + # status = match.group('status') + filename = match.group('filename') + filenames.append(filename) + return filenames + + @staticmethod + def in_working_directory(path): + raise NotImplementedError, "subclasses must implement" + + @staticmethod + def find_checkout_root(path): + raise NotImplementedError, "subclasses must implement" + + @staticmethod + def commit_success_regexp(): + raise NotImplementedError, "subclasses must implement" + + def working_directory_is_clean(self): + raise NotImplementedError, "subclasses must implement" + + def clean_working_directory(self): + raise NotImplementedError, "subclasses must implement" + + def update_webkit(self): + raise NotImplementedError, "subclasses must implement" + + def status_command(self): + raise NotImplementedError, "subclasses must implement" + + def changed_files(self): + raise NotImplementedError, "subclasses must implement" + + def display_name(self): + raise NotImplementedError, "subclasses must implement" + + def create_patch(self): + raise NotImplementedError, "subclasses must implement" + + def commit_with_message(self, message): + raise NotImplementedError, "subclasses must implement" + + # Subclasses must indicate if they support local commits, + # but the SCM baseclass will only call local_commits methods when this is true. + @staticmethod + def supports_local_commits(): + raise NotImplementedError, "subclasses must implement" + + def create_patch_from_local_commit(self, commit_id): + pass + + def commit_locally_with_message(self, message): + pass + + def discard_local_commits(self): + pass + + def local_commits(self): + return [] + + +class SVN(SCM): + def __init__(self, cwd, dryrun=False): + SCM.__init__(self, cwd, dryrun) + self.cached_version = None + + @staticmethod + def in_working_directory(path): + return os.path.isdir(os.path.join(path, '.svn')) + + @staticmethod + def find_uuid(path): + if not SVN.in_working_directory(path): + return None + info = SVN.run_command(['svn', 'info', path]) + match = re.search("^Repository UUID: (?P<uuid>.+)$", info, re.MULTILINE) + if not match: + raise ScriptError('svn info did not contain a UUID.') + return match.group('uuid') + + @staticmethod + def find_checkout_root(path): + uuid = SVN.find_uuid(path) + # If |path| is not in a working directory, we're supposed to return |path|. + if not uuid: + return path + # Search up the directory hierarchy until we find a different UUID. + last_path = None + while True: + if uuid != SVN.find_uuid(path): + return last_path + last_path = path + (path, last_component) = os.path.split(path) + if last_path == path: + return None + + @staticmethod + def commit_success_regexp(): + return "^Committed revision (?P<svn_revision>\d+)\.$" + + def svn_version(self): + if not self.cached_version: + self.cached_version = self.run_command(['svn', '--version', '--quiet']) + + return self.cached_version + + def working_directory_is_clean(self): + return self.run_command(['svn', 'diff']) == "" + + def clean_working_directory(self): + self.run_command(['svn', 'revert', '-R', '.']) + + def update_webkit(self): + self.run_command(self.script_path("update-webkit")) + + def status_command(self): + return ['svn', 'status'] + + def changed_files(self): + if self.svn_version() > "1.6": + status_regexp = "^(?P<status>[ACDMR]).{6} (?P<filename>.+)$" + else: + status_regexp = "^(?P<status>[ACDMR]).{5} (?P<filename>.+)$" + return self.run_status_and_extract_filenames(self.status_command(), status_regexp) + + @staticmethod + def supports_local_commits(): + return False + + def display_name(self): + return "svn" + + def create_patch(self): + return self.run_command(self.script_path("svn-create-patch")) + + def commit_with_message(self, message): + if self.dryrun: + return "Dry run, no remote commit." + return self.run_command(['svn', 'commit', '-m', message]) + + +# All git-specific logic should go here. +class Git(SCM): + def __init__(self, cwd, dryrun=False): + SCM.__init__(self, cwd, dryrun) + + @classmethod + def in_working_directory(cls, path): + return cls.run_command(['git', 'rev-parse', '--is-inside-work-tree'], raise_on_failure=False, cwd=path) == "true" + + @classmethod + def find_checkout_root(cls, path): + # "git rev-parse --show-cdup" would be another way to get to the root + (checkout_root, dot_git) = os.path.split(cls.run_command(['git', 'rev-parse', '--git-dir'], cwd=path)) + # If we were using 2.6 # checkout_root = os.path.relpath(checkout_root, path) + if not os.path.isabs(checkout_root): # Sometimes git returns relative paths + checkout_root = os.path.join(path, checkout_root) + return checkout_root + + @staticmethod + def commit_success_regexp(): + return "^Committed r(?P<svn_revision>\d+)$" + + def discard_local_commits(self): + self.run_command(['git', 'reset', '--hard', 'trunk']) + + def local_commits(self): + return self.run_command(['git', 'log', '--pretty=oneline', 'HEAD...trunk']).splitlines() + + def working_directory_is_clean(self): + return self.run_command(['git', 'diff-index', 'HEAD']) == "" + + def clean_working_directory(self): + # Could run git clean here too, but that wouldn't match working_directory_is_clean + self.run_command(['git', 'reset', '--hard', 'HEAD']) + + def update_webkit(self): + # FIXME: Should probably call update-webkit, no? + log("Updating working directory") + self.run_command(['git', 'svn', 'rebase']) + + def status_command(self): + return ['git', 'status'] + + def changed_files(self): + status_command = ['git', 'diff', '-r', '--name-status', '-C', '-M', 'HEAD'] + status_regexp = '^(?P<status>[ADM])\t(?P<filename>.+)$' + return self.run_status_and_extract_filenames(status_command, status_regexp) + + @staticmethod + def supports_local_commits(): + return True + + def display_name(self): + return "git" + + def create_patch(self): + return self.run_command(['git', 'diff', 'HEAD']) + + def commit_with_message(self, message): + self.commit_locally_with_message(message) + return self.push_local_commits_to_server() + + # Git-specific methods: + + def create_patch_from_local_commit(self, commit_id): + return self.run_command(['git', 'diff', commit_id + "^.." + commit_id]) + + def commit_locally_with_message(self, message): + self.run_command(['git', 'commit', '--all', '-F', '-'], input=message) + + def push_local_commits_to_server(self): + if self.dryrun: + return "Dry run, no remote commit." + return self.run_command(['git', 'svn', 'dcommit']) + + def commit_ids_from_range_arguments(self, args, cherry_pick=False): + # First get the commit-ids for the passed in revisions. + revisions = self.run_command(['git', 'rev-parse', '--revs-only'] + args).splitlines() + + if cherry_pick: + return revisions + + # If we're not cherry picking and were only passed one revision, assume "^revision head" aka "revision..head". + if len(revisions) < 2: + revisions[0] = "^" + revisions[0] + revisions.append("HEAD") + + return self.run_command(['git', 'rev-list'] + revisions).splitlines() + + def commit_message_for_local_commit(self, commit_id): + commit_lines = self.run_command(['git', 'cat-file', 'commit', commit_id]).splitlines() + + # Skip the git headers. + first_line_after_headers = 0 + for line in commit_lines: + first_line_after_headers += 1 + if line == "": + break + return CommitMessage(commit_lines[first_line_after_headers:]) + + def files_changed_summary_for_commit(self, commit_id): + return self.run_command(['git', 'diff-tree', '--shortstat', '--no-commit-id', commit_id]) + + +# All hg-specific logic should go here. +class HG(SCM): + def __init__(self, cwd, dryrun=False): + SCM.__init__(self, cwd, dryrun) + + @classmethod + def in_working_directory(cls, path): + return cls.run_command(['hg', 'status'], cwd=path, return_exit_code=True) == 0 + + @classmethod + def find_checkout_root(cls, path): + checkout_root = cls.run_command(['hg', 'root'], cwd=path) + return checkout_root + + def status_command(self): + return ['hg', 'status'] + + def display_name(self): + return "hg" + + def create_patch(self): + if self.run_command(['hg', 'diff']) != "": + sys.stderr.write("Warning: outstanding changes not include in style check.\n") + return self.run_command(['hg', 'export', 'tip']) diff --git a/tools/check-moz-style/run_tests.py b/tools/check-moz-style/run_tests.py new file mode 100755 index 000000000..5ef3fa311 --- /dev/null +++ b/tools/check-moz-style/run_tests.py @@ -0,0 +1,78 @@ +#!/usr/bin/python +# +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ +# + +from __future__ import print_function +from modules.scm import detect_scm_system +from contextlib import closing +import checkmozstyle +import os +import modules.cpplint as cpplint +import StringIO + +TESTS = [ + # Empty patch + { + "patch": "tests/test1.patch", + "cpp": "tests/test1.cpp", + "out": "tests/test1.out" + }, + # Bad header + { + "patch": "tests/test2.patch", + "cpp": "tests/test2.cpp", + "out": "tests/test2.out" + }, + # Bad Description + { + "patch": "tests/test3.patch", + "cpp": "tests/test3.cpp", + "out": "tests/test3.out" + }, + # readability tests + { + "patch": "tests/test4.patch", + "cpp": "tests/test4.cpp", + "out": "tests/test4.out" + }, + # runtime tests + { + "patch": "tests/test5.patch", + "cpp": "tests/test5.cpp", + "out": "tests/test5.out" + }, +] + + +def main(): + cwd = os.path.abspath('.') + scm = detect_scm_system(cwd) + cpplint.use_mozilla_styles() + (args, flags) = cpplint.parse_arguments([]) + + for test in TESTS: + with open(test["patch"]) as fh: + patch = fh.read() + + with closing(StringIO.StringIO()) as output: + cpplint.set_stream(output) + checkmozstyle.process_patch(patch, cwd, cwd, scm) + result = output.getvalue() + + with open(test["out"]) as fh: + expected_output = fh.read() + + test_status = "PASSED" + if result != expected_output: + test_status = "FAILED" + print("TEST " + test["patch"] + " " + test_status) + print("Got result:\n" + result + "Expected:\n" + expected_output) + else: + print("TEST " + test["patch"] + " " + test_status) + + +if __name__ == "__main__": + main() + diff --git a/tools/check-moz-style/tests/test1.cpp b/tools/check-moz-style/tests/test1.cpp new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tools/check-moz-style/tests/test1.cpp diff --git a/tools/check-moz-style/tests/test1.out b/tools/check-moz-style/tests/test1.out new file mode 100644 index 000000000..3c313c18a --- /dev/null +++ b/tools/check-moz-style/tests/test1.out @@ -0,0 +1 @@ +patch:0: Patch does not appear to diff against any file. [patch/notempty] [3] diff --git a/tools/check-moz-style/tests/test1.patch b/tools/check-moz-style/tests/test1.patch new file mode 100644 index 000000000..8c56b2416 --- /dev/null +++ b/tools/check-moz-style/tests/test1.patch @@ -0,0 +1 @@ +Bad patch that doesn't diff any files diff --git a/tools/check-moz-style/tests/test2.cpp b/tools/check-moz-style/tests/test2.cpp new file mode 100644 index 000000000..4cce7f667 --- /dev/null +++ b/tools/check-moz-style/tests/test2.cpp @@ -0,0 +1,3 @@ +int main() { + return 0; +} diff --git a/tools/check-moz-style/tests/test2.out b/tools/check-moz-style/tests/test2.out new file mode 100644 index 000000000..0648d555e --- /dev/null +++ b/tools/check-moz-style/tests/test2.out @@ -0,0 +1,4 @@ +patch:0: Patch does not have a summary. [patch/nosummary] [3] +patch:0: Patch does not have a description. [patch/nodescription] [3] +tests/test2.cpp:1: No copyright message found. [legal/copyright] [3] +Done processing tests/test2.cpp diff --git a/tools/check-moz-style/tests/test2.patch b/tools/check-moz-style/tests/test2.patch new file mode 100644 index 000000000..e497ec0cf --- /dev/null +++ b/tools/check-moz-style/tests/test2.patch @@ -0,0 +1,9 @@ +# Test +diff --git a/tests/test2.cpp b/tests/test2.cpp +new file mode 100644 +--- /dev/null ++++ b/tests/test2.cpp +@@ -0,0 +1,3 @@ ++int main() { ++ return 0; ++} diff --git a/tools/check-moz-style/tests/test3.out b/tools/check-moz-style/tests/test3.out new file mode 100644 index 000000000..5a2cc7385 --- /dev/null +++ b/tools/check-moz-style/tests/test3.out @@ -0,0 +1,3 @@ +patch:0: Patch summary should begin with 'Bug XXXXX - ' or 'No bug -'. [patch/bugnumber] [3] +tests/test2.cpp:1: No copyright message found. [legal/copyright] [3] +Done processing tests/test2.cpp diff --git a/tools/check-moz-style/tests/test3.patch b/tools/check-moz-style/tests/test3.patch new file mode 100644 index 000000000..d0443fa52 --- /dev/null +++ b/tools/check-moz-style/tests/test3.patch @@ -0,0 +1,12 @@ +# Test +patch summary with no bug number + +Some bogus patch description +diff --git a/tests/test2.cpp b/tests/test2.cpp +new file mode 100644 +--- /dev/null ++++ b/tests/test2.cpp +@@ -0,0 +1,3 @@ ++int main() { ++ return 0; ++} diff --git a/tools/check-moz-style/tests/test4.cpp b/tools/check-moz-style/tests/test4.cpp new file mode 100644 index 000000000..64ceae0b6 --- /dev/null +++ b/tools/check-moz-style/tests/test4.cpp @@ -0,0 +1,40 @@ +class ShouldUseExplicit { + // runtime/explicit + ShouldUseExplicit(int i); +}; + +// readability/function +int foo(int) { +} + +int main() { + int i = 0; + + // readability/control_flow + // XXX This doesn't trigger it. It needs to be fixed. + if (i) { + return; + } else { + i++; + } + + // whitespace/parens + if(i){} + + // readability/casting + void* bad = (void*)i; + + // readability/comparison_to_zero + if (i == true) {} + if (i == false) {} + if (i != true) {} + if (i != false) {} + if (i == NULL) {} + if (i != NULL) {} + if (i == nullptr) {} + if (i != nullptr) {} + if (i) {} + if (!i) {} + + return 0; +} diff --git a/tools/check-moz-style/tests/test4.out b/tools/check-moz-style/tests/test4.out new file mode 100644 index 000000000..21eadcca1 --- /dev/null +++ b/tools/check-moz-style/tests/test4.out @@ -0,0 +1,13 @@ +tests/test4.cpp:1: No copyright message found. [legal/copyright] [3] +tests/test4.cpp:3: Single-argument constructors should be marked explicit. [runtime/explicit] [5] +tests/test4.cpp:7: All parameters should be named in a function [readability/function] [3] +tests/test4.cpp:22: Missing space before ( in if( [whitespace/parens] [5] +tests/test4.cpp:22: Missing space before { [whitespace/braces] [5] +tests/test4.cpp:25: Using C-style cast. Use reinterpret_cast<void*>(...) instead [readability/casting] [4] +tests/test4.cpp:28: Tests for true/false, null/non-null, and zero/non-zero should all be done without equality comparisons. [readability/comparison_to_zero] [5] +tests/test4.cpp:29: Tests for true/false, null/non-null, and zero/non-zero should all be done without equality comparisons. [readability/comparison_to_zero] [5] +tests/test4.cpp:30: Tests for true/false, null/non-null, and zero/non-zero should all be done without equality comparisons. [readability/comparison_to_zero] [5] +tests/test4.cpp:31: Tests for true/false, null/non-null, and zero/non-zero should all be done without equality comparisons. [readability/comparison_to_zero] [5] +tests/test4.cpp:32: Tests for true/false, null/non-null, and zero/non-zero should all be done without equality comparisons. [readability/comparison_to_zero] [5] +tests/test4.cpp:33: Tests for true/false, null/non-null, and zero/non-zero should all be done without equality comparisons. [readability/comparison_to_zero] [5] +Done processing tests/test4.cpp diff --git a/tools/check-moz-style/tests/test4.patch b/tools/check-moz-style/tests/test4.patch new file mode 100644 index 000000000..735ef82ca --- /dev/null +++ b/tools/check-moz-style/tests/test4.patch @@ -0,0 +1,49 @@ +# Test +Bug 12 - patch summary with no bug number + +Some bogus patch description +diff --git a/tests/test4.cpp b/tests/test4.cpp +new file mode 100644 +--- /dev/null ++++ b/tests/test4.cpp +@@ -0,0 +1,49 @@ ++class ShouldUseExplicit { ++ // runtime/explicit ++ ShouldUseExplicit(int i); ++}; ++ ++// readability/function ++int foo(int) { ++} ++ ++int main() { ++ int i = 0; ++ ++ // readability/control_flow ++ // XXX This doesn't trigger it. It needs to be fixed. ++ if (i) { ++ return; ++ } else { ++ i++; ++ } ++ ++ // whitespace/parens ++ if(i){} ++ ++ // readability/casting ++ void* bad = (void*)i; ++ ++ // readability/comparison_to_zero ++ if (i == true) {} ++ if (i == false) {} ++ if (i != true) {} ++ if (i != false) {} ++ if (i == NULL) {} ++ if (i != NULL) {} ++ if (i == nullptr) {} ++ if (i != nullptr) {} ++ if (i) {} ++ if (!i) {} ++ ++ return 0; ++} diff --git a/tools/check-moz-style/tests/test5.cpp b/tools/check-moz-style/tests/test5.cpp new file mode 100644 index 000000000..95c756151 --- /dev/null +++ b/tools/check-moz-style/tests/test5.cpp @@ -0,0 +1,24 @@ +// License bogus + +// runtime/virtual +class ShouldHaveVirtualDes { + virtual foo(); +}; + +int main() { + // runtime/memset + memset(blah, sizeof(blah), 0); + + // runtime/rtti + dynamic_cast<Derived*>(obj); + + // runtime/sizeof + int varname = 0; + int mySize = sizeof(int); + + // runtime/threadsafe_fn + getpwuid(); + strtok(); + + return 0; +} diff --git a/tools/check-moz-style/tests/test5.out b/tools/check-moz-style/tests/test5.out new file mode 100644 index 000000000..68c0eab8a --- /dev/null +++ b/tools/check-moz-style/tests/test5.out @@ -0,0 +1,7 @@ +tests/test5.cpp:4: The class ShouldHaveVirtualDes probably needs a virtual destructor due to having virtual method(s), one declared at line 5. [runtime/virtual] [4] +tests/test5.cpp:10: Did you mean "memset(blah, 0, sizeof(blah))"? [runtime/memset] [4] +tests/test5.cpp:13: Do not use dynamic_cast<>. If you need to cast within a class hierarchy, use static_cast<> to upcast. Mozilla doesn't support RTTI. [runtime/rtti] [5] +tests/test5.cpp:17: Using sizeof(type). Use sizeof(varname) instead if possible [runtime/sizeof] [1] +tests/test5.cpp:20: Consider using getpwuid_r(...) instead of getpwuid(...) for improved thread safety. [runtime/threadsafe_fn] [2] +tests/test5.cpp:21: Consider using strtok_r(...) instead of strtok(...) for improved thread safety. [runtime/threadsafe_fn] [2] +Done processing tests/test5.cpp diff --git a/tools/check-moz-style/tests/test5.patch b/tools/check-moz-style/tests/test5.patch new file mode 100644 index 000000000..19f7bee8c --- /dev/null +++ b/tools/check-moz-style/tests/test5.patch @@ -0,0 +1,33 @@ +# Test +Bug 12 - patch summary with no bug number + +Some bogus patch description +diff --git a/tests/test5.cpp b/tests/test5.cpp +new file mode 100644 +--- /dev/null ++++ b/tests/test5.cpp +@@ -0,0 +1,24 @@ ++// License bogus ++ ++// runtime/virtual ++class ShouldHaveVirtualDes { ++ virtual foo(); ++}; ++ ++int main() { ++ // runtime/memset ++ memset(blah, sizeof(blah), 0); ++ ++ // runtime/rtti ++ dynamic_cast<Derived*>(obj); ++ ++ // runtime/sizeof ++ int varname = 0; ++ int mySize = sizeof(int); ++ ++ // runtime/threadsafe_fn ++ getpwuid(); ++ strtok(); ++ ++ return 0; ++} diff --git a/tools/coverity/model.cpp b/tools/coverity/model.cpp new file mode 100644 index 000000000..32dc5a427 --- /dev/null +++ b/tools/coverity/model.cpp @@ -0,0 +1,128 @@ +/* +Coverity model file in order to avoid false-positive +*/ + +#define NULL (void *)0 + +typedef unsigned char jsbytecode; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned int int32_t; +typedef unsigned char uint8_t; + +static const uint16_t CHUNK_HEAD_SIZE = 8; + +void assert(bool expr) { + if (!expr) { + __coverity_panic__(); + } +} + +#define ERREXIT(cinfo, err) __coverity_panic__(); + +void MOZ_ASSUME_UNREACHABLE(char * str) { + __coverity_panic__(); +} + +static void MOZ_ReportAssertionFailure(const char* aStr, const char* aFilename, + int aLine) { + __coverity_panic__(); +} + +static void MOZ_ReportCrash(const char* aStr, const char* aFilename, + int aLine) { + __coverity_panic__(); +} + +#define MOZ_ASSERT(expr, msg) assert(!!(expr)) + +#define MOZ_ASSERT(expr) assert(!!(expr)) + +#define NS_ASSERTION(expr, msg) assert(!!(expr)) + +#define PORT_Assert(expr) assert(!!(expr)) + +#define PR_ASSERT(expr) assert(!!(expr)) + +int GET_JUMP_OFFSET(jsbytecode* pc) { + __coverity_tainted_data_sanitize__(&pc[1]); + __coverity_tainted_data_sanitize__(&pc[2]); + __coverity_tainted_data_sanitize__(&pc[3]); + __coverity_tainted_data_sanitize__(&pc[4]); + + return 0; +} + + +// Data sanity checkers +#define XPT_SWAB16(data) __coverity_tainted_data_sanitize__(&data) + +#define XPT_SWAB32(data) __coverity_tainted_data_sanitize__(&data) + + +static unsigned GET_UINT24(const jsbytecode* pc) { + __coverity_tainted_data_sanitize__(static_cast<void*>(pc)); + //return unsigned((pc[1] << 16) | (pc[2] << 8) | pc[3]); + return 0; +} + + +class HeaderParser { + +private: + class ChunkHeader { + + uint8_t mRaw[CHUNK_HEAD_SIZE]; + + HeaderParser::ChunkHeader::ChunkSize() const { + __coverity_tainted_data_sanitize__(static_cast<void*>(&mRaw[4])); + __coverity_tainted_data_sanitize__(static_cast<void*>(&mRaw[5])); + __coverity_tainted_data_sanitize__(static_cast<void*>(&mRaw[6])); + __coverity_tainted_data_sanitize__(static_cast<void*>(&mRaw[7])); + + return ((mRaw[7] << 24) | (mRaw[6] << 16) | (mRaw[5] << 8) | (mRaw[4])); + } + }; +}; + +void NS_DebugBreak(uint32_t aSeverity, const char* aStr, const char* aExpr, + const char* aFile, int32_t aLine) { + __coverity_panic__(); +} + +static inline void Swap(uint32_t* value) { + __coverity_tainted_data_sanitize__(static_cast<void*>(&value)); + *value = (*value >> 24) | + ((*value >> 8) & 0x0000ff00) | + ((*value << 8) & 0x00ff0000) | + (*value << 24); +} + +static uint32_t xtolong (const uint8_t *ll) { + __coverity_tainted_data_sanitize__(static_cast<void*>(&ll[0])); + __coverity_tainted_data_sanitize__(static_cast<void*>(&ll[1])); + __coverity_tainted_data_sanitize__(static_cast<void*>(&ll[2])); + __coverity_tainted_data_sanitize__(static_cast<void*>(&ll[3])); + + return (uint32_t)( (ll [0] << 0) | + (ll [1] << 8) | + (ll [2] << 16) | + (ll [3] << 24) ); +} + +class ByteReader { +public: + const uint8_t* Read(size_t aCount); + uint32_t ReadU24() { + const uint8_t *ptr = Read(3); + if (!ptr) { + MOZ_ASSERT(false); + return 0; + } + __coverity_tainted_data_sanitize__(static_cast<void*>(&ptr[0])); + __coverity_tainted_data_sanitize__(static_cast<void*>(&ptr[1])); + __coverity_tainted_data_sanitize__(static_cast<void*>(&ptr[2])); + return ptr[0] << 16 | ptr[1] << 8 | ptr[2]; + } +}; + diff --git a/tools/docs/Vagrantfile b/tools/docs/Vagrantfile new file mode 100644 index 000000000..247afe1b3 --- /dev/null +++ b/tools/docs/Vagrantfile @@ -0,0 +1,13 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# We intentionally use the old config format because Mozilla's Jenkins +# server doesn't run a modern Vagrant. +Vagrant::Config.run do |config| + config.vm.box = "precise64" + config.vm.box_url = "http://files.vagrantup.com/precise64.box" + config.vm.share_folder("gecko", "/gecko", "../..") + # Doxygen needs more than the default memory or it will swap and be + # extremely slow. + config.vm.customize ["modifyvm", :id, "--memory", 2048] +end diff --git a/tools/docs/conf.py b/tools/docs/conf.py new file mode 100644 index 000000000..1a919063f --- /dev/null +++ b/tools/docs/conf.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import re +import sys + +from datetime import datetime + +# Set up Python environment to load build system packages. +OUR_DIR = os.path.dirname(__file__) +topsrcdir = os.path.normpath(os.path.join(OUR_DIR, '..', '..')) + +EXTRA_PATHS = ( + 'layout/tools/reftest', + 'python/futures', + 'python/jsmin', + 'python/mach', + 'python/mozbuild', + 'python/mozversioncontrol', + 'python/which', + 'testing/mozbase/manifestparser', + 'testing/mozbase/mozfile', + 'testing/mozbase/mozprocess', +) + +sys.path[:0] = [os.path.join(topsrcdir, p) for p in EXTRA_PATHS] + +sys.path.insert(0, OUR_DIR) + +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.graphviz', + 'sphinx.ext.todo', + 'mozbuild.sphinx', +] + +templates_path = ['_templates'] +source_suffix = '.rst' +master_doc = 'index' +project = u'Mozilla Source Tree Docs' +year = datetime.utcnow().year + +# Grab the version from the source tree's milestone. +# FUTURE Use Python API from bug 941299. +with open(os.path.join(topsrcdir, 'config', 'milestone.txt'), 'rt') as fh: + for line in fh: + line = line.strip() + + if not line or line.startswith('#'): + continue + + release = line + break + +version = re.sub(r'[ab]\d+$', '', release) + +exclude_patterns = ['_build', '_staging', '_venv'] +pygments_style = 'sphinx' + +# We need to perform some adjustment of the settings and environment +# when running on Read The Docs. +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' + +if on_rtd: + # SHELL isn't set on RTD and mach.mixin.process's import raises if a + # shell-related environment variable can't be found. Set the variable here + # to hack us into working on RTD. + assert 'SHELL' not in os.environ + os.environ['SHELL'] = '/bin/bash' +else: + # We only need to set the RTD theme when not on RTD because the RTD + # environment handles this otherwise. + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + + +html_static_path = ['_static'] +htmlhelp_basename = 'MozillaTreeDocs' diff --git a/tools/docs/index.rst b/tools/docs/index.rst new file mode 100644 index 000000000..ae2a9d593 --- /dev/null +++ b/tools/docs/index.rst @@ -0,0 +1,58 @@ +================================= +Mozilla Source Tree Documentation +================================= + +.. toctree:: + :maxdepth: 1 + + {indexes} + +Python Packages +=============== + +.. toctree:: + :maxdepth: 2 + + {python_packages} + +Managing Documentation +====================== + +This documentation is generated via the +`Sphinx <http://sphinx-doc.org/>`_ tool from sources in the tree. + +To build the documentation, run ``mach doc``. Run +``mach help doc`` to see configurable options. + +Adding Documentation +-------------------- + +To add new documentation, define the ``SPHINX_TREES`` and +``SPHINX_PYTHON_PACKAGE_DIRS`` variables in ``moz.build`` files in +the tree and documentation will automatically get picked up. + +Say you have a directory ``featureX`` you would like to write some +documentation for. Here are the steps to create Sphinx documentation +for it: + +1. Create a directory for the docs. This is typically ``docs``. e.g. + ``featureX/docs``. +2. Create an ``index.rst`` file in this directory. The ``index.rst`` file + is the root documentation for that section. See ``build/docs/index.rst`` + for an example file. +3. In a ``moz.build`` file (typically the one in the parent directory of + the ``docs`` directory), define ``SPHINX_TREES`` to hook up the plumbing. + e.g. ``SPHINX_TREES['featureX'] = 'docs'``. This says *the ``docs`` + directory under the current directory should be installed into the + Sphinx documentation tree under ``/featureX``*. +4. If you have Python packages you would like to generate Python API + documentation for, you can use ``SPHINX_PYTHON_PACKAGE_DIRS`` to + declare directories containing Python packages. e.g. + ``SPHINX_PYTHON_PACKAGE_DIRS += ['mozpackage']``. + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/tools/docs/mach_commands.py b/tools/docs/mach_commands.py new file mode 100644 index 000000000..825e5ec38 --- /dev/null +++ b/tools/docs/mach_commands.py @@ -0,0 +1,117 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import sys + +from mach.decorators import ( + Command, + CommandArgument, + CommandProvider, +) + +import mozhttpd + +from mozbuild.base import MachCommandBase + + +@CommandProvider +class Documentation(MachCommandBase): + """Helps manage in-tree documentation.""" + + @Command('doc', category='devenv', + description='Generate and display documentation from the tree.') + @CommandArgument('what', nargs='*', metavar='DIRECTORY [, DIRECTORY]', + help='Path(s) to documentation to build and display.') + @CommandArgument('--format', default='html', + help='Documentation format to write.') + @CommandArgument('--outdir', default=None, metavar='DESTINATION', + help='Where to write output.') + @CommandArgument('--no-open', dest='auto_open', default=True, action='store_false', + help="Don't automatically open HTML docs in a browser.") + @CommandArgument('--http', const=':6666', metavar='ADDRESS', nargs='?', + help='Serve documentation on an HTTP server, e.g. ":6666".') + def build_docs(self, what=None, format=None, outdir=None, auto_open=True, http=None): + self._activate_virtualenv() + self.virtualenv_manager.install_pip_package('sphinx_rtd_theme==0.1.6') + + import sphinx + import webbrowser + + if not outdir: + outdir = os.path.join(self.topobjdir, 'docs') + if not what: + what = [os.path.join(self.topsrcdir, 'tools')] + outdir = os.path.join(outdir, format) + + generated = [] + failed = [] + for path in what: + path = os.path.normpath(os.path.abspath(path)) + docdir = self._find_doc_dir(path) + + if not docdir: + failed.append((path, 'could not find docs at this location')) + continue + + # find project name to use as a namespace within `outdir` + project = self._find_project_name(docdir) + savedir = os.path.join(outdir, project) + + args = [ + 'sphinx', + '-b', format, + docdir, + savedir, + ] + result = sphinx.build_main(args) + if result != 0: + failed.append((path, 'sphinx return code %d' % result)) + else: + generated.append(savedir) + + index_path = os.path.join(savedir, 'index.html') + if not http and auto_open and os.path.isfile(index_path): + webbrowser.open(index_path) + + if generated: + print('\nGenerated documentation:\n%s\n' % '\n'.join(generated)) + + if failed: + failed = ['%s: %s' % (f[0], f[1]) for f in failed] + return die('failed to generate documentation:\n%s' % '\n'.join(failed)) + + if http is not None: + host, port = http.split(':', 1) + addr = (host, int(port)) + if len(addr) != 2: + return die('invalid address: %s' % http) + + httpd = mozhttpd.MozHttpd(host=addr[0], port=addr[1], docroot=outdir) + print('listening on %s:%d' % addr) + httpd.start(block=True) + + def _find_project_name(self, path): + import imp + path = os.path.join(path, 'conf.py') + with open(path, 'r') as fh: + conf = imp.load_module('doc_conf', fh, path, + ('.py', 'r', imp.PY_SOURCE)) + + return conf.project.replace(' ', '_') + + def _find_doc_dir(self, path): + search_dirs = ('doc', 'docs') + for d in search_dirs: + p = os.path.join(path, d) + if os.path.isfile(os.path.join(p, 'conf.py')): + return p + + +def die(msg, exit_code=1): + msg = '%s: %s' % (sys.argv[0], msg) + print(msg, file=sys.stderr) + return exit_code diff --git a/tools/docs/moztreedocs/__init__.py b/tools/docs/moztreedocs/__init__.py new file mode 100644 index 000000000..a67edbded --- /dev/null +++ b/tools/docs/moztreedocs/__init__.py @@ -0,0 +1,126 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os + +from mozbuild.frontend.reader import BuildReader +from mozpack.copier import FileCopier +from mozpack.files import FileFinder +from mozpack.manifests import InstallManifest + +import sphinx +import sphinx.apidoc + + +class SphinxManager(object): + """Manages the generation of Sphinx documentation for the tree.""" + + def __init__(self, topsrcdir, main_path, output_dir): + self._topsrcdir = topsrcdir + self._output_dir = output_dir + self._docs_dir = os.path.join(output_dir, '_staging') + self._conf_py_path = os.path.join(main_path, 'conf.py') + self._index_path = os.path.join(main_path, 'index.rst') + self._trees = {} + self._python_package_dirs = set() + + def read_build_config(self): + """Read the active build config and add docs to this instance.""" + + # Reading the Sphinx variables doesn't require a full build context. + # Only define the parts we need. + class fakeconfig(object): + def __init__(self, topsrcdir): + self.topsrcdir = topsrcdir + + config = fakeconfig(self._topsrcdir) + reader = BuildReader(config) + + for path, name, key, value in reader.find_sphinx_variables(): + reldir = os.path.dirname(path) + + if name == 'SPHINX_TREES': + assert key + self.add_tree(os.path.join(reldir, value), + os.path.join(reldir, key)) + + if name == 'SPHINX_PYTHON_PACKAGE_DIRS': + self.add_python_package_dir(os.path.join(reldir, value)) + + def add_tree(self, source_dir, dest_dir): + """Add a directory from where docs should be sourced.""" + if dest_dir in self._trees: + raise Exception('%s has already been registered as a destination.' + % dest_dir) + + self._trees[dest_dir] = source_dir + + def add_python_package_dir(self, source_dir): + """Add a directory containing Python packages. + + Added directories will have Python API docs generated automatically. + """ + self._python_package_dirs.add(source_dir) + + def generate_docs(self, app): + """Generate/stage documentation.""" + app.info('Reading Sphinx metadata from build configuration') + self.read_build_config() + app.info('Staging static documentation') + self._synchronize_docs() + app.info('Generating Python API documentation') + self._generate_python_api_docs() + + def _generate_python_api_docs(self): + """Generate Python API doc files.""" + out_dir = os.path.join(self._docs_dir, 'python') + base_args = ['sphinx', '--no-toc', '-o', out_dir] + + for p in sorted(self._python_package_dirs): + full = os.path.join(self._topsrcdir, p) + + finder = FileFinder(full, find_executables=False) + dirs = {os.path.dirname(f[0]) for f in finder.find('**')} + + excludes = {d for d in dirs if d.endswith('test')} + + args = list(base_args) + args.append(full) + args.extend(excludes) + + sphinx.apidoc.main(args) + + def _synchronize_docs(self): + m = InstallManifest() + + m.add_symlink(self._conf_py_path, 'conf.py') + + for dest, source in sorted(self._trees.items()): + source_dir = os.path.join(self._topsrcdir, source) + for root, dirs, files in os.walk(source_dir): + for f in files: + source_path = os.path.join(root, f) + rel_source = source_path[len(source_dir) + 1:] + + m.add_symlink(source_path, os.path.join(dest, rel_source)) + + copier = FileCopier() + m.populate_registry(copier) + copier.copy(self._docs_dir) + + with open(self._index_path, 'rb') as fh: + data = fh.read() + + indexes = ['%s/index' % p for p in sorted(self._trees.keys())] + indexes = '\n '.join(indexes) + + packages = [os.path.basename(p) for p in self._python_package_dirs] + packages = ['python/%s' % p for p in packages] + packages = '\n '.join(sorted(packages)) + data = data.format(indexes=indexes, python_packages=packages) + + with open(os.path.join(self._docs_dir, 'index.rst'), 'wb') as fh: + fh.write(data) diff --git a/tools/fuzzing/interface/FuzzingInterface.cpp b/tools/fuzzing/interface/FuzzingInterface.cpp new file mode 100644 index 000000000..59077f382 --- /dev/null +++ b/tools/fuzzing/interface/FuzzingInterface.cpp @@ -0,0 +1,67 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Interface implementation for the unified fuzzing interface + */ + +#include "FuzzingInterface.h" + +#include "nsNetUtil.h" + +namespace mozilla { + +#ifdef __AFL_COMPILER +void afl_interface_stream(const char* testFile, FuzzingTestFuncStream testFunc) { + nsresult rv; + nsCOMPtr<nsIProperties> dirService = + do_GetService(NS_DIRECTORY_SERVICE_CONTRACTID); + MOZ_RELEASE_ASSERT(dirService != nullptr); + nsCOMPtr<nsIFile> file; + rv = dirService->Get(NS_OS_CURRENT_WORKING_DIR, + NS_GET_IID(nsIFile), getter_AddRefs(file)); + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); + file->AppendNative(nsDependentCString(testFile)); + while(__AFL_LOOP(1000)) { + nsCOMPtr<nsIInputStream> inputStream; + rv = NS_NewLocalFileInputStream(getter_AddRefs(inputStream), file); + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); + if (!NS_InputStreamIsBuffered(inputStream)) { + nsCOMPtr<nsIInputStream> bufStream; + rv = NS_NewBufferedInputStream(getter_AddRefs(bufStream), + inputStream, 1024); + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); + inputStream = bufStream; + } + testFunc(inputStream.forget()); + } +} + +void afl_interface_raw(const char* testFile, FuzzingTestFuncRaw testFunc) { + char* buf = NULL; + + while(__AFL_LOOP(1000)) { + std::ifstream is; + is.open (testFile, std::ios::binary); + is.seekg (0, std::ios::end); + int len = is.tellg(); + is.seekg (0, std::ios::beg); + MOZ_RELEASE_ASSERT(len >= 0); + if (!len) { + is.close(); + continue; + } + buf = (char*)realloc(buf, len); + MOZ_RELEASE_ASSERT(buf); + is.read(buf,len); + is.close(); + testFunc((uint8_t*)buf, (size_t)len); + } + + free(buf); +} +#endif + +} // namespace mozilla diff --git a/tools/fuzzing/interface/FuzzingInterface.h b/tools/fuzzing/interface/FuzzingInterface.h new file mode 100644 index 000000000..c2838e807 --- /dev/null +++ b/tools/fuzzing/interface/FuzzingInterface.h @@ -0,0 +1,100 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Interface definitions for the unified fuzzing interface + */ + +#ifndef FuzzingInterface_h__ +#define FuzzingInterface_h__ + +#include "gtest/gtest.h" +#include "nsComponentManagerUtils.h" +#include "nsCOMPtr.h" +#include "nsIInputStream.h" + +#include "nsDirectoryServiceDefs.h" +#include "nsIDirectoryService.h" +#include "nsIFile.h" +#include "nsStreamUtils.h" +#include "nsStringStream.h" + +#include <fstream> + +namespace mozilla { + +typedef int(*FuzzingTestFuncRaw)(const uint8_t*, size_t); +typedef int(*FuzzingTestFuncStream)(nsCOMPtr<nsIInputStream>); + +#ifdef __AFL_COMPILER +void afl_interface_stream(const char* testFile, FuzzingTestFuncStream testFunc); +void afl_interface_raw(const char* testFile, FuzzingTestFuncRaw testFunc); + +#define MOZ_AFL_INTERFACE_COMMON(initFunc) \ + initFunc(NULL, NULL); \ + char* testFilePtr = getenv("MOZ_FUZZ_TESTFILE"); \ + if (!testFilePtr) { \ + EXPECT_TRUE(false) << "Must specify testfile in MOZ_FUZZ_TESTFILE environment variable."; \ + return; \ + } \ + /* Make a copy of testFilePtr so the testing function can safely call getenv */ \ + std::string testFile(testFilePtr); + +#define MOZ_AFL_INTERFACE_STREAM(initFunc, testFunc, moduleName) \ + TEST(AFL, moduleName) { \ + MOZ_AFL_INTERFACE_COMMON(initFunc); \ + ::mozilla::afl_interface_stream(testFile.c_str(), testFunc); \ + } + +#define MOZ_AFL_INTERFACE_RAW(initFunc, testFunc, moduleName) \ + TEST(AFL, moduleName) { \ + MOZ_AFL_INTERFACE_COMMON(initFunc); \ + ::mozilla::afl_interface_raw(testFile.c_str(), testFunc); \ + } +#else +#define MOZ_AFL_INTERFACE_STREAM(initFunc, testFunc, moduleName) /* Nothing */ +#define MOZ_AFL_INTERFACE_RAW(initFunc, testFunc, moduleName) /* Nothing */ +#endif + +#ifdef LIBFUZZER +#define MOZ_LIBFUZZER_INTERFACE_STREAM(initFunc, testFunc, moduleName) \ + static int LibFuzzerTest##moduleName (const uint8_t *data, size_t size) { \ + if (size > INT32_MAX) \ + return 0; \ + nsCOMPtr<nsIInputStream> stream; \ + nsresult rv = NS_NewByteInputStream(getter_AddRefs(stream), \ + (const char*)data, size, NS_ASSIGNMENT_DEPEND); \ + MOZ_RELEASE_ASSERT(NS_SUCCEEDED(rv)); \ + testFunc(stream.forget()); \ + return 0; \ + } \ + static void __attribute__ ((constructor)) LibFuzzerRegister() { \ + ::mozilla::LibFuzzerRegistry::getInstance().registerModule( \ + #moduleName, initFunc, LibFuzzerTest##moduleName \ + ); \ + } + +#define MOZ_LIBFUZZER_INTERFACE_RAW(initFunc, testFunc, moduleName) \ + static void __attribute__ ((constructor)) LibFuzzerRegister() { \ + ::mozilla::LibFuzzerRegistry::getInstance().registerModule( \ + #moduleName, initFunc, testFunc \ + ); \ + } +#else +#define MOZ_LIBFUZZER_INTERFACE_STREAM(initFunc, testFunc, moduleName) /* Nothing */ +#define MOZ_LIBFUZZER_INTERFACE_RAW(initFunc, testFunc, moduleName) /* Nothing */ +#endif + +#define MOZ_FUZZING_INTERFACE_STREAM(initFunc, testFunc, moduleName) \ + MOZ_LIBFUZZER_INTERFACE_STREAM(initFunc, testFunc, moduleName); \ + MOZ_AFL_INTERFACE_STREAM(initFunc, testFunc, moduleName); + +#define MOZ_FUZZING_INTERFACE_RAW(initFunc, testFunc, moduleName) \ + MOZ_LIBFUZZER_INTERFACE_RAW(initFunc, testFunc, moduleName); \ + MOZ_AFL_INTERFACE_RAW(initFunc, testFunc, moduleName); + +} // namespace mozilla + +#endif // FuzzingInterface_h__ diff --git a/tools/fuzzing/interface/moz.build b/tools/fuzzing/interface/moz.build new file mode 100644 index 000000000..1f79f920f --- /dev/null +++ b/tools/fuzzing/interface/moz.build @@ -0,0 +1,14 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS += [ + 'FuzzingInterface.h', +] + +SOURCES += [ + 'FuzzingInterface.cpp', +] + +FINAL_LIBRARY = 'xul-gtest' diff --git a/tools/fuzzing/libfuzzer/FuzzerCustomMain.cpp b/tools/fuzzing/libfuzzer/FuzzerCustomMain.cpp new file mode 100644 index 000000000..2293efd95 --- /dev/null +++ b/tools/fuzzing/libfuzzer/FuzzerCustomMain.cpp @@ -0,0 +1,37 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <cstdlib> + +#include "FuzzerInterface.h" +#include "FuzzerInternal.h" +#include "harness/LibFuzzerRegistry.h" + +/* This is a wrapper defined in browser/app/nsBrowserApp.cpp, + * encapsulating the XRE_ equivalent defined in libxul */ +extern void libFuzzerGetFuncs(const char*, LibFuzzerInitFunc*, + LibFuzzerTestingFunc*); + +int libfuzzer_main(int argc, char **argv) { + LibFuzzerInitFunc initFunc = nullptr; + LibFuzzerTestingFunc testingFunc = nullptr; + + libFuzzerGetFuncs(getenv("LIBFUZZER"), &initFunc, &testingFunc); + + if (initFunc) { + int ret = initFunc(&argc, &argv); + if (ret) { + fprintf(stderr, "LibFuzzer: Error: Initialize callback failed\n"); + return ret; + } + } + + if (!testingFunc) { + fprintf(stderr, "LibFuzzer: Error: No testing callback found\n"); + return 1; + } + + return fuzzer::FuzzerDriver(&argc, &argv, testingFunc); +} diff --git a/tools/fuzzing/libfuzzer/Makefile.in b/tools/fuzzing/libfuzzer/Makefile.in new file mode 100644 index 000000000..7ffe87685 --- /dev/null +++ b/tools/fuzzing/libfuzzer/Makefile.in @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +include $(topsrcdir)/config/rules.mk + +# According to the LLVM docs, LibFuzzer isn't supposed to be built with any +# sanitizer flags and in fact, building it with ASan coverage currently causes +# Clang 3.9+ to crash, so we filter out all sanitizer-related flags here. +CXXFLAGS := $(filter-out -fsanitize%,$(CXXFLAGS)) +CFLAGS := $(filter-out -fsanitize%,$(CFLAGS)) +LDFLAGS := $(filter-out -fsanitize%,$(LDFLAGS)) diff --git a/tools/fuzzing/libfuzzer/clone_libfuzzer.sh b/tools/fuzzing/libfuzzer/clone_libfuzzer.sh new file mode 100755 index 000000000..6170362ac --- /dev/null +++ b/tools/fuzzing/libfuzzer/clone_libfuzzer.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +mkdir tmp/ +git clone --no-checkout --depth 1 https://chromium.googlesource.com/chromium/llvm-project/llvm/lib/Fuzzer tmp/ +mv tmp/.git . +rm -Rf tmp +git reset --hard HEAD diff --git a/tools/fuzzing/libfuzzer/harness/LibFuzzerRegistry.cpp b/tools/fuzzing/libfuzzer/harness/LibFuzzerRegistry.cpp new file mode 100644 index 000000000..5390c91c2 --- /dev/null +++ b/tools/fuzzing/libfuzzer/harness/LibFuzzerRegistry.cpp @@ -0,0 +1,32 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "LibFuzzerRegistry.h" + +extern "C" { + void MOZ_EXPORT XRE_LibFuzzerGetFuncs(const char* moduleName, LibFuzzerInitFunc* initFunc, LibFuzzerTestingFunc* testingFunc) { + std::string moduleNameStr(moduleName); + mozilla::LibFuzzerFunctions funcs = mozilla::LibFuzzerRegistry::getInstance().getModuleFunctions(moduleNameStr); + *initFunc = funcs.first; + *testingFunc = funcs.second; + } +} + +namespace mozilla { + +LibFuzzerRegistry& LibFuzzerRegistry::getInstance() { + static LibFuzzerRegistry instance; + return instance; +} + +void LibFuzzerRegistry::registerModule(std::string moduleName, LibFuzzerInitFunc initFunc, LibFuzzerTestingFunc testingFunc) { + moduleMap.insert(std::pair<std::string, LibFuzzerFunctions>(moduleName,LibFuzzerFunctions(initFunc, testingFunc))); +} + +LibFuzzerFunctions LibFuzzerRegistry::getModuleFunctions(std::string& moduleName) { + return moduleMap[moduleName]; +} + +} // namespace mozilla diff --git a/tools/fuzzing/libfuzzer/harness/LibFuzzerRegistry.h b/tools/fuzzing/libfuzzer/harness/LibFuzzerRegistry.h new file mode 100644 index 000000000..e459ade33 --- /dev/null +++ b/tools/fuzzing/libfuzzer/harness/LibFuzzerRegistry.h @@ -0,0 +1,41 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _LibFuzzerRegistry_h__ +#define _LibFuzzerRegistry_h__ + +#include <cstdint> +#include <map> +#include <string> +#include <utility> + +#include "mozilla/Attributes.h" + +typedef int(*LibFuzzerMain)(int, char**); +typedef int(*LibFuzzerInitFunc)(int*, char***); +typedef int(*LibFuzzerTestingFunc)(const uint8_t*, size_t); + +namespace mozilla { + +typedef std::pair<LibFuzzerInitFunc, LibFuzzerTestingFunc> LibFuzzerFunctions; + +class LibFuzzerRegistry { + public: + MOZ_EXPORT static LibFuzzerRegistry& getInstance(); + MOZ_EXPORT void registerModule(std::string moduleName, LibFuzzerInitFunc initFunc, LibFuzzerTestingFunc testingFunc); + MOZ_EXPORT LibFuzzerFunctions getModuleFunctions(std::string& moduleName); + + LibFuzzerRegistry(LibFuzzerRegistry const&) = delete; + void operator=(LibFuzzerRegistry const&) = delete; + + private: + LibFuzzerRegistry() {}; + std::map<std::string, LibFuzzerFunctions> moduleMap; +}; + +} // namespace mozilla + + +#endif // _LibFuzzerRegistry_h__ diff --git a/tools/fuzzing/libfuzzer/harness/LibFuzzerRunner.cpp b/tools/fuzzing/libfuzzer/harness/LibFuzzerRunner.cpp new file mode 100644 index 000000000..2a57ddac8 --- /dev/null +++ b/tools/fuzzing/libfuzzer/harness/LibFuzzerRunner.cpp @@ -0,0 +1,38 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "LibFuzzerRunner.h" +#include "mozilla/Attributes.h" +#include "prenv.h" + +#include "LibFuzzerTestHarness.h" + +namespace mozilla { + +// We use a static var 'libFuzzerRunner' defined in nsAppRunner.cpp. +// libFuzzerRunner is initialized to nullptr but if LibFuzzer (this file) +// is linked in then libFuzzerRunner will be set here indicating that +// we want to call into LibFuzzer's main. +class _InitLibFuzzer { +public: + _InitLibFuzzer() { + libFuzzerRunner = new LibFuzzerRunner(); + } +} InitLibFuzzer; + +int LibFuzzerRunner::Run() { + ScopedXPCOM xpcom("LibFuzzer"); + return mFuzzerMain(mArgc, mArgv); +} + +typedef int(*LibFuzzerMain)(int, char**); + +void LibFuzzerRunner::setParams(int argc, char** argv, LibFuzzerMain main) { + mArgc = argc; + mArgv = argv; + mFuzzerMain = main; +} + +} // namespace mozilla diff --git a/tools/fuzzing/libfuzzer/harness/LibFuzzerRunner.h b/tools/fuzzing/libfuzzer/harness/LibFuzzerRunner.h new file mode 100644 index 000000000..c2362f4e9 --- /dev/null +++ b/tools/fuzzing/libfuzzer/harness/LibFuzzerRunner.h @@ -0,0 +1,23 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * * This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +namespace mozilla { + +typedef int(*LibFuzzerMain)(int, char**); + +class LibFuzzerRunner { +public: + int Run(); + void setParams(int argc, char** argv, LibFuzzerMain main); + +private: + int mArgc; + char** mArgv; + LibFuzzerMain mFuzzerMain; +}; + +extern LibFuzzerRunner* libFuzzerRunner; + +} // namespace mozilla diff --git a/tools/fuzzing/libfuzzer/harness/LibFuzzerTestHarness.h b/tools/fuzzing/libfuzzer/harness/LibFuzzerTestHarness.h new file mode 100644 index 000000000..70321a574 --- /dev/null +++ b/tools/fuzzing/libfuzzer/harness/LibFuzzerTestHarness.h @@ -0,0 +1,290 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* + * Test harness for XPCOM objects, providing a scoped XPCOM initializer, + * nsCOMPtr, nsRefPtr, do_CreateInstance, do_GetService, ns(Auto|C|)String, + * and stdio.h/stdlib.h. + */ + +#ifndef LibFuzzerTestHarness_h__ +#define LibFuzzerTestHarness_h__ + +#include "mozilla/ArrayUtils.h" + +#include "prenv.h" +#include "nsComponentManagerUtils.h" +#include "nsServiceManagerUtils.h" +#include "nsCOMPtr.h" +#include "nsAutoPtr.h" +#include "nsStringGlue.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsDirectoryServiceDefs.h" +#include "nsDirectoryServiceUtils.h" +#include "nsIDirectoryService.h" +#include "nsIFile.h" +#include "nsIProperties.h" +#include "nsIObserverService.h" +#include "nsXULAppAPI.h" +#include <stdio.h> +#include <stdlib.h> +#include <stdarg.h> + +namespace { + +static uint32_t gFailCount = 0; + +/** + * Prints the given failure message and arguments using printf, prepending + * "TEST-UNEXPECTED-FAIL " for the benefit of the test harness and + * appending "\n" to eliminate having to type it at each call site. + */ +void fail(const char* msg, ...) +{ + va_list ap; + + printf("TEST-UNEXPECTED-FAIL | "); + + va_start(ap, msg); + vprintf(msg, ap); + va_end(ap); + + putchar('\n'); + ++gFailCount; +} + +/** + * Prints the given success message and arguments using printf, prepending + * "TEST-PASS " for the benefit of the test harness and + * appending "\n" to eliminate having to type it at each call site. + */ +void passed(const char* msg, ...) +{ + va_list ap; + + printf("TEST-PASS | "); + + va_start(ap, msg); + vprintf(msg, ap); + va_end(ap); + + putchar('\n'); +} + +//----------------------------------------------------------------------------- + +static class ScopedXPCOM : public nsIDirectoryServiceProvider2 +{ + public: + NS_DECL_ISUPPORTS + + explicit ScopedXPCOM(const char* testName, + nsIDirectoryServiceProvider *dirSvcProvider = nullptr) + : mDirSvcProvider(dirSvcProvider) + { + mTestName = testName; + printf("Running %s tests...\n", mTestName); + + nsresult rv = NS_InitXPCOM2(&mServMgr, nullptr, this); + if (NS_FAILED(rv)) + { + fail("NS_InitXPCOM2 returned failure code 0x%x", rv); + mServMgr = nullptr; + return; + } + } + + ~ScopedXPCOM() + { + // If we created a profile directory, we need to remove it. + if (mProfD) { + nsCOMPtr<nsIObserverService> os = + do_GetService(NS_OBSERVERSERVICE_CONTRACTID); + MOZ_ASSERT(os); + if (os) { + MOZ_ALWAYS_SUCCEEDS(os->NotifyObservers(nullptr, "profile-change-net-teardown", nullptr)); + MOZ_ALWAYS_SUCCEEDS(os->NotifyObservers(nullptr, "profile-change-teardown", nullptr)); + MOZ_ALWAYS_SUCCEEDS(os->NotifyObservers(nullptr, "profile-before-change", nullptr)); + MOZ_ALWAYS_SUCCEEDS(os->NotifyObservers(nullptr, "profile-before-change-qm", nullptr)); + MOZ_ALWAYS_SUCCEEDS(os->NotifyObservers(nullptr, "profile-before-change-telemetry", nullptr)); + } + + if (NS_FAILED(mProfD->Remove(true))) { + NS_WARNING("Problem removing profile directory"); + } + + mProfD = nullptr; + } + + if (mServMgr) + { + NS_RELEASE(mServMgr); + nsresult rv = NS_ShutdownXPCOM(nullptr); + if (NS_FAILED(rv)) + { + fail("XPCOM shutdown failed with code 0x%x", rv); + exit(1); + } + } + + printf("Finished running %s tests.\n", mTestName); + } + + bool failed() + { + return mServMgr == nullptr; + } + + already_AddRefed<nsIFile> GetProfileDirectory() + { + if (mProfD) { + nsCOMPtr<nsIFile> copy = mProfD; + return copy.forget(); + } + + // Create a unique temporary folder to use for this test. + // Note that runcppunittests.py will run tests with a temp + // directory as the cwd, so just put something under that. + nsCOMPtr<nsIFile> profD; + nsresult rv = NS_GetSpecialDirectory(NS_OS_CURRENT_PROCESS_DIR, + getter_AddRefs(profD)); + NS_ENSURE_SUCCESS(rv, nullptr); + + rv = profD->Append(NS_LITERAL_STRING("cpp-unit-profd")); + NS_ENSURE_SUCCESS(rv, nullptr); + + rv = profD->CreateUnique(nsIFile::DIRECTORY_TYPE, 0755); + NS_ENSURE_SUCCESS(rv, nullptr); + + mProfD = profD; + return profD.forget(); + } + + already_AddRefed<nsIFile> GetGREDirectory() + { + if (mGRED) { + nsCOMPtr<nsIFile> copy = mGRED; + return copy.forget(); + } + + char* env = PR_GetEnv("MOZ_XRE_DIR"); + nsCOMPtr<nsIFile> greD; + if (env) { + NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false, + getter_AddRefs(greD)); + } + + mGRED = greD; + return greD.forget(); + } + + already_AddRefed<nsIFile> GetGREBinDirectory() + { + if (mGREBinD) { + nsCOMPtr<nsIFile> copy = mGREBinD; + return copy.forget(); + } + + nsCOMPtr<nsIFile> greD = GetGREDirectory(); + if (!greD) { + return greD.forget(); + } + greD->Clone(getter_AddRefs(mGREBinD)); + + nsCOMPtr<nsIFile> copy = mGREBinD; + return copy.forget(); + } + + //////////////////////////////////////////////////////////////////////////// + //// nsIDirectoryServiceProvider + + NS_IMETHODIMP GetFile(const char *aProperty, bool *_persistent, + nsIFile **_result) override + { + // If we were supplied a directory service provider, ask it first. + if (mDirSvcProvider && + NS_SUCCEEDED(mDirSvcProvider->GetFile(aProperty, _persistent, + _result))) { + return NS_OK; + } + + // Otherwise, the test harness provides some directories automatically. + if (0 == strcmp(aProperty, NS_APP_USER_PROFILE_50_DIR) || + 0 == strcmp(aProperty, NS_APP_USER_PROFILE_LOCAL_50_DIR) || + 0 == strcmp(aProperty, NS_APP_PROFILE_LOCAL_DIR_STARTUP)) { + nsCOMPtr<nsIFile> profD = GetProfileDirectory(); + NS_ENSURE_TRUE(profD, NS_ERROR_FAILURE); + + nsCOMPtr<nsIFile> clone; + nsresult rv = profD->Clone(getter_AddRefs(clone)); + NS_ENSURE_SUCCESS(rv, rv); + + *_persistent = true; + clone.forget(_result); + return NS_OK; + } else if (0 == strcmp(aProperty, NS_GRE_DIR)) { + nsCOMPtr<nsIFile> greD = GetGREDirectory(); + NS_ENSURE_TRUE(greD, NS_ERROR_FAILURE); + + *_persistent = true; + greD.forget(_result); + return NS_OK; + } else if (0 == strcmp(aProperty, NS_GRE_BIN_DIR)) { + nsCOMPtr<nsIFile> greBinD = GetGREBinDirectory(); + NS_ENSURE_TRUE(greBinD, NS_ERROR_FAILURE); + + *_persistent = true; + greBinD.forget(_result); + return NS_OK; + } + + return NS_ERROR_FAILURE; + } + + //////////////////////////////////////////////////////////////////////////// + //// nsIDirectoryServiceProvider2 + + NS_IMETHODIMP GetFiles(const char *aProperty, nsISimpleEnumerator **_enum) override + { + // If we were supplied a directory service provider, ask it first. + nsCOMPtr<nsIDirectoryServiceProvider2> provider = + do_QueryInterface(mDirSvcProvider); + if (provider && NS_SUCCEEDED(provider->GetFiles(aProperty, _enum))) { + return NS_OK; + } + + return NS_ERROR_FAILURE; + } + + private: + const char* mTestName; + nsIServiceManager* mServMgr; + nsCOMPtr<nsIDirectoryServiceProvider> mDirSvcProvider; + nsCOMPtr<nsIFile> mProfD; + nsCOMPtr<nsIFile> mGRED; + nsCOMPtr<nsIFile> mGREBinD; +}; + +NS_IMPL_QUERY_INTERFACE( + ScopedXPCOM, + nsIDirectoryServiceProvider, + nsIDirectoryServiceProvider2 +) + +NS_IMETHODIMP_(MozExternalRefCountType) +ScopedXPCOM::AddRef() +{ + return 2; +} + +NS_IMETHODIMP_(MozExternalRefCountType) +ScopedXPCOM::Release() +{ + return 1; +} + +} // namespace + +#endif // LibFuzzerTestHarness_h__ diff --git a/tools/fuzzing/libfuzzer/harness/moz.build b/tools/fuzzing/libfuzzer/harness/moz.build new file mode 100644 index 000000000..f634bd6f8 --- /dev/null +++ b/tools/fuzzing/libfuzzer/harness/moz.build @@ -0,0 +1,18 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library('fuzzer-runner') + +SOURCES += [ + 'LibFuzzerRegistry.cpp', + 'LibFuzzerRunner.cpp', +] + +EXPORTS += [ + 'LibFuzzerRegistry.h', + 'LibFuzzerRunner.h', +] + +FINAL_LIBRARY = "xul" diff --git a/tools/fuzzing/libfuzzer/moz.build b/tools/fuzzing/libfuzzer/moz.build new file mode 100644 index 000000000..69eee9afa --- /dev/null +++ b/tools/fuzzing/libfuzzer/moz.build @@ -0,0 +1,25 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +Library('fuzzer') + +DIRS += [ + 'harness', +] + +SOURCES += [ + 'FuzzerCrossOver.cpp', + 'FuzzerCustomMain.cpp', + 'FuzzerDriver.cpp', + 'FuzzerExtFunctionsDlsym.cpp', + 'FuzzerExtFunctionsWeak.cpp', + 'FuzzerIO.cpp', + 'FuzzerLoop.cpp', + 'FuzzerMutate.cpp', + 'FuzzerSHA1.cpp', + 'FuzzerTracePC.cpp', + 'FuzzerTraceState.cpp', + 'FuzzerUtil.cpp', +] diff --git a/tools/fuzzing/moz.build b/tools/fuzzing/moz.build new file mode 100644 index 000000000..26cc7a7d0 --- /dev/null +++ b/tools/fuzzing/moz.build @@ -0,0 +1,13 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += [ + 'interface', +] + +if CONFIG['LIBFUZZER']: + DIRS += [ + 'libfuzzer', + ] diff --git a/tools/leak-gauge/leak-gauge.html b/tools/leak-gauge/leak-gauge.html new file mode 100644 index 000000000..74f24fd40 --- /dev/null +++ b/tools/leak-gauge/leak-gauge.html @@ -0,0 +1,302 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" + "http://www.w3.org/TR/html4/strict.dtd"> +<!-- + vim:sw=4:ts=4:et: + This Source Code Form is subject to the terms of the Mozilla Public + - License, v. 2.0. If a copy of the MPL was not distributed with this + - file, You can obtain one at http://mozilla.org/MPL/2.0/. +--> +<html lang="en-US"> +<head> +<meta charset="UTF-8" /> +<title>Leak Gauge</title> + +<style type="text/css"> +pre { margin: 0; } +pre.output { border: medium solid; padding: 1em; margin: 1em; } +</style> +<script type="text/javascript"> + +function runfile(file) { + var result = "Results of processing log " + file.fileName + " :\n"; + + var fileReader = new FileReader(); + fileReader.onload = function(e) + { + runContents(result, e.target.result); + } + fileReader.readAsText(file, "iso-8859-1"); +} + +function runContents(result, contents) { + // A hash of objects (keyed by the first word of the line in the log) + // that have two public methods, handle_line and dump (to be called using + // call, above), along with any private data they need. + var handlers = { + "DOMWINDOW": { + count: 0, + windows: {}, + handle_line: function(line) { + var match = line.match(/^([0-9a-f]*) (\S*)(.*)/); + if (match) { + var addr = match[1]; + var verb = match[2]; + var rest = match[3]; + if (verb == "created") { + var m = rest.match(/ outer=([0-9a-f]*)$/); + if (!m) + throw "outer expected"; + this.windows[addr] = { outer: m[1] }; + ++this.count; + } else if (verb == "destroyed") { + delete this.windows[addr]; + } else if (verb == "SetNewDocument") { + var m = rest.match(/^ (.*)$/); + if (!m) + throw "URI expected"; + this.windows[addr][m[1]] = true; + } + } + }, + dump: function() { + for (var addr in this.windows) { + var winobj = this.windows[addr]; + var outer = winobj.outer; + delete winobj.outer; + result += "Leaked " + (outer == "0" ? "outer" : "inner") + + " window " + addr + " " + + (outer == "0" ? "" : "(outer " + outer + ") ") + + "at address " + addr + ".\n"; + for (var uri in winobj) { + result += " ... with URI \"" + uri + "\".\n"; + } + } + }, + summary: function() { + var len = 0; + for (var w in this.windows) + ++len; + result += 'Leaked ' + len + ' out of ' + + this.count + " DOM Windows\n"; + } + }, + "DOCUMENT": { + count: 0, + docs: {}, + handle_line: function(line) { + var match = line.match(/^([0-9a-f]*) (\S*)(.*)/); + if (match) { + var addr = match[1]; + var verb = match[2]; + var rest = match[3]; + if (verb == "created") { + this.docs[addr] = {}; + ++this.count; + } else if (verb == "destroyed") { + delete this.docs[addr]; + } else if (verb == "ResetToURI" || + verb == "StartDocumentLoad") { + var m = rest.match(/^ (.*)$/); + if (!m) + throw "URI expected"; + var uri = m[1]; + var doc_info = this.docs[addr]; + doc_info[uri] = true; + if ("nim" in doc_info) { + doc_info["nim"][uri] = true; + } + } + } + }, + dump: function() { + for (var addr in this.docs) { + var doc = this.docs[addr]; + result += "Leaked document at address " + addr + ".\n"; + for (var uri in doc) { + if (uri != "nim") { + result += " ... with URI \"" + uri + "\".\n"; + } + } + } + }, + summary: function() { + var len = 0; + for (var w in this.docs) + ++len; + result += 'Leaked ' + len + ' out of ' + + this.count + " documents\n"; + } + }, + "DOCSHELL": { + count: 0, + shells: {}, + handle_line: function(line) { + var match = line.match(/^([0-9a-f]*) (\S*)(.*)/); + if (match) { + var addr = match[1]; + var verb = match[2]; + var rest = match[3]; + if (verb == "created") { + this.shells[addr] = {}; + ++this.count; + } else if (verb == "destroyed") { + delete this.shells[addr]; + } else if (verb == "InternalLoad" || + verb == "SetCurrentURI") { + var m = rest.match(/^ (.*)$/); + if (!m) + throw "URI expected"; + this.shells[addr][m[1]] = true; + } + } + }, + dump: function() { + for (var addr in this.shells) { + var doc = this.shells[addr]; + result += "Leaked docshell at address " + addr + ".\n"; + for (var uri in doc) { + result += " ... which loaded URI \"" + uri + "\".\n"; + } + } + }, + summary: function() { + var len = 0; + for (var w in this.shells) + ++len; + result += 'Leaked ' + len + ' out of ' + + this.count + " docshells\n"; + } + }, + "NODEINFOMANAGER": { + count: 0, + nims: {}, + handle_line: function(line) { + var match = line.match(/^([0-9a-f]*) (\S*)(.*)/); + if (match) { + var addr = match[1]; + var verb = match[2]; + var rest = match[3]; + if (verb == "created") { + this.nims[addr] = {}; + ++this.count; + } else if (verb == "destroyed") { + delete this.nims[addr]; + } else if (verb == "Init") { + var m = rest.match(/^ document=(.*)$/); + if (!m) + throw "document pointer expected"; + var nim_info = this.nims[addr]; + var doc = m[1]; + if (doc != "0") { + var doc_info = handlers["DOCUMENT"].docs[doc]; + for (var uri in doc_info) { + nim_info[uri] = true; + } + doc_info["nim"] = nim_info; + } + } + } + }, + dump: function() { + for (var addr in this.nims) { + var nim = this.nims[addr]; + result += "Leaked content nodes associated with node info manager at address " + addr + ".\n"; + for (var uri in nim) { + result += " ... with document URI \"" + uri + "\".\n"; + } + } + }, + summary: function() { + var len = 0; + for (var w in this.nims) + ++len; + result += 'Leaked content nodes in ' + len + ' out of ' + + this.count + " documents\n"; + } + } + }; + + var lines = contents.split(/[\r\n]+/); + for (var j in lines) { + var line = lines[j]; + // strip off initial "-", thread id, and thread pointer; separate + // first word and rest + var matches = line.match(/^\-?[0-9]*\[[0-9a-f]*\]: (\S*) (.*)$/); + if (matches) { + var handler = matches[1]; + var data = matches[2]; + if (typeof(handlers[handler]) != "undefined") { + handlers[handler].handle_line(data); + } + } + } + + for (var handler in handlers) + handlers[handler].dump(); + if (result.length) + result += "\n"; + result += "Summary:\n"; + for (var handler in handlers) + handlers[handler].summary(); + result += "\n"; + + var out = document.createElement("pre"); + out.className = "output"; + out.appendChild(document.createTextNode(result)); + document.body.appendChild(out); +} + +function run() { + var input = document.getElementById("fileinput"); + var files = input.files; + for (var i = 0; i < files.length; ++i) + runfile(files[i]); + // So the user can process the same filename again (after + // overwriting the log), clear the value on the form input so we + // will always get an onchange event. + input.value = ""; +} + +</script> +</head> +<body> + +<h1>Leak Gauge</h1> + +<pre>$Id: leak-gauge.html,v 1.8 2008/02/08 19:55:34 dbaron%dbaron.org Exp $</pre> + +<p>This script is designed to help testers isolate and simplify testcases +for many classes of leaks (those that involve large graphs of core +data structures) in Mozilla-based browsers. It is designed to print +information about what has leaked by processing a log taken while +running the browser. Such a log can be taken over a long session of +normal browsing and then the log can be processed to find sites that +leak. Once a site is known to leak, the logging can then be repeated +to figure out under what conditions the leak occurs.</p> + +<p>The way to create this log is to set the environment variables:</p> +<pre> MOZ_LOG=DOMLeak:5,DocumentLeak:5,nsDocShellLeak:5,NodeInfoManagerLeak:5 + MOZ_LOG_FILE=nspr.log <i>(or any other filename of your choice)</i></pre> +<p>in your shell and then run the program.</p> +<ul> +<li>In a Windows command prompt, set environment variables with +<pre> set VAR=value</pre></li> +<li> In an sh-based shell such as bash, set environment variables with +<pre> export VAR=value</pre></li> +<li>In a csh-based shell such as tcsh, set environment variables with +<pre> setenv VAR value</pre></li> +</ul> + +<p>Once you have this log from a complete run of the browser (you have +to exit; otherwise it will look like everything leaked), you can load +this page (be careful not to overwrite the log when starting the browser +to load this page) and enter the filename of the log:</p> + +<p><input type="file" id="fileinput" onchange="run()"></p> + +<p>Then you'll see the output below, which will tell you which of +certain core objects leaked and the URLs associated with those +objects.</p> + +</body> +</html> diff --git a/tools/leak-gauge/leak-gauge.pl b/tools/leak-gauge/leak-gauge.pl new file mode 100755 index 000000000..76ac597df --- /dev/null +++ b/tools/leak-gauge/leak-gauge.pl @@ -0,0 +1,239 @@ +#!/usr/bin/perl -w +# vim:sw=4:ts=4:et: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# $Id: leak-gauge.pl,v 1.8 2008/02/08 19:55:03 dbaron%dbaron.org Exp $ +# This script is designed to help testers isolate and simplify testcases +# for many classes of leaks (those that involve large graphs of core +# data structures) in Mozilla-based browsers. It is designed to print +# information about what has leaked by processing a log taken while +# running the browser. Such a log can be taken over a long session of +# normal browsing and then the log can be processed to find sites that +# leak. Once a site is known to leak, the logging can then be repeated +# to figure out under what conditions the leak occurs. +# +# The way to create this log is to set the environment variables: +# MOZ_LOG=DOMLeak:5,DocumentLeak:5,nsDocShellLeak:5,NodeInfoManagerLeak:5 +# MOZ_LOG_FILE=nspr.log (or any other filename of your choice) +# in your shell and then run the program. +# * In a Windows command prompt, set environment variables with +# set VAR=value +# * In an sh-based shell such as bash, set environment variables with +# export VAR=value +# * In a csh-based shell such as tcsh, set environment variables with +# setenv VAR value +# +# Then, after you have exited the browser, run this perl script over the +# log. Either of the following commands should work: +# perl ./path/to/leak-gauge.pl nspr.log +# cat nspr.log | perl ./path/to/leak-gauge.pl +# and it will tell you which of certain core objects leaked and the URLs +# associated with those objects. + + +# Nobody said I'm not allowed to write my own object system in perl. No +# classes here. Just objects and methods. +sub call { + my $func = shift; + my $obj = shift; + my $funcref = ${$obj}{$func}; + &$funcref($obj, @_); +} + +# A hash of objects (keyed by the first word of the line in the log) +# that have two public methods, handle_line and dump (to be called using +# call, above), along with any private data they need. +my $handlers = { + "DOMWINDOW" => { + count => 0, + windows => {}, + handle_line => sub($$) { + my ($self, $line) = @_; + my $windows = ${$self}{windows}; + if ($line =~ /^([0-9a-f]*) (\S*)/) { + my ($addr, $verb, $rest) = ($1, $2, $'); + if ($verb eq "created") { + $rest =~ / outer=([0-9a-f]*)$/ || die "outer expected"; + my $outer = $1; + ${$windows}{$addr} = { outer => $1 }; + ++${$self}{count}; + } elsif ($verb eq "destroyed") { + delete ${$windows}{$addr}; + } elsif ($verb eq "SetNewDocument") { + $rest =~ /^ (.*)$/ || die "URI expected"; + my $uri = ($1); + ${${$windows}{$addr}}{$uri} = 1; + } + } + }, + dump => sub ($) { + my ($self) = @_; + my $windows = ${$self}{windows}; + foreach my $addr (keys(%{$windows})) { + my $winobj = ${$windows}{$addr}; + my $outer = delete ${$winobj}{outer}; + print "Leaked " . ($outer eq "0" ? "outer" : "inner") . + " window $addr " . + ($outer eq "0" ? "" : "(outer $outer) ") . + "at address $addr.\n"; + foreach my $uri (keys(%{$winobj})) { + print " ... with URI \"$uri\".\n"; + } + } + }, + summary => sub($) { + my ($self) = @_; + my $windows = ${$self}{windows}; + print 'Leaked ' . keys(%{$windows}) . ' out of ' . + ${$self}{count} . " DOM Windows\n"; + } + }, + "DOCUMENT" => { + count => 0, + docs => {}, + handle_line => sub($$) { + my ($self, $line) = @_; + # This doesn't work; I don't have time to figure out why not. + # my $docs = ${$self}{docs}; + my $docs = ${$handlers}{"DOCUMENT"}{docs}; + if ($line =~ /^([0-9a-f]*) (\S*)/) { + my ($addr, $verb, $rest) = ($1, $2, $'); + if ($verb eq "created") { + ${$docs}{$addr} = {}; + ++${$self}{count}; + } elsif ($verb eq "destroyed") { + delete ${$docs}{$addr}; + } elsif ($verb eq "ResetToURI" || + $verb eq "StartDocumentLoad") { + $rest =~ /^ (.*)$/ || die "URI expected"; + my $uri = $1; + my $doc_info = ${$docs}{$addr}; + ${$doc_info}{$uri} = 1; + if (exists(${$doc_info}{"nim"})) { + ${$doc_info}{"nim"}{$uri} = 1; + } + } + } + }, + dump => sub ($) { + my ($self) = @_; + my $docs = ${$self}{docs}; + foreach my $addr (keys(%{$docs})) { + print "Leaked document at address $addr.\n"; + foreach my $uri (keys(%{${$docs}{$addr}})) { + print " ... with URI \"$uri\".\n" unless $uri eq "nim"; + } + } + }, + summary => sub($) { + my ($self) = @_; + my $docs = ${$self}{docs}; + print 'Leaked ' . keys(%{$docs}) . ' out of ' . + ${$self}{count} . " documents\n"; + } + }, + "DOCSHELL" => { + count => 0, + shells => {}, + handle_line => sub($$) { + my ($self, $line) = @_; + my $shells = ${$self}{shells}; + if ($line =~ /^([0-9a-f]*) (\S*)/) { + my ($addr, $verb, $rest) = ($1, $2, $'); + if ($verb eq "created") { + ${$shells}{$addr} = {}; + ++${$self}{count}; + } elsif ($verb eq "destroyed") { + delete ${$shells}{$addr}; + } elsif ($verb eq "InternalLoad" || + $verb eq "SetCurrentURI") { + $rest =~ /^ (.*)$/ || die "URI expected"; + my $uri = $1; + ${${$shells}{$addr}}{$uri} = 1; + } + } + }, + dump => sub ($) { + my ($self) = @_; + my $shells = ${$self}{shells}; + foreach my $addr (keys(%{$shells})) { + print "Leaked docshell at address $addr.\n"; + foreach my $uri (keys(%{${$shells}{$addr}})) { + print " ... which loaded URI \"$uri\".\n"; + } + } + }, + summary => sub($) { + my ($self) = @_; + my $shells = ${$self}{shells}; + print 'Leaked ' . keys(%{$shells}) . ' out of ' . + ${$self}{count} . " docshells\n"; + } + }, + "NODEINFOMANAGER" => { + count => 0, + nims => {}, + handle_line => sub($$) { + my ($self, $line) = @_; + my $nims = ${$self}{nims}; + if ($line =~ /^([0-9a-f]*) (\S*)/) { + my ($addr, $verb, $rest) = ($1, $2, $'); + if ($verb eq "created") { + ${$nims}{$addr} = {}; + ++${$self}{count}; + } elsif ($verb eq "destroyed") { + delete ${$nims}{$addr}; + } elsif ($verb eq "Init") { + $rest =~ /^ document=(.*)$/ || + die "document pointer expected"; + my $doc = $1; + if ($doc ne "0") { + my $nim_info = ${$nims}{$addr}; + my $doc_info = ${$handlers}{"DOCUMENT"}{docs}{$doc}; + foreach my $uri (keys(%{$doc_info})) { + ${$nim_info}{$uri} = 1; + } + ${$doc_info}{"nim"} = $nim_info; + } + } + } + }, + dump => sub ($) { + my ($self) = @_; + my $nims = ${$self}{nims}; + foreach my $addr (keys(%{$nims})) { + print "Leaked content nodes associated with node info manager at address $addr.\n"; + foreach my $uri (keys(%{${$nims}{$addr}})) { + print " ... with document URI \"$uri\".\n"; + } + } + }, + summary => sub($) { + my ($self) = @_; + my $nims = ${$self}{nims}; + print 'Leaked content nodes within ' . keys(%{$nims}) . ' out of ' . + ${$self}{count} . " documents\n"; + } + } +}; + +while (<>) { + # strip off initial "-", thread id, and thread pointer; separate + # first word and rest + if (/^\-?[0-9]*\[[0-9a-f]*\]: (\S*) ([^\n\r]*)[\n\r]*$/) { + my ($handler, $data) = ($1, $2); + if (defined(${$handlers}{$handler})) { + call("handle_line", ${$handlers}{$handler}, $data); + } + } +} + +foreach my $key (keys(%{$handlers})) { + call("dump", ${$handlers}{$key}); +} +print "Summary:\n"; +foreach my $key (keys(%{$handlers})) { + call("summary", ${$handlers}{$key}); +} diff --git a/tools/lint/docs/Makefile b/tools/lint/docs/Makefile new file mode 100644 index 000000000..1d97fa065 --- /dev/null +++ b/tools/lint/docs/Makefile @@ -0,0 +1,192 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/mozlint.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/mozlint.qhc" + +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/mozlint" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/mozlint" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/tools/lint/docs/conf.py b/tools/lint/docs/conf.py new file mode 100644 index 000000000..31ebb6dcc --- /dev/null +++ b/tools/lint/docs/conf.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- +# +# mozlint documentation build configuration file, created by +# sphinx-quickstart on Fri Nov 27 17:38:49 2015. +# +# This file is execfile()d with the current directory set to its +# containing dir. + +import os + +# -- General configuration ------------------------------------------------ + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'mozlint' +copyright = u'2015, Andrew Halberstadt' +author = u'Andrew Halberstadt' + +# The short X.Y version. +version = '0.1.0' +# The full version, including alpha/beta/rc tags. +release = '0.1.0' + +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +if os.environ.get('READTHEDOCS', None) != 'True': + try: + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + except ImportError: + pass + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Output file base name for HTML help builder. +htmlhelp_basename = 'mozlintdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = {} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'mozlint.tex', u'mozlint Documentation', + u'Andrew Halberstadt', 'manual'), +] + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'mozlint', u'mozlint Documentation', + [author], 1) +] + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'mozlint', u'mozlint Documentation', + author, 'mozlint', 'One line description of project.', + 'Miscellaneous'), +] + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/': None} diff --git a/tools/lint/docs/create.rst b/tools/lint/docs/create.rst new file mode 100644 index 000000000..a132417a8 --- /dev/null +++ b/tools/lint/docs/create.rst @@ -0,0 +1,153 @@ +Adding a New Linter to the Tree +=============================== + +A linter is a python file with a ``.lint`` extension and a global dict called LINTER. Depending on how +complex it is, there may or may not be any actual python code alongside the LINTER definition. + +Here's a trivial example: + +no-eval.lint + +.. code-block:: python + + LINTER = { + 'name': 'EvalLinter', + 'description': "Ensures the string 'eval' doesn't show up." + 'include': "**/*.js", + 'type': 'string', + 'payload': 'eval', + } + +Now ``no-eval.lint`` gets passed into :func:`LintRoller.read`. + + +Linter Types +------------ + +There are three types of linters, though more may be added in the future. + +1. string - fails if substring is found +2. regex - fails if regex matches +3. external - fails if a python function returns a non-empty result list +4. structured_log - fails if a mozlog logger emits any lint_error or lint_warning log messages + +As seen from the example above, string and regex linters are very easy to create, but they +should be avoided if possible. It is much better to use a context aware linter for the language you +are trying to lint. For example, use eslint to lint JavaScript files, use flake8 to lint python +files, etc. + +Which brings us to the third and most interesting type of linter, +external. External linters call an arbitrary python function which is +responsible for not only running the linter, but ensuring the results +are structured properly. For example, an external type could shell out +to a 3rd party linter, collect the output and format it into a list of +:class:`ResultContainer` objects. The signature for this python +function is ``lint(files, **kwargs)``, where ``files`` is a list of +files to lint. + +Structured log linters are much like external linters, but suitable +for cases where the linter code is using mozlog and emits +``lint_error`` or ``lint_warning`` logging messages when the lint +fails. This is recommended for writing novel gecko-specific lints. In +this case the signature for lint functions is ``lint(files, logger, +**kwargs)``. + +LINTER Definition +----------------- + +Each ``.lint`` file must have a variable called LINTER which is a dict containing metadata about the +linter. Here are the supported keys: + +* name - The name of the linter (required) +* description - A brief description of the linter's purpose (required) +* type - One of 'string', 'regex' or 'external' (required) +* payload - The actual linting logic, depends on the type (required) +* include - A list of glob patterns that must be matched (optional) +* exclude - A list of glob patterns that must not be matched (optional) +* extensions - A list of file extensions to be considered (optional) +* setup - A function that sets up external dependencies (optional) + +In addition to the above, some ``.lint`` files correspond to a single lint rule. For these, the +following additional keys may be specified: + +* message - A string to print on infraction (optional) +* hint - A string with a clue on how to fix the infraction (optional) +* rule - An id string for the lint rule (optional) +* level - The severity of the infraction, either 'error' or 'warning' (optional) + +For structured_log lints the following additional keys apply: + +* logger - A StructuredLog object to use for logging. If not supplied + one will be created (optional) + +Example +------- + +Here is an example of an external linter that shells out to the python flake8 linter: + +.. code-block:: python + + import json + import os + import subprocess + from collections import defaultdict + + from mozlint import result + + + FLAKE8_NOT_FOUND = """ + Could not find flake8! Install flake8 and try again. + """.strip() + + + def lint(files, **lintargs): + import which + + binary = os.environ.get('FLAKE8') + if not binary: + try: + binary = which.which('flake8') + except which.WhichError: + print(FLAKE8_NOT_FOUND) + return 1 + + # Flake8 allows passing in a custom format string. We use + # this to help mold the default flake8 format into what + # mozlint's ResultContainer object expects. + cmdargs = [ + binary, + '--format', + '{"path":"%(path)s","lineno":%(row)s,"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}', + ] + files + + proc = subprocess.Popen(cmdargs, stdout=subprocess.PIPE, env=os.environ) + output = proc.communicate()[0] + + # all passed + if not output: + return [] + + results = [] + for line in output.splitlines(): + # res is a dict of the form specified by --format above + res = json.loads(line) + + # parse level out of the id string + if 'code' in res and res['code'].startswith('W'): + res['level'] = 'warning' + + # result.from_linter is a convenience method that + # creates a ResultContainer using a LINTER definition + # to populate some defaults. + results.append(result.from_linter(LINTER, **res)) + + return results + + + LINTER = { + 'name': "flake8", + 'description': "Python linter", + 'include': ['**/*.py'], + 'type': 'external', + 'payload': lint, + } diff --git a/tools/lint/docs/index.rst b/tools/lint/docs/index.rst new file mode 100644 index 000000000..54bc404a3 --- /dev/null +++ b/tools/lint/docs/index.rst @@ -0,0 +1,37 @@ +Linting +======= + +Linters are used in mozilla-central to help enforce coding style and avoid bad practices. Due to the +wide variety of languages in use and the varying style preferences per team, this is not an easy +task. In addition, linters should be runnable from editors, from the command line, from review tools +and from continuous integration. It's easy to see how the complexity of running all of these +different kinds of linters in all of these different places could quickly balloon out of control. + +``Mozlint`` is a library that accomplishes two goals: + +1. It provides a standard method for adding new linters to the tree, which can be as easy as + defining a json object in a ``.lint`` file. This helps keep lint related code localized, and + prevents different teams from coming up with their own unique lint implementations. +2. It provides a streamlined interface for running all linters at once. Instead of running N + different lint commands to test your patch, a single ``mach lint`` command will automatically run + all applicable linters. This means there is a single API surface that other tools can use to + invoke linters. + +``Mozlint`` isn't designed to be used directly by end users. Instead, it can be consumed by things +like mach, mozreview and taskcluster. + +.. toctree:: + :caption: Linting User Guide + :maxdepth: 2 + + usage + create + linters/eslint + linters/flake8 + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/tools/lint/docs/linters/eslint-plugin-mozilla.rst b/tools/lint/docs/linters/eslint-plugin-mozilla.rst new file mode 100644 index 000000000..e75864238 --- /dev/null +++ b/tools/lint/docs/linters/eslint-plugin-mozilla.rst @@ -0,0 +1,174 @@ +===================== +Mozilla ESLint Plugin +===================== + + +balanced-listeners +------------------ + +Checks that for every occurence of 'addEventListener' or 'on' there is an +occurence of 'removeEventListener' or 'off' with the same event name. + + +components-imports +------------------ + +Checks the filename of imported files e.g. ``Cu.import("some/path/Blah.jsm")`` +adds Blah to the global scope. + + +import-browserjs-globals +------------------------ + +When included files from the main browser UI scripts will be loaded and any +declared globals will be defined for the current file. This is mostly useful for +browser-chrome mochitests that call browser functions. + + +import-globals-from +------------------- + +Parses a file for globals defined in various unique Mozilla ways. + +When a "import-globals-from <path>" comment is found in a file, then all globals +from the file at <path> will be imported in the current scope. This will also +operate recursively. + +This is useful for scripts that are loaded as <script> tag in a window and rely +on each other's globals. + +If <path> is a relative path, then it must be relative to the file being +checked by the rule. + + +import-headjs-globals +--------------------- + +Import globals from head.js and from any files that were imported by +head.js (as far as we can correctly resolve the path). + +The following file import patterns are supported: + +- ``Services.scriptloader.loadSubScript(path)`` +- ``loader.loadSubScript(path)`` +- ``loadSubScript(path)`` +- ``loadHelperScript(path)`` +- ``import-globals-from path`` + +If path does not exist because it is generated e.g. +``testdir + "/somefile.js"`` we do our best to resolve it. + +The following patterns are supported: + +- ``Cu.import("resource://devtools/client/shared/widgets/ViewHelpers.jsm");`` +- ``loader.lazyImporter(this, "name1");`` +- ``loader.lazyRequireGetter(this, "name2"`` +- ``loader.lazyServiceGetter(this, "name3"`` +- ``XPCOMUtils.defineLazyModuleGetter(this, "setNamedTimeout", ...)`` +- ``loader.lazyGetter(this, "toolboxStrings"`` +- ``XPCOMUtils.defineLazyGetter(this, "clipboardHelper"`` + + +mark-test-function-used +----------------------- + +Simply marks `test` (the test method) or `run_test` as used when in mochitests +or xpcshell tests respectively. This avoids ESLint telling us that the function +is never called. + + +no-aArgs +-------- + +Checks that function argument names don't start with lowercase 'a' followed by +a capital letter. This is to prevent the use of Hungarian notation whereby the +first letter is a prefix that indicates the type or intended use of a variable. + + +no-cpows-in-tests +----------------- + +This rule checks if the file is a browser mochitest and, if so, checks for +possible CPOW usage by checking for the following strings: + +- "gBrowser.contentWindow" +- "gBrowser.contentDocument" +- "gBrowser.selectedBrowser.contentWindow" +- "browser.contentDocument" +- "window.content" +- "content" +- "content." + +Note: These are string matches so we will miss situations where the parent +object is assigned to another variable e.g.:: + + var b = gBrowser; + b.content // Would not be detected as a CPOW. + + +no-single-arg-cu-import +----------------------- + +Rejects calls to "Cu.import" that do not supply a second argument (meaning they +add the exported properties into global scope). + + +reject-importGlobalProperties +----------------------------- + +Rejects calls to ``Cu.importGlobalProperties``. Use of this function is +undesirable in some parts of the tree. + + +reject-some-requires +-------------------- + +This takes an option, a regular expression. Invocations of +``require`` with a string literal argument are matched against this +regexp; and if it matches, the ``require`` use is flagged. + + +this-top-level-scope +-------------------- + +Treats top-level assignments like ``this.mumble = value`` as declaring a global. + +Note: These are string matches so we will miss situations where the parent +object is assigned to another variable e.g.:: + + var b = gBrowser; + b.content // Would not be detected as a CPOW. + + +var-only-at-top-level +--------------------- + +Marks all var declarations that are not at the top level invalid. + + +Example +======= + ++-------+-----------------------+ +| Possible values for all rules | ++-------+-----------------------+ +| Value | Meaning | ++-------+-----------------------+ +| 0 | Deactivated | ++-------+-----------------------+ +| 1 | Warning | ++-------+-----------------------+ +| 2 | Error | ++-------+-----------------------+ + +Example configuration:: + + "rules": { + "mozilla/balanced-listeners": 2, + "mozilla/components-imports": 1, + "mozilla/import-globals-from": 1, + "mozilla/import-headjs-globals": 1, + "mozilla/mark-test-function-used": 1, + "mozilla/var-only-at-top-level": 1, + "mozilla/no-cpows-in-tests": 1, + } diff --git a/tools/lint/docs/linters/eslint.rst b/tools/lint/docs/linters/eslint.rst new file mode 100644 index 000000000..31f7f8121 --- /dev/null +++ b/tools/lint/docs/linters/eslint.rst @@ -0,0 +1,45 @@ +ESLint +====== + +`ESLint`_ is a popular linter for JavaScript. + +Run Locally +----------- + +The mozlint integration of `ESLint`_ can be run using mach: + +.. parsed-literal:: + + $ mach lint --linter eslint <file paths> + +Alternatively, omit the ``--linter eslint`` and run all configured linters, which will include +ESLint. + + +Configuration +------------- + +The `ESLint`_ mozilla-central integration uses a blacklist to exclude certain directories from being +linted. This lives in ``topsrcdir/.eslintignore``. If you don't wish your directory to be linted, it +must be added here. + +The global configuration file lives in ``topsrcdir/.eslintrc``. This global configuration can be +overridden by including an ``.eslintrc`` in the appropriate subdirectory. For an overview of the +supported configuration, see `ESLint's documentation`_. + + +ESLint Plugin Mozilla +--------------------- + +In addition to default ESLint rules, there are several Mozilla-specific rules that are defined in +the :doc:`Mozilla ESLint Plugin <eslint-plugin-mozilla>`. + + +.. _ESLint: http://eslint.org/ +.. _ESLint's documentation: http://eslint.org/docs/user-guide/configuring + + +.. toctree:: + :hidden: + + eslint-plugin-mozilla diff --git a/tools/lint/docs/linters/flake8.rst b/tools/lint/docs/linters/flake8.rst new file mode 100644 index 000000000..5ef17d41d --- /dev/null +++ b/tools/lint/docs/linters/flake8.rst @@ -0,0 +1,50 @@ +Flake8 +====== + +`Flake8`_ is a popular lint wrapper for python. Under the hood, it runs three other tools and +combines their results: + +* `pep8`_ for checking style +* `pyflakes`_ for checking syntax +* `mccabe`_ for checking complexity + + +Run Locally +----------- + +The mozlint integration of flake8 can be run using mach: + +.. parsed-literal:: + + $ mach lint --linter flake8 <file paths> + +Alternatively, omit the ``--linter flake8`` and run all configured linters, which will include +flake8. + + +Configuration +------------- + +Only directories explicitly whitelisted will have flake8 run against them. To enable flake8 linting +in a source directory, it must be added to the ``include`` directive in ```tools/lint/flake8.lint``. +If you wish to exclude a subdirectory of an included one, you can add it to the ``exclude`` +directive. + +The default configuration file lives in ``topsrcdir/.flake8``. The default configuration can be +overriden for a given subdirectory by creating a new ``.flake8`` file in the subdirectory. Be warned +that ``.flake8`` files cannot inherit from one another, so all configuration you wish to keep must +be re-defined. + +.. warning:: + + Only ``.flake8`` files that live in a directory that is explicitly included in the ``include`` + directive will be considered. See `bug 1277851`_ for more details. + +For an overview of the supported configuration, see `flake8's documentation`_. + +.. _Flake8: https://flake8.readthedocs.io/en/latest/ +.. _pep8: http://pep8.readthedocs.io/en/latest/ +.. _pyflakes: https://github.com/pyflakes/pyflakes +.. _mccabe: https://github.com/pycqa/mccabe +.. _bug 1277851: https://bugzilla.mozilla.org/show_bug.cgi?id=1277851 +.. _flake8's documentation: https://flake8.readthedocs.io/en/latest/config.html diff --git a/tools/lint/docs/make.bat b/tools/lint/docs/make.bat new file mode 100644 index 000000000..be5489633 --- /dev/null +++ b/tools/lint/docs/make.bat @@ -0,0 +1,263 @@ +@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+ set SPHINXBUILD=sphinx-build
+)
+set BUILDDIR=_build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
+set I18NSPHINXOPTS=%SPHINXOPTS% .
+if NOT "%PAPER%" == "" (
+ set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+ set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+ :help
+ echo.Please use `make ^<target^>` where ^<target^> is one of
+ echo. html to make standalone HTML files
+ echo. dirhtml to make HTML files named index.html in directories
+ echo. singlehtml to make a single large HTML file
+ echo. pickle to make pickle files
+ echo. json to make JSON files
+ echo. htmlhelp to make HTML files and a HTML help project
+ echo. qthelp to make HTML files and a qthelp project
+ echo. devhelp to make HTML files and a Devhelp project
+ echo. epub to make an epub
+ echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+ echo. text to make text files
+ echo. man to make manual pages
+ echo. texinfo to make Texinfo files
+ echo. gettext to make PO message catalogs
+ echo. changes to make an overview over all changed/added/deprecated items
+ echo. xml to make Docutils-native XML files
+ echo. pseudoxml to make pseudoxml-XML files for display purposes
+ echo. linkcheck to check all external links for integrity
+ echo. doctest to run all doctests embedded in the documentation if enabled
+ echo. coverage to run coverage check of the documentation if enabled
+ goto end
+)
+
+if "%1" == "clean" (
+ for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+ del /q /s %BUILDDIR%\*
+ goto end
+)
+
+
+REM Check if sphinx-build is available and fallback to Python version if any
+%SPHINXBUILD% 2> nul
+if errorlevel 9009 goto sphinx_python
+goto sphinx_ok
+
+:sphinx_python
+
+set SPHINXBUILD=python -m sphinx.__init__
+%SPHINXBUILD% 2> nul
+if errorlevel 9009 (
+ echo.
+ echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+ echo.installed, then set the SPHINXBUILD environment variable to point
+ echo.to the full path of the 'sphinx-build' executable. Alternatively you
+ echo.may add the Sphinx directory to PATH.
+ echo.
+ echo.If you don't have Sphinx installed, grab it from
+ echo.http://sphinx-doc.org/
+ exit /b 1
+)
+
+:sphinx_ok
+
+
+if "%1" == "html" (
+ %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+ goto end
+)
+
+if "%1" == "dirhtml" (
+ %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+ goto end
+)
+
+if "%1" == "singlehtml" (
+ %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
+ goto end
+)
+
+if "%1" == "pickle" (
+ %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the pickle files.
+ goto end
+)
+
+if "%1" == "json" (
+ %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the JSON files.
+ goto end
+)
+
+if "%1" == "htmlhelp" (
+ %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+ goto end
+)
+
+if "%1" == "qthelp" (
+ %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+ echo.^> qcollectiongenerator %BUILDDIR%\qthelp\mozlint.qhcp
+ echo.To view the help file:
+ echo.^> assistant -collectionFile %BUILDDIR%\qthelp\mozlint.ghc
+ goto end
+)
+
+if "%1" == "devhelp" (
+ %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished.
+ goto end
+)
+
+if "%1" == "epub" (
+ %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The epub file is in %BUILDDIR%/epub.
+ goto end
+)
+
+if "%1" == "latex" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "latexpdf" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ cd %BUILDDIR%/latex
+ make all-pdf
+ cd %~dp0
+ echo.
+ echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "latexpdfja" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ cd %BUILDDIR%/latex
+ make all-pdf-ja
+ cd %~dp0
+ echo.
+ echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "text" (
+ %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The text files are in %BUILDDIR%/text.
+ goto end
+)
+
+if "%1" == "man" (
+ %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The manual pages are in %BUILDDIR%/man.
+ goto end
+)
+
+if "%1" == "texinfo" (
+ %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
+ goto end
+)
+
+if "%1" == "gettext" (
+ %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
+ goto end
+)
+
+if "%1" == "changes" (
+ %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.The overview file is in %BUILDDIR%/changes.
+ goto end
+)
+
+if "%1" == "linkcheck" (
+ %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+ goto end
+)
+
+if "%1" == "doctest" (
+ %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+ goto end
+)
+
+if "%1" == "coverage" (
+ %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Testing of coverage in the sources finished, look at the ^
+results in %BUILDDIR%/coverage/python.txt.
+ goto end
+)
+
+if "%1" == "xml" (
+ %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The XML files are in %BUILDDIR%/xml.
+ goto end
+)
+
+if "%1" == "pseudoxml" (
+ %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
+ goto end
+)
+
+:end
diff --git a/tools/lint/docs/usage.rst b/tools/lint/docs/usage.rst new file mode 100644 index 000000000..0eb5f3d22 --- /dev/null +++ b/tools/lint/docs/usage.rst @@ -0,0 +1,41 @@ +Running Linters Locally +======================= + +You can run all the various linters in the tree using the ``mach lint`` command. Simply pass in the +directory or file you wish to lint (defaults to current working directory): + +.. parsed-literal:: + + ./mach lint path/to/files + +Multiple paths are allowed: + +.. parsed-literal:: + + ./mach lint path/to/foo.js path/to/bar.py path/to/dir + +``Mozlint`` will automatically determine which types of files exist, and which linters need to be run +against them. For example, if the directory contains both JavaScript and Python files then mozlint +will automatically run both ESLint and Flake8 against those files respectively. + +To restrict which linters are invoked manually, pass in ``-l/--linter``: + +.. parsed-literal:: + + ./mach lint -l eslint path/to/files + +Finally, ``mozlint`` can lint the files touched by a set of revisions or the working directory using +the ``-r/--rev`` and ``-w/--workdir`` arguments respectively. These work both with mercurial and +git. In the case of ``--rev`` the value is passed directly to the underlying vcs, so normal revision +specifiers will work. For example, say we want to lint all files touched by the last three commits. +In mercurial, this would be: + +.. parsed-literal:: + + ./mach lint -r ".~2::." + +In git, this would be: + +.. parsed-literal:: + + ./mach lint -r "HEAD~2 HEAD" diff --git a/tools/lint/eslint.lint b/tools/lint/eslint.lint new file mode 100644 index 000000000..a2d864f75 --- /dev/null +++ b/tools/lint/eslint.lint @@ -0,0 +1,367 @@ +# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import platform +import re +import signal +import subprocess +import sys +from distutils.version import LooseVersion + +import which +from mozprocess import ProcessHandler + +from mozlint import result + +ESLINT_ERROR_MESSAGE = """ +An error occurred running eslint. Please check the following error messages: + +{} +""".strip() + +ESLINT_NOT_FOUND_MESSAGE = """ +Could not find eslint! We looked at the --binary option, at the ESLINT +environment variable, and then at your local node_modules path. Please Install +eslint and needed plugins with: + +mach eslint --setup + +and try again. +""".strip() + +NODE_NOT_FOUND_MESSAGE = """ +nodejs v4.2.3 is either not installed or is installed to a non-standard path. +Please install nodejs from https://nodejs.org and try again. + +Valid installation paths: +""".strip() + +NPM_NOT_FOUND_MESSAGE = """ +Node Package Manager (npm) is either not installed or installed to a +non-standard path. Please install npm from https://nodejs.org (it comes as an +option in the node installation) and try again. + +Valid installation paths: +""".strip() + + +VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$") +CARET_VERSION_RANGE_RE = re.compile(r"^\^((\d+)\.\d+\.\d+)$") + +EXTENSIONS = ['.js', '.jsm', '.jsx', '.xml', '.html', '.xhtml'] + +project_root = None + + +def eslint_setup(): + """Ensure eslint is optimally configured. + + This command will inspect your eslint configuration and + guide you through an interactive wizard helping you configure + eslint for optimal use on Mozilla projects. + """ + orig_cwd = os.getcwd() + sys.path.append(os.path.dirname(__file__)) + + module_path = get_eslint_module_path() + + # npm sometimes fails to respect cwd when it is run using check_call so + # we manually switch folders here instead. + os.chdir(module_path) + + npm_path = get_node_or_npm_path("npm") + if not npm_path: + return 1 + + # Install ESLint and external plugins + cmd = [npm_path, "install"] + print("Installing eslint for mach using \"%s\"..." % (" ".join(cmd))) + if not call_process("eslint", cmd): + return 1 + + # Install in-tree ESLint plugin + cmd = [npm_path, "install", + os.path.join(module_path, "eslint-plugin-mozilla")] + print("Installing eslint-plugin-mozilla using \"%s\"..." % (" ".join(cmd))) + if not call_process("eslint-plugin-mozilla", cmd): + return 1 + + eslint_path = os.path.join(module_path, "node_modules", ".bin", "eslint") + + print("\nESLint and approved plugins installed successfully!") + print("\nNOTE: Your local eslint binary is at %s\n" % eslint_path) + + os.chdir(orig_cwd) + + +def call_process(name, cmd, cwd=None): + try: + with open(os.devnull, "w") as fnull: + subprocess.check_call(cmd, cwd=cwd, stdout=fnull) + except subprocess.CalledProcessError: + if cwd: + print("\nError installing %s in the %s folder, aborting." % (name, cwd)) + else: + print("\nError installing %s, aborting." % name) + + return False + + return True + + +def expected_eslint_modules(): + # Read the expected version of ESLint and external modules + expected_modules_path = os.path.join(get_eslint_module_path(), "package.json") + with open(expected_modules_path, "r") as f: + expected_modules = json.load(f)["dependencies"] + + # Also read the in-tree ESLint plugin version + mozilla_json_path = os.path.join(get_eslint_module_path(), + "eslint-plugin-mozilla", "package.json") + with open(mozilla_json_path, "r") as f: + expected_modules["eslint-plugin-mozilla"] = json.load(f)["version"] + + return expected_modules + + +def eslint_module_has_issues(): + has_issues = False + node_modules_path = os.path.join(get_eslint_module_path(), "node_modules") + + for name, version_range in expected_eslint_modules().iteritems(): + path = os.path.join(node_modules_path, name, "package.json") + + if not os.path.exists(path): + print("%s v%s needs to be installed locally." % (name, version_range)) + has_issues = True + continue + + data = json.load(open(path)) + + if not version_in_range(data["version"], version_range): + print("%s v%s should be v%s." % (name, data["version"], version_range)) + has_issues = True + + return has_issues + + +def version_in_range(version, version_range): + """ + Check if a module version is inside a version range. Only supports explicit versions and + caret ranges for the moment, since that's all we've used so far. + """ + if version == version_range: + return True + + version_match = VERSION_RE.match(version) + if not version_match: + raise RuntimeError("mach eslint doesn't understand module version %s" % version) + version = LooseVersion(version) + + # Caret ranges as specified by npm allow changes that do not modify the left-most non-zero + # digit in the [major, minor, patch] tuple. The code below assumes the major digit is + # non-zero. + range_match = CARET_VERSION_RANGE_RE.match(version_range) + if range_match: + range_version = range_match.group(1) + range_major = int(range_match.group(2)) + + range_min = LooseVersion(range_version) + range_max = LooseVersion("%d.0.0" % (range_major + 1)) + + return range_min <= version < range_max + + return False + + +def get_possible_node_paths_win(): + """ + Return possible nodejs paths on Windows. + """ + if platform.system() != "Windows": + return [] + + return list({ + "%s\\nodejs" % os.environ.get("SystemDrive"), + os.path.join(os.environ.get("ProgramFiles"), "nodejs"), + os.path.join(os.environ.get("PROGRAMW6432"), "nodejs"), + os.path.join(os.environ.get("PROGRAMFILES"), "nodejs") + }) + + +def get_node_or_npm_path(filename, minversion=None): + """ + Return the nodejs or npm path. + """ + if platform.system() == "Windows": + for ext in [".cmd", ".exe", ""]: + try: + node_or_npm_path = which.which(filename + ext, + path=get_possible_node_paths_win()) + if is_valid(node_or_npm_path, minversion): + return node_or_npm_path + except which.WhichError: + pass + else: + try: + node_or_npm_path = which.which(filename) + if is_valid(node_or_npm_path, minversion): + return node_or_npm_path + except which.WhichError: + pass + + if filename == "node": + print(NODE_NOT_FOUND_MESSAGE) + elif filename == "npm": + print(NPM_NOT_FOUND_MESSAGE) + + if platform.system() == "Windows": + app_paths = get_possible_node_paths_win() + + for p in app_paths: + print(" - %s" % p) + elif platform.system() == "Darwin": + print(" - /usr/local/bin/node") + elif platform.system() == "Linux": + print(" - /usr/bin/nodejs") + + return None + + +def is_valid(path, minversion=None): + try: + version_str = subprocess.check_output([path, "--version"], + stderr=subprocess.STDOUT) + if minversion: + # nodejs prefixes its version strings with "v" + version = LooseVersion(version_str.lstrip('v')) + return version >= minversion + return True + except (subprocess.CalledProcessError, OSError): + return False + + +def get_project_root(): + global project_root + return project_root + + +def get_eslint_module_path(): + return os.path.join(get_project_root(), "tools", "lint", "eslint") + + +def lint(paths, binary=None, fix=None, setup=None, **lintargs): + """Run eslint.""" + global project_root + project_root = lintargs['root'] + + module_path = get_eslint_module_path() + + # eslint requires at least node 4.2.3 + node_path = get_node_or_npm_path("node", LooseVersion("4.2.3")) + if not node_path: + return 1 + + if setup: + return eslint_setup() + + npm_path = get_node_or_npm_path("npm") + if not npm_path: + return 1 + + if eslint_module_has_issues(): + eslint_setup() + + # Valid binaries are: + # - Any provided by the binary argument. + # - Any pointed at by the ESLINT environmental variable. + # - Those provided by mach eslint --setup. + # + # eslint --setup installs some mozilla specific plugins and installs + # all node modules locally. This is the preferred method of + # installation. + + if not binary: + binary = os.environ.get('ESLINT', None) + + if not binary: + binary = os.path.join(module_path, "node_modules", ".bin", "eslint") + if not os.path.isfile(binary): + binary = None + + if not binary: + print(ESLINT_NOT_FOUND_MESSAGE) + return 1 + + extra_args = lintargs.get('extra_args') or [] + cmd_args = [binary, + # Enable the HTML plugin. + # We can't currently enable this in the global config file + # because it has bad interactions with the SublimeText + # ESLint plugin (bug 1229874). + '--plugin', 'html', + # This keeps ext as a single argument. + '--ext', '[{}]'.format(','.join(EXTENSIONS)), + '--format', 'json', + ] + extra_args + paths + + # eslint requires that --fix be set before the --ext argument. + if fix: + cmd_args.insert(1, '--fix') + + shell = False + if os.environ.get('MSYSTEM') in ('MINGW32', 'MINGW64'): + # The eslint binary needs to be run from a shell with msys + shell = True + + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + proc = ProcessHandler(cmd_args, env=os.environ, stream=None, shell=shell) + proc.run() + signal.signal(signal.SIGINT, orig) + + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + return [] + + if not proc.output: + return [] # no output means success + + try: + jsonresult = json.loads(proc.output[0]) + except ValueError: + print(ESLINT_ERROR_MESSAGE.format("\n".join(proc.output))) + return 1 + + results = [] + for obj in jsonresult: + errors = obj['messages'] + + for err in errors: + err.update({ + 'hint': err.get('fix'), + 'level': 'error' if err['severity'] == 2 else 'warning', + 'lineno': err.get('line'), + 'path': obj['filePath'], + 'rule': err.get('ruleId'), + }) + results.append(result.from_linter(LINTER, **err)) + + return results + + +LINTER = { + 'name': "eslint", + 'description': "JavaScript linter", + # ESLint infra handles its own path filtering, so just include cwd + 'include': ['.'], + 'exclude': [], + 'extensions': EXTENSIONS, + 'type': 'external', + 'payload': lint, +} diff --git a/tools/lint/eslint/eslint-plugin-mozilla/LICENSE b/tools/lint/eslint/eslint-plugin-mozilla/LICENSE new file mode 100644 index 000000000..e87a115e4 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/LICENSE @@ -0,0 +1,363 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/globals.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/globals.js new file mode 100644 index 000000000..acbfa0684 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/globals.js @@ -0,0 +1,188 @@ +/** + * @fileoverview functions for scanning an AST for globals including + * traversing referenced scripts. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const path = require("path"); +const fs = require("fs"); +const helpers = require("./helpers"); +const escope = require("escope"); +const estraverse = require("estraverse"); + +/** + * Parses a list of "name:boolean_value" or/and "name" options divided by comma or + * whitespace. + * + * This function was copied from eslint.js + * + * @param {string} string The string to parse. + * @param {Comment} comment The comment node which has the string. + * @returns {Object} Result map object of names and boolean values + */ +function parseBooleanConfig(string, comment) { + let items = {}; + + // Collapse whitespace around : to make parsing easier + string = string.replace(/\s*:\s*/g, ":"); + // Collapse whitespace around , + string = string.replace(/\s*,\s*/g, ","); + + string.split(/\s|,+/).forEach(function(name) { + if (!name) { + return; + } + + let pos = name.indexOf(":"); + let value = undefined; + if (pos !== -1) { + value = name.substring(pos + 1, name.length); + name = name.substring(0, pos); + } + + items[name] = { + value: (value === "true"), + comment: comment + }; + }); + + return items; +} + +/** + * Global discovery can require parsing many files. This map of + * {String} => {Object} caches what globals were discovered for a file path. + */ +const globalCache = new Map(); + +/** + * An object that returns found globals for given AST node types. Each prototype + * property should be named for a node type and accepts a node parameter and a + * parents parameter which is a list of the parent nodes of the current node. + * Each returns an array of globals found. + * + * @param {String} path + * The absolute path of the file being parsed. + */ +function GlobalsForNode(path) { + this.path = path; + this.root = helpers.getRootDir(path); +} + +GlobalsForNode.prototype = { + BlockComment(node, parents) { + let value = node.value.trim(); + let match = /^import-globals-from\s+(.+)$/.exec(value); + if (!match) { + return []; + } + + let filePath = match[1].trim(); + + if (!path.isAbsolute(filePath)) { + let dirName = path.dirname(this.path); + filePath = path.resolve(dirName, filePath); + } + + return module.exports.getGlobalsForFile(filePath); + }, + + ExpressionStatement(node, parents) { + let isGlobal = helpers.getIsGlobalScope(parents); + let names = helpers.convertExpressionToGlobals(node, isGlobal, this.root); + return names.map(name => { return { name, writable: true }}); + }, +}; + +module.exports = { + /** + * Returns all globals for a given file. Recursively searches through + * import-globals-from directives and also includes globals defined by + * standard eslint directives. + * + * @param {String} path + * The absolute path of the file to be parsed. + */ + getGlobalsForFile(path) { + if (globalCache.has(path)) { + return globalCache.get(path); + } + + let content = fs.readFileSync(path, "utf8"); + + // Parse the content into an AST + let ast = helpers.getAST(content); + + // Discover global declarations + let scopeManager = escope.analyze(ast); + let globalScope = scopeManager.acquire(ast); + + let globals = Object.keys(globalScope.variables).map(v => ({ + name: globalScope.variables[v].name, + writable: true, + })); + + // Walk over the AST to find any of our custom globals + let handler = new GlobalsForNode(path); + + helpers.walkAST(ast, (type, node, parents) => { + // We have to discover any globals that ESLint would have defined through + // comment directives + if (type == "BlockComment") { + let value = node.value.trim(); + let match = /^globals?\s+(.+)$/.exec(value); + if (match) { + let values = parseBooleanConfig(match[1].trim(), node); + for (let name of Object.keys(values)) { + globals.push({ + name, + writable: values[name].value + }) + } + } + } + + if (type in handler) { + let newGlobals = handler[type](node, parents); + globals.push.apply(globals, newGlobals); + } + }); + + globalCache.set(path, globals); + + return globals; + }, + + /** + * Intended to be used as-is for an ESLint rule that parses for globals in + * the current file and recurses through import-globals-from directives. + * + * @param {Object} context + * The ESLint parsing context. + */ + getESLintGlobalParser(context) { + let globalScope; + + let parser = { + Program(node) { + globalScope = context.getScope(); + } + }; + + // Install thin wrappers around GlobalsForNode + let handler = new GlobalsForNode(helpers.getAbsoluteFilePath(context)); + + for (let type of Object.keys(GlobalsForNode.prototype)) { + parser[type] = function(node) { + let globals = handler[type](node, context.getAncestors()); + helpers.addGlobals(globals, globalScope); + } + } + + return parser; + } +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/helpers.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/helpers.js new file mode 100644 index 000000000..50e00ab97 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/helpers.js @@ -0,0 +1,524 @@ +/** + * @fileoverview A collection of helper functions. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +"use strict"; + +var escope = require("escope"); +var espree = require("espree"); +var estraverse = require("estraverse"); +var path = require("path"); +var fs = require("fs"); +var ini = require("ini-parser"); + +var modules = null; +var directoryManifests = new Map(); + +var definitions = [ + /^loader\.lazyGetter\(this, "(\w+)"/, + /^loader\.lazyImporter\(this, "(\w+)"/, + /^loader\.lazyServiceGetter\(this, "(\w+)"/, + /^loader\.lazyRequireGetter\(this, "(\w+)"/, + /^XPCOMUtils\.defineLazyGetter\(this, "(\w+)"/, + /^XPCOMUtils\.defineLazyModuleGetter\(this, "(\w+)"/, + /^XPCOMUtils\.defineLazyServiceGetter\(this, "(\w+)"/, + /^XPCOMUtils\.defineConstant\(this, "(\w+)"/, + /^DevToolsUtils\.defineLazyModuleGetter\(this, "(\w+)"/, + /^DevToolsUtils\.defineLazyGetter\(this, "(\w+)"/, + /^Object\.defineProperty\(this, "(\w+)"/, + /^Reflect\.defineProperty\(this, "(\w+)"/, + /^this\.__defineGetter__\("(\w+)"/, + /^this\.(\w+) =/ +]; + +var imports = [ + /^(?:Cu|Components\.utils)\.import\(".*\/((.*?)\.jsm?)"(?:, this)?\)/, +]; + +module.exports = { + /** + * Gets the abstract syntax tree (AST) of the JavaScript source code contained + * in sourceText. + * + * @param {String} sourceText + * Text containing valid JavaScript. + * + * @return {Object} + * The resulting AST. + */ + getAST: function(sourceText) { + // Use a permissive config file to allow parsing of anything that Espree + // can parse. + var config = this.getPermissiveConfig(); + + return espree.parse(sourceText, config); + }, + + /** + * A simplistic conversion of some AST nodes to a standard string form. + * + * @param {Object} node + * The AST node to convert. + * + * @return {String} + * The JS source for the node. + */ + getASTSource: function(node) { + switch (node.type) { + case "MemberExpression": + if (node.computed) + throw new Error("getASTSource unsupported computed MemberExpression"); + return this.getASTSource(node.object) + "." + this.getASTSource(node.property); + case "ThisExpression": + return "this"; + case "Identifier": + return node.name; + case "Literal": + return JSON.stringify(node.value); + case "CallExpression": + var args = node.arguments.map(a => this.getASTSource(a)).join(", "); + return this.getASTSource(node.callee) + "(" + args + ")"; + case "ObjectExpression": + return "{}"; + case "ExpressionStatement": + return this.getASTSource(node.expression) + ";"; + case "FunctionExpression": + return "function() {}"; + case "ArrowFunctionExpression": + return "() => {}"; + case "AssignmentExpression": + return this.getASTSource(node.left) + " = " + this.getASTSource(node.right); + default: + throw new Error("getASTSource unsupported node type: " + node.type); + } + }, + + /** + * This walks an AST in a manner similar to ESLint passing node and comment + * events to the listener. The listener is expected to be a simple function + * which accepts node type, node and parents arguments. + * + * @param {Object} ast + * The AST to walk. + * @param {Function} listener + * A callback function to call for the nodes. Passed three arguments, + * event type, node and an array of parent nodes for the current node. + */ + walkAST(ast, listener) { + let parents = []; + + let seenComments = new Set(); + function sendCommentEvents(comments) { + if (!comments) { + return; + } + + for (let comment of comments) { + if (seenComments.has(comment)) { + return; + } + seenComments.add(comment); + + listener(comment.type + "Comment", comment, parents); + } + } + + estraverse.traverse(ast, { + enter(node, parent) { + // Comments are held in node.comments for empty programs + let leadingComments = node.leadingComments; + if (node.type === "Program" && node.body.length == 0) { + leadingComments = node.comments; + } + + sendCommentEvents(leadingComments); + listener(node.type, node, parents); + sendCommentEvents(node.trailingComments); + + parents.push(node); + }, + + leave(node, parent) { + // TODO send comment exit events + listener(node.type + ":exit", node, parents); + + if (parents.length == 0) { + throw new Error("Left more nodes than entered."); + } + parents.pop(); + } + }); + if (parents.length) { + throw new Error("Entered more nodes than left."); + } + }, + + /** + * Attempts to convert an ExpressionStatement to likely global variable + * definitions. + * + * @param {Object} node + * The AST node to convert. + * @param {boolean} isGlobal + * True if the current node is in the global scope. + * @param {String} repository + * The root of the repository. + * + * @return {Array} + * An array of variable names defined. + */ + convertExpressionToGlobals: function(node, isGlobal, repository) { + if (!modules) { + modules = require(path.join(repository, "tools", "lint", "eslint", "modules.json")); + } + + try { + var source = this.getASTSource(node); + } + catch (e) { + return []; + } + + for (var reg of definitions) { + var match = source.match(reg); + if (match) { + // Must be in the global scope + if (!isGlobal) { + return []; + } + + return [match[1]]; + } + } + + for (reg of imports) { + var match = source.match(reg); + if (match) { + // The two argument form is only acceptable in the global scope + if (node.expression.arguments.length > 1 && !isGlobal) { + return []; + } + + if (match[1] in modules) { + return modules[match[1]]; + } + + return [match[2]]; + } + } + + return []; + }, + + /** + * Add a variable to the current scope. + * HACK: This relies on eslint internals so it could break at any time. + * + * @param {String} name + * The variable name to add to the scope. + * @param {ASTScope} scope + * The scope to add to. + * @param {boolean} writable + * Whether the global can be overwritten. + */ + addVarToScope: function(name, scope, writable) { + scope.__defineGeneric(name, scope.set, scope.variables, null, null); + + let variable = scope.set.get(name); + variable.eslintExplicitGlobal = false; + variable.writeable = writable; + + // Walk to the global scope which holds all undeclared variables. + while (scope.type != "global") { + scope = scope.upper; + } + + // "through" contains all references with no found definition. + scope.through = scope.through.filter(function(reference) { + if (reference.identifier.name != name) { + return true; + } + + // Links the variable and the reference. + // And this reference is removed from `Scope#through`. + reference.resolved = variable; + variable.references.push(reference); + return false; + }); + }, + + /** + * Adds a set of globals to a scope. + * + * @param {Array} globalVars + * An array of global variable names. + * @param {ASTScope} scope + * The scope. + */ + addGlobals: function(globalVars, scope) { + globalVars.forEach(v => this.addVarToScope(v.name, scope, v.writable)); + }, + + /** + * To allow espree to parse almost any JavaScript we need as many features as + * possible turned on. This method returns that config. + * + * @return {Object} + * Espree compatible permissive config. + */ + getPermissiveConfig: function() { + return { + range: true, + loc: true, + comment: true, + attachComment: true, + ecmaVersion: 8, + sourceType: "script", + ecmaFeatures: { + experimentalObjectRestSpread: true, + globalReturn: true, + } + }; + }, + + /** + * Check whether the context is the global scope. + * + * @param {Array} ancestors + * The parents of the current node. + * + * @return {Boolean} + * True or false + */ + getIsGlobalScope: function(ancestors) { + for (let parent of ancestors) { + if (parent.type == "FunctionExpression" || + parent.type == "FunctionDeclaration") { + return false; + } + } + return true; + }, + + /** + * Check whether we might be in a test head file. + * + * @param {RuleContext} scope + * You should pass this from within a rule + * e.g. helpers.getIsHeadFile(this) + * + * @return {Boolean} + * True or false + */ + getIsHeadFile: function(scope) { + var pathAndFilename = this.cleanUpPath(scope.getFilename()); + + return /.*[\\/]head(_.+)?\.js$/.test(pathAndFilename); + }, + + /** + * Gets the head files for a potential test file + * + * @param {RuleContext} scope + * You should pass this from within a rule + * e.g. helpers.getIsHeadFile(this) + * + * @return {String[]} + * Paths to head files to load for the test + */ + getTestHeadFiles: function(scope) { + if (!this.getIsTest(scope)) { + return []; + } + + let filepath = this.cleanUpPath(scope.getFilename()); + let dir = path.dirname(filepath); + + let names = fs.readdirSync(dir) + .filter(name => name.startsWith("head") && name.endsWith(".js")) + .map(name => path.join(dir, name)); + return names; + }, + + /** + * Gets all the test manifest data for a directory + * + * @param {String} dir + * The directory + * + * @return {Array} + * An array of objects with file and manifest properties + */ + getManifestsForDirectory: function(dir) { + if (directoryManifests.has(dir)) { + return directoryManifests.get(dir); + } + + let manifests = []; + + let names = fs.readdirSync(dir); + for (let name of names) { + if (!name.endsWith(".ini")) { + continue; + } + + try { + let manifest = ini.parse(fs.readFileSync(path.join(dir, name), 'utf8')); + + manifests.push({ + file: path.join(dir, name), + manifest + }) + } catch (e) { + } + } + + directoryManifests.set(dir, manifests); + return manifests; + }, + + /** + * Gets the manifest file a test is listed in + * + * @param {RuleContext} scope + * You should pass this from within a rule + * e.g. helpers.getIsHeadFile(this) + * + * @return {String} + * The path to the test manifest file + */ + getTestManifest: function(scope) { + let filepath = this.cleanUpPath(scope.getFilename()); + + let dir = path.dirname(filepath); + let filename = path.basename(filepath); + + for (let manifest of this.getManifestsForDirectory(dir)) { + if (filename in manifest.manifest) { + return manifest.file; + } + } + + return null; + }, + + /** + * Check whether we are in a test of some kind. + * + * @param {RuleContext} scope + * You should pass this from within a rule + * e.g. helpers.getIsTest(this) + * + * @return {Boolean} + * True or false + */ + getIsTest: function(scope) { + // Regardless of the manifest name being in a manifest means we're a test. + let manifest = this.getTestManifest(scope); + if (manifest) { + return true; + } + + return !!this.getTestType(scope); + }, + + /** + * Gets the type of test or null if this isn't a test. + * + * @param {RuleContext} scope + * You should pass this from within a rule + * e.g. helpers.getIsHeadFile(this) + * + * @return {String or null} + * Test type: xpcshell, browser, chrome, mochitest + */ + getTestType: function(scope) { + let manifest = this.getTestManifest(scope); + if (manifest) { + let name = path.basename(manifest); + for (let testType of ["browser", "xpcshell", "chrome", "mochitest"]) { + if (name.startsWith(testType)) { + return testType; + } + } + } + + let filepath = this.cleanUpPath(scope.getFilename()); + let filename = path.basename(filepath); + + if (filename.startsWith("browser_")) { + return "browser"; + } + + if (filename.startsWith("test_")) { + return "xpcshell"; + } + + return null; + }, + + /** + * Gets the root directory of the repository by walking up directories until + * a .eslintignore file is found. + * @param {String} fileName + * The absolute path of a file in the repository + * + * @return {String} The absolute path of the repository directory + */ + getRootDir: function(fileName) { + var dirName = path.dirname(fileName); + + while (dirName && !fs.existsSync(path.join(dirName, ".eslintignore"))) { + dirName = path.dirname(dirName); + } + + if (!dirName) { + throw new Error("Unable to find root of repository"); + } + + return dirName; + }, + + /** + * ESLint may be executed from various places: from mach, at the root of the + * repository, or from a directory in the repository when, for instance, + * executed by a text editor's plugin. + * The value returned by context.getFileName() varies because of this. + * This helper function makes sure to return an absolute file path for the + * current context, by looking at process.cwd(). + * @param {Context} context + * @return {String} The absolute path + */ + getAbsoluteFilePath: function(context) { + var fileName = this.cleanUpPath(context.getFilename()); + var cwd = process.cwd(); + + if (path.isAbsolute(fileName)) { + // Case 2: executed from the repo's root with mach: + // fileName: /path/to/mozilla/repo/a/b/c/d.js + // cwd: /path/to/mozilla/repo + return fileName; + } else if (path.basename(fileName) == fileName) { + // Case 1b: executed from a nested directory, fileName is the base name + // without any path info (happens in Atom with linter-eslint) + return path.join(cwd, fileName); + } else { + // Case 1: executed form in a nested directory, e.g. from a text editor: + // fileName: a/b/c/d.js + // cwd: /path/to/mozilla/repo/a/b/c + var dirName = path.dirname(fileName); + return cwd.slice(0, cwd.length - dirName.length) + fileName; + } + }, + + /** + * When ESLint is run from SublimeText, paths retrieved from + * context.getFileName contain leading and trailing double-quote characters. + * These characters need to be removed. + */ + cleanUpPath: function(path) { + return path.replace(/^"/, "").replace(/"$/, ""); + } +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/index.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/index.js new file mode 100644 index 000000000..e1f694c36 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/index.js @@ -0,0 +1,45 @@ +/** + * @fileoverview A collection of rules that help enforce JavaScript coding + * standard and avoid common errors in the Mozilla project. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +//------------------------------------------------------------------------------ +// Plugin Definition +//------------------------------------------------------------------------------ + +module.exports = { + processors: { + ".xml": require("../lib/processors/xbl-bindings"), + }, + rules: { + "balanced-listeners": require("../lib/rules/balanced-listeners"), + "import-globals": require("../lib/rules/import-globals"), + "import-headjs-globals": require("../lib/rules/import-headjs-globals"), + "import-browserjs-globals": require("../lib/rules/import-browserjs-globals"), + "mark-test-function-used": require("../lib/rules/mark-test-function-used"), + "no-aArgs": require("../lib/rules/no-aArgs"), + "no-cpows-in-tests": require("../lib/rules/no-cpows-in-tests"), + "no-single-arg-cu-import": require("../lib/rules/no-single-arg-cu-import"), + "reject-importGlobalProperties": require("../lib/rules/reject-importGlobalProperties"), + "reject-some-requires": require("../lib/rules/reject-some-requires"), + "var-only-at-top-level": require("../lib/rules/var-only-at-top-level") + }, + rulesConfig: { + "balanced-listeners": 0, + "import-globals": 0, + "import-headjs-globals": 0, + "import-browserjs-globals": 0, + "mark-test-function-used": 0, + "no-aArgs": 0, + "no-cpows-in-tests": 0, + "no-single-arg-cu-import": 0, + "reject-importGlobalProperties": 0, + "reject-some-requires": 0, + "var-only-at-top-level": 0 + } +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/processors/xbl-bindings.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/processors/xbl-bindings.js new file mode 100644 index 000000000..dc09550f2 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/processors/xbl-bindings.js @@ -0,0 +1,363 @@ +/** + * @fileoverview Converts functions and handlers from XBL bindings into JS + * functions + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +const NS_XBL = "http://www.mozilla.org/xbl"; + +let sax = require("sax"); + +// Converts sax's error message to something that eslint will understand +let errorRegex = /(.*)\nLine: (\d+)\nColumn: (\d+)\nChar: (.*)/ +function parseError(err) { + let matches = err.message.match(errorRegex); + if (!matches) + return null; + + return { + fatal: true, + message: matches[1], + line: parseInt(matches[2]) + 1, + column: parseInt(matches[3]) + } +} + +let entityRegex = /&[\w][\w-\.]*;/g; + +// A simple sax listener that generates a tree of element information +function XMLParser(parser) { + this.parser = parser; + parser.onopentag = this.onOpenTag.bind(this); + parser.onclosetag = this.onCloseTag.bind(this); + parser.ontext = this.onText.bind(this); + parser.onopencdata = this.onOpenCDATA.bind(this); + parser.oncdata = this.onCDATA.bind(this); + parser.oncomment = this.onComment.bind(this); + + this.document = { + local: "#document", + uri: null, + children: [], + comments: [], + } + this._currentNode = this.document; +} + +XMLParser.prototype = { + parser: null, + + onOpenTag: function(tag) { + let node = { + parentNode: this._currentNode, + local: tag.local, + namespace: tag.uri, + attributes: {}, + children: [], + comments: [], + textContent: "", + textLine: this.parser.line, + textColumn: this.parser.column, + textEndLine: this.parser.line + } + + for (let attr of Object.keys(tag.attributes)) { + if (tag.attributes[attr].uri == "") { + node.attributes[attr] = tag.attributes[attr].value; + } + } + + this._currentNode.children.push(node); + this._currentNode = node; + }, + + onCloseTag: function(tagname) { + this._currentNode.textEndLine = this.parser.line; + this._currentNode = this._currentNode.parentNode; + }, + + addText: function(text) { + this._currentNode.textContent += text; + }, + + onText: function(text) { + // Replace entities with some valid JS token. + this.addText(text.replace(entityRegex, "null")); + }, + + onOpenCDATA: function() { + // Turn the CDATA opening tag into whitespace for indent alignment + this.addText(" ".repeat("<![CDATA[".length)); + }, + + onCDATA: function(text) { + this.addText(text); + }, + + onComment: function(text) { + this._currentNode.comments.push(text); + } +} + +// ----------------------------------------------------------------------------- +// Processor Definition +// ----------------------------------------------------------------------------- + +const INDENT_LEVEL = 2; + +function indent(count) { + return " ".repeat(count * INDENT_LEVEL); +} + +// Stores any XML parse error +let xmlParseError = null; + +// Stores the lines of JS code generated from the XBL +let scriptLines = []; +// Stores a map from the synthetic line number to the real line number +// and column offset. +let lineMap = []; + +function addSyntheticLine(line, linePos, addDisableLine) { + lineMap[scriptLines.length] = { line: linePos, offset: null }; + scriptLines.push(line + (addDisableLine ? "" : " // eslint-disable-line")); +} + +/** + * Adds generated lines from an XBL node to the script to be passed back to eslint. + */ +function addNodeLines(node, reindent) { + let lines = node.textContent.split("\n"); + let startLine = node.textLine; + let startColumn = node.textColumn; + + // The case where there are no whitespace lines before the first text is + // treated differently for indentation + let indentFirst = false; + + // Strip off any preceeding whitespace only lines. These are often used to + // format the XML and CDATA blocks. + while (lines.length && lines[0].trim() == "") { + indentFirst = true; + startLine++; + lines.shift(); + } + + // Strip off any whitespace lines at the end. These are often used to line + // up the closing tags + while (lines.length && lines[lines.length - 1].trim() == "") { + lines.pop(); + } + + if (!indentFirst) { + let firstLine = lines.shift(); + firstLine = " ".repeat(reindent * INDENT_LEVEL) + firstLine; + // ESLint counts columns starting at 1 rather than 0 + lineMap[scriptLines.length] = { line: startLine, offset: reindent * INDENT_LEVEL - (startColumn - 1) }; + scriptLines.push(firstLine); + startLine++; + } + + // Find the preceeding whitespace for all lines that aren't entirely whitespace + let indents = lines.filter(s => s.trim().length > 0) + .map(s => s.length - s.trimLeft().length); + // Find the smallest indent level in use + let minIndent = Math.min.apply(null, indents); + + for (let line of lines) { + if (line.trim().length == 0) { + // Don't offset lines that are only whitespace, the only possible JS error + // is trailing whitespace and we want it to point at the right place + lineMap[scriptLines.length] = { line: startLine, offset: 0 }; + } else { + line = " ".repeat(reindent * INDENT_LEVEL) + line.substring(minIndent); + lineMap[scriptLines.length] = { line: startLine, offset: reindent * INDENT_LEVEL - (minIndent - 1) }; + } + + scriptLines.push(line); + startLine++; + } +} + +module.exports = { + preprocess: function(text, filename) { + xmlParseError = null; + scriptLines = []; + lineMap = []; + + // Non-strict allows us to ignore many errors from entities and + // preprocessing at the expense of failing to report some XML errors. + // Unfortunately it also throws away the case of tagnames and attributes + let parser = sax.parser(false, { + lowercase: true, + xmlns: true, + }); + + parser.onerror = function(err) { + xmlParseError = parseError(err); + } + + let xp = new XMLParser(parser); + parser.write(text); + + // Sanity checks to make sure we're dealing with an XBL document + let document = xp.document; + if (document.children.length != 1) { + return []; + } + + let bindings = document.children[0]; + if (bindings.local != "bindings" || bindings.namespace != NS_XBL) { + return []; + } + + for (let comment of document.comments) { + addSyntheticLine(`/*`, 0, true); + for (let line of comment.split("\n")) { + addSyntheticLine(`${line.trim()}`, 0, true); + } + addSyntheticLine(`*/`, 0, true); + } + + addSyntheticLine(`this.bindings = {`, bindings.textLine); + + for (let binding of bindings.children) { + if (binding.local != "binding" || binding.namespace != NS_XBL) { + continue; + } + + addSyntheticLine(indent(1) + `"${binding.attributes.id}": {`, binding.textLine); + + for (let part of binding.children) { + if (part.namespace != NS_XBL) { + continue; + } + + if (part.local == "implementation") { + addSyntheticLine(indent(2) + `implementation: {`, part.textLine); + } else if (part.local == "handlers") { + addSyntheticLine(indent(2) + `handlers: [`, part.textLine); + } else { + continue; + } + + for (let item of part.children) { + if (item.namespace != NS_XBL) { + continue; + } + + switch (item.local) { + case "field": { + // Fields are something like lazy getter functions + + // Ignore empty fields + if (item.textContent.trim().length == 0) { + continue; + } + + addSyntheticLine(indent(3) + `get ${item.attributes.name}() {`, item.textLine); + addSyntheticLine(indent(4) + `return (`, item.textLine); + + // Remove trailing semicolons, as we are adding our own + item.textContent = item.textContent.replace(/;(?=\s*$)/, ""); + addNodeLines(item, 5); + + addSyntheticLine(indent(4) + `);`, item.textLine); + addSyntheticLine(indent(3) + `},`, item.textEndLine); + break; + } + case "constructor": + case "destructor": { + // Constructors and destructors become function declarations + addSyntheticLine(indent(3) + `${item.local}() {`, item.textLine); + addNodeLines(item, 4); + addSyntheticLine(indent(3) + `},`, item.textEndLine); + break; + } + case "method": { + // Methods become function declarations with the appropriate params + + let params = item.children.filter(n => n.local == "parameter" && n.namespace == NS_XBL) + .map(n => n.attributes.name) + .join(", "); + let body = item.children.filter(n => n.local == "body" && n.namespace == NS_XBL)[0]; + + addSyntheticLine(indent(3) + `${item.attributes.name}(${params}) {`, item.textLine); + addNodeLines(body, 4); + addSyntheticLine(indent(3) + `},`, item.textEndLine); + break; + } + case "property": { + // Properties become one or two function declarations + for (let propdef of item.children) { + if (propdef.namespace != NS_XBL) { + continue; + } + + if (propdef.local == "setter") { + addSyntheticLine(indent(3) + `set ${item.attributes.name}(val) {`, propdef.textLine); + } else if (propdef.local == "getter") { + addSyntheticLine(indent(3) + `get ${item.attributes.name}() {`, propdef.textLine); + } else { + continue; + } + addNodeLines(propdef, 4); + addSyntheticLine(indent(3) + `},`, propdef.textEndLine); + } + break; + } + case "handler": { + // Handlers become a function declaration with an `event` parameter + addSyntheticLine(indent(3) + `function(event) {`, item.textLine); + addNodeLines(item, 4); + addSyntheticLine(indent(3) + `},`, item.textEndLine); + break; + } + default: + continue; + } + } + + addSyntheticLine(indent(2) + (part.local == "implementation" ? `},` : `],`), part.textEndLine); + } + addSyntheticLine(indent(1) + `},`, binding.textEndLine); + } + addSyntheticLine(`};`, bindings.textEndLine); + + let script = scriptLines.join("\n") + "\n"; + return [script]; + }, + + postprocess: function(messages, filename) { + // If there was an XML parse error then just return that + if (xmlParseError) { + return [xmlParseError]; + } + + // For every message from every script block update the line to point to the + // correct place. + let errors = []; + for (let i = 0; i < messages.length; i++) { + for (let message of messages[i]) { + // ESLint indexes lines starting at 1 but our arrays start at 0 + let mapped = lineMap[message.line - 1]; + + message.line = mapped.line + 1; + if (mapped.offset) { + message.column -= mapped.offset; + } else { + message.column = NaN; + } + + errors.push(message); + } + } + + return errors; + } +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/.eslintrc.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/.eslintrc.js new file mode 100644 index 000000000..505a3ea82 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/.eslintrc.js @@ -0,0 +1,51 @@ +"use strict"; + +/** + * Based on npm coding standards at https://docs.npmjs.com/misc/coding-style. + * + * The places we differ from the npm coding style: + * - Commas should be at the end of a line. + * - Always use semicolons. + * - Functions should not have whitespace before params. + */ + +module.exports = { + "env": { + "node": true + }, + + "rules": { + "brace-style": ["error", "1tbs"], + "camelcase": "error", + "comma-dangle": ["error", "never"], + "comma-spacing": "error", + "comma-style": ["error", "last"], + "curly": ["error", "multi-line"], + "handle-callback-err": ["error", "er"], + "indent": ["error", 2, {"SwitchCase": 1}], + "max-len": ["error", 80, "error"], + "no-multiple-empty-lines": ["error", {"max": 1}], + "no-undef": "error", + "no-undef-init": "error", + "no-unexpected-multiline": "error", + "object-curly-spacing": "off", + "one-var": ["error", "never"], + "operator-linebreak": ["error", "after"], + "semi": ["error", "always"], + "space-before-blocks": "error", + "space-before-function-paren": ["error", "never"], + "keyword-spacing": "error", + "strict": ["error", "global"], + }, + + // Globals accessible within node modules. + "globals": { + "DTRACE_HTTP_CLIENT_REQUEST": true, + "DTRACE_HTTP_CLIENT_RESPONSE": true, + "DTRACE_HTTP_SERVER_REQUEST": true, + "DTRACE_HTTP_SERVER_RESPONSE": true, + "DTRACE_NET_SERVER_CONNECTION": true, + "DTRACE_NET_STREAM_END": true, + "Intl": true, + }, +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/balanced-listeners.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/balanced-listeners.js new file mode 100644 index 000000000..c658a6b44 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/balanced-listeners.js @@ -0,0 +1,113 @@ +/**
+ * @fileoverview Check that there's a removeEventListener for each
+ * addEventListener and an off for each on.
+ * Note that for now, this rule is rather simple in that it only checks that
+ * for each event name there is both an add and remove listener. It doesn't
+ * check that these are called on the right objects or with the same callback.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+"use strict";
+
+// -----------------------------------------------------------------------------
+// Rule Definition
+// -----------------------------------------------------------------------------
+
+module.exports = function(context) {
+ // ---------------------------------------------------------------------------
+ // Helpers
+ // ---------------------------------------------------------------------------
+
+ var DICTIONARY = {
+ "addEventListener": "removeEventListener",
+ "on": "off"
+ };
+ // Invert this dictionary to make it easy later.
+ var INVERTED_DICTIONARY = {};
+ for (var i in DICTIONARY) {
+ INVERTED_DICTIONARY[DICTIONARY[i]] = i;
+ }
+
+ // Collect the add/remove listeners in these 2 arrays.
+ var addedListeners = [];
+ var removedListeners = [];
+
+ function addAddedListener(node) {
+ addedListeners.push({
+ functionName: node.callee.property.name,
+ type: node.arguments[0].value,
+ node: node.callee.property,
+ useCapture: node.arguments[2] ? node.arguments[2].value : null
+ });
+ }
+
+ function addRemovedListener(node) {
+ removedListeners.push({
+ functionName: node.callee.property.name,
+ type: node.arguments[0].value,
+ useCapture: node.arguments[2] ? node.arguments[2].value : null
+ });
+ }
+
+ function getUnbalancedListeners() {
+ var unbalanced = [];
+
+ for (var j = 0; j < addedListeners.length; j++) {
+ if (!hasRemovedListener(addedListeners[j])) {
+ unbalanced.push(addedListeners[j]);
+ }
+ }
+ addedListeners = removedListeners = [];
+
+ return unbalanced;
+ }
+
+ function hasRemovedListener(addedListener) {
+ for (var k = 0; k < removedListeners.length; k++) {
+ var listener = removedListeners[k];
+ if (DICTIONARY[addedListener.functionName] === listener.functionName &&
+ addedListener.type === listener.type &&
+ addedListener.useCapture === listener.useCapture) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ // ---------------------------------------------------------------------------
+ // Public
+ // ---------------------------------------------------------------------------
+
+ return {
+ CallExpression: function(node) {
+ if (node.arguments.length === 0) {
+ return;
+ }
+
+ if (node.callee.type === "MemberExpression") {
+ var listenerMethodName = node.callee.property.name;
+
+ if (DICTIONARY.hasOwnProperty(listenerMethodName)) {
+ addAddedListener(node);
+ } else if (INVERTED_DICTIONARY.hasOwnProperty(listenerMethodName)) {
+ addRemovedListener(node);
+ }
+ }
+ },
+
+ "Program:exit": function() {
+ getUnbalancedListeners().forEach(function(listener) {
+ context.report(listener.node,
+ "No corresponding '{{functionName}}({{type}})' was found.",
+ {
+ functionName: DICTIONARY[listener.functionName],
+ type: listener.type
+ });
+ });
+ }
+ };
+};
diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-browserjs-globals.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-browserjs-globals.js new file mode 100644 index 000000000..e449931bd --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-browserjs-globals.js @@ -0,0 +1,81 @@ +/** + * @fileoverview Import globals from browser.js. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +var fs = require("fs"); +var path = require("path"); +var helpers = require("../helpers"); +var globals = require("../globals"); + +const SCRIPTS = [ + //"browser/base/content/nsContextMenu.js", + "toolkit/content/contentAreaUtils.js", + "browser/components/places/content/editBookmarkOverlay.js", + "toolkit/components/printing/content/printUtils.js", + "toolkit/content/viewZoomOverlay.js", + "browser/components/places/content/browserPlacesViews.js", + "browser/base/content/browser.js", + "browser/components/downloads/content/downloads.js", + "browser/components/downloads/content/indicator.js", + "browser/components/customizableui/content/panelUI.js", + "toolkit/components/viewsource/content/viewSourceUtils.js", + "browser/base/content/browser-addons.js", + "browser/base/content/browser-ctrlTab.js", + "browser/base/content/browser-customization.js", + "browser/base/content/browser-feeds.js", + "browser/base/content/browser-fullScreenAndPointerLock.js", + "browser/base/content/browser-fullZoom.js", + "browser/base/content/browser-gestureSupport.js", + "browser/base/content/browser-media.js", + "browser/base/content/browser-places.js", + "browser/base/content/browser-plugins.js", + "browser/base/content/browser-refreshblocker.js", + "browser/base/content/browser-safebrowsing.js", + "browser/base/content/browser-sidebar.js", + "browser/base/content/browser-social.js", + "browser/base/content/browser-syncui.js", + "browser/base/content/browser-tabsintitlebar.js", + "browser/base/content/browser-thumbnails.js", + "browser/base/content/browser-trackingprotection.js", + "browser/base/content/browser-data-submission-info-bar.js" +]; + +module.exports = function(context) { + return { + Program: function(node) { + if (helpers.getTestType(this) != "browser" && + !helpers.getIsHeadFile(this)) { + return; + } + + let filepath = helpers.getAbsoluteFilePath(context); + let root = helpers.getRootDir(filepath); + for (let script of SCRIPTS) { + let fileName = path.join(root, script); + try { + let newGlobals = globals.getGlobalsForFile(fileName); + helpers.addGlobals(newGlobals, context.getScope()); + } catch (e) { + context.report( + node, + "Could not load globals from file {{filePath}}: {{error}}", + { + filePath: path.relative(root, fileName), + error: e + } + ); + } + } + } + }; +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-globals.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-globals.js new file mode 100644 index 000000000..053a9e702 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-globals.js @@ -0,0 +1,15 @@ +/** + * @fileoverview Discovers all globals for the current file. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +module.exports = require("../globals").getESLintGlobalParser; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-headjs-globals.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-headjs-globals.js new file mode 100644 index 000000000..783642f58 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/import-headjs-globals.js @@ -0,0 +1,49 @@ +/** + * @fileoverview Import globals from head.js and from any files that were + * imported by head.js (as far as we can correctly resolve the path). + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +var fs = require("fs"); +var path = require("path"); +var helpers = require("../helpers"); +var globals = require("../globals"); + +module.exports = function(context) { + + function importHead(path, node) { + try { + let stats = fs.statSync(path); + if (!stats.isFile()) { + return; + } + } catch (e) { + return; + } + + let newGlobals = globals.getGlobalsForFile(path); + helpers.addGlobals(newGlobals, context.getScope()); + } + + // --------------------------------------------------------------------------- + // Public + // --------------------------------------------------------------------------- + + return { + Program: function(node) { + let heads = helpers.getTestHeadFiles(this); + for (let head of heads) { + importHead(head, node); + } + } + }; +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/mark-test-function-used.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/mark-test-function-used.js new file mode 100644 index 000000000..b2e8ec294 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/mark-test-function-used.js @@ -0,0 +1,37 @@ +/** + * @fileoverview Simply marks `test` (the test method) or `run_test` as used when + * in mochitests or xpcshell tests respectively. This avoids ESLint telling us + * that the function is never called. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +var helpers = require("../helpers"); + +module.exports = function(context) { + // --------------------------------------------------------------------------- + // Public + // --------------------------------------------------------------------------- + + return { + Program: function() { + if (helpers.getTestType(this) == "browser") { + context.markVariableAsUsed("test"); + return; + } + + if (helpers.getTestType(this) == "xpcshell") { + context.markVariableAsUsed("run_test"); + return; + } + } + }; +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-aArgs.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-aArgs.js new file mode 100644 index 000000000..267f6836f --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-aArgs.js @@ -0,0 +1,55 @@ +/**
+ * @fileoverview warns against using hungarian notation in function arguments
+ * (i.e. aArg).
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+"use strict";
+
+// -----------------------------------------------------------------------------
+// Rule Definition
+// -----------------------------------------------------------------------------
+
+module.exports = function(context) {
+ // ---------------------------------------------------------------------------
+ // Helpers
+ // ---------------------------------------------------------------------------
+
+ function isPrefixed(name) {
+ return name.length >= 2 && /^a[A-Z]/.test(name);
+ }
+
+ function deHungarianize(name) {
+ return name.substring(1, 2).toLowerCase() +
+ name.substring(2, name.length);
+ }
+
+ function checkFunction(node) {
+ for (var i = 0; i < node.params.length; i++) {
+ var param = node.params[i];
+ if (param.name && isPrefixed(param.name)) {
+ var errorObj = {
+ name: param.name,
+ suggestion: deHungarianize(param.name)
+ };
+ context.report(param,
+ "Parameter '{{name}}' uses Hungarian Notation, " +
+ "consider using '{{suggestion}}' instead.",
+ errorObj);
+ }
+ }
+ }
+
+ // ---------------------------------------------------------------------------
+ // Public
+ // ---------------------------------------------------------------------------
+
+ return {
+ "FunctionDeclaration": checkFunction,
+ "ArrowFunctionExpression": checkFunction,
+ "FunctionExpression": checkFunction
+ };
+};
diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-cpows-in-tests.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-cpows-in-tests.js new file mode 100644 index 000000000..415cb2fc9 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-cpows-in-tests.js @@ -0,0 +1,112 @@ +/** + * @fileoverview Prevent access to CPOWs in browser mochitests. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +var helpers = require("../helpers"); + +var cpows = [ + /^gBrowser\.contentWindow/, + /^gBrowser\.contentDocument/, + /^gBrowser\.selectedBrowser.contentWindow/, + /^browser\.contentDocument/, + /^window\.content/ +]; + +var isInContentTask = false; + +module.exports = function(context) { + // --------------------------------------------------------------------------- + // Helpers + // --------------------------------------------------------------------------- + + function showError(node, identifier) { + if (isInContentTask) { + return; + } + + context.report({ + node: node, + message: identifier + + " is a possible Cross Process Object Wrapper (CPOW)." + }); + } + + function isContentTask(node) { + return node && + node.type === "MemberExpression" && + node.property.type === "Identifier" && + node.property.name === "spawn" && + node.object.type === "Identifier" && + node.object.name === "ContentTask"; + } + + // --------------------------------------------------------------------------- + // Public + // --------------------------------------------------------------------------- + + return { + CallExpression: function(node) { + if (isContentTask(node.callee)) { + isInContentTask = true; + } + }, + + "CallExpression:exit": function(node) { + if (isContentTask(node.callee)) { + isInContentTask = false; + } + }, + + MemberExpression: function(node) { + if (helpers.getTestType(this) != "browser") { + return; + } + + var expression = context.getSource(node); + + // Only report a single CPOW error per node -- so if checking + // |cpows| reports one, don't report another below. + var someCpowFound = cpows.some(function(cpow) { + if (cpow.test(expression)) { + showError(node, expression); + return true; + } + return false; + }); + if (!someCpowFound && helpers.getIsGlobalScope(context.getAncestors())) { + if (/^content\./.test(expression)) { + showError(node, expression); + return; + } + } + }, + + Identifier: function(node) { + if (helpers.getTestType(this) != "browser") { + return; + } + + var expression = context.getSource(node); + if (expression == "content" || /^content\./.test(expression)) { + if (node.parent.type === "MemberExpression" && + node.parent.object && + node.parent.object.type === "Identifier" && + node.parent.object.name != "content") { + return; + } + showError(node, expression); + return; + } + } + }; +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-single-arg-cu-import.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-single-arg-cu-import.js new file mode 100644 index 000000000..b295f3555 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/no-single-arg-cu-import.js @@ -0,0 +1,39 @@ +/** + * @fileoverview Reject use of single argument Cu.import + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +var helpers = require("../helpers"); + +module.exports = function(context) { + + // --------------------------------------------------------------------------- + // Public + // -------------------------------------------------------------------------- + + return { + "CallExpression": function(node) { + if (node.callee.type === "MemberExpression") { + let memexp = node.callee; + if (memexp.object.type === "Identifier" && + // Only Cu, not Components.utils; see bug 1230369. + memexp.object.name === "Cu" && + memexp.property.type === "Identifier" && + memexp.property.name === "import" && + node.arguments.length === 1) { + context.report(node, "Single argument Cu.import exposes new " + + "globals to all modules"); + } + } + } + }; +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-importGlobalProperties.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-importGlobalProperties.js new file mode 100644 index 000000000..0661c91d4 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-importGlobalProperties.js @@ -0,0 +1,37 @@ +/** + * @fileoverview Reject use of Cu.importGlobalProperties + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +var helpers = require("../helpers"); + +module.exports = function(context) { + + // --------------------------------------------------------------------------- + // Public + // -------------------------------------------------------------------------- + + return { + "CallExpression": function(node) { + if (node.callee.type === "MemberExpression") { + let memexp = node.callee; + if (memexp.object.type === "Identifier" && + // Only Cu, not Components.utils; see bug 1230369. + memexp.object.name === "Cu" && + memexp.property.type === "Identifier" && + memexp.property.name === "importGlobalProperties") { + context.report(node, "Unexpected call to Cu.importGlobalProperties"); + } + } + } + }; +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-some-requires.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-some-requires.js new file mode 100644 index 000000000..746f98a1f --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/reject-some-requires.js @@ -0,0 +1,48 @@ +/** + * @fileoverview Reject some uses of require. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +module.exports = function(context) { + + // --------------------------------------------------------------------------- + // Public + // -------------------------------------------------------------------------- + + if (typeof(context.options[0]) !== "string") { + throw new Error("reject-some-requires expects a regexp"); + } + const RX = new RegExp(context.options[0]); + + const checkPath = function(node, path) { + if (RX.test(path)) { + context.report(node, `require(${path}) is not allowed`); + } + }; + + return { + "CallExpression": function(node) { + if (node.callee.type == "Identifier" && + node.callee.name == "require" && + node.arguments.length == 1 && + node.arguments[0].type == "Literal") { + checkPath(node, node.arguments[0].value); + } else if (node.callee.type == "MemberExpression" && + node.callee.property.type == "Identifier" && + node.callee.property.name == "lazyRequireGetter" && + node.arguments.length >= 3 && + node.arguments[2].type == "Literal") { + checkPath(node, node.arguments[2].value); + } + } + }; +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/var-only-at-top-level.js b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/var-only-at-top-level.js new file mode 100644 index 000000000..a1e14e166 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/lib/rules/var-only-at-top-level.js @@ -0,0 +1,34 @@ +/** + * @fileoverview Marks all var declarations that are not at the top level + * invalid. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +"use strict"; + +// ----------------------------------------------------------------------------- +// Rule Definition +// ----------------------------------------------------------------------------- + +var helpers = require("../helpers"); + +module.exports = function(context) { + // --------------------------------------------------------------------------- + // Public + // -------------------------------------------------------------------------- + + return { + "VariableDeclaration": function(node) { + if (node.kind === "var") { + if (helpers.getIsGlobalScope(context.getAncestors())) { + return; + } + + context.report(node, "Unexpected var, use let or const instead."); + } + } + }; +}; diff --git a/tools/lint/eslint/eslint-plugin-mozilla/package.json b/tools/lint/eslint/eslint-plugin-mozilla/package.json new file mode 100644 index 000000000..2f4a85172 --- /dev/null +++ b/tools/lint/eslint/eslint-plugin-mozilla/package.json @@ -0,0 +1,29 @@ +{ + "name": "eslint-plugin-mozilla", + "version": "0.2.5", + "description": "A collection of rules that help enforce JavaScript coding standard in the Mozilla project.", + "keywords": [ + "eslint", + "eslintplugin", + "eslint-plugin", + "mozilla", + "firefox" + ], + "bugs": { + "url": "https://bugzilla.mozilla.org/enter_bug.cgi?product=Firefox&component=Developer%20Tools" + }, + "homepage": "https://bugzilla.mozilla.org/show_bug.cgi?id=1203520", + "author": "Mike Ratcliffe", + "main": "lib/index.js", + "dependencies": { + "escope": "^3.6.0", + "espree": "^3.2.0", + "estraverse": "^4.2.0", + "ini-parser": "^0.0.2", + "sax": "^1.1.4" + }, + "engines": { + "node": ">=0.10.0" + }, + "license": "MPL-2.0" +} diff --git a/tools/lint/eslint/manifest.tt b/tools/lint/eslint/manifest.tt new file mode 100644 index 000000000..c98f6455f --- /dev/null +++ b/tools/lint/eslint/manifest.tt @@ -0,0 +1,9 @@ +[ +{ +"size": 2768586, +"visibility": "public", +"digest": "1b9a7c5b1ca8d7d2aeb803055129d1cb0fa0d17b90dd3cf852ea77d86d1b1d9d09f34007a71908074afef9ce1ab87972a5cf16a36952e1a6159f7abfc6fa15b3", +"algorithm": "sha512", +"filename": "eslint.tar.gz" +} +] diff --git a/tools/lint/eslint/modules.json b/tools/lint/eslint/modules.json new file mode 100644 index 000000000..0b4401636 --- /dev/null +++ b/tools/lint/eslint/modules.json @@ -0,0 +1,233 @@ +{ + "AboutHome.jsm": ["AboutHomeUtils", "AboutHome"], + "AddonLogging.jsm": ["LogManager"], + "AddonManager.jsm": ["AddonManager", "AddonManagerPrivate"], + "addons.js": ["AddonsEngine", "AddonValidator"], + "addons.jsm": ["Addon", "STATE_ENABLED", "STATE_DISABLED"], + "addonsreconciler.js": ["AddonsReconciler", "CHANGE_INSTALLED", "CHANGE_UNINSTALLED", "CHANGE_ENABLED", "CHANGE_DISABLED"], + "AddonTestUtils.jsm": ["AddonTestUtils", "MockAsyncShutdown"], + "addonutils.js": ["AddonUtils"], + "ajv-4.1.1.js": ["Ajv"], + "AlertsHelper.jsm": [], + "AppData.jsm": ["makeFakeAppDir"], + "AppInfo.jsm": ["newAppInfo", "getAppInfo", "updateAppInfo"], + "AppsServiceChild.jsm": ["DOMApplicationRegistry", "WrappedManifestCache"], + "arrays.js": ["inArray", "getSet", "indexOf", "remove", "rindexOf", "compare"], + "assertions.js": ["Assert", "Expect"], + "async.js": ["Async"], + "AsyncSpellCheckTestHelper.jsm": ["onSpellCheck"], + "Battery.jsm": ["GetBattery", "Battery"], + "blocklist-clients.js": ["AddonBlocklistClient", "GfxBlocklistClient", "OneCRLBlocklistClient", "PluginBlocklistClient", "FILENAME_ADDONS_JSON", "FILENAME_GFX_JSON", "FILENAME_PLUGINS_JSON"], + "blocklist-updater.js": ["checkVersions", "addTestBlocklistClient"], + "bogus_element_type.jsm": [], + "bookmark_validator.js": ["BookmarkValidator", "BookmarkProblemData"], + "bookmarks.js": ["BookmarksEngine", "PlacesItem", "Bookmark", "BookmarkFolder", "BookmarkQuery", "Livemark", "BookmarkSeparator"], + "bookmarks.jsm": ["PlacesItem", "Bookmark", "Separator", "Livemark", "BookmarkFolder", "DumpBookmarks"], + "BootstrapMonitor.jsm": ["monitor"], + "browser-loader.js": ["BrowserLoader"], + "browserid_identity.js": ["BrowserIDManager", "AuthenticationError"], + "CertUtils.jsm": ["BadCertHandler", "checkCert", "readCertPrefs", "validateCert"], + "clients.js": ["ClientEngine", "ClientsRec"], + "CloudSyncAdapters.jsm": ["Adapters"], + "CloudSyncBookmarks.jsm": ["Bookmarks"], + "CloudSyncBookmarksFolderCache.jsm": ["FolderCache"], + "CloudSyncEventSource.jsm": ["EventSource"], + "CloudSyncLocal.jsm": ["Local"], + "CloudSyncPlacesWrapper.jsm": ["PlacesWrapper"], + "CloudSyncTabs.jsm": ["Tabs"], + "cluster.js": ["ClusterManager"], + "collection_validator.js": ["CollectionValidator", "CollectionProblemData"], + "Console.jsm": ["console", "ConsoleAPI"], + "Constants.jsm": ["Roles", "Events", "Relations", "Filters", "States", "Prefilters"], + "ContactDB.jsm": ["ContactDB", "DB_NAME", "STORE_NAME", "SAVED_GETALL_STORE_NAME", "REVISION_STORE", "DB_VERSION"], + "content-server.jsm": ["init"], + "content.jsm": ["registerContentFrame"], + "ContentCrashHandlers.jsm": ["TabCrashHandler", "PluginCrashReporter", "UnsubmittedCrashHandler"], + "ContentObservers.jsm": [], + "ContentPrefUtils.jsm": ["ContentPref", "cbHandleResult", "cbHandleError", "cbHandleCompletion", "safeCallback"], + "controller.js": ["MozMillController", "globalEventRegistry", "sleep", "windowMap"], + "cookies.js": ["Cookies"], + "CoverageUtils.jsm": ["CoverageCollector"], + "CrashManagerTest.jsm": ["configureLogging", "getManager", "sleep", "TestingCrashManager"], + "dbg-client.jsm": ["DebuggerTransport", "DebuggerClient", "RootClient", "LongStringClient", "EnvironmentClient", "ObjectClient"], + "dbg-server.jsm": ["DebuggerServer", "ActorPool", "OriginalLocation"], + "debug.js": ["NS_ASSERT"], + "declined.js": ["DeclinedEngines"], + "dispatcher.js": ["Dispatcher"], + "distribution.js": ["DistributionCustomizer"], + "DNSTypes.jsm": ["DNS_QUERY_RESPONSE_CODES", "DNS_AUTHORITATIVE_ANSWER_CODES", "DNS_CLASS_CODES", "DNS_RECORD_TYPES"], + "dom.js": ["getAttributes"], + "DOMRequestHelper.jsm": ["DOMRequestIpcHelper"], + "DownloadCore.jsm": ["Download", "DownloadSource", "DownloadTarget", "DownloadError", "DownloadSaver", "DownloadCopySaver", "DownloadLegacySaver", "DownloadPDFSaver"], + "DownloadList.jsm": ["DownloadList", "DownloadCombinedList", "DownloadSummary"], + "E10SAddonsRollout.jsm": ["isAddonPartOfE10SRollout"], + "elementslib.js": ["ID", "Link", "XPath", "Selector", "Name", "Anon", "AnonXPath", "Lookup", "_byID", "_byName", "_byAttrib", "_byAnonAttrib"], + "engines.js": ["EngineManager", "Engine", "SyncEngine", "Tracker", "Store", "Changeset"], + "enginesync.js": ["EngineSynchronizer"], + "errors.js": ["BaseError", "ApplicationQuitError", "AssertionError", "TimeoutError"], + "evaluate.js": ["evaluate", "sandbox", "Sandboxes"], + "event-emitter.js": ["EventEmitter"], + "EventUtils.js": ["disableNonTestMouseEvents", "sendMouseEvent", "sendChar", "sendString", "sendKey", "synthesizeMouse", "synthesizeTouch", "synthesizeMouseAtPoint", "synthesizeTouchAtPoint", "synthesizeMouseAtCenter", "synthesizeTouchAtCenter", "synthesizeWheel", "synthesizeKey", "synthesizeMouseExpectEvent", "synthesizeKeyExpectEvent", "synthesizeText", "synthesizeComposition", "synthesizeQuerySelectedText"], + "Extension.jsm": ["Extension", "ExtensionData"], + "ExtensionAPI.jsm": ["ExtensionAPI", "ExtensionAPIs"], + "ExtensionXPCShellUtils.jsm": ["ExtensionTestUtils"], + "fakeservices.js": ["FakeCryptoService", "FakeFilesystemService", "FakeGUIDService", "fakeSHA256HMAC"], + "file_expandosharing.jsm": ["checkFromJSM"], + "file_stringencoding.jsm": ["checkFromJSM"], + "file_url.jsm": ["checkFromJSM"], + "file_worker_url.jsm": ["checkFromJSM"], + "Finder.jsm": ["Finder", "GetClipboardSearchString"], + "FormAutofillContent.jsm": ["FormAutofillHandler"], + "forms.js": ["FormEngine", "FormRec", "FormValidator"], + "forms.jsm": ["FormData"], + "frame.js": ["Collector", "Runner", "events", "runTestFile", "log", "timers", "persisted", "shutdownApplication"], + "FrameScriptManager.jsm": ["getNewLoaderID"], + "gDevTools.jsm": ["gDevTools", "gDevToolsBrowser"], + "gDevTools.jsm": ["gDevTools", "gDevToolsBrowser"], + "Geometry.jsm": ["Point", "Rect"], + "Gestures.jsm": ["GestureSettings", "GestureTracker"], + "GMPInstallManager.jsm": ["GMPInstallManager", "GMPExtractor", "GMPDownloader", "GMPAddon"], + "GMPProvider.jsm": [], + "GMPUtils.jsm": ["GMP_PLUGIN_IDS", "GMPPrefs", "GMPUtils", "OPEN_H264_ID", "WIDEVINE_ID"], + "hawkclient.js": ["HawkClient"], + "hawkrequest.js": ["HAWKAuthenticatedRESTRequest", "deriveHawkCredentials"], + "HelperApps.jsm": ["App", "HelperApps"], + "history.js": ["HistoryEngine", "HistoryRec"], + "history.jsm": ["HistoryEntry", "DumpHistory"], + "Http.jsm": ["httpRequest", "percentEncode"], + "httpd.js": ["HTTP_400", "HTTP_401", "HTTP_402", "HTTP_403", "HTTP_404", "HTTP_405", "HTTP_406", "HTTP_407", "HTTP_408", "HTTP_409", "HTTP_410", "HTTP_411", "HTTP_412", "HTTP_413", "HTTP_414", "HTTP_415", "HTTP_417", "HTTP_500", "HTTP_501", "HTTP_502", "HTTP_503", "HTTP_504", "HTTP_505", "HttpError", "HttpServer"], + "identity.js": ["IdentityManager"], + "Identity.jsm": ["IdentityService"], + "IdentityUtils.jsm": ["checkDeprecated", "checkRenamed", "getRandomId", "objectCopy", "makeMessageObject"], + "import_module.jsm": ["MODULE_IMPORTED", "MODULE_URI", "SUBMODULE_IMPORTED", "same_scope", "SUBMODULE_IMPORTED_TO_SCOPE"], + "import_sub_module.jsm": ["SUBMODULE_IMPORTED", "test_obj"], + "InlineSpellChecker.jsm": ["InlineSpellChecker", "SpellCheckHelper"], + "jpakeclient.js": ["JPAKEClient", "SendCredentialsController"], + "Jsbeautify.jsm": ["jsBeautify"], + "jsdebugger.jsm": ["addDebuggerToGlobal"], + "json2.js": ["JSON"], + "keys.js": ["BulkKeyBundle", "SyncKeyBundle"], + "KeyValueParser.jsm": ["parseKeyValuePairsFromLines", "parseKeyValuePairs", "parseKeyValuePairsFromFile"], + "kinto-http-client.js": ["KintoHttpClient"], + "kinto-offline-client.js": ["loadKinto"], + "loader-plugin-raw.jsm": ["requireRawId"], + "loader.js": ["WorkerDebuggerLoader", "worker"], + "Loader.jsm": ["DevToolsLoader", "devtools", "BuiltinProvider", "require", "loader"], + "logger.jsm": ["Logger"], + "logging.js": ["getTestLogger", "initTestLogging"], + "LoginManagerContent.jsm": ["LoginManagerContent", "LoginFormFactory", "UserAutoCompleteResult"], + "LoginRecipes.jsm": ["LoginRecipesContent", "LoginRecipesParent"], + "logmanager.js": ["LogManager"], + "LogUtils.jsm": ["Logger"], + "lz4.js": ["Lz4"], + "lz4_internal.js": ["Primitives"], + "main.js": ["Weave"], + "MatchPattern.jsm": ["MatchPattern", "MatchGlobs", "MatchURLFilters"], + "mcc_iso3166_table.jsm": ["MCC_ISO3166_TABLE"], + "message.js": ["Command", "Message", "MessageOrigin", "Response"], + "Messaging.jsm": ["sendMessageToJava", "Messaging"], + "microformat-shiv.js": ["Microformats"], + "MigrationUtils.jsm": ["MigrationUtils", "MigratorPrototype"], + "MinimalIdentity.jsm": ["IdentityService"], + "mozelement.js": ["Elem", "Selector", "ID", "Link", "XPath", "Name", "Lookup", "MozMillElement", "MozMillCheckBox", "MozMillRadio", "MozMillDropList", "MozMillTextBox", "subclasses"], + "mozmill.js": ["controller", "utils", "elementslib", "os", "getBrowserController", "newBrowserController", "getAddonsController", "getPreferencesController", "newMail3PaneController", "getMail3PaneController", "wm", "platform", "getAddrbkController", "getMsgComposeController", "getDownloadsController", "Application", "findElement", "getPlacesController", "isMac", "isLinux", "isWindows", "firePythonCallback", "getAddons"], + "msgbroker.js": ["addListener", "addObject", "removeListener", "sendMessage", "log", "pass", "fail"], + "NativeMessaging.jsm": ["HostManifestManager", "NativeApp"], + "NetworkPrioritizer.jsm": ["trackBrowserWindow"], + "NotificationDB.jsm": [], + "nsFormAutoCompleteResult.jsm": ["FormAutoCompleteResult"], + "objects.js": ["getLength"], + "observers.js": ["Observers"], + "offlineAppCache.jsm": ["OfflineAppCacheHelper"], + "OrientationChangeHandler.jsm": [], + "os.js": ["listDirectory", "getFileForPath", "abspath", "getPlatform"], + "osfile.jsm": ["OS"], + "osfile_async_front.jsm": ["OS"], + "osfile_native.jsm": ["read"], + "osfile_shared_allthreads.jsm": ["LOG", "clone", "Config", "Constants", "Type", "HollowStructure", "OSError", "Library", "declareFFI", "declareLazy", "declareLazyFFI", "normalizeBufferArgs", "projectValue", "isArrayBuffer", "isTypedArray", "defineLazyGetter", "OS"], + "osfile_unix_allthreads.jsm": ["declareFFI", "libc", "Error", "AbstractInfo", "AbstractEntry", "Type", "POS_START", "POS_CURRENT", "POS_END"], + "osfile_win_allthreads.jsm": ["declareFFI", "libc", "Error", "AbstractInfo", "AbstractEntry", "Type", "POS_START", "POS_CURRENT", "POS_END"], + "ospath_unix.jsm": ["basename", "dirname", "join", "normalize", "split", "toFileURI", "fromFileURI"], + "ospath_win.jsm": ["basename", "dirname", "join", "normalize", "split", "winGetDrive", "winIsAbsolute", "toFileURI", "fromFileURI"], + "OutputGenerator.jsm": ["UtteranceGenerator", "BrailleGenerator"], + "PageMenu.jsm": ["PageMenuParent", "PageMenuChild"], + "PageThumbs.jsm": ["PageThumbs", "PageThumbsStorage"], + "Parser.jsm": ["Parser", "ParserHelpers", "SyntaxTreeVisitor"], + "parsingTestHelpers.jsm": ["generateURIsFromDirTree"], + "passwords.js": ["PasswordEngine", "LoginRec", "PasswordValidator"], + "passwords.jsm": ["Password", "DumpPasswords"], + "PdfJsNetwork.jsm": ["NetworkManager"], + "PermissionSettings.jsm": ["PermissionSettingsModule"], + "PermissionsTable.jsm": ["PermissionsTable", "PermissionsReverseTable", "expandPermissions", "appendAccessToPermName", "isExplicitInPermissionsTable", "AllPossiblePermissions"], + "PhoneNumberMetaData.jsm": ["PHONE_NUMBER_META_DATA"], + "PlacesUtils.jsm": ["PlacesUtils", "PlacesAggregatedTransaction", "PlacesCreateFolderTransaction", "PlacesCreateBookmarkTransaction", "PlacesCreateSeparatorTransaction", "PlacesCreateLivemarkTransaction", "PlacesMoveItemTransaction", "PlacesRemoveItemTransaction", "PlacesEditItemTitleTransaction", "PlacesEditBookmarkURITransaction", "PlacesSetItemAnnotationTransaction", "PlacesSetPageAnnotationTransaction", "PlacesEditBookmarkKeywordTransaction", "PlacesEditBookmarkPostDataTransaction", "PlacesEditItemDateAddedTransaction", "PlacesEditItemLastModifiedTransaction", "PlacesSortFolderByNameTransaction", "PlacesTagURITransaction", "PlacesUntagURITransaction"], + "PluginProvider.jsm": [], + "PointerAdapter.jsm": ["PointerRelay", "PointerAdapter"], + "policies.js": ["ErrorHandler", "SyncScheduler"], + "prefs.js": ["PrefsEngine", "PrefRec"], + "prefs.jsm": ["Preference"], + "PresentationDeviceInfoManager.jsm": ["PresentationDeviceInfoService"], + "PromiseWorker.jsm": ["BasePromiseWorker"], + "PushCrypto.jsm": ["PushCrypto", "concatArray"], + "quit.js": ["goQuitApplication"], + "record.js": ["WBORecord", "RecordManager", "CryptoWrapper", "CollectionKeyManager", "Collection"], + "recursive_importA.jsm": ["foo", "bar"], + "recursive_importB.jsm": ["baz", "qux"], + "reflect.jsm": ["Reflect"], + "RemoteFinder.jsm": ["RemoteFinder", "RemoteFinderListener"], + "RemotePageManager.jsm": ["RemotePages", "RemotePageManager", "PageListener"], + "RemoteWebProgress.jsm": ["RemoteWebProgressManager"], + "resource.js": ["AsyncResource", "Resource"], + "responsivedesign.jsm": ["ResponsiveUIManager"], + "rest.js": ["RESTRequest", "RESTResponse", "TokenAuthenticatedRESTRequest", "SyncStorageRequest"], + "rotaryengine.js": ["RotaryEngine", "RotaryRecord", "RotaryStore", "RotaryTracker"], + "RTCStatsReport.jsm": ["convertToRTCStatsReport"], + "scratchpad-manager.jsm": ["ScratchpadManager"], + "server.js": ["MarionetteServer"], + "service.js": ["Service"], + "SettingsDB.jsm": ["SettingsDB", "SETTINGSDB_NAME", "SETTINGSSTORE_NAME"], + "SharedPromptUtils.jsm": ["PromptUtils", "EnableDelayHelper"], + "ShutdownLeaksCollector.jsm": ["ContentCollector"], + "SignInToWebsite.jsm": ["SignInToWebsiteController"], + "SpecialPowersObserver.jsm": ["SpecialPowersObserver", "SpecialPowersObserverFactory"], + "stack.js": ["findCallerFrame"], + "StateMachineHelper.jsm": ["State", "CommandType"], + "status.js": ["Status"], + "storageserver.js": ["ServerBSO", "StorageServerCallback", "StorageServerCollection", "StorageServer", "storageServerForUsers"], + "stringbundle.js": ["StringBundle"], + "strings.js": ["trim", "vslice"], + "StructuredLog.jsm": ["StructuredLogger", "StructuredFormatter"], + "StyleEditorUtil.jsm": ["getString", "assert", "log", "text", "wire", "showFilePicker"], + "subprocess_common.jsm": ["BaseProcess", "PromiseWorker", "SubprocessConstants"], + "subprocess_unix.jsm": ["SubprocessImpl"], + "subprocess_win.jsm": ["SubprocessImpl"], + "sync.jsm": ["Authentication"], + "tabs.js": ["TabEngine", "TabSetRecord"], + "tabs.jsm": ["BrowserTabs"], + "tcpsocket_test.jsm": ["createSocket", "createServer", "enablePrefsAndPermissions", "socketCompartmentInstanceOfArrayBuffer"], + "telemetry.js": ["SyncTelemetry"], + "test.jsm": ["Foo"], + "test2.jsm": ["Bar"], + "test_bug883784.jsm": ["Test"], + "Timer.jsm": ["setTimeout", "clearTimeout", "setInterval", "clearInterval"], + "tokenserverclient.js": ["TokenServerClient", "TokenServerClientError", "TokenServerClientNetworkError", "TokenServerClientServerError"], + "ToolboxProcess.jsm": ["BrowserToolboxProcess"], + "tps.jsm": ["ACTIONS", "TPS"], + "Traversal.jsm": ["TraversalRules", "TraversalHelper"], + "userapi.js": ["UserAPI10Client"], + "util.js": ["getChromeWindow", "XPCOMUtils", "Services", "Utils", "Async", "Svc", "Str"], + "utils.js": ["applicationName", "assert", "Copy", "getBrowserObject", "getChromeWindow", "getWindows", "getWindowByTitle", "getWindowByType", "getWindowId", "getMethodInWindows", "getPreference", "saveDataURL", "setPreference", "sleep", "startTimer", "stopTimer", "takeScreenshot", "unwrapNode", "waitFor", "btoa", "encryptPayload", "isConfiguredWithLegacyIdentity", "ensureLegacyIdentityManager", "setBasicCredentials", "makeIdentityConfig", "configureIdentity", "SyncTestingInfrastructure", "waitForZeroTimer", "Promise", "add_identity_test", "MockFxaStorageManager", "AccountState", "sumHistogram", "CommonUtils", "CryptoUtils", "TestingUtils"], + "Utils.jsm": ["Utils", "Logger", "PivotContext", "PrefCache", "SettingCache"], + "VariablesView.jsm": ["VariablesView", "escapeHTML"], + "VariablesViewController.jsm": ["VariablesViewController", "StackFrameUtils"], + "version.jsm": ["VERSION"], + "vtt.jsm": ["WebVTT"], + "WebChannel.jsm": ["WebChannel", "WebChannelBroker"], + "WindowDraggingUtils.jsm": ["WindowDraggingElement"], + "windows.js": ["init", "map"], + "windows.jsm": ["BrowserWindows"], + "WindowsJumpLists.jsm": ["WinTaskbarJumpList"], + "WindowsPreviewPerTab.jsm": ["AeroPeek"], + "withs.js": ["startsWith", "endsWith"], + "xul-app.jsm": ["XulApp"] +} diff --git a/tools/lint/eslint/npm-shrinkwrap.json b/tools/lint/eslint/npm-shrinkwrap.json new file mode 100644 index 000000000..4421c4cd3 --- /dev/null +++ b/tools/lint/eslint/npm-shrinkwrap.json @@ -0,0 +1,718 @@ +{ + "name": "mach-eslint", + "dependencies": { + "acorn": { + "version": "4.0.3", + "from": "acorn@>=4.0.1 <5.0.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-4.0.3.tgz" + }, + "acorn-jsx": { + "version": "3.0.1", + "from": "acorn-jsx@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-3.0.1.tgz", + "dependencies": { + "acorn": { + "version": "3.3.0", + "from": "acorn@>=3.0.4 <4.0.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz" + } + } + }, + "ajv": { + "version": "4.8.2", + "from": "ajv@>=4.7.0 <5.0.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.8.2.tgz" + }, + "ajv-keywords": { + "version": "1.1.1", + "from": "ajv-keywords@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-1.1.1.tgz" + }, + "ansi-escapes": { + "version": "1.4.0", + "from": "ansi-escapes@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-1.4.0.tgz" + }, + "ansi-regex": { + "version": "2.0.0", + "from": "ansi-regex@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" + }, + "ansi-styles": { + "version": "2.2.1", + "from": "ansi-styles@>=2.2.1 <3.0.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz" + }, + "argparse": { + "version": "1.0.9", + "from": "argparse@>=1.0.7 <2.0.0", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.9.tgz" + }, + "array-union": { + "version": "1.0.2", + "from": "array-union@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz" + }, + "array-uniq": { + "version": "1.0.3", + "from": "array-uniq@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz" + }, + "arrify": { + "version": "1.0.1", + "from": "arrify@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz" + }, + "balanced-match": { + "version": "0.4.2", + "from": "balanced-match@>=0.4.1 <0.5.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.4.2.tgz" + }, + "brace-expansion": { + "version": "1.1.6", + "from": "brace-expansion@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.6.tgz" + }, + "caller-path": { + "version": "0.1.0", + "from": "caller-path@>=0.1.0 <0.2.0", + "resolved": "https://registry.npmjs.org/caller-path/-/caller-path-0.1.0.tgz" + }, + "callsites": { + "version": "0.2.0", + "from": "callsites@>=0.2.0 <0.3.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-0.2.0.tgz" + }, + "chalk": { + "version": "1.1.3", + "from": "chalk@>=1.1.3 <2.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz" + }, + "circular-json": { + "version": "0.3.1", + "from": "circular-json@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.3.1.tgz" + }, + "cli-cursor": { + "version": "1.0.2", + "from": "cli-cursor@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-1.0.2.tgz" + }, + "cli-width": { + "version": "2.1.0", + "from": "cli-width@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.1.0.tgz" + }, + "co": { + "version": "4.6.0", + "from": "co@>=4.6.0 <5.0.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz" + }, + "code-point-at": { + "version": "1.1.0", + "from": "code-point-at@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz" + }, + "concat-map": { + "version": "0.0.1", + "from": "concat-map@0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" + }, + "concat-stream": { + "version": "1.5.2", + "from": "concat-stream@>=1.4.6 <2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.2.tgz" + }, + "core-util-is": { + "version": "1.0.2", + "from": "core-util-is@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" + }, + "d": { + "version": "0.1.1", + "from": "d@>=0.1.1 <0.2.0", + "resolved": "https://registry.npmjs.org/d/-/d-0.1.1.tgz" + }, + "debug": { + "version": "2.3.0", + "from": "debug@>=2.1.1 <3.0.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.3.0.tgz" + }, + "deep-is": { + "version": "0.1.3", + "from": "deep-is@>=0.1.3 <0.2.0", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz" + }, + "del": { + "version": "2.2.2", + "from": "del@>=2.0.2 <3.0.0", + "resolved": "https://registry.npmjs.org/del/-/del-2.2.2.tgz" + }, + "doctrine": { + "version": "1.5.0", + "from": "doctrine@>=1.2.2 <2.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.5.0.tgz" + }, + "dom-serializer": { + "version": "0.1.0", + "from": "dom-serializer@>=0.0.0 <1.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", + "dependencies": { + "domelementtype": { + "version": "1.1.3", + "from": "domelementtype@>=1.1.1 <1.2.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz" + } + } + }, + "domelementtype": { + "version": "1.3.0", + "from": "domelementtype@>=1.3.0 <2.0.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz" + }, + "domhandler": { + "version": "2.3.0", + "from": "domhandler@>=2.3.0 <3.0.0", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.3.0.tgz" + }, + "domutils": { + "version": "1.5.1", + "from": "domutils@>=1.5.1 <2.0.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz" + }, + "entities": { + "version": "1.1.1", + "from": "entities@>=1.1.1 <2.0.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz" + }, + "es5-ext": { + "version": "0.10.12", + "from": "es5-ext@>=0.10.11 <0.11.0", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.12.tgz" + }, + "es6-iterator": { + "version": "2.0.0", + "from": "es6-iterator@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.0.tgz" + }, + "es6-map": { + "version": "0.1.4", + "from": "es6-map@>=0.1.3 <0.2.0", + "resolved": "https://registry.npmjs.org/es6-map/-/es6-map-0.1.4.tgz" + }, + "es6-set": { + "version": "0.1.4", + "from": "es6-set@>=0.1.3 <0.2.0", + "resolved": "https://registry.npmjs.org/es6-set/-/es6-set-0.1.4.tgz" + }, + "es6-symbol": { + "version": "3.1.0", + "from": "es6-symbol@>=3.1.0 <3.2.0", + "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.0.tgz" + }, + "es6-weak-map": { + "version": "2.0.1", + "from": "es6-weak-map@>=2.0.1 <3.0.0", + "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.1.tgz" + }, + "escape-string-regexp": { + "version": "1.0.5", + "from": "escape-string-regexp@>=1.0.2 <2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" + }, + "escope": { + "version": "3.6.0", + "from": "escope@>=3.6.0 <4.0.0", + "resolved": "https://registry.npmjs.org/escope/-/escope-3.6.0.tgz" + }, + "eslint": { + "version": "3.8.1", + "from": "eslint@3.8.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-3.8.1.tgz" + }, + "eslint-plugin-html": { + "version": "1.5.2", + "from": "eslint-plugin-html@1.5.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-html/-/eslint-plugin-html-1.5.2.tgz" + }, + "eslint-plugin-react": { + "version": "4.2.3", + "from": "eslint-plugin-react@4.2.3", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-4.2.3.tgz" + }, + "espree": { + "version": "3.3.2", + "from": "espree@>=3.2.0 <4.0.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-3.3.2.tgz" + }, + "esprima": { + "version": "2.7.3", + "from": "esprima@>=2.6.0 <3.0.0", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz" + }, + "esrecurse": { + "version": "4.1.0", + "from": "esrecurse@>=4.1.0 <5.0.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.1.0.tgz", + "dependencies": { + "estraverse": { + "version": "4.1.1", + "from": "estraverse@>=4.1.0 <4.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.1.1.tgz" + } + } + }, + "estraverse": { + "version": "4.2.0", + "from": "estraverse@>=4.2.0 <5.0.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz" + }, + "esutils": { + "version": "2.0.2", + "from": "esutils@>=2.0.2 <3.0.0", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz" + }, + "event-emitter": { + "version": "0.3.4", + "from": "event-emitter@>=0.3.4 <0.4.0", + "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.4.tgz" + }, + "exit-hook": { + "version": "1.1.1", + "from": "exit-hook@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/exit-hook/-/exit-hook-1.1.1.tgz" + }, + "fast-levenshtein": { + "version": "2.0.5", + "from": "fast-levenshtein@>=2.0.4 <2.1.0", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.5.tgz" + }, + "figures": { + "version": "1.7.0", + "from": "figures@>=1.3.5 <2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-1.7.0.tgz" + }, + "file-entry-cache": { + "version": "2.0.0", + "from": "file-entry-cache@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-2.0.0.tgz" + }, + "flat-cache": { + "version": "1.2.1", + "from": "flat-cache@>=1.2.1 <2.0.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-1.2.1.tgz" + }, + "fs.realpath": { + "version": "1.0.0", + "from": "fs.realpath@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" + }, + "generate-function": { + "version": "2.0.0", + "from": "generate-function@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz" + }, + "generate-object-property": { + "version": "1.2.0", + "from": "generate-object-property@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz" + }, + "glob": { + "version": "7.1.1", + "from": "glob@>=7.0.3 <8.0.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.1.tgz" + }, + "globals": { + "version": "9.12.0", + "from": "globals@>=9.2.0 <10.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-9.12.0.tgz" + }, + "globby": { + "version": "5.0.0", + "from": "globby@>=5.0.0 <6.0.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-5.0.0.tgz" + }, + "graceful-fs": { + "version": "4.1.10", + "from": "graceful-fs@>=4.1.2 <5.0.0", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.10.tgz" + }, + "has-ansi": { + "version": "2.0.0", + "from": "has-ansi@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz" + }, + "htmlparser2": { + "version": "3.9.2", + "from": "htmlparser2@>=3.8.2 <4.0.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz" + }, + "ignore": { + "version": "3.2.0", + "from": "ignore@>=3.1.5 <4.0.0", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.2.0.tgz" + }, + "imurmurhash": { + "version": "0.1.4", + "from": "imurmurhash@>=0.1.4 <0.2.0", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz" + }, + "inflight": { + "version": "1.0.6", + "from": "inflight@>=1.0.4 <2.0.0", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz" + }, + "inherits": { + "version": "2.0.3", + "from": "inherits@>=2.0.1 <2.1.0", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" + }, + "ini-parser": { + "version": "0.0.2", + "from": "ini-parser@>=0.0.2 <0.0.3", + "resolved": "https://registry.npmjs.org/ini-parser/-/ini-parser-0.0.2.tgz" + }, + "inquirer": { + "version": "0.12.0", + "from": "inquirer@>=0.12.0 <0.13.0", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-0.12.0.tgz" + }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "from": "is-fullwidth-code-point@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz" + }, + "is-my-json-valid": { + "version": "2.15.0", + "from": "is-my-json-valid@>=2.10.0 <3.0.0", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.15.0.tgz" + }, + "is-path-cwd": { + "version": "1.0.0", + "from": "is-path-cwd@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-1.0.0.tgz" + }, + "is-path-in-cwd": { + "version": "1.0.0", + "from": "is-path-in-cwd@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-1.0.0.tgz" + }, + "is-path-inside": { + "version": "1.0.0", + "from": "is-path-inside@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.0.tgz" + }, + "is-property": { + "version": "1.0.2", + "from": "is-property@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz" + }, + "is-resolvable": { + "version": "1.0.0", + "from": "is-resolvable@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/is-resolvable/-/is-resolvable-1.0.0.tgz" + }, + "isarray": { + "version": "1.0.0", + "from": "isarray@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" + }, + "js-yaml": { + "version": "3.6.1", + "from": "js-yaml@>=3.5.1 <4.0.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.6.1.tgz" + }, + "json-stable-stringify": { + "version": "1.0.1", + "from": "json-stable-stringify@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz" + }, + "jsonify": { + "version": "0.0.0", + "from": "jsonify@>=0.0.0 <0.1.0", + "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz" + }, + "jsonpointer": { + "version": "4.0.0", + "from": "jsonpointer@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.0.0.tgz" + }, + "levn": { + "version": "0.3.0", + "from": "levn@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz" + }, + "lodash": { + "version": "4.16.6", + "from": "lodash@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.16.6.tgz" + }, + "minimatch": { + "version": "3.0.3", + "from": "minimatch@>=3.0.2 <4.0.0", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.3.tgz" + }, + "minimist": { + "version": "0.0.8", + "from": "minimist@0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz" + }, + "mkdirp": { + "version": "0.5.1", + "from": "mkdirp@>=0.5.0 <0.6.0", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz" + }, + "ms": { + "version": "0.7.2", + "from": "ms@0.7.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz" + }, + "mute-stream": { + "version": "0.0.5", + "from": "mute-stream@0.0.5", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.5.tgz" + }, + "natural-compare": { + "version": "1.4.0", + "from": "natural-compare@>=1.4.0 <2.0.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz" + }, + "number-is-nan": { + "version": "1.0.1", + "from": "number-is-nan@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz" + }, + "object-assign": { + "version": "4.1.0", + "from": "object-assign@>=4.0.1 <5.0.0", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.0.tgz" + }, + "once": { + "version": "1.4.0", + "from": "once@>=1.3.0 <2.0.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz" + }, + "onetime": { + "version": "1.1.0", + "from": "onetime@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz" + }, + "optionator": { + "version": "0.8.2", + "from": "optionator@>=0.8.2 <0.9.0", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz" + }, + "os-homedir": { + "version": "1.0.2", + "from": "os-homedir@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz" + }, + "path-is-absolute": { + "version": "1.0.1", + "from": "path-is-absolute@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz" + }, + "path-is-inside": { + "version": "1.0.2", + "from": "path-is-inside@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz" + }, + "pify": { + "version": "2.3.0", + "from": "pify@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz" + }, + "pinkie": { + "version": "2.0.4", + "from": "pinkie@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz" + }, + "pinkie-promise": { + "version": "2.0.1", + "from": "pinkie-promise@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz" + }, + "pluralize": { + "version": "1.2.1", + "from": "pluralize@>=1.2.1 <2.0.0", + "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-1.2.1.tgz" + }, + "prelude-ls": { + "version": "1.1.2", + "from": "prelude-ls@>=1.1.2 <1.2.0", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz" + }, + "process-nextick-args": { + "version": "1.0.7", + "from": "process-nextick-args@>=1.0.6 <1.1.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz" + }, + "progress": { + "version": "1.1.8", + "from": "progress@>=1.1.8 <2.0.0", + "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz" + }, + "readable-stream": { + "version": "2.0.6", + "from": "readable-stream@>=2.0.0 <2.1.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz" + }, + "readline2": { + "version": "1.0.1", + "from": "readline2@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/readline2/-/readline2-1.0.1.tgz" + }, + "require-uncached": { + "version": "1.0.3", + "from": "require-uncached@>=1.0.2 <2.0.0", + "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz" + }, + "resolve-from": { + "version": "1.0.1", + "from": "resolve-from@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-1.0.1.tgz" + }, + "restore-cursor": { + "version": "1.0.1", + "from": "restore-cursor@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-1.0.1.tgz" + }, + "rimraf": { + "version": "2.5.4", + "from": "rimraf@>=2.2.8 <3.0.0", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.5.4.tgz" + }, + "run-async": { + "version": "0.1.0", + "from": "run-async@>=0.1.0 <0.2.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-0.1.0.tgz" + }, + "rx-lite": { + "version": "3.1.2", + "from": "rx-lite@>=3.1.2 <4.0.0", + "resolved": "https://registry.npmjs.org/rx-lite/-/rx-lite-3.1.2.tgz" + }, + "sax": { + "version": "1.2.1", + "from": "sax@>=1.1.4 <2.0.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz" + }, + "shelljs": { + "version": "0.6.1", + "from": "shelljs@>=0.6.0 <0.7.0", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.6.1.tgz" + }, + "slice-ansi": { + "version": "0.0.4", + "from": "slice-ansi@0.0.4", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-0.0.4.tgz" + }, + "sprintf-js": { + "version": "1.0.3", + "from": "sprintf-js@>=1.0.2 <1.1.0", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz" + }, + "string_decoder": { + "version": "0.10.31", + "from": "string_decoder@>=0.10.0 <0.11.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" + }, + "string-width": { + "version": "1.0.2", + "from": "string-width@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz" + }, + "strip-ansi": { + "version": "3.0.1", + "from": "strip-ansi@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz" + }, + "strip-bom": { + "version": "3.0.0", + "from": "strip-bom@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz" + }, + "strip-json-comments": { + "version": "1.0.4", + "from": "strip-json-comments@>=1.0.1 <1.1.0", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-1.0.4.tgz" + }, + "supports-color": { + "version": "2.0.0", + "from": "supports-color@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz" + }, + "table": { + "version": "3.8.3", + "from": "table@>=3.7.8 <4.0.0", + "resolved": "https://registry.npmjs.org/table/-/table-3.8.3.tgz", + "dependencies": { + "is-fullwidth-code-point": { + "version": "2.0.0", + "from": "is-fullwidth-code-point@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz" + }, + "string-width": { + "version": "2.0.0", + "from": "string-width@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.0.0.tgz" + } + } + }, + "text-table": { + "version": "0.2.0", + "from": "text-table@>=0.2.0 <0.3.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz" + }, + "through": { + "version": "2.3.8", + "from": "through@>=2.3.6 <3.0.0", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz" + }, + "tryit": { + "version": "1.0.3", + "from": "tryit@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/tryit/-/tryit-1.0.3.tgz" + }, + "type-check": { + "version": "0.3.2", + "from": "type-check@>=0.3.2 <0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz" + }, + "typedarray": { + "version": "0.0.6", + "from": "typedarray@>=0.0.5 <0.1.0", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz" + }, + "user-home": { + "version": "2.0.0", + "from": "user-home@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/user-home/-/user-home-2.0.0.tgz" + }, + "util-deprecate": { + "version": "1.0.2", + "from": "util-deprecate@>=1.0.1 <1.1.0", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" + }, + "wordwrap": { + "version": "1.0.0", + "from": "wordwrap@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz" + }, + "wrappy": { + "version": "1.0.2", + "from": "wrappy@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz" + }, + "write": { + "version": "0.2.1", + "from": "write@>=0.2.1 <0.3.0", + "resolved": "https://registry.npmjs.org/write/-/write-0.2.1.tgz" + }, + "xtend": { + "version": "4.0.1", + "from": "xtend@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz" + } + } +} diff --git a/tools/lint/eslint/package.json b/tools/lint/eslint/package.json new file mode 100644 index 000000000..9c0a8f803 --- /dev/null +++ b/tools/lint/eslint/package.json @@ -0,0 +1,16 @@ +{ + "name": "mach-eslint", + "description": "ESLint and external plugins for use with mach", + "repository": {}, + "license": "MPL-2.0", + "dependencies": { + "eslint": "3.8.1", + "eslint-plugin-html": "1.5.2", + "eslint-plugin-react": "4.2.3", + "escope": "^3.6.0", + "espree": "^3.2.0", + "estraverse": "^4.2.0", + "ini-parser": "^0.0.2", + "sax": "^1.1.4" + } +} diff --git a/tools/lint/eslint/update b/tools/lint/eslint/update new file mode 100755 index 000000000..477584236 --- /dev/null +++ b/tools/lint/eslint/update @@ -0,0 +1,70 @@ +#!/bin/sh +# Force the scripts working directory to be projdir/tools/lint/eslint. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $DIR + +echo "To complete this script you will need the following tokens from https://api.pub.build.mozilla.org/tokenauth/" +echo " - tooltool.upload.public" +echo " - tooltool.download.public" +echo "" +read -p "Are these tokens visible at the above URL (y/n)?" choice +case "$choice" in + y|Y ) + echo "" + echo "1. Go to https://api.pub.build.mozilla.org/" + echo "2. Log in using your Mozilla LDAP account." + echo "3. Click on \"Tokens.\"" + echo "4. Issue a user token with the permissions tooltool.upload.public and tooltool.download.public." + echo "" + echo "When you click issue you will be presented with a long string. Paste the string into a temporary file called ~/.tooltool-token." + echo "" + read -rsp $'Press any key to continue...\n' -n 1 + ;; + n|N ) + echo "" + echo "You will need to contact somebody that has these permissions... people most likely to have these permissions are members of the releng, ateam, a sheriff, mratcliffe, or jryans" + exit 1 + ;; + * ) + echo "" + echo "Invalid input." + continue + ;; +esac + +echo "" +echo "Removing node_modules and npm_shrinkwrap.json..." +rm -rf node_modules/ +rm npm-shrinkwrap.json + +echo "Installing eslint and external plugins..." +# ESLint and all _external_ plugins are listed in this directory's package.json, +# so a regular `npm install` will install them at the specified versions. +# The in-tree eslint-plugin-mozilla is kept out of this tooltool archive on +# purpose so that it can be changed by any developer without requiring tooltool +# access to make changes. +npm install + +echo "Creating npm shrinkwrap..." +npm shrinkwrap + +echo "Creating eslint.tar.gz..." +tar cvfz eslint.tar.gz node_modules + +echo "Downloading tooltool..." +wget https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py +chmod +x tooltool.py + +echo "Adding eslint.tar.gz to tooltool..." +rm manifest.tt +./tooltool.py add --visibility public eslint.tar.gz + +echo "Uploading eslint.tar.gz to tooltool..." +./tooltool.py upload --authentication-file=~/.tooltool-token --message "node_modules folder update for tools/lint/eslint" + +echo "Cleaning up..." +rm eslint.tar.gz +rm tooltool.py + +echo "" +echo "Update complete, please commit and check in your changes." diff --git a/tools/lint/flake8.lint b/tools/lint/flake8.lint new file mode 100644 index 000000000..2609bfbb2 --- /dev/null +++ b/tools/lint/flake8.lint @@ -0,0 +1,194 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import signal +import subprocess + +import which +from mozprocess import ProcessHandler + +from mozlint import result + + +here = os.path.abspath(os.path.dirname(__file__)) +FLAKE8_REQUIREMENTS_PATH = os.path.join(here, 'flake8', 'flake8_requirements.txt') + +FLAKE8_NOT_FOUND = """ +Could not find flake8! Install flake8 and try again. + + $ pip install -U --require-hashes -r {} +""".strip().format(FLAKE8_REQUIREMENTS_PATH) + + +FLAKE8_INSTALL_ERROR = """ +Unable to install correct version of flake8 +Try to install it manually with: + $ pip install -U --require-hashes -r {} +""".strip().format(FLAKE8_REQUIREMENTS_PATH) + +LINE_OFFSETS = { + # continuation line under-indented for hanging indent + 'E121': (-1, 2), + # continuation line missing indentation or outdented + 'E122': (-1, 2), + # continuation line over-indented for hanging indent + 'E126': (-1, 2), + # continuation line over-indented for visual indent + 'E127': (-1, 2), + # continuation line under-indented for visual indent + 'E128': (-1, 2), + # continuation line unaligned for hanging indend + 'E131': (-1, 2), + # expected 1 blank line, found 0 + 'E301': (-1, 2), + # expected 2 blank lines, found 1 + 'E302': (-2, 3), +} +"""Maps a flake8 error to a lineoffset tuple. + +The offset is of the form (lineno_offset, num_lines) and is passed +to the lineoffset property of `ResultContainer`. +""" + +EXTENSIONS = ['.py', '.lint'] +results = [] + + +def process_line(line): + # Escape slashes otherwise JSON conversion will not work + line = line.replace('\\', '\\\\') + try: + res = json.loads(line) + except ValueError: + print('Non JSON output from linter, will not be processed: {}'.format(line)) + return + + if 'code' in res: + if res['code'].startswith('W'): + res['level'] = 'warning' + + if res['code'] in LINE_OFFSETS: + res['lineoffset'] = LINE_OFFSETS[res['code']] + + results.append(result.from_linter(LINTER, **res)) + + +def run_process(cmdargs): + # flake8 seems to handle SIGINT poorly. Handle it here instead + # so we can kill the process without a cryptic traceback. + orig = signal.signal(signal.SIGINT, signal.SIG_IGN) + proc = ProcessHandler(cmdargs, env=os.environ, + processOutputLine=process_line) + proc.run() + signal.signal(signal.SIGINT, orig) + + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + + +def get_flake8_binary(): + """ + Returns the path of the first flake8 binary available + if not found returns None + """ + binary = os.environ.get('FLAKE8') + if binary: + return binary + + try: + return which.which('flake8') + except which.WhichError: + return None + + +def _run_pip(*args): + """ + Helper function that runs pip with subprocess + """ + try: + subprocess.check_output(['pip'] + list(args), + stderr=subprocess.STDOUT) + return True + except subprocess.CalledProcessError as e: + print(e.output) + return False + + +def reinstall_flake8(): + """ + Try to install flake8 at the target version, returns True on success + otherwise prints the otuput of the pip command and returns False + """ + if _run_pip('install', '-U', + '--require-hashes', '-r', + FLAKE8_REQUIREMENTS_PATH): + return True + + return False + + +def lint(files, **lintargs): + + if not reinstall_flake8(): + print(FLAKE8_INSTALL_ERROR) + return 1 + + binary = get_flake8_binary() + + cmdargs = [ + binary, + '--format', '{"path":"%(path)s","lineno":%(row)s,' + '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}', + ] + + # Run any paths with a .flake8 file in the directory separately so + # it gets picked up. This means only .flake8 files that live in + # directories that are explicitly included will be considered. + # See bug 1277851 + no_config = [] + for f in files: + if not os.path.isfile(os.path.join(f, '.flake8')): + no_config.append(f) + continue + run_process(cmdargs+[f]) + + # XXX For some reason passing in --exclude results in flake8 not using + # the local .flake8 file. So for now only pass in --exclude if there + # is no local config. + exclude = lintargs.get('exclude') + if exclude: + cmdargs += ['--exclude', ','.join(lintargs['exclude'])] + + if no_config: + run_process(cmdargs+no_config) + + return results + + +LINTER = { + 'name': "flake8", + 'description': "Python linter", + 'include': [ + 'python/mozlint', + 'taskcluster', + 'testing/firefox-ui', + 'testing/marionette/client', + 'testing/marionette/harness', + 'testing/marionette/puppeteer', + 'testing/mozbase', + 'testing/mochitest', + 'testing/talos/', + 'tools/lint', + ], + 'exclude': ["testing/mozbase/mozdevice/mozdevice/Zeroconf.py", + 'testing/mochitest/pywebsocket'], + 'extensions': EXTENSIONS, + 'type': 'external', + 'payload': lint, +} diff --git a/tools/lint/flake8/flake8_requirements.txt b/tools/lint/flake8/flake8_requirements.txt new file mode 100644 index 000000000..df927d826 --- /dev/null +++ b/tools/lint/flake8/flake8_requirements.txt @@ -0,0 +1,4 @@ +flake8==2.5.4 --hash=sha256:fb5a67af4024622287a76abf6b7fe4fb3cfacf765a790976ce64f52c44c88e4a +mccabe==0.4.0 --hash=sha256:cbc2938f6c01061bc6d21d0c838c2489664755cb18676f0734d7617f4577d09e +pep8==1.7.0 --hash=sha256:4fc2e478addcf17016657dff30b2d8d611e8341fac19ccf2768802f6635d7b8a +pyflakes==1.2.3 --hash=sha256:e87bac26c62ea5b45067cc89e4a12f56e1483f1f2cda17e7c9b375b9fd2f40da diff --git a/tools/lint/mach_commands.py b/tools/lint/mach_commands.py new file mode 100644 index 000000000..f0f3c9bdf --- /dev/null +++ b/tools/lint/mach_commands.py @@ -0,0 +1,62 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import os + +from mozbuild.base import ( + MachCommandBase, +) + + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) + + +here = os.path.abspath(os.path.dirname(__file__)) + + +def setup_argument_parser(): + from mozlint import cli + return cli.MozlintParser() + + +@CommandProvider +class MachCommands(MachCommandBase): + + @Command( + 'lint', category='devenv', + description='Run linters.', + parser=setup_argument_parser) + def lint(self, *runargs, **lintargs): + """Run linters.""" + from mozlint import cli + lintargs['exclude'] = ['obj*'] + cli.SEARCH_PATHS.append(here) + self._activate_virtualenv() + return cli.run(*runargs, **lintargs) + + @Command('eslint', category='devenv', + description='Run eslint or help configure eslint for optimal development.') + @CommandArgument('paths', default=None, nargs='*', + help="Paths to file or directories to lint, like " + "'browser/components/loop' Defaults to the " + "current directory if not given.") + @CommandArgument('-s', '--setup', default=False, action='store_true', + help='Configure eslint for optimal development.') + @CommandArgument('-b', '--binary', default=None, + help='Path to eslint binary.') + @CommandArgument('--fix', default=False, action='store_true', + help='Request that eslint automatically fix errors, where possible.') + @CommandArgument('extra_args', nargs=argparse.REMAINDER, + help='Extra args that will be forwarded to eslint.') + def eslint(self, paths, extra_args=[], **kwargs): + self._mach_context.commands.dispatch('lint', self._mach_context, + linters=['eslint'], paths=paths, + argv=extra_args, **kwargs) diff --git a/tools/lint/wpt.lint b/tools/lint/wpt.lint new file mode 100644 index 000000000..2b8e94421 --- /dev/null +++ b/tools/lint/wpt.lint @@ -0,0 +1,54 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os + +from mozprocess import ProcessHandler + +from mozlint import result + +top_src_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir) +tests_dir = os.path.join(top_src_dir, "testing", "web-platform", "tests") + +results = [] + + +def process_line(line): + try: + data = json.loads(line) + except ValueError: + return + data["level"] = "error" + data["path"] = os.path.relpath(os.path.join(tests_dir, data["path"]), top_src_dir) + results.append(result.from_linter(LINTER, **data)) + + +def run_process(): + path = os.path.join(tests_dir, "lint") + proc = ProcessHandler([path, "--json"], env=os.environ, + processOutputLine=process_line) + proc.run() + try: + proc.wait() + except KeyboardInterrupt: + proc.kill() + + +def lint(files, **kwargs): + run_process() + return results + + +LINTER = { + 'name': "wpt", + 'description': "web-platform-tests lint", + 'include': [ + 'testing/web-platform/tests', + ], + 'exclude': [], + 'type': 'external', + 'payload': lint, +} diff --git a/tools/lint/wpt_manifest.lint b/tools/lint/wpt_manifest.lint new file mode 100644 index 000000000..334d69689 --- /dev/null +++ b/tools/lint/wpt_manifest.lint @@ -0,0 +1,33 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import imp +import json +import os +import sys + +from mozprocess import ProcessHandler + +from mozlint import result + + +def lint(files, logger, **kwargs): + wpt_dir = os.path.join(kwargs["root"], "testing", "web-platform") + manifestupdate = imp.load_source("manifestupdate", + os.path.join(wpt_dir, "manifestupdate.py")) + manifestupdate.update(logger, wpt_dir, True) + + +LINTER = { + 'name': "wpt_manifest", + 'description': "web-platform-tests manifest lint", + 'include': [ + 'testing/web-platform/tests', + 'testing/web-platform/mozilla/tests', + ], + 'exclude': [], + 'type': 'structured_log', + 'payload': lint, +} diff --git a/tools/mach_commands.py b/tools/mach_commands.py new file mode 100644 index 000000000..898073bb6 --- /dev/null +++ b/tools/mach_commands.py @@ -0,0 +1,364 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import sys +import os +import stat +import platform +import errno +import subprocess + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) + +from mozbuild.base import MachCommandBase, MozbuildObject + + +@CommandProvider +class SearchProvider(object): + @Command('dxr', category='misc', + description='Search for something in DXR.') + @CommandArgument('term', nargs='+', help='Term(s) to search for.') + def dxr(self, term): + import webbrowser + term = ' '.join(term) + uri = 'http://dxr.mozilla.org/mozilla-central/search?q=%s&redirect=true' % term + webbrowser.open_new_tab(uri) + + @Command('mdn', category='misc', + description='Search for something on MDN.') + @CommandArgument('term', nargs='+', help='Term(s) to search for.') + def mdn(self, term): + import webbrowser + term = ' '.join(term) + uri = 'https://developer.mozilla.org/search?q=%s' % term + webbrowser.open_new_tab(uri) + + @Command('google', category='misc', + description='Search for something on Google.') + @CommandArgument('term', nargs='+', help='Term(s) to search for.') + def google(self, term): + import webbrowser + term = ' '.join(term) + uri = 'https://www.google.com/search?q=%s' % term + webbrowser.open_new_tab(uri) + + @Command('search', category='misc', + description='Search for something on the Internets. ' + 'This will open 3 new browser tabs and search for the term on Google, ' + 'MDN, and DXR.') + @CommandArgument('term', nargs='+', help='Term(s) to search for.') + def search(self, term): + self.google(term) + self.mdn(term) + self.dxr(term) + + +@CommandProvider +class UUIDProvider(object): + @Command('uuid', category='misc', + description='Generate a uuid.') + @CommandArgument('--format', '-f', choices=['idl', 'cpp', 'c++'], + help='Output format for the generated uuid.') + def uuid(self, format=None): + import uuid + u = uuid.uuid4() + if format in [None, 'idl']: + print(u) + if format is None: + print('') + if format in [None, 'cpp', 'c++']: + u = u.hex + print('{ 0x%s, 0x%s, 0x%s, \\' % (u[0:8], u[8:12], u[12:16])) + pairs = tuple(map(lambda n: u[n:n+2], range(16, 32, 2))) + print((' { ' + '0x%s, ' * 7 + '0x%s } }') % pairs) + + +@CommandProvider +class RageProvider(MachCommandBase): + @Command('rage', category='misc', + description='Express your frustration') + def rage(self): + """Have a bad experience developing Firefox? Run this command to + express your frustration. + + This command will open your default configured web browser to a short + form where you can submit feedback. Just close the tab when done. + """ + import getpass + import urllib + import webbrowser + + # Try to resolve the current user. + user = None + with open(os.devnull, 'wb') as null: + if os.path.exists(os.path.join(self.topsrcdir, '.hg')): + try: + user = subprocess.check_output(['hg', 'config', + 'ui.username'], + cwd=self.topsrcdir, + stderr=null) + + i = user.find('<') + if i >= 0: + user = user[i + 1:-2] + except subprocess.CalledProcessError: + pass + elif os.path.exists(os.path.join(self.topsrcdir, '.git')): + try: + user = subprocess.check_output(['git', 'config', '--get', + 'user.email'], + cwd=self.topsrcdir, + stderr=null) + except subprocess.CalledProcessError: + pass + + if not user: + try: + user = getpass.getuser() + except Exception: + pass + + url = 'https://docs.google.com/a/mozilla.com/forms/d/e/1FAIpQLSeDVC3IXJu5d33Hp_ZTCOw06xEUiYH1pBjAqJ1g_y63sO2vvA/viewform' + if user: + url += '?entry.1281044204=%s' % urllib.quote(user) + + print('Please leave your feedback in the opened web form') + webbrowser.open_new_tab(url) + + +@CommandProvider +class PastebinProvider(object): + @Command('pastebin', category='misc', + description='Command line interface to pastebin.mozilla.org.') + @CommandArgument('--language', default=None, + help='Language to use for syntax highlighting') + @CommandArgument('--poster', default='', + help='Specify your name for use with pastebin.mozilla.org') + @CommandArgument('--duration', default='day', + choices=['d', 'day', 'm', 'month', 'f', 'forever'], + help='Keep for specified duration (default: %(default)s)') + @CommandArgument('file', nargs='?', default=None, + help='Specify the file to upload to pastebin.mozilla.org') + + def pastebin(self, language, poster, duration, file): + import urllib + import urllib2 + + URL = 'https://pastebin.mozilla.org/' + + FILE_TYPES = [{'value': 'text', 'name': 'None', 'extension': 'txt'}, + {'value': 'bash', 'name': 'Bash', 'extension': 'sh'}, + {'value': 'c', 'name': 'C', 'extension': 'c'}, + {'value': 'cpp', 'name': 'C++', 'extension': 'cpp'}, + {'value': 'html4strict', 'name': 'HTML', 'extension': 'html'}, + {'value': 'javascript', 'name': 'Javascript', 'extension': 'js'}, + {'value': 'javascript', 'name': 'Javascript', 'extension': 'jsm'}, + {'value': 'lua', 'name': 'Lua', 'extension': 'lua'}, + {'value': 'perl', 'name': 'Perl', 'extension': 'pl'}, + {'value': 'php', 'name': 'PHP', 'extension': 'php'}, + {'value': 'python', 'name': 'Python', 'extension': 'py'}, + {'value': 'ruby', 'name': 'Ruby', 'extension': 'rb'}, + {'value': 'css', 'name': 'CSS', 'extension': 'css'}, + {'value': 'diff', 'name': 'Diff', 'extension': 'diff'}, + {'value': 'ini', 'name': 'INI file', 'extension': 'ini'}, + {'value': 'java', 'name': 'Java', 'extension': 'java'}, + {'value': 'xml', 'name': 'XML', 'extension': 'xml'}, + {'value': 'xml', 'name': 'XML', 'extension': 'xul'}] + + lang = '' + + if file: + try: + with open(file, 'r') as f: + content = f.read() + # TODO: Use mime-types instead of extensions; suprocess('file <f_name>') + # Guess File-type based on file extension + extension = file.split('.')[-1] + for l in FILE_TYPES: + if extension == l['extension']: + print('Identified file as %s' % l['name']) + lang = l['value'] + except IOError: + print('ERROR. No such file') + return 1 + else: + content = sys.stdin.read() + duration = duration[0] + + if language: + lang = language + + + params = [ + ('parent_pid', ''), + ('format', lang), + ('code2', content), + ('poster', poster), + ('expiry', duration), + ('paste', 'Send')] + + data = urllib.urlencode(params) + print('Uploading ...') + try: + req = urllib2.Request(URL, data) + response = urllib2.urlopen(req) + http_response_code = response.getcode() + if http_response_code == 200: + print(response.geturl()) + else: + print('Could not upload the file, ' + 'HTTP Response Code %s' %(http_response_code)) + except urllib2.URLError: + print('ERROR. Could not connect to pastebin.mozilla.org.') + return 1 + return 0 + + +@CommandProvider +class FormatProvider(MachCommandBase): + @Command('clang-format', category='misc', + description='Run clang-format on current changes') + @CommandArgument('--show', '-s', action = 'store_true', + help = 'Show diff output on instead of applying changes') + def clang_format(self, show=False): + import urllib2 + + plat = platform.system() + fmt = plat.lower() + "/clang-format-3.5" + fmt_diff = "clang-format-diff-3.5" + + # We are currently using a modified version of clang-format hosted on people.mozilla.org. + # This is a temporary work around until we upstream the necessary changes and we can use + # a system version of clang-format. See bug 961541. + if plat == "Windows": + fmt += ".exe" + else: + arch = os.uname()[4] + if (plat != "Linux" and plat != "Darwin") or arch != 'x86_64': + print("Unsupported platform " + plat + "/" + arch + + ". Supported platforms are Windows/*, Linux/x86_64 and Darwin/x86_64") + return 1 + + os.chdir(self.topsrcdir) + self.prompt = True + + try: + if not self.locate_or_fetch(fmt): + return 1 + clang_format_diff = self.locate_or_fetch(fmt_diff) + if not clang_format_diff: + return 1 + + except urllib2.HTTPError as e: + print("HTTP error {0}: {1}".format(e.code, e.reason)) + return 1 + + from subprocess import Popen, PIPE + + if os.path.exists(".hg"): + diff_process = Popen(["hg", "diff", "-U0", "-r", "tip^", + "--include", "glob:**.c", "--include", "glob:**.cpp", "--include", "glob:**.h", + "--exclude", "listfile:.clang-format-ignore"], stdout=PIPE) + else: + git_process = Popen(["git", "diff", "-U0", "HEAD^"], stdout=PIPE) + try: + diff_process = Popen(["filterdiff", "--include=*.h", "--include=*.cpp", + "--exclude-from-file=.clang-format-ignore"], + stdin=git_process.stdout, stdout=PIPE) + except OSError as e: + if e.errno == errno.ENOENT: + print("Can't find filterdiff. Please install patchutils.") + else: + print("OSError {0}: {1}".format(e.code, e.reason)) + return 1 + + + args = [sys.executable, clang_format_diff, "-p1"] + if not show: + args.append("-i") + cf_process = Popen(args, stdin=diff_process.stdout) + return cf_process.communicate()[0] + + def locate_or_fetch(self, root): + target = os.path.join(self._mach_context.state_dir, os.path.basename(root)) + if not os.path.exists(target): + site = "https://people.mozilla.org/~ajones/clang-format/" + if self.prompt and raw_input("Download clang-format executables from {0} (yN)? ".format(site)).lower() != 'y': + print("Download aborted.") + return 1 + self.prompt = False + + u = site + root + print("Downloading {0} to {1}".format(u, target)) + data = urllib2.urlopen(url=u).read() + temp = target + ".tmp" + with open(temp, "wb") as fh: + fh.write(data) + fh.close() + os.chmod(temp, os.stat(temp).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + os.rename(temp, target) + return target + +def mozregression_import(): + # Lazy loading of mozregression. + # Note that only the mach_interface module should be used from this file. + try: + import mozregression.mach_interface + except ImportError: + return None + return mozregression.mach_interface + + +def mozregression_create_parser(): + # Create the mozregression command line parser. + # if mozregression is not installed, or not up to date, it will + # first be installed. + cmd = MozbuildObject.from_environment() + cmd._activate_virtualenv() + mozregression = mozregression_import() + if not mozregression: + # mozregression is not here at all, install it + cmd.virtualenv_manager.install_pip_package('mozregression') + print("mozregression was installed. please re-run your" + " command. If you keep getting this message please " + " manually run: 'pip install -U mozregression'.") + else: + # check if there is a new release available + release = mozregression.new_release_on_pypi() + if release: + print(release) + # there is one, so install it. Note that install_pip_package + # does not work here, so just run pip directly. + cmd.virtualenv_manager._run_pip([ + 'install', + 'mozregression==%s' % release + ]) + print("mozregression was updated to version %s. please" + " re-run your command." % release) + else: + # mozregression is up to date, return the parser. + return mozregression.parser() + # exit if we updated or installed mozregression because + # we may have already imported mozregression and running it + # as this may cause issues. + sys.exit(0) + + +@CommandProvider +class MozregressionCommand(MachCommandBase): + @Command('mozregression', + category='misc', + description=("Regression range finder for nightly" + " and inbound builds."), + parser=mozregression_create_parser) + def run(self, **options): + self._activate_virtualenv() + mozregression = mozregression_import() + mozregression.run(options) diff --git a/tools/memory-profiler/CompactTraceTable.h b/tools/memory-profiler/CompactTraceTable.h new file mode 100644 index 000000000..45c47eccb --- /dev/null +++ b/tools/memory-profiler/CompactTraceTable.h @@ -0,0 +1,115 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef memory_profiler_CompactTraceTable_h +#define memory_profiler_CompactTraceTable_h + +#include "mozilla/HashFunctions.h" + +#include "nsDataHashtable.h" +#include "nsTArray.h" + +namespace mozilla { + +struct TrieNode final +{ + uint32_t parentIdx; + uint32_t nameIdx; + bool operator==(const TrieNode t) const + { + return parentIdx == t.parentIdx && nameIdx == t.nameIdx; + } + uint32_t Hash() const + { + return HashGeneric(parentIdx, nameIdx); + } +}; + +// This class maps a Node of type T to its parent's index in the +// map. When serializing, the map is traversed and put into an ordered +// array of Nodes. +template<typename KeyClass, typename T> +class NodeIndexMap final +{ +public: + uint32_t Insert(const T& e) + { + uint32_t index = mMap.Count(); + if (!mMap.Get(e, &index)) { + mMap.Put(e, index); + } + return index; + } + + nsTArray<T> Serialize() const + { + nsTArray<T> v; + v.SetLength(mMap.Count()); + for (auto iter = mMap.ConstIter(); !iter.Done(); iter.Next()) { + v[iter.Data()] = iter.Key(); + } + return v; + } + + uint32_t Size() const + { + return mMap.Count(); + } + + void Clear() + { + mMap.Clear(); + } +private: + nsDataHashtable<KeyClass, uint32_t> mMap; +}; + +// Backtraces are stored in a trie to save spaces. +// Function names are stored in an unique table and TrieNodes contain indexes +// into that table. +// The trie is implemented with a hash table; children are stored in +// traces[TrieNode{parent node index, branch/function name index}]. +class CompactTraceTable final +{ +public: + CompactTraceTable() + { + mNames.Insert(nsAutoCString("(unknown)")); + mTraces.Insert(TrieNode{0, 0}); + } + + nsTArray<nsCString> GetNames() const + { + return mNames.Serialize(); + } + + nsTArray<TrieNode> GetTraces() const + { + return mTraces.Serialize(); + } + + // Returns an ID to a stacktrace. + uint32_t Insert(const nsTArray<nsCString>& aRawStacktrace) + { + uint32_t parent = 0; + for (auto& frame: aRawStacktrace) { + parent = mTraces.Insert(TrieNode{parent, mNames.Insert(frame)}); + } + return parent; + } + + void Reset() + { + mNames.Clear(); + mTraces.Clear(); + } +private: + NodeIndexMap<nsCStringHashKey, nsCString> mNames; + NodeIndexMap<nsGenericHashKey<TrieNode>, TrieNode> mTraces; +}; + +} // namespace mozilla + +#endif // memory_profiler_CompactTraceTable_h diff --git a/tools/memory-profiler/moz.build b/tools/memory-profiler/moz.build new file mode 100644 index 000000000..434617f06 --- /dev/null +++ b/tools/memory-profiler/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +if CONFIG['GNU_CXX']: + CXXFLAGS += ['-Wno-error=shadow'] diff --git a/tools/memory/collect_b2g_uss_data.sh b/tools/memory/collect_b2g_uss_data.sh new file mode 100644 index 000000000..15b911b8f --- /dev/null +++ b/tools/memory/collect_b2g_uss_data.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +if [ $# -ne 2 ] +then + echo "Usage: `basename $0` <app title> <output_file.csv>" + exit 1 +fi + +while true +do + sample=`adb shell b2g-procrank | grep "^${1}" | awk '{ print $6 }' | sed 's/.$//'` + echo "$sample" + echo "$sample" >> "$2" + sleep 1 +done + diff --git a/tools/mercurial/eslintvalidate.py b/tools/mercurial/eslintvalidate.py new file mode 100644 index 000000000..5de0f9416 --- /dev/null +++ b/tools/mercurial/eslintvalidate.py @@ -0,0 +1,76 @@ +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +import os +import sys +import re +import json +from subprocess import check_output, CalledProcessError + +lintable = re.compile(r'.+\.(?:js|jsm|jsx|xml|html)$') +ignored = 'File ignored because of a matching ignore pattern. Use "--no-ignore" to override.' + +def is_lintable(filename): + return lintable.match(filename) + +def display(ui, output): + results = json.loads(output) + for file in results: + path = os.path.relpath(file["filePath"]) + for message in file["messages"]: + if message["message"] == ignored: + continue + + if "line" in message: + ui.warn("%s:%d:%d %s\n" % (path, message["line"], message["column"], message["message"])) + else: + ui.warn("%s: %s\n" % (path, message["message"])) + +def eslinthook(ui, repo, node=None, **opts): + ctx = repo[node] + if len(ctx.parents()) > 1: + return 0 + + deleted = repo.status(ctx.p1().node(), ctx.node()).deleted + files = [f for f in ctx.files() if f not in deleted and is_lintable(f)] + + if len(files) == 0: + return + + try: + basepath = get_project_root() + + if not basepath: + return + + dir = os.path.join(basepath, "tools", "lint", "eslint", "node_modules", ".bin") + + eslint_path = os.path.join(dir, "eslint") + if os.path.exists(os.path.join(dir, "eslint.cmd")): + eslint_path = os.path.join(dir, "eslint.cmd") + output = check_output([eslint_path, + "--format", "json", "--plugin", "html"] + files, + cwd=basepath) + display(ui, output) + except CalledProcessError as ex: + display(ui, ex.output) + ui.warn("ESLint found problems in your changes, please correct them.\n") + +def reposetup(ui, repo): + ui.setconfig('hooks', 'commit.eslint', eslinthook) + +def get_project_root(): + file_found = False + folder = os.getcwd() + + while (folder): + if os.path.exists(os.path.join(folder, 'mach')): + file_found = True + break + else: + folder = os.path.dirname(folder) + + if file_found: + return os.path.abspath(folder) + + return None diff --git a/tools/moz.build b/tools/moz.build new file mode 100644 index 000000000..b41094f6d --- /dev/null +++ b/tools/moz.build @@ -0,0 +1,6 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SPHINX_TREES['lint'] = 'lint/docs' diff --git a/tools/power/mach_commands.py b/tools/power/mach_commands.py new file mode 100644 index 000000000..281e7a868 --- /dev/null +++ b/tools/power/mach_commands.py @@ -0,0 +1,142 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function + +from distutils.version import StrictVersion + +from mach.decorators import ( + Command, + CommandArgument, + CommandProvider, +) +from mozbuild.base import ( + MachCommandBase, + MachCommandConditions as conditions, +) + + +def is_osx_10_10_or_greater(cls): + import platform + release = platform.mac_ver()[0] + return release and StrictVersion(release) >= StrictVersion('10.10') + + +@CommandProvider +class MachCommands(MachCommandBase): + ''' + Get system power consumption and related measurements. + ''' + def __init__(self, context): + MachCommandBase.__init__(self, context) + + @Command('power', category='misc', + conditions=[is_osx_10_10_or_greater], + description='Get system power consumption and related measurements for ' + 'all running browsers. Available only on Mac OS X 10.10 and above. ' + 'Requires root access.') + @CommandArgument('-i', '--interval', type=int, default=30000, + help='The sample period, measured in milliseconds. Defaults to 30000.') + def power(self, interval): + import os + import re + import subprocess + + rapl = os.path.join(self.topobjdir, 'dist', 'bin', 'rapl') + + interval = str(interval) + + # Run a trivial command with |sudo| to gain temporary root privileges + # before |rapl| and |powermetrics| are called. This ensures that |rapl| + # doesn't start measuring while |powermetrics| is waiting for the root + # password to be entered. + try: + subprocess.check_call(['sudo', 'true']) + except: + print('\nsudo failed; aborting') + return 1 + + # This runs rapl in the background because nothing in this script + # depends on the output. This is good because we want |rapl| and + # |powermetrics| to run at the same time. + subprocess.Popen([rapl, '-n', '1', '-i', interval]) + + lines = subprocess.check_output(['sudo', 'powermetrics', + '--samplers', 'tasks', + '--show-process-coalition', + '--show-process-gpu', + '-n', '1', + '-i', interval]) + + # When run with --show-process-coalition, |powermetrics| groups outputs + # into process coalitions, each of which has a leader. + # + # For example, when Firefox runs from the dock, its coalition looks + # like this: + # + # org.mozilla.firefox + # firefox + # plugin-container + # + # When Safari runs from the dock: + # + # com.apple.Safari + # Safari + # com.apple.WebKit.Networking + # com.apple.WebKit.WebContent + # com.apple.WebKit.WebContent + # + # When Chrome runs from the dock: + # + # com.google.Chrome + # Google Chrome + # Google Chrome Helper + # Google Chrome Helper + # + # In these cases, we want to print the whole coalition. + # + # Also, when you run any of them from the command line, things are the + # same except that the leader is com.apple.Terminal and there may be + # non-browser processes in the coalition, e.g.: + # + # com.apple.Terminal + # firefox + # plugin-container + # <and possibly other, non-browser processes> + # + # Also, the WindowServer and kernel coalitions and processes are often + # relevant. + # + # We want to print all these but omit uninteresting coalitions. We + # could do this by properly parsing powermetrics output, but it's + # simpler and more robust to just grep for a handful of identifying + # strings. + + print() # blank line between |rapl| output and |powermetrics| output + + for line in lines.splitlines(): + # Search for the following things. + # + # - '^Name' is for the columns headings line. + # + # - 'firefox' and 'plugin-container' are for Firefox + # + # - 'Safari\b' and 'WebKit' are for Safari. The '\b' excludes + # SafariCloudHistoryPush, which is a process that always + # runs, even when Safari isn't open. + # + # - 'Chrome' is for Chrome. + # + # - 'Terminal' is for the terminal. If no browser is running from + # within the terminal, it will show up unnecessarily. This is a + # minor disadvantage of this very simple parsing strategy. + # + # - 'WindowServer' is for the WindowServer. + # + # - 'kernel' is for the kernel. + # + if re.search(r'(^Name|firefox|plugin-container|Safari\b|WebKit|Chrome|Terminal|WindowServer|kernel)', line): + print(line) + + return 0 diff --git a/tools/power/moz.build b/tools/power/moz.build new file mode 100644 index 000000000..004dd3354 --- /dev/null +++ b/tools/power/moz.build @@ -0,0 +1,19 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +do_rapl = False + +if CONFIG['OS_ARCH'] == 'Darwin' and CONFIG['CPU_ARCH'] == 'x86_64': + do_rapl = True + +if CONFIG['OS_ARCH'] == 'Linux' and CONFIG['CPU_ARCH'] in ('x86', 'x86_64'): + do_rapl = True + +if do_rapl: + SimplePrograms([ + 'rapl', + ]) + +DISABLE_STL_WRAPPING = True diff --git a/tools/power/rapl.cpp b/tools/power/rapl.cpp new file mode 100644 index 000000000..54c00f79f --- /dev/null +++ b/tools/power/rapl.cpp @@ -0,0 +1,899 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// This program provides processor power estimates. It does this by reading +// model-specific registers (MSRs) that are part Intel's Running Average Power +// Limit (RAPL) interface. These MSRs provide good quality estimates of the +// energy consumption of up to four system components: +// - PKG: the entire processor package; +// - PP0: the cores (a subset of the package); +// - PP1: the GPU (a subset of the package); +// - DRAM: main memory. +// +// For more details about RAPL, see section 14.9 of Volume 3 of the "Intel 64 +// and IA-32 Architecture's Software Developer's Manual", Order Number 325384. +// +// This program exists because there are no existing tools on Mac that can +// obtain all four RAPL estimates. (|powermetrics| can obtain the package +// estimate, but not the others. Intel Power Gadget can obtain the package and +// cores estimates.) +// +// On Linux |perf| can obtain all four estimates (as Joules, which are easily +// converted to Watts), but this program is implemented for Linux because it's +// not too hard to do, and that gives us multi-platform consistency. +// +// This program does not support Windows, unfortunately. It's not obvious how +// to access the RAPL MSRs on Windows. +// +// This program deliberately uses only standard libraries and avoids +// Mozilla-specific code, to make it easy to compile and test on different +// machines. + +#include <assert.h> +#include <getopt.h> +#include <math.h> +#include <signal.h> +#include <stdarg.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/time.h> +#include <unistd.h> + +#include <algorithm> +#include <numeric> +#include <vector> + +//--------------------------------------------------------------------------- +// Utilities +//--------------------------------------------------------------------------- + +// MOZ_FALLTHROUGH is an annotation to suppress compiler warnings about switch +// cases that fall through without a break or return statement. MOZ_FALLTHROUGH +// is only needed on cases that have code. This definition of MOZ_FALLTHROUGH +// is identical to the one in mfbt/Attributes.h, which we don't use here because +// this file avoids depending on Mozilla headers. +#if defined(__clang__) && __cplusplus >= 201103L + /* clang's fallthrough annotations are only available starting in C++11. */ +# define MOZ_FALLTHROUGH [[clang::fallthrough]] +#elif defined(_MSC_VER) + /* + * MSVC's __fallthrough annotations are checked by /analyze (Code Analysis): + * https://msdn.microsoft.com/en-us/library/ms235402%28VS.80%29.aspx + */ +# include <sal.h> +# define MOZ_FALLTHROUGH __fallthrough +#else +# define MOZ_FALLTHROUGH /* FALLTHROUGH */ +#endif + +// The value of argv[0] passed to main(). Used in error messages. +static const char* gArgv0; + +static void +Abort(const char* aFormat, ...) +{ + va_list vargs; + va_start(vargs, aFormat); + fprintf(stderr, "%s: ", gArgv0); + vfprintf(stderr, aFormat, vargs); + fprintf(stderr, "\n"); + va_end(vargs); + + exit(1); +} + +static void +CmdLineAbort(const char* aMsg) +{ + if (aMsg) { + fprintf(stderr, "%s: %s\n", gArgv0, aMsg); + } + fprintf(stderr, "Use --help for more information.\n"); + exit(1); +} + +// A special value that represents an estimate from an unsupported RAPL domain. +static const double kUnsupported_j = -1.0; + +// Print to stdout and flush it, so that the output appears immediately even if +// being redirected through |tee| or anything like that. +static void +PrintAndFlush(const char* aFormat, ...) +{ + va_list vargs; + va_start(vargs, aFormat); + vfprintf(stdout, aFormat, vargs); + va_end(vargs); + + fflush(stdout); +} + +//--------------------------------------------------------------------------- +// Mac-specific code +//--------------------------------------------------------------------------- + +#if defined(__APPLE__) + +// Because of the pkg_energy_statistics_t::pkes_version check below, the +// earliest OS X version this code will work with is 10.9.0 (xnu-2422.1.72). + +#include <sys/types.h> +#include <sys/sysctl.h> + +// OS X has four kinds of system calls: +// +// 1. Mach traps; +// 2. UNIX system calls; +// 3. machine-dependent calls; +// 4. diagnostic calls. +// +// (See "Mac OS X and iOS Internals" by Jonathan Levin for more details.) +// +// The last category has a single call named diagCall() or diagCall64(). Its +// mode is controlled by its first argument, and one of the modes allows access +// to the Intel RAPL MSRs. +// +// The interface to diagCall64() is not exported, so we have to import some +// definitions from the XNU kernel. All imported definitions are annotated with +// the XNU source file they come from, and information about what XNU versions +// they were introduced in and (if relevant) modified. + +// The diagCall64() mode. +// From osfmk/i386/Diagnostics.h +// - In 10.8.4 (xnu-2050.24.15) this value was introduced. (In 10.8.3 the value +// 17 was used for dgGzallocTest.) +#define dgPowerStat 17 + +// From osfmk/i386/cpu_data.h +// - In 10.8.5 these values were introduced, along with core_energy_stat_t. +#define CPU_RTIME_BINS (12) +#define CPU_ITIME_BINS (CPU_RTIME_BINS) + +// core_energy_stat_t and pkg_energy_statistics_t are both from +// osfmk/i386/Diagnostics.c. +// - In 10.8.4 (xnu-2050.24.15) both structs were introduced, but with many +// fewer fields. +// - In 10.8.5 (xnu-2050.48.11) both structs were substantially expanded, with +// numerous new fields. +// - In 10.9.0 (xnu-2422.1.72) pkg_energy_statistics_t::pkes_version was added. +// diagCall64(dgPowerStat) fills it with '1' in all versions since (up to +// 10.10.2 at time of writing). +// - in 10.10.2 (xnu-2782.10.72) core_energy_stat_t::gpmcs was conditionally +// added, if DIAG_ALL_PMCS is true. (DIAG_ALL_PMCS is not even defined in the +// source code, but it could be defined at compile-time via compiler flags.) +// pkg_energy_statistics_t::pkes_version did not change, though. + +typedef struct { + uint64_t caperf; + uint64_t cmperf; + uint64_t ccres[6]; + uint64_t crtimes[CPU_RTIME_BINS]; + uint64_t citimes[CPU_ITIME_BINS]; + uint64_t crtime_total; + uint64_t citime_total; + uint64_t cpu_idle_exits; + uint64_t cpu_insns; + uint64_t cpu_ucc; + uint64_t cpu_urc; +#if DIAG_ALL_PMCS // Added in 10.10.2 (xnu-2782.10.72). + uint64_t gpmcs[4]; // Added in 10.10.2 (xnu-2782.10.72). +#endif /* DIAG_ALL_PMCS */ // Added in 10.10.2 (xnu-2782.10.72). +} core_energy_stat_t; + +typedef struct { + uint64_t pkes_version; // Added in 10.9.0 (xnu-2422.1.72). + uint64_t pkg_cres[2][7]; + + // This is read from MSR 0x606, which Intel calls MSR_RAPL_POWER_UNIT + // and XNU calls MSR_IA32_PKG_POWER_SKU_UNIT. + uint64_t pkg_power_unit; + + // These are the four fields for the four RAPL domains. For each field + // we list: + // + // - the corresponding MSR number; + // - Intel's name for that MSR; + // - XNU's name for that MSR; + // - which Intel processors the MSR is supported on. + // + // The last of these is determined from chapter 35 of Volume 3 of the + // "Intel 64 and IA-32 Architecture's Software Developer's Manual", + // Order Number 325384. (Note that chapter 35 contradicts section 14.9 + // to some degree.) + + // 0x611 == MSR_PKG_ENERGY_STATUS == MSR_IA32_PKG_ENERGY_STATUS + // Atom (various), Sandy Bridge, Next Gen Xeon Phi (model 0x57). + uint64_t pkg_energy; + + // 0x639 == MSR_PP0_ENERGY_STATUS == MSR_IA32_PP0_ENERGY_STATUS + // Atom (various), Sandy Bridge, Next Gen Xeon Phi (model 0x57). + uint64_t pp0_energy; + + // 0x641 == MSR_PP1_ENERGY_STATUS == MSR_PP1_ENERGY_STATUS + // Sandy Bridge, Haswell. + uint64_t pp1_energy; + + // 0x619 == MSR_DRAM_ENERGY_STATUS == MSR_IA32_DDR_ENERGY_STATUS + // Xeon E5, Xeon E5 v2, Haswell/Haswell-E, Next Gen Xeon Phi (model + // 0x57) + uint64_t ddr_energy; + + uint64_t llc_flushed_cycles; + uint64_t ring_ratio_instantaneous; + uint64_t IA_frequency_clipping_cause; + uint64_t GT_frequency_clipping_cause; + uint64_t pkg_idle_exits; + uint64_t pkg_rtimes[CPU_RTIME_BINS]; + uint64_t pkg_itimes[CPU_ITIME_BINS]; + uint64_t mbus_delay_time; + uint64_t mint_delay_time; + uint32_t ncpus; + core_energy_stat_t cest[]; +} pkg_energy_statistics_t; + +static int +diagCall64(uint64_t aMode, void* aBuf) +{ + // We cannot use syscall() here because it doesn't work with diagnostic + // system calls -- it raises SIGSYS if you try. So we have to use asm. + +#ifdef __x86_64__ + // The 0x40000 prefix indicates it's a diagnostic system call. The 0x01 + // suffix indicates the syscall number is 1, which also happens to be the + // only diagnostic system call. See osfmk/mach/i386/syscall_sw.h for more + // details. + static const uint64_t diagCallNum = 0x4000001; + uint64_t rv; + + __asm__ __volatile__( + "syscall" + + // Return value goes in "a" (%rax). + : /* outputs */ "=a"(rv) + + // The syscall number goes in "0", a synonym (from outputs) for "a" (%rax). + // The syscall arguments go in "D" (%rdi) and "S" (%rsi). + : /* inputs */ "0"(diagCallNum), "D"(aMode), "S"(aBuf) + + // The |syscall| instruction clobbers %rcx, %r11, and %rflags ("cc"). And + // this particular syscall also writes memory (aBuf). + : /* clobbers */ "rcx", "r11", "cc", "memory" + ); + return rv; +#else +#error Sorry, only x86-64 is supported +#endif +} + +static void +diagCall64_dgPowerStat(pkg_energy_statistics_t* aPkes) +{ + static const uint64_t supported_version = 1; + + // Write an unsupported version number into pkes_version so that the check + // below cannot succeed by dumb luck. + aPkes->pkes_version = supported_version - 1; + + // diagCall64() returns 1 on success, and 0 on failure (which can only happen + // if the mode is unrecognized, e.g. in 10.7.x or earlier versions). + if (diagCall64(dgPowerStat, aPkes) != 1) { + Abort("diagCall64() failed"); + } + + if (aPkes->pkes_version != 1) { + Abort("unexpected pkes_version: %llu", aPkes->pkes_version); + } +} + +class RAPL +{ + bool mIsGpuSupported; // Is the GPU domain supported by the processor? + bool mIsRamSupported; // Is the RAM domain supported by the processor? + + // The DRAM domain on Haswell servers has a fixed energy unit (1/65536 J == + // 15.3 microJoules) which is different to the power unit MSR. (See the + // "Intel Xeon Processor E5-1600 and E5-2600 v3 Product Families, Volume 2 of + // 2, Registers" datasheet, September 2014, Reference Number: 330784-001.) + // This field records whether the quirk is present. + bool mHasRamUnitsQuirk; + + // The abovementioned 15.3 microJoules value. + static const double kQuirkyRamJoulesPerTick; + + // The previous sample's MSR values. + uint64_t mPrevPkgTicks; + uint64_t mPrevPp0Ticks; + uint64_t mPrevPp1Ticks; + uint64_t mPrevDdrTicks; + + // The struct passed to diagCall64(). + pkg_energy_statistics_t* mPkes; + +public: + RAPL() + : mHasRamUnitsQuirk(false) + { + // Work out which RAPL MSRs this CPU model supports. + int cpuModel; + size_t size = sizeof(cpuModel); + if (sysctlbyname("machdep.cpu.model", &cpuModel, &size, NULL, 0) != 0) { + Abort("sysctlbyname(\"machdep.cpu.model\") failed"); + } + + // This is similar to arch/x86/kernel/cpu/perf_event_intel_rapl.c in + // linux-4.1.5/. + switch (cpuModel) { + case 60: // 0x3c: Haswell + case 69: // 0x45: Haswell-Celeron + case 70: // 0x46: Haswell + case 61: // 0x3d: Broadwell + // Supports package, cores, GPU, RAM. + mIsGpuSupported = true; + mIsRamSupported = true; + break; + + case 42: // 0x2a: Sandy Bridge + case 58: // 0x3a: Ivy Bridge + // Supports package, cores, GPU. + mIsGpuSupported = true; + mIsRamSupported = false; + break; + + case 63: // 0x3f: Haswell-Server + mHasRamUnitsQuirk = true; + MOZ_FALLTHROUGH; + case 45: // 0x2d: Sandy Bridge-EP + case 62: // 0x3e: Ivy Bridge-E + // Supports package, cores, RAM. + mIsGpuSupported = false; + mIsRamSupported = true; + break; + + default: + Abort("unknown CPU model: %d", cpuModel); + break; + } + + // Get the maximum number of logical CPUs so that we know how big to make + // |mPkes|. + int logicalcpu_max; + size = sizeof(logicalcpu_max); + if (sysctlbyname("hw.logicalcpu_max", + &logicalcpu_max, &size, NULL, 0) != 0) { + Abort("sysctlbyname(\"hw.logicalcpu_max\") failed"); + } + + // Over-allocate by 1024 bytes per CPU to allow for the uncertainty around + // core_energy_stat_t::gpmcs and for any other future extensions to that + // struct. (The fields we read all come before the core_energy_stat_t + // array, so it won't matter to us whether gpmcs is present or not.) + size_t pkesSize = sizeof(pkg_energy_statistics_t) + + logicalcpu_max * sizeof(core_energy_stat_t) + + logicalcpu_max * 1024; + mPkes = (pkg_energy_statistics_t*) malloc(pkesSize); + if (!mPkes) { + Abort("malloc() failed"); + } + + // Do an initial measurement so that the first sample's diffs are sensible. + double dummy1, dummy2, dummy3, dummy4; + EnergyEstimates(dummy1, dummy2, dummy3, dummy4); + } + + ~RAPL() + { + free(mPkes); + } + + static double Joules(uint64_t aTicks, double aJoulesPerTick) + { + return double(aTicks) * aJoulesPerTick; + } + + void EnergyEstimates(double& aPkg_J, double& aCores_J, double& aGpu_J, + double& aRam_J) + { + diagCall64_dgPowerStat(mPkes); + + // Bits 12:8 are the ESU. + // Energy measurements come in multiples of 1/(2^ESU). + uint32_t energyStatusUnits = (mPkes->pkg_power_unit >> 8) & 0x1f; + double joulesPerTick = ((double)1 / (1 << energyStatusUnits)); + + aPkg_J = Joules(mPkes->pkg_energy - mPrevPkgTicks, joulesPerTick); + aCores_J = Joules(mPkes->pp0_energy - mPrevPp0Ticks, joulesPerTick); + aGpu_J = mIsGpuSupported + ? Joules(mPkes->pp1_energy - mPrevPp1Ticks, joulesPerTick) + : kUnsupported_j; + aRam_J = mIsRamSupported + ? Joules(mPkes->ddr_energy - mPrevDdrTicks, + mHasRamUnitsQuirk ? kQuirkyRamJoulesPerTick + : joulesPerTick) + : kUnsupported_j; + + mPrevPkgTicks = mPkes->pkg_energy; + mPrevPp0Ticks = mPkes->pp0_energy; + if (mIsGpuSupported) { + mPrevPp1Ticks = mPkes->pp1_energy; + } + if (mIsRamSupported) { + mPrevDdrTicks = mPkes->ddr_energy; + } + } +}; + +/* static */ const double RAPL::kQuirkyRamJoulesPerTick = (double)1 / 65536; + +//--------------------------------------------------------------------------- +// Linux-specific code +//--------------------------------------------------------------------------- + +#elif defined(__linux__) + +#include <linux/perf_event.h> +#include <sys/syscall.h> + +// There is no glibc wrapper for this system call so we provide our own. +static int +perf_event_open(struct perf_event_attr* aAttr, pid_t aPid, int aCpu, + int aGroupFd, unsigned long aFlags) +{ + return syscall(__NR_perf_event_open, aAttr, aPid, aCpu, aGroupFd, aFlags); +} + +// Returns false if the file cannot be opened. +template <typename T> +static bool +ReadValueFromPowerFile(const char* aStr1, const char* aStr2, const char* aStr3, + const char* aScanfString, T* aOut) +{ + // The filenames going into this buffer are under our control and the longest + // one is "/sys/bus/event_source/devices/power/events/energy-cores.scale". + // So 256 chars is plenty. + char filename[256]; + + sprintf(filename, "/sys/bus/event_source/devices/power/%s%s%s", + aStr1, aStr2, aStr3); + FILE* fp = fopen(filename, "r"); + if (!fp) { + return false; + } + if (fscanf(fp, aScanfString, aOut) != 1) { + Abort("fscanf() failed"); + } + fclose(fp); + + return true; +} + +// This class encapsulates the reading of a single RAPL domain. +class Domain +{ + bool mIsSupported; // Is the domain supported by the processor? + + // These three are only set if |mIsSupported| is true. + double mJoulesPerTick; // How many Joules each tick of the MSR represents. + int mFd; // The fd through which the MSR is read. + double mPrevTicks; // The previous sample's MSR value. + +public: + enum IsOptional { Optional, NonOptional }; + + Domain(const char* aName, uint32_t aType, IsOptional aOptional = NonOptional) + { + uint64_t config; + if (!ReadValueFromPowerFile("events/energy-", aName, "", "event=%llx", + &config)) { + // Failure is allowed for optional domains. + if (aOptional == NonOptional) { + Abort("failed to open file for non-optional domain '%s'\n" + "- Is your kernel version 3.14 or later, as required? " + "Run |uname -r| to see.", aName); + } + mIsSupported = false; + return; + } + + mIsSupported = true; + + ReadValueFromPowerFile("events/energy-", aName, ".scale", "%lf", + &mJoulesPerTick); + + // The unit should be "Joules", so 128 chars should be plenty. + char unit[128]; + ReadValueFromPowerFile("events/energy-", aName, ".unit", "%127s", unit); + if (strcmp(unit, "Joules") != 0) { + Abort("unexpected unit '%s' in .unit file", unit); + } + + struct perf_event_attr attr; + memset(&attr, 0, sizeof(attr)); + attr.type = aType; + attr.size = uint32_t(sizeof(attr)); + attr.config = config; + + // Measure all processes/threads. The specified CPU doesn't matter. + mFd = perf_event_open(&attr, /* aPid = */ -1, /* aCpu = */ 0, + /* aGroupFd = */ -1, /* aFlags = */ 0); + if (mFd < 0) { + Abort("perf_event_open() failed\n" + "- Did you run as root (e.g. with |sudo|) or set\n" + " /proc/sys/kernel/perf_event_paranoid to 0, as required?"); + } + + mPrevTicks = 0; + } + + ~Domain() + { + if (mIsSupported) { + close(mFd); + } + } + + double EnergyEstimate() + { + if (!mIsSupported) { + return kUnsupported_j; + } + + uint64_t thisTicks; + if (read(mFd, &thisTicks, sizeof(uint64_t)) != sizeof(uint64_t)) { + Abort("read() failed"); + } + + uint64_t ticks = thisTicks - mPrevTicks; + mPrevTicks = thisTicks; + double joules = ticks * mJoulesPerTick; + return joules; + } +}; + +class RAPL +{ + Domain* mPkg; + Domain* mCores; + Domain* mGpu; + Domain* mRam; + +public: + RAPL() + { + uint32_t type; + ReadValueFromPowerFile("type", "", "", "%u", &type); + + mPkg = new Domain("pkg", type); + mCores = new Domain("cores", type); + mGpu = new Domain("gpu", type, Domain::Optional); + mRam = new Domain("ram", type, Domain::Optional); + if (!mPkg || !mCores || !mGpu || !mRam) { + Abort("new Domain() failed"); + } + } + + ~RAPL() + { + delete mPkg; + delete mCores; + delete mGpu; + delete mRam; + } + + void EnergyEstimates(double& aPkg_J, double& aCores_J, double& aGpu_J, + double& aRam_J) + { + aPkg_J = mPkg->EnergyEstimate(); + aCores_J = mCores->EnergyEstimate(); + aGpu_J = mGpu->EnergyEstimate(); + aRam_J = mRam->EnergyEstimate(); + } +}; + +#else + +//--------------------------------------------------------------------------- +// Unsupported platforms +//--------------------------------------------------------------------------- + +#error Sorry, this platform is not supported + +#endif // platform + +//--------------------------------------------------------------------------- +// The main loop +//--------------------------------------------------------------------------- + +// The sample interval, measured in seconds. +static double gSampleInterval_sec; + +// The platform-specific RAPL-reading machinery. +static RAPL* gRapl; + +// All the sampled "total" values, in Watts. +static std::vector<double> gTotals_W; + +// Power = Energy / Time, where power is measured in Watts, Energy is measured +// in Joules, and Time is measured in seconds. +static double +JoulesToWatts(double aJoules) +{ + return aJoules / gSampleInterval_sec; +} + +// "Normalize" here means convert kUnsupported_j to zero so it can be used in +// additive expressions. All printed values are 5 or maybe 6 chars (though 6 +// chars would require a value > 100 W, which is unlikely). +static void +NormalizeAndPrintAsWatts(char* aBuf, double& aValue_J) +{ + if (aValue_J == kUnsupported_j) { + aValue_J = 0; + sprintf(aBuf, "%s", " n/a "); + } else { + sprintf(aBuf, "%5.2f", JoulesToWatts(aValue_J)); + } +} + +static void +SigAlrmHandler(int aSigNum, siginfo_t* aInfo, void* aContext) +{ + static int sampleNumber = 1; + + double pkg_J, cores_J, gpu_J, ram_J; + gRapl->EnergyEstimates(pkg_J, cores_J, gpu_J, ram_J); + + // We should have pkg and cores estimates, but might not have gpu and ram + // estimates. + assert(pkg_J != kUnsupported_j); + assert(cores_J != kUnsupported_j); + + // This needs to be big enough to print watt values to two decimal places. 16 + // should be plenty. + static const size_t kNumStrLen = 16; + + static char pkgStr[kNumStrLen], coresStr[kNumStrLen], gpuStr[kNumStrLen], + ramStr[kNumStrLen]; + NormalizeAndPrintAsWatts(pkgStr, pkg_J); + NormalizeAndPrintAsWatts(coresStr, cores_J); + NormalizeAndPrintAsWatts(gpuStr, gpu_J); + NormalizeAndPrintAsWatts(ramStr, ram_J); + + // Core and GPU power are a subset of the package power. + assert(pkg_J >= cores_J + gpu_J); + + // Compute "other" (i.e. rest of the package) and "total" only after the + // other values have been normalized. + + char otherStr[kNumStrLen]; + double other_J = pkg_J - cores_J - gpu_J; + NormalizeAndPrintAsWatts(otherStr, other_J); + + char totalStr[kNumStrLen]; + double total_J = pkg_J + ram_J; + NormalizeAndPrintAsWatts(totalStr, total_J); + + gTotals_W.push_back(JoulesToWatts(total_J)); + + // Print and flush so that the output appears immediately even if being + // redirected through |tee| or anything like that. + PrintAndFlush("#%02d %s W = %s (%s + %s + %s) + %s W\n", + sampleNumber++, totalStr, pkgStr, coresStr, gpuStr, otherStr, + ramStr); +} + +static void +Finish() +{ + size_t n = gTotals_W.size(); + + // This time calculation assumes that the timers are perfectly accurate which + // is not true but the inaccuracy should be small in practice. + double time = n * gSampleInterval_sec; + + printf("\n"); + printf("%d sample%s taken over a period of %.3f second%s\n", + int(n), n == 1 ? "" : "s", + n * gSampleInterval_sec, time == 1.0 ? "" : "s"); + + if (n == 0 || n == 1) { + exit(0); + } + + // Compute the mean. + double sum = std::accumulate(gTotals_W.begin(), gTotals_W.end(), 0.0); + double mean = sum / n; + + // Compute the *population* standard deviation: + // + // popStdDev = sqrt(Sigma(x - m)^2 / n) + // + // where |x| is the sum variable, |m| is the mean, and |n| is the + // population size. + // + // This is different from the *sample* standard deviation, which divides by + // |n - 1|, and would be appropriate if we were using a random sample of a + // larger population. + double sumOfSquaredDeviations = 0; + for (auto iter = gTotals_W.begin(); iter != gTotals_W.end(); ++iter) { + double deviation = (*iter - mean); + sumOfSquaredDeviations += deviation * deviation; + } + double popStdDev = sqrt(sumOfSquaredDeviations / n); + + // Sort so that percentiles can be determined. We use the "Nearest Rank" + // method of determining percentiles, which is simplest to compute and which + // chooses values from those that appear in the input set. + std::sort(gTotals_W.begin(), gTotals_W.end()); + + printf("\n"); + printf("Distribution of 'total' values:\n"); + printf(" mean = %5.2f W\n", mean); + printf(" std dev = %5.2f W\n", popStdDev); + printf(" 0th percentile = %5.2f W (min)\n", gTotals_W[0]); + printf(" 5th percentile = %5.2f W\n", gTotals_W[ceil(0.05 * n) - 1]); + printf(" 25th percentile = %5.2f W\n", gTotals_W[ceil(0.25 * n) - 1]); + printf(" 50th percentile = %5.2f W\n", gTotals_W[ceil(0.50 * n) - 1]); + printf(" 75th percentile = %5.2f W\n", gTotals_W[ceil(0.75 * n) - 1]); + printf(" 95th percentile = %5.2f W\n", gTotals_W[ceil(0.95 * n) - 1]); + printf("100th percentile = %5.2f W (max)\n", gTotals_W[n - 1]); + + exit(0); +} + +static void +SigIntHandler(int aSigNum, siginfo_t* aInfo, void *aContext) +{ + Finish(); +} + +static void +PrintUsage() +{ + printf( +"usage: rapl [options]\n" +"\n" +"Options:\n" +"\n" +" -h --help show this message\n" +" -i --sample-interval <N> sample every N ms [default=1000]\n" +" -n --sample-count <N> get N samples (0 means unlimited) [default=0]\n" +"\n" +#if defined(__APPLE__) +"On Mac this program can be run by any user.\n" +#elif defined(__linux__) +"On Linux this program can only be run by the super-user unless the contents\n" +"of /proc/sys/kernel/perf_event_paranoid is set to 0 or lower.\n" +#else +#error Sorry, this platform is not supported +#endif +"\n" + ); +} + +int +main(int argc, char** argv) +{ + // Process command line options. + + gArgv0 = argv[0]; + + // Default values. + int sampleInterval_msec = 1000; + int sampleCount = 0; + + struct option longOptions[] = { + { "help", no_argument, NULL, 'h' }, + { "sample-interval", required_argument, NULL, 'i' }, + { "sample-count", required_argument, NULL, 'n' }, + { NULL, 0, NULL, 0 } + }; + const char* shortOptions = "hi:n:"; + + int c; + char* endPtr; + while ((c = getopt_long(argc, argv, shortOptions, longOptions, NULL)) != -1) { + switch (c) { + case 'h': + PrintUsage(); + exit(0); + + case 'i': + sampleInterval_msec = strtol(optarg, &endPtr, /* base = */ 10); + if (*endPtr) { + CmdLineAbort("sample interval is not an integer"); + } + if (sampleInterval_msec < 1 || sampleInterval_msec > 3600000) { + CmdLineAbort("sample interval must be in the range 1..3600000 ms"); + } + break; + + case 'n': + sampleCount = strtol(optarg, &endPtr, /* base = */ 10); + if (*endPtr) { + CmdLineAbort("sample count is not an integer"); + } + if (sampleCount < 0 || sampleCount > 1000000) { + CmdLineAbort("sample count must be in the range 0..1000000"); + } + break; + + default: + CmdLineAbort(NULL); + } + } + + // The RAPL MSRs update every ~1 ms, but the measurement period isn't exactly + // 1 ms, which means the sample periods are not exact. "Power Measurement + // Techniques on Standard Compute Nodes: A Quantitative Comparison" by + // Hackenberg et al. suggests the following. + // + // "RAPL provides energy (and not power) consumption data without + // timestamps associated to each counter update. This makes sampling rates + // above 20 Samples/s unfeasible if the systematic error should be below + // 5%... Constantly polling the RAPL registers will both occupy a processor + // core and distort the measurement itself." + // + // So warn about this case. + if (sampleInterval_msec < 50) { + fprintf(stderr, + "\nWARNING: sample intervals < 50 ms are likely to produce " + "inaccurate estimates\n\n"); + } + gSampleInterval_sec = double(sampleInterval_msec) / 1000; + + // Initialize the platform-specific RAPL reading machinery. + gRapl = new RAPL(); + if (!gRapl) { + Abort("new RAPL() failed"); + } + + // Install the signal handlers. + + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_flags = SA_RESTART | SA_SIGINFO; + // The extra parens around (0) suppress a -Wunreachable-code warning on OS X + // where sigemptyset() is a macro that can never fail and always returns 0. + if (sigemptyset(&sa.sa_mask) < (0)) { + Abort("sigemptyset() failed"); + } + sa.sa_sigaction = SigAlrmHandler; + if (sigaction(SIGALRM, &sa, NULL) < 0) { + Abort("sigaction(SIGALRM) failed"); + } + sa.sa_sigaction = SigIntHandler; + if (sigaction(SIGINT, &sa, NULL) < 0) { + Abort("sigaction(SIGINT) failed"); + } + + // Set up the timer. + struct itimerval timer; + timer.it_interval.tv_sec = sampleInterval_msec / 1000; + timer.it_interval.tv_usec = (sampleInterval_msec % 1000) * 1000; + timer.it_value = timer.it_interval; + if (setitimer(ITIMER_REAL, &timer, NULL) < 0) { + Abort("setitimer() failed"); + } + + // Print header. + PrintAndFlush(" total W = _pkg_ (cores + _gpu_ + other) + _ram_ W\n"); + + // Take samples. + if (sampleCount == 0) { + while (true) { + pause(); + } + } else { + for (int i = 0; i < sampleCount; i++) { + pause(); + } + } + + Finish(); + + return 0; +} diff --git a/tools/profiler/core/PlatformMacros.h b/tools/profiler/core/PlatformMacros.h new file mode 100644 index 000000000..de74531e8 --- /dev/null +++ b/tools/profiler/core/PlatformMacros.h @@ -0,0 +1,64 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef SPS_PLATFORM_MACROS_H +#define SPS_PLATFORM_MACROS_H + +/* Define platform selection macros in a consistent way. Don't add + anything else to this file, so it can remain freestanding. The + primary factorisation is on (ARCH,OS) pairs ("PLATforms") but ARCH_ + and OS_ macros are defined too, since they are sometimes + convenient. */ + +#undef SPS_PLAT_amd64_linux +#undef SPS_PLAT_x86_linux +#undef SPS_PLAT_amd64_darwin +#undef SPS_PLAT_x86_darwin +#undef SPS_PLAT_x86_windows +#undef SPS_PLAT_amd64_windows + +#undef SPS_ARCH_arm +#undef SPS_ARCH_x86 +#undef SPS_ARCH_amd64 + +#undef SPS_OS_linux +#undef SPS_OS_darwin +#undef SPS_OS_windows + +#if defined(__linux__) && defined(__x86_64__) +# define SPS_PLAT_amd64_linux 1 +# define SPS_ARCH_amd64 1 +# define SPS_OS_linux 1 + +#elif defined(__linux__) && defined(__i386__) +# define SPS_PLAT_x86_linux 1 +# define SPS_ARCH_x86 1 +# define SPS_OS_linux 1 + +#elif defined(__APPLE__) && defined(__x86_64__) +# define SPS_PLAT_amd64_darwin 1 +# define SPS_ARCH_amd64 1 +# define SPS_OS_darwin 1 + +#elif defined(__APPLE__) && defined(__i386__) +# define SPS_PLAT_x86_darwin 1 +# define SPS_ARCH_x86 1 +# define SPS_OS_darwin 1 + +#elif (defined(_MSC_VER) || defined(__MINGW32__)) && (defined(_M_IX86) || defined(__i386__)) +# define SPS_PLAT_x86_windows 1 +# define SPS_ARCH_x86 1 +# define SPS_OS_windows 1 + +#elif (defined(_MSC_VER) || defined(__MINGW32__)) && (defined(_M_X64) || defined(__x86_64__)) +# define SPS_PLAT_amd64_windows 1 +# define SPS_ARCH_amd64 1 +# define SPS_OS_windows 1 + +#else +# error "Unsupported platform" +#endif + +#endif /* ndef SPS_PLATFORM_MACROS_H */ diff --git a/tools/profiler/core/v8-support.h b/tools/profiler/core/v8-support.h new file mode 100644 index 000000000..391069dcc --- /dev/null +++ b/tools/profiler/core/v8-support.h @@ -0,0 +1,48 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* This contains stubs and infrastructure to support code from v8 */ + +#ifndef V8_SUPPORT_H_ +#define V8_SUPPORT_H_ + +#if defined(_M_X64) || defined(__x86_64__) +#define V8_HOST_ARCH_X64 1 +#elif defined(_M_IX86) || defined(__i386__) || defined(__i386) +#define V8_HOST_ARCH_IA32 1 +#elif defined(__ARMEL__) +#define V8_HOST_ARCH_ARM 1 +#else +#warning Please add support for your architecture in chromium_types.h +#endif + +typedef int32_t Atomic32; + +#if defined(V8_HOST_ARCH_X64) || defined(V8_HOST_ARCH_IA32) || defined(V8_HOST_ARCH_ARM) +inline void NoBarrier_Store(volatile Atomic32* ptr, Atomic32 value) { + *ptr = value; +} +#endif + + +const int kMaxInt = 0x7FFFFFFF; +const int kMinInt = -kMaxInt - 1; + +// A macro to disallow the evil copy constructor and operator= functions +// This should be used in the private: declarations for a class +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&); \ + void operator=(const TypeName&) + + +// The USE(x) template is used to silence C++ compiler warnings +// issued for (yet) unused variables (typically parameters). +template <typename T> +static inline void USE(T) { } + +class Malloced { +}; + +#endif // V8_SUPPORT_H_ diff --git a/tools/profiler/gecko/ProfilerIOInterposeObserver.h b/tools/profiler/gecko/ProfilerIOInterposeObserver.h new file mode 100644 index 000000000..fd47e48bd --- /dev/null +++ b/tools/profiler/gecko/ProfilerIOInterposeObserver.h @@ -0,0 +1,10 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef PROFILERIOINTERPOSEOBSERVER_H +#define PROFILERIOINTERPOSEOBSERVER_H + + /*** STUB ***/ + +#endif // PROFILERIOINTERPOSEOBSERVER_H diff --git a/tools/profiler/gecko/ProfilerTypes.ipdlh b/tools/profiler/gecko/ProfilerTypes.ipdlh new file mode 100644 index 000000000..1ef670b03 --- /dev/null +++ b/tools/profiler/gecko/ProfilerTypes.ipdlh @@ -0,0 +1,16 @@ +/* -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil; tab-width: 8 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +namespace mozilla { + +struct ProfilerInitParams { + bool enabled; + uint32_t entries; + double interval; + nsCString[] threadFilters; + nsCString[] features; +}; + +} // namespace mozilla
\ No newline at end of file diff --git a/tools/profiler/gecko/nsProfilerCIID.h b/tools/profiler/gecko/nsProfilerCIID.h new file mode 100644 index 000000000..3057a6ae0 --- /dev/null +++ b/tools/profiler/gecko/nsProfilerCIID.h @@ -0,0 +1,14 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsProfilerCIID_h__ +#define nsProfilerCIID_h__ + +#define NS_PROFILER_CID \ +{ 0x25db9b8e, 0x8123, 0x4de1, \ +{ 0xb6, 0x6d, 0x8b, 0xbb, 0xed, 0xf2, 0xcd, 0xf4 } } + +#endif + diff --git a/tools/profiler/lul/LulCommonExt.h b/tools/profiler/lul/LulCommonExt.h new file mode 100644 index 000000000..2491bc4b8 --- /dev/null +++ b/tools/profiler/lul/LulCommonExt.h @@ -0,0 +1,553 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ + +// Copyright (c) 2006, 2010, 2012, 2013 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Original author: Jim Blandy <jimb@mozilla.com> <jimb@red-bean.com> + +// module.h: Define google_breakpad::Module. A Module holds debugging +// information, and can write that information out as a Breakpad +// symbol file. + + +// (C) Copyright Greg Colvin and Beman Dawes 1998, 1999. +// Copyright (c) 2001, 2002 Peter Dimov +// +// Permission to copy, use, modify, sell and distribute this software +// is granted provided this copyright notice appears in all copies. +// This software is provided "as is" without express or implied +// warranty, and with no claim as to its suitability for any purpose. +// +// See http://www.boost.org/libs/smart_ptr/scoped_ptr.htm for documentation. +// + + +// This file is derived from the following files in +// toolkit/crashreporter/google-breakpad: +// src/common/unique_string.h +// src/common/scoped_ptr.h +// src/common/module.h + +// External interface for the "Common" component of LUL. + +#ifndef LulCommonExt_h +#define LulCommonExt_h + +#include <stdlib.h> +#include <stdio.h> +#include <stdint.h> + +#include <string> +#include <map> +#include <vector> +#include <cstddef> // for std::ptrdiff_t + +#include "mozilla/Assertions.h" + +namespace lul { + +using std::string; +using std::map; + + +//////////////////////////////////////////////////////////////// +// UniqueString +// + +// Abstract type +class UniqueString; + +// Get the contained C string (debugging only) +const char* FromUniqueString(const UniqueString*); + +// Is the given string empty (that is, "") ? +bool IsEmptyUniqueString(const UniqueString*); + + +//////////////////////////////////////////////////////////////// +// UniqueStringUniverse +// + +// All UniqueStrings live in some specific UniqueStringUniverse. +class UniqueStringUniverse { +public: + UniqueStringUniverse() {} + ~UniqueStringUniverse(); + // Convert a |string| to a UniqueString, that lives in this universe. + const UniqueString* ToUniqueString(string str); +private: + map<string, UniqueString*> map_; +}; + + +//////////////////////////////////////////////////////////////// +// GUID +// + +typedef struct { + uint32_t data1; + uint16_t data2; + uint16_t data3; + uint8_t data4[8]; +} MDGUID; // GUID + +typedef MDGUID GUID; + + +//////////////////////////////////////////////////////////////// +// scoped_ptr +// + +// scoped_ptr mimics a built-in pointer except that it guarantees deletion +// of the object pointed to, either on destruction of the scoped_ptr or via +// an explicit reset(). scoped_ptr is a simple solution for simple needs; +// use shared_ptr or std::auto_ptr if your needs are more complex. + +// *** NOTE *** +// If your scoped_ptr is a class member of class FOO pointing to a +// forward declared type BAR (as shown below), then you MUST use a non-inlined +// version of the destructor. The destructor of a scoped_ptr (called from +// FOO's destructor) must have a complete definition of BAR in order to +// destroy it. Example: +// +// -- foo.h -- +// class BAR; +// +// class FOO { +// public: +// FOO(); +// ~FOO(); // Required for sources that instantiate class FOO to compile! +// +// private: +// scoped_ptr<BAR> bar_; +// }; +// +// -- foo.cc -- +// #include "foo.h" +// FOO::~FOO() {} // Empty, but must be non-inlined to FOO's class definition. + +// scoped_ptr_malloc added by Google +// When one of these goes out of scope, instead of doing a delete or +// delete[], it calls free(). scoped_ptr_malloc<char> is likely to see +// much more use than any other specializations. + +// release() added by Google +// Use this to conditionally transfer ownership of a heap-allocated object +// to the caller, usually on method success. + +template <typename T> +class scoped_ptr { + private: + + T* ptr; + + scoped_ptr(scoped_ptr const &); + scoped_ptr & operator=(scoped_ptr const &); + + public: + + typedef T element_type; + + explicit scoped_ptr(T* p = 0): ptr(p) {} + + ~scoped_ptr() { + delete ptr; + } + + void reset(T* p = 0) { + if (ptr != p) { + delete ptr; + ptr = p; + } + } + + T& operator*() const { + MOZ_ASSERT(ptr != 0); + return *ptr; + } + + T* operator->() const { + MOZ_ASSERT(ptr != 0); + return ptr; + } + + bool operator==(T* p) const { + return ptr == p; + } + + bool operator!=(T* p) const { + return ptr != p; + } + + T* get() const { + return ptr; + } + + void swap(scoped_ptr & b) { + T* tmp = b.ptr; + b.ptr = ptr; + ptr = tmp; + } + + T* release() { + T* tmp = ptr; + ptr = 0; + return tmp; + } + + private: + + // no reason to use these: each scoped_ptr should have its own object + template <typename U> bool operator==(scoped_ptr<U> const& p) const; + template <typename U> bool operator!=(scoped_ptr<U> const& p) const; +}; + +template<typename T> inline +void swap(scoped_ptr<T>& a, scoped_ptr<T>& b) { + a.swap(b); +} + +template<typename T> inline +bool operator==(T* p, const scoped_ptr<T>& b) { + return p == b.get(); +} + +template<typename T> inline +bool operator!=(T* p, const scoped_ptr<T>& b) { + return p != b.get(); +} + +// scoped_array extends scoped_ptr to arrays. Deletion of the array pointed to +// is guaranteed, either on destruction of the scoped_array or via an explicit +// reset(). Use shared_array or std::vector if your needs are more complex. + +template<typename T> +class scoped_array { + private: + + T* ptr; + + scoped_array(scoped_array const &); + scoped_array & operator=(scoped_array const &); + + public: + + typedef T element_type; + + explicit scoped_array(T* p = 0) : ptr(p) {} + + ~scoped_array() { + delete[] ptr; + } + + void reset(T* p = 0) { + if (ptr != p) { + delete [] ptr; + ptr = p; + } + } + + T& operator[](std::ptrdiff_t i) const { + MOZ_ASSERT(ptr != 0); + MOZ_ASSERT(i >= 0); + return ptr[i]; + } + + bool operator==(T* p) const { + return ptr == p; + } + + bool operator!=(T* p) const { + return ptr != p; + } + + T* get() const { + return ptr; + } + + void swap(scoped_array & b) { + T* tmp = b.ptr; + b.ptr = ptr; + ptr = tmp; + } + + T* release() { + T* tmp = ptr; + ptr = 0; + return tmp; + } + + private: + + // no reason to use these: each scoped_array should have its own object + template <typename U> bool operator==(scoped_array<U> const& p) const; + template <typename U> bool operator!=(scoped_array<U> const& p) const; +}; + +template<class T> inline +void swap(scoped_array<T>& a, scoped_array<T>& b) { + a.swap(b); +} + +template<typename T> inline +bool operator==(T* p, const scoped_array<T>& b) { + return p == b.get(); +} + +template<typename T> inline +bool operator!=(T* p, const scoped_array<T>& b) { + return p != b.get(); +} + + +// This class wraps the c library function free() in a class that can be +// passed as a template argument to scoped_ptr_malloc below. +class ScopedPtrMallocFree { + public: + inline void operator()(void* x) const { + free(x); + } +}; + +// scoped_ptr_malloc<> is similar to scoped_ptr<>, but it accepts a +// second template argument, the functor used to free the object. + +template<typename T, typename FreeProc = ScopedPtrMallocFree> +class scoped_ptr_malloc { + private: + + T* ptr; + + scoped_ptr_malloc(scoped_ptr_malloc const &); + scoped_ptr_malloc & operator=(scoped_ptr_malloc const &); + + public: + + typedef T element_type; + + explicit scoped_ptr_malloc(T* p = 0): ptr(p) {} + + ~scoped_ptr_malloc() { + free_((void*) ptr); + } + + void reset(T* p = 0) { + if (ptr != p) { + free_((void*) ptr); + ptr = p; + } + } + + T& operator*() const { + MOZ_ASSERT(ptr != 0); + return *ptr; + } + + T* operator->() const { + MOZ_ASSERT(ptr != 0); + return ptr; + } + + bool operator==(T* p) const { + return ptr == p; + } + + bool operator!=(T* p) const { + return ptr != p; + } + + T* get() const { + return ptr; + } + + void swap(scoped_ptr_malloc & b) { + T* tmp = b.ptr; + b.ptr = ptr; + ptr = tmp; + } + + T* release() { + T* tmp = ptr; + ptr = 0; + return tmp; + } + + private: + + // no reason to use these: each scoped_ptr_malloc should have its own object + template <typename U, typename GP> + bool operator==(scoped_ptr_malloc<U, GP> const& p) const; + template <typename U, typename GP> + bool operator!=(scoped_ptr_malloc<U, GP> const& p) const; + + static FreeProc const free_; +}; + +template<typename T, typename FP> +FP const scoped_ptr_malloc<T,FP>::free_ = FP(); + +template<typename T, typename FP> inline +void swap(scoped_ptr_malloc<T,FP>& a, scoped_ptr_malloc<T,FP>& b) { + a.swap(b); +} + +template<typename T, typename FP> inline +bool operator==(T* p, const scoped_ptr_malloc<T,FP>& b) { + return p == b.get(); +} + +template<typename T, typename FP> inline +bool operator!=(T* p, const scoped_ptr_malloc<T,FP>& b) { + return p != b.get(); +} + + +//////////////////////////////////////////////////////////////// +// Module +// + +// A Module represents the contents of a module, and supports methods +// for adding information produced by parsing STABS or DWARF data +// --- possibly both from the same file --- and then writing out the +// unified contents as a Breakpad-format symbol file. +class Module { +public: + // The type of addresses and sizes in a symbol table. + typedef uint64_t Address; + + // Representation of an expression. This can either be a postfix + // expression, in which case it is stored as a string, or a simple + // expression of the form (identifier + imm) or *(identifier + imm). + // It can also be invalid (denoting "no value"). + enum ExprHow { + kExprInvalid = 1, + kExprPostfix, + kExprSimple, + kExprSimpleMem + }; + + struct Expr { + // Construct a simple-form expression + Expr(const UniqueString* ident, long offset, bool deref) { + if (IsEmptyUniqueString(ident)) { + Expr(); + } else { + postfix_ = ""; + ident_ = ident; + offset_ = offset; + how_ = deref ? kExprSimpleMem : kExprSimple; + } + } + + // Construct an invalid expression + Expr() { + postfix_ = ""; + ident_ = nullptr; + offset_ = 0; + how_ = kExprInvalid; + } + + // Return the postfix expression string, either directly, + // if this is a postfix expression, or by synthesising it + // for a simple expression. + std::string getExprPostfix() const { + switch (how_) { + case kExprPostfix: + return postfix_; + case kExprSimple: + case kExprSimpleMem: { + char buf[40]; + sprintf(buf, " %ld %c%s", labs(offset_), offset_ < 0 ? '-' : '+', + how_ == kExprSimple ? "" : " ^"); + return std::string(FromUniqueString(ident_)) + std::string(buf); + } + case kExprInvalid: + default: + MOZ_ASSERT(0 && "getExprPostfix: invalid Module::Expr type"); + return "Expr::genExprPostfix: kExprInvalid"; + } + } + + // The identifier that gives the starting value for simple expressions. + const UniqueString* ident_; + // The offset to add for simple expressions. + long offset_; + // The Postfix expression string to evaluate for non-simple expressions. + std::string postfix_; + // The operation expressed by this expression. + ExprHow how_; + }; + + // A map from register names to expressions that recover + // their values. This can represent a complete set of rules to + // follow at some address, or a set of changes to be applied to an + // extant set of rules. + // NOTE! there are two completely different types called RuleMap. This + // is one of them. + typedef std::map<const UniqueString*, Expr> RuleMap; + + // A map from addresses to RuleMaps, representing changes that take + // effect at given addresses. + typedef std::map<Address, RuleMap> RuleChangeMap; + + // A range of 'STACK CFI' stack walking information. An instance of + // this structure corresponds to a 'STACK CFI INIT' record and the + // subsequent 'STACK CFI' records that fall within its range. + struct StackFrameEntry { + // The starting address and number of bytes of machine code this + // entry covers. + Address address, size; + + // The initial register recovery rules, in force at the starting + // address. + RuleMap initial_rules; + + // A map from addresses to rule changes. To find the rules in + // force at a given address, start with initial_rules, and then + // apply the changes given in this map for all addresses up to and + // including the address you're interested in. + RuleChangeMap rule_changes; + }; + + // Create a new module with the given name, operating system, + // architecture, and ID string. + Module(const std::string &name, const std::string &os, + const std::string &architecture, const std::string &id); + ~Module(); + +private: + + // Module header entries. + std::string name_, os_, architecture_, id_; +}; + + +} // namespace lul + +#endif // LulCommonExt_h diff --git a/tools/profiler/lul/LulDwarfExt.h b/tools/profiler/lul/LulDwarfExt.h new file mode 100644 index 000000000..2f8851d68 --- /dev/null +++ b/tools/profiler/lul/LulDwarfExt.h @@ -0,0 +1,1286 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ + +// Copyright 2006, 2010 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Original author: Jim Blandy <jimb@mozilla.com> <jimb@red-bean.com> + +// This file is derived from the following files in +// toolkit/crashreporter/google-breakpad: +// src/common/dwarf/types.h +// src/common/dwarf/dwarf2enums.h +// src/common/dwarf/bytereader.h +// src/common/dwarf_cfi_to_module.h +// src/common/dwarf/dwarf2reader.h + +#ifndef LulDwarfExt_h +#define LulDwarfExt_h + +#include <stdint.h> + +#include "mozilla/Assertions.h" + +#include "LulDwarfSummariser.h" + +typedef signed char int8; +typedef short int16; +typedef int int32; +typedef long long int64; + +typedef unsigned char uint8; +typedef unsigned short uint16; +typedef unsigned int uint32; +typedef unsigned long long uint64; + +#ifdef __PTRDIFF_TYPE__ +typedef __PTRDIFF_TYPE__ intptr; +typedef unsigned __PTRDIFF_TYPE__ uintptr; +#else +#error "Can't find pointer-sized integral types." +#endif + + +namespace lul { + +// Exception handling frame description pointer formats, as described +// by the Linux Standard Base Core Specification 4.0, section 11.5, +// DWARF Extensions. +enum DwarfPointerEncoding + { + DW_EH_PE_absptr = 0x00, + DW_EH_PE_omit = 0xff, + DW_EH_PE_uleb128 = 0x01, + DW_EH_PE_udata2 = 0x02, + DW_EH_PE_udata4 = 0x03, + DW_EH_PE_udata8 = 0x04, + DW_EH_PE_sleb128 = 0x09, + DW_EH_PE_sdata2 = 0x0A, + DW_EH_PE_sdata4 = 0x0B, + DW_EH_PE_sdata8 = 0x0C, + DW_EH_PE_pcrel = 0x10, + DW_EH_PE_textrel = 0x20, + DW_EH_PE_datarel = 0x30, + DW_EH_PE_funcrel = 0x40, + DW_EH_PE_aligned = 0x50, + + // The GNU toolchain sources define this enum value as well, + // simply to help classify the lower nybble values into signed and + // unsigned groups. + DW_EH_PE_signed = 0x08, + + // This is not documented in LSB 4.0, but it is used in both the + // Linux and OS X toolchains. It can be added to any other + // encoding (except DW_EH_PE_aligned), and indicates that the + // encoded value represents the address at which the true address + // is stored, not the true address itself. + DW_EH_PE_indirect = 0x80 + }; + + +// We can't use the obvious name of LITTLE_ENDIAN and BIG_ENDIAN +// because it conflicts with a macro +enum Endianness { + ENDIANNESS_BIG, + ENDIANNESS_LITTLE +}; + +// A ByteReader knows how to read single- and multi-byte values of +// various endiannesses, sizes, and encodings, as used in DWARF +// debugging information and Linux C++ exception handling data. +class ByteReader { + public: + // Construct a ByteReader capable of reading one-, two-, four-, and + // eight-byte values according to ENDIANNESS, absolute machine-sized + // addresses, DWARF-style "initial length" values, signed and + // unsigned LEB128 numbers, and Linux C++ exception handling data's + // encoded pointers. + explicit ByteReader(enum Endianness endianness); + virtual ~ByteReader(); + + // Read a single byte from BUFFER and return it as an unsigned 8 bit + // number. + uint8 ReadOneByte(const char* buffer) const; + + // Read two bytes from BUFFER and return them as an unsigned 16 bit + // number, using this ByteReader's endianness. + uint16 ReadTwoBytes(const char* buffer) const; + + // Read four bytes from BUFFER and return them as an unsigned 32 bit + // number, using this ByteReader's endianness. This function returns + // a uint64 so that it is compatible with ReadAddress and + // ReadOffset. The number it returns will never be outside the range + // of an unsigned 32 bit integer. + uint64 ReadFourBytes(const char* buffer) const; + + // Read eight bytes from BUFFER and return them as an unsigned 64 + // bit number, using this ByteReader's endianness. + uint64 ReadEightBytes(const char* buffer) const; + + // Read an unsigned LEB128 (Little Endian Base 128) number from + // BUFFER and return it as an unsigned 64 bit integer. Set LEN to + // the number of bytes read. + // + // The unsigned LEB128 representation of an integer N is a variable + // number of bytes: + // + // - If N is between 0 and 0x7f, then its unsigned LEB128 + // representation is a single byte whose value is N. + // + // - Otherwise, its unsigned LEB128 representation is (N & 0x7f) | + // 0x80, followed by the unsigned LEB128 representation of N / + // 128, rounded towards negative infinity. + // + // In other words, we break VALUE into groups of seven bits, put + // them in little-endian order, and then write them as eight-bit + // bytes with the high bit on all but the last. + uint64 ReadUnsignedLEB128(const char* buffer, size_t* len) const; + + // Read a signed LEB128 number from BUFFER and return it as an + // signed 64 bit integer. Set LEN to the number of bytes read. + // + // The signed LEB128 representation of an integer N is a variable + // number of bytes: + // + // - If N is between -0x40 and 0x3f, then its signed LEB128 + // representation is a single byte whose value is N in two's + // complement. + // + // - Otherwise, its signed LEB128 representation is (N & 0x7f) | + // 0x80, followed by the signed LEB128 representation of N / 128, + // rounded towards negative infinity. + // + // In other words, we break VALUE into groups of seven bits, put + // them in little-endian order, and then write them as eight-bit + // bytes with the high bit on all but the last. + int64 ReadSignedLEB128(const char* buffer, size_t* len) const; + + // Indicate that addresses on this architecture are SIZE bytes long. SIZE + // must be either 4 or 8. (DWARF allows addresses to be any number of + // bytes in length from 1 to 255, but we only support 32- and 64-bit + // addresses at the moment.) You must call this before using the + // ReadAddress member function. + // + // For data in a .debug_info section, or something that .debug_info + // refers to like line number or macro data, the compilation unit + // header's address_size field indicates the address size to use. Call + // frame information doesn't indicate its address size (a shortcoming of + // the spec); you must supply the appropriate size based on the + // architecture of the target machine. + void SetAddressSize(uint8 size); + + // Return the current address size, in bytes. This is either 4, + // indicating 32-bit addresses, or 8, indicating 64-bit addresses. + uint8 AddressSize() const { return address_size_; } + + // Read an address from BUFFER and return it as an unsigned 64 bit + // integer, respecting this ByteReader's endianness and address size. You + // must call SetAddressSize before calling this function. + uint64 ReadAddress(const char* buffer) const; + + // DWARF actually defines two slightly different formats: 32-bit DWARF + // and 64-bit DWARF. This is *not* related to the size of registers or + // addresses on the target machine; it refers only to the size of section + // offsets and data lengths appearing in the DWARF data. One only needs + // 64-bit DWARF when the debugging data itself is larger than 4GiB. + // 32-bit DWARF can handle x86_64 or PPC64 code just fine, unless the + // debugging data itself is very large. + // + // DWARF information identifies itself as 32-bit or 64-bit DWARF: each + // compilation unit and call frame information entry begins with an + // "initial length" field, which, in addition to giving the length of the + // data, also indicates the size of section offsets and lengths appearing + // in that data. The ReadInitialLength member function, below, reads an + // initial length and sets the ByteReader's offset size as a side effect. + // Thus, in the normal process of reading DWARF data, the appropriate + // offset size is set automatically. So, you should only need to call + // SetOffsetSize if you are using the same ByteReader to jump from the + // midst of one block of DWARF data into another. + + // Read a DWARF "initial length" field from START, and return it as + // an unsigned 64 bit integer, respecting this ByteReader's + // endianness. Set *LEN to the length of the initial length in + // bytes, either four or twelve. As a side effect, set this + // ByteReader's offset size to either 4 (if we see a 32-bit DWARF + // initial length) or 8 (if we see a 64-bit DWARF initial length). + // + // A DWARF initial length is either: + // + // - a byte count stored as an unsigned 32-bit value less than + // 0xffffff00, indicating that the data whose length is being + // measured uses the 32-bit DWARF format, or + // + // - The 32-bit value 0xffffffff, followed by a 64-bit byte count, + // indicating that the data whose length is being measured uses + // the 64-bit DWARF format. + uint64 ReadInitialLength(const char* start, size_t* len); + + // Read an offset from BUFFER and return it as an unsigned 64 bit + // integer, respecting the ByteReader's endianness. In 32-bit DWARF, the + // offset is 4 bytes long; in 64-bit DWARF, the offset is eight bytes + // long. You must call ReadInitialLength or SetOffsetSize before calling + // this function; see the comments above for details. + uint64 ReadOffset(const char* buffer) const; + + // Return the current offset size, in bytes. + // A return value of 4 indicates that we are reading 32-bit DWARF. + // A return value of 8 indicates that we are reading 64-bit DWARF. + uint8 OffsetSize() const { return offset_size_; } + + // Indicate that section offsets and lengths are SIZE bytes long. SIZE + // must be either 4 (meaning 32-bit DWARF) or 8 (meaning 64-bit DWARF). + // Usually, you should not call this function yourself; instead, let a + // call to ReadInitialLength establish the data's offset size + // automatically. + void SetOffsetSize(uint8 size); + + // The Linux C++ ABI uses a variant of DWARF call frame information + // for exception handling. This data is included in the program's + // address space as the ".eh_frame" section, and intepreted at + // runtime to walk the stack, find exception handlers, and run + // cleanup code. The format is mostly the same as DWARF CFI, with + // some adjustments made to provide the additional + // exception-handling data, and to make the data easier to work with + // in memory --- for example, to allow it to be placed in read-only + // memory even when describing position-independent code. + // + // In particular, exception handling data can select a number of + // different encodings for pointers that appear in the data, as + // described by the DwarfPointerEncoding enum. There are actually + // four axes(!) to the encoding: + // + // - The pointer size: pointers can be 2, 4, or 8 bytes long, or use + // the DWARF LEB128 encoding. + // + // - The pointer's signedness: pointers can be signed or unsigned. + // + // - The pointer's base address: the data stored in the exception + // handling data can be the actual address (that is, an absolute + // pointer), or relative to one of a number of different base + // addreses --- including that of the encoded pointer itself, for + // a form of "pc-relative" addressing. + // + // - The pointer may be indirect: it may be the address where the + // true pointer is stored. (This is used to refer to things via + // global offset table entries, program linkage table entries, or + // other tricks used in position-independent code.) + // + // There are also two options that fall outside that matrix + // altogether: the pointer may be omitted, or it may have padding to + // align it on an appropriate address boundary. (That last option + // may seem like it should be just another axis, but it is not.) + + // Indicate that the exception handling data is loaded starting at + // SECTION_BASE, and that the start of its buffer in our own memory + // is BUFFER_BASE. This allows us to find the address that a given + // byte in our buffer would have when loaded into the program the + // data describes. We need this to resolve DW_EH_PE_pcrel pointers. + void SetCFIDataBase(uint64 section_base, const char *buffer_base); + + // Indicate that the base address of the program's ".text" section + // is TEXT_BASE. We need this to resolve DW_EH_PE_textrel pointers. + void SetTextBase(uint64 text_base); + + // Indicate that the base address for DW_EH_PE_datarel pointers is + // DATA_BASE. The proper value depends on the ABI; it is usually the + // address of the global offset table, held in a designated register in + // position-independent code. You will need to look at the startup code + // for the target system to be sure. I tried; my eyes bled. + void SetDataBase(uint64 data_base); + + // Indicate that the base address for the FDE we are processing is + // FUNCTION_BASE. This is the start address of DW_EH_PE_funcrel + // pointers. (This encoding does not seem to be used by the GNU + // toolchain.) + void SetFunctionBase(uint64 function_base); + + // Indicate that we are no longer processing any FDE, so any use of + // a DW_EH_PE_funcrel encoding is an error. + void ClearFunctionBase(); + + // Return true if ENCODING is a valid pointer encoding. + bool ValidEncoding(DwarfPointerEncoding encoding) const; + + // Return true if we have all the information we need to read a + // pointer that uses ENCODING. This checks that the appropriate + // SetFooBase function for ENCODING has been called. + bool UsableEncoding(DwarfPointerEncoding encoding) const; + + // Read an encoded pointer from BUFFER using ENCODING; return the + // absolute address it represents, and set *LEN to the pointer's + // length in bytes, including any padding for aligned pointers. + // + // This function calls 'abort' if ENCODING is invalid or refers to a + // base address this reader hasn't been given, so you should check + // with ValidEncoding and UsableEncoding first if you would rather + // die in a more helpful way. + uint64 ReadEncodedPointer(const char *buffer, DwarfPointerEncoding encoding, + size_t *len) const; + + private: + + // Function pointer type for our address and offset readers. + typedef uint64 (ByteReader::*AddressReader)(const char*) const; + + // Read an offset from BUFFER and return it as an unsigned 64 bit + // integer. DWARF2/3 define offsets as either 4 or 8 bytes, + // generally depending on the amount of DWARF2/3 info present. + // This function pointer gets set by SetOffsetSize. + AddressReader offset_reader_; + + // Read an address from BUFFER and return it as an unsigned 64 bit + // integer. DWARF2/3 allow addresses to be any size from 0-255 + // bytes currently. Internally we support 4 and 8 byte addresses, + // and will CHECK on anything else. + // This function pointer gets set by SetAddressSize. + AddressReader address_reader_; + + Endianness endian_; + uint8 address_size_; + uint8 offset_size_; + + // Base addresses for Linux C++ exception handling data's encoded pointers. + bool have_section_base_, have_text_base_, have_data_base_; + bool have_function_base_; + uint64 section_base_; + uint64 text_base_, data_base_, function_base_; + const char *buffer_base_; +}; + + +inline uint8 ByteReader::ReadOneByte(const char* buffer) const { + return buffer[0]; +} + +inline uint16 ByteReader::ReadTwoBytes(const char* signed_buffer) const { + const unsigned char *buffer + = reinterpret_cast<const unsigned char *>(signed_buffer); + const uint16 buffer0 = buffer[0]; + const uint16 buffer1 = buffer[1]; + if (endian_ == ENDIANNESS_LITTLE) { + return buffer0 | buffer1 << 8; + } else { + return buffer1 | buffer0 << 8; + } +} + +inline uint64 ByteReader::ReadFourBytes(const char* signed_buffer) const { + const unsigned char *buffer + = reinterpret_cast<const unsigned char *>(signed_buffer); + const uint32 buffer0 = buffer[0]; + const uint32 buffer1 = buffer[1]; + const uint32 buffer2 = buffer[2]; + const uint32 buffer3 = buffer[3]; + if (endian_ == ENDIANNESS_LITTLE) { + return buffer0 | buffer1 << 8 | buffer2 << 16 | buffer3 << 24; + } else { + return buffer3 | buffer2 << 8 | buffer1 << 16 | buffer0 << 24; + } +} + +inline uint64 ByteReader::ReadEightBytes(const char* signed_buffer) const { + const unsigned char *buffer + = reinterpret_cast<const unsigned char *>(signed_buffer); + const uint64 buffer0 = buffer[0]; + const uint64 buffer1 = buffer[1]; + const uint64 buffer2 = buffer[2]; + const uint64 buffer3 = buffer[3]; + const uint64 buffer4 = buffer[4]; + const uint64 buffer5 = buffer[5]; + const uint64 buffer6 = buffer[6]; + const uint64 buffer7 = buffer[7]; + if (endian_ == ENDIANNESS_LITTLE) { + return buffer0 | buffer1 << 8 | buffer2 << 16 | buffer3 << 24 | + buffer4 << 32 | buffer5 << 40 | buffer6 << 48 | buffer7 << 56; + } else { + return buffer7 | buffer6 << 8 | buffer5 << 16 | buffer4 << 24 | + buffer3 << 32 | buffer2 << 40 | buffer1 << 48 | buffer0 << 56; + } +} + +// Read an unsigned LEB128 number. Each byte contains 7 bits of +// information, plus one bit saying whether the number continues or +// not. + +inline uint64 ByteReader::ReadUnsignedLEB128(const char* buffer, + size_t* len) const { + uint64 result = 0; + size_t num_read = 0; + unsigned int shift = 0; + unsigned char byte; + + do { + byte = *buffer++; + num_read++; + + result |= (static_cast<uint64>(byte & 0x7f)) << shift; + + shift += 7; + + } while (byte & 0x80); + + *len = num_read; + + return result; +} + +// Read a signed LEB128 number. These are like regular LEB128 +// numbers, except the last byte may have a sign bit set. + +inline int64 ByteReader::ReadSignedLEB128(const char* buffer, + size_t* len) const { + int64 result = 0; + unsigned int shift = 0; + size_t num_read = 0; + unsigned char byte; + + do { + byte = *buffer++; + num_read++; + result |= (static_cast<uint64>(byte & 0x7f) << shift); + shift += 7; + } while (byte & 0x80); + + if ((shift < 8 * sizeof (result)) && (byte & 0x40)) + result |= -((static_cast<int64>(1)) << shift); + *len = num_read; + return result; +} + +inline uint64 ByteReader::ReadOffset(const char* buffer) const { + MOZ_ASSERT(this->offset_reader_); + return (this->*offset_reader_)(buffer); +} + +inline uint64 ByteReader::ReadAddress(const char* buffer) const { + MOZ_ASSERT(this->address_reader_); + return (this->*address_reader_)(buffer); +} + +inline void ByteReader::SetCFIDataBase(uint64 section_base, + const char *buffer_base) { + section_base_ = section_base; + buffer_base_ = buffer_base; + have_section_base_ = true; +} + +inline void ByteReader::SetTextBase(uint64 text_base) { + text_base_ = text_base; + have_text_base_ = true; +} + +inline void ByteReader::SetDataBase(uint64 data_base) { + data_base_ = data_base; + have_data_base_ = true; +} + +inline void ByteReader::SetFunctionBase(uint64 function_base) { + function_base_ = function_base; + have_function_base_ = true; +} + +inline void ByteReader::ClearFunctionBase() { + have_function_base_ = false; +} + + +// (derived from) +// dwarf_cfi_to_module.h: Define the DwarfCFIToModule class, which +// accepts parsed DWARF call frame info and adds it to a Summariser object. + +// This class is a reader for DWARF's Call Frame Information. CFI +// describes how to unwind stack frames --- even for functions that do +// not follow fixed conventions for saving registers, whose frame size +// varies as they execute, etc. +// +// CFI describes, at each machine instruction, how to compute the +// stack frame's base address, how to find the return address, and +// where to find the saved values of the caller's registers (if the +// callee has stashed them somewhere to free up the registers for its +// own use). +// +// For example, suppose we have a function whose machine code looks +// like this (imagine an assembly language that looks like C, for a +// machine with 32-bit registers, and a stack that grows towards lower +// addresses): +// +// func: ; entry point; return address at sp +// func+0: sp = sp - 16 ; allocate space for stack frame +// func+1: sp[12] = r0 ; save r0 at sp+12 +// ... ; other code, not frame-related +// func+10: sp -= 4; *sp = x ; push some x on the stack +// ... ; other code, not frame-related +// func+20: r0 = sp[16] ; restore saved r0 +// func+21: sp += 20 ; pop whole stack frame +// func+22: pc = *sp; sp += 4 ; pop return address and jump to it +// +// DWARF CFI is (a very compressed representation of) a table with a +// row for each machine instruction address and a column for each +// register showing how to restore it, if possible. +// +// A special column named "CFA", for "Canonical Frame Address", tells how +// to compute the base address of the frame; registers' entries may +// refer to the CFA in describing where the registers are saved. +// +// Another special column, named "RA", represents the return address. +// +// For example, here is a complete (uncompressed) table describing the +// function above: +// +// insn cfa r0 r1 ... ra +// ======================================= +// func+0: sp cfa[0] +// func+1: sp+16 cfa[0] +// func+2: sp+16 cfa[-4] cfa[0] +// func+11: sp+20 cfa[-4] cfa[0] +// func+21: sp+20 cfa[0] +// func+22: sp cfa[0] +// +// Some things to note here: +// +// - Each row describes the state of affairs *before* executing the +// instruction at the given address. Thus, the row for func+0 +// describes the state before we allocate the stack frame. In the +// next row, the formula for computing the CFA has changed, +// reflecting that allocation. +// +// - The other entries are written in terms of the CFA; this allows +// them to remain unchanged as the stack pointer gets bumped around. +// For example, the rule for recovering the return address (the "ra" +// column) remains unchanged throughout the function, even as the +// stack pointer takes on three different offsets from the return +// address. +// +// - Although we haven't shown it, most calling conventions designate +// "callee-saves" and "caller-saves" registers. The callee must +// preserve the values of callee-saves registers; if it uses them, +// it must save their original values somewhere, and restore them +// before it returns. In contrast, the callee is free to trash +// caller-saves registers; if the callee uses these, it will +// probably not bother to save them anywhere, and the CFI will +// probably mark their values as "unrecoverable". +// +// (However, since the caller cannot assume the callee was going to +// save them, caller-saves registers are probably dead in the caller +// anyway, so compilers usually don't generate CFA for caller-saves +// registers.) +// +// - Exactly where the CFA points is a matter of convention that +// depends on the architecture and ABI in use. In the example, the +// CFA is the value the stack pointer had upon entry to the +// function, pointing at the saved return address. But on the x86, +// the call frame information generated by GCC follows the +// convention that the CFA is the address *after* the saved return +// address. +// +// But by definition, the CFA remains constant throughout the +// lifetime of the frame. This makes it a useful value for other +// columns to refer to. It is also gives debuggers a useful handle +// for identifying a frame. +// +// If you look at the table above, you'll notice that a given entry is +// often the same as the one immediately above it: most instructions +// change only one or two aspects of the stack frame, if they affect +// it at all. The DWARF format takes advantage of this fact, and +// reduces the size of the data by mentioning only the addresses and +// columns at which changes take place. So for the above, DWARF CFI +// data would only actually mention the following: +// +// insn cfa r0 r1 ... ra +// ======================================= +// func+0: sp cfa[0] +// func+1: sp+16 +// func+2: cfa[-4] +// func+11: sp+20 +// func+21: r0 +// func+22: sp +// +// In fact, this is the way the parser reports CFI to the consumer: as +// a series of statements of the form, "At address X, column Y changed +// to Z," and related conventions for describing the initial state. +// +// Naturally, it would be impractical to have to scan the entire +// program's CFI, noting changes as we go, just to recover the +// unwinding rules in effect at one particular instruction. To avoid +// this, CFI data is grouped into "entries", each of which covers a +// specified range of addresses and begins with a complete statement +// of the rules for all recoverable registers at that starting +// address. Each entry typically covers a single function. +// +// Thus, to compute the contents of a given row of the table --- that +// is, rules for recovering the CFA, RA, and registers at a given +// instruction --- the consumer should find the entry that covers that +// instruction's address, start with the initial state supplied at the +// beginning of the entry, and work forward until it has processed all +// the changes up to and including those for the present instruction. +// +// There are seven kinds of rules that can appear in an entry of the +// table: +// +// - "undefined": The given register is not preserved by the callee; +// its value cannot be recovered. +// +// - "same value": This register has the same value it did in the callee. +// +// - offset(N): The register is saved at offset N from the CFA. +// +// - val_offset(N): The value the register had in the caller is the +// CFA plus offset N. (This is usually only useful for describing +// the stack pointer.) +// +// - register(R): The register's value was saved in another register R. +// +// - expression(E): Evaluating the DWARF expression E using the +// current frame's registers' values yields the address at which the +// register was saved. +// +// - val_expression(E): Evaluating the DWARF expression E using the +// current frame's registers' values yields the value the register +// had in the caller. + +class CallFrameInfo { + public: + // The different kinds of entries one finds in CFI. Used internally, + // and for error reporting. + enum EntryKind { kUnknown, kCIE, kFDE, kTerminator }; + + // The handler class to which the parser hands the parsed call frame + // information. Defined below. + class Handler; + + // A reporter class, which CallFrameInfo uses to report errors + // encountered while parsing call frame information. Defined below. + class Reporter; + + // Create a DWARF CFI parser. BUFFER points to the contents of the + // .debug_frame section to parse; BUFFER_LENGTH is its length in bytes. + // REPORTER is an error reporter the parser should use to report + // problems. READER is a ByteReader instance that has the endianness and + // address size set properly. Report the data we find to HANDLER. + // + // This class can also parse Linux C++ exception handling data, as found + // in '.eh_frame' sections. This data is a variant of DWARF CFI that is + // placed in loadable segments so that it is present in the program's + // address space, and is interpreted by the C++ runtime to search the + // call stack for a handler interested in the exception being thrown, + // actually pop the frames, and find cleanup code to run. + // + // There are two differences between the call frame information described + // in the DWARF standard and the exception handling data Linux places in + // the .eh_frame section: + // + // - Exception handling data uses uses a different format for call frame + // information entry headers. The distinguished CIE id, the way FDEs + // refer to their CIEs, and the way the end of the series of entries is + // determined are all slightly different. + // + // If the constructor's EH_FRAME argument is true, then the + // CallFrameInfo parses the entry headers as Linux C++ exception + // handling data. If EH_FRAME is false or omitted, the CallFrameInfo + // parses standard DWARF call frame information. + // + // - Linux C++ exception handling data uses CIE augmentation strings + // beginning with 'z' to specify the presence of additional data after + // the CIE and FDE headers and special encodings used for addresses in + // frame description entries. + // + // CallFrameInfo can handle 'z' augmentations in either DWARF CFI or + // exception handling data if you have supplied READER with the base + // addresses needed to interpret the pointer encodings that 'z' + // augmentations can specify. See the ByteReader interface for details + // about the base addresses. See the CallFrameInfo::Handler interface + // for details about the additional information one might find in + // 'z'-augmented data. + // + // Thus: + // + // - If you are parsing standard DWARF CFI, as found in a .debug_frame + // section, you should pass false for the EH_FRAME argument, or omit + // it, and you need not worry about providing READER with the + // additional base addresses. + // + // - If you want to parse Linux C++ exception handling data from a + // .eh_frame section, you should pass EH_FRAME as true, and call + // READER's Set*Base member functions before calling our Start method. + // + // - If you want to parse DWARF CFI that uses the 'z' augmentations + // (although I don't think any toolchain ever emits such data), you + // could pass false for EH_FRAME, but call READER's Set*Base members. + // + // The extensions the Linux C++ ABI makes to DWARF for exception + // handling are described here, rather poorly: + // http://refspecs.linux-foundation.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/dwarfext.html + // http://refspecs.linux-foundation.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html + // + // The mechanics of C++ exception handling, personality routines, + // and language-specific data areas are described here, rather nicely: + // http://www.codesourcery.com/public/cxx-abi/abi-eh.html + + CallFrameInfo(const char *buffer, size_t buffer_length, + ByteReader *reader, Handler *handler, Reporter *reporter, + bool eh_frame = false) + : buffer_(buffer), buffer_length_(buffer_length), + reader_(reader), handler_(handler), reporter_(reporter), + eh_frame_(eh_frame) { } + + ~CallFrameInfo() { } + + // Parse the entries in BUFFER, reporting what we find to HANDLER. + // Return true if we reach the end of the section successfully, or + // false if we encounter an error. + bool Start(); + + // Return the textual name of KIND. For error reporting. + static const char *KindName(EntryKind kind); + + private: + + struct CIE; + + // A CFI entry, either an FDE or a CIE. + struct Entry { + // The starting offset of the entry in the section, for error + // reporting. + size_t offset; + + // The start of this entry in the buffer. + const char *start; + + // Which kind of entry this is. + // + // We want to be able to use this for error reporting even while we're + // in the midst of parsing. Error reporting code may assume that kind, + // offset, and start fields are valid, although kind may be kUnknown. + EntryKind kind; + + // The end of this entry's common prologue (initial length and id), and + // the start of this entry's kind-specific fields. + const char *fields; + + // The start of this entry's instructions. + const char *instructions; + + // The address past the entry's last byte in the buffer. (Note that + // since offset points to the entry's initial length field, and the + // length field is the number of bytes after that field, this is not + // simply buffer_ + offset + length.) + const char *end; + + // For both DWARF CFI and .eh_frame sections, this is the CIE id in a + // CIE, and the offset of the associated CIE in an FDE. + uint64 id; + + // The CIE that applies to this entry, if we've parsed it. If this is a + // CIE, then this field points to this structure. + CIE *cie; + }; + + // A common information entry (CIE). + struct CIE: public Entry { + uint8 version; // CFI data version number + std::string augmentation; // vendor format extension markers + uint64 code_alignment_factor; // scale for code address adjustments + int data_alignment_factor; // scale for stack pointer adjustments + unsigned return_address_register; // which register holds the return addr + + // True if this CIE includes Linux C++ ABI 'z' augmentation data. + bool has_z_augmentation; + + // Parsed 'z' augmentation data. These are meaningful only if + // has_z_augmentation is true. + bool has_z_lsda; // The 'z' augmentation included 'L'. + bool has_z_personality; // The 'z' augmentation included 'P'. + bool has_z_signal_frame; // The 'z' augmentation included 'S'. + + // If has_z_lsda is true, this is the encoding to be used for language- + // specific data area pointers in FDEs. + DwarfPointerEncoding lsda_encoding; + + // If has_z_personality is true, this is the encoding used for the + // personality routine pointer in the augmentation data. + DwarfPointerEncoding personality_encoding; + + // If has_z_personality is true, this is the address of the personality + // routine --- or, if personality_encoding & DW_EH_PE_indirect, the + // address where the personality routine's address is stored. + uint64 personality_address; + + // This is the encoding used for addresses in the FDE header and + // in DW_CFA_set_loc instructions. This is always valid, whether + // or not we saw a 'z' augmentation string; its default value is + // DW_EH_PE_absptr, which is what normal DWARF CFI uses. + DwarfPointerEncoding pointer_encoding; + }; + + // A frame description entry (FDE). + struct FDE: public Entry { + uint64 address; // start address of described code + uint64 size; // size of described code, in bytes + + // If cie->has_z_lsda is true, then this is the language-specific data + // area's address --- or its address's address, if cie->lsda_encoding + // has the DW_EH_PE_indirect bit set. + uint64 lsda_address; + }; + + // Internal use. + class Rule; + class UndefinedRule; + class SameValueRule; + class OffsetRule; + class ValOffsetRule; + class RegisterRule; + class ExpressionRule; + class ValExpressionRule; + class RuleMap; + class State; + + // Parse the initial length and id of a CFI entry, either a CIE, an FDE, + // or a .eh_frame end-of-data mark. CURSOR points to the beginning of the + // data to parse. On success, populate ENTRY as appropriate, and return + // true. On failure, report the problem, and return false. Even if we + // return false, set ENTRY->end to the first byte after the entry if we + // were able to figure that out, or NULL if we weren't. + bool ReadEntryPrologue(const char *cursor, Entry *entry); + + // Parse the fields of a CIE after the entry prologue, including any 'z' + // augmentation data. Assume that the 'Entry' fields of CIE are + // populated; use CIE->fields and CIE->end as the start and limit for + // parsing. On success, populate the rest of *CIE, and return true; on + // failure, report the problem and return false. + bool ReadCIEFields(CIE *cie); + + // Parse the fields of an FDE after the entry prologue, including any 'z' + // augmentation data. Assume that the 'Entry' fields of *FDE are + // initialized; use FDE->fields and FDE->end as the start and limit for + // parsing. Assume that FDE->cie is fully initialized. On success, + // populate the rest of *FDE, and return true; on failure, report the + // problem and return false. + bool ReadFDEFields(FDE *fde); + + // Report that ENTRY is incomplete, and return false. This is just a + // trivial wrapper for invoking reporter_->Incomplete; it provides a + // little brevity. + bool ReportIncomplete(Entry *entry); + + // Return true if ENCODING has the DW_EH_PE_indirect bit set. + static bool IsIndirectEncoding(DwarfPointerEncoding encoding) { + return encoding & DW_EH_PE_indirect; + } + + // The contents of the DWARF .debug_info section we're parsing. + const char *buffer_; + size_t buffer_length_; + + // For reading multi-byte values with the appropriate endianness. + ByteReader *reader_; + + // The handler to which we should report the data we find. + Handler *handler_; + + // For reporting problems in the info we're parsing. + Reporter *reporter_; + + // True if we are processing .eh_frame-format data. + bool eh_frame_; +}; + + +// The handler class for CallFrameInfo. The a CFI parser calls the +// member functions of a handler object to report the data it finds. +class CallFrameInfo::Handler { + public: + // The pseudo-register number for the canonical frame address. + enum { kCFARegister = DW_REG_CFA }; + + Handler() { } + virtual ~Handler() { } + + // The parser has found CFI for the machine code at ADDRESS, + // extending for LENGTH bytes. OFFSET is the offset of the frame + // description entry in the section, for use in error messages. + // VERSION is the version number of the CFI format. AUGMENTATION is + // a string describing any producer-specific extensions present in + // the data. RETURN_ADDRESS is the number of the register that holds + // the address to which the function should return. + // + // Entry should return true to process this CFI, or false to skip to + // the next entry. + // + // The parser invokes Entry for each Frame Description Entry (FDE) + // it finds. The parser doesn't report Common Information Entries + // to the handler explicitly; instead, if the handler elects to + // process a given FDE, the parser reiterates the appropriate CIE's + // contents at the beginning of the FDE's rules. + virtual bool Entry(size_t offset, uint64 address, uint64 length, + uint8 version, const std::string &augmentation, + unsigned return_address) = 0; + + // When the Entry function returns true, the parser calls these + // handler functions repeatedly to describe the rules for recovering + // registers at each instruction in the given range of machine code. + // Immediately after a call to Entry, the handler should assume that + // the rule for each callee-saves register is "unchanged" --- that + // is, that the register still has the value it had in the caller. + // + // If a *Rule function returns true, we continue processing this entry's + // instructions. If a *Rule function returns false, we stop evaluating + // instructions, and skip to the next entry. Either way, we call End + // before going on to the next entry. + // + // In all of these functions, if the REG parameter is kCFARegister, then + // the rule describes how to find the canonical frame address. + // kCFARegister may be passed as a BASE_REGISTER argument, meaning that + // the canonical frame address should be used as the base address for the + // computation. All other REG values will be positive. + + // At ADDRESS, register REG's value is not recoverable. + virtual bool UndefinedRule(uint64 address, int reg) = 0; + + // At ADDRESS, register REG's value is the same as that it had in + // the caller. + virtual bool SameValueRule(uint64 address, int reg) = 0; + + // At ADDRESS, register REG has been saved at offset OFFSET from + // BASE_REGISTER. + virtual bool OffsetRule(uint64 address, int reg, + int base_register, long offset) = 0; + + // At ADDRESS, the caller's value of register REG is the current + // value of BASE_REGISTER plus OFFSET. (This rule doesn't provide an + // address at which the register's value is saved.) + virtual bool ValOffsetRule(uint64 address, int reg, + int base_register, long offset) = 0; + + // At ADDRESS, register REG has been saved in BASE_REGISTER. This differs + // from ValOffsetRule(ADDRESS, REG, BASE_REGISTER, 0), in that + // BASE_REGISTER is the "home" for REG's saved value: if you want to + // assign to a variable whose home is REG in the calling frame, you + // should put the value in BASE_REGISTER. + virtual bool RegisterRule(uint64 address, int reg, int base_register) = 0; + + // At ADDRESS, the DWARF expression EXPRESSION yields the address at + // which REG was saved. + virtual bool ExpressionRule(uint64 address, int reg, + const std::string &expression) = 0; + + // At ADDRESS, the DWARF expression EXPRESSION yields the caller's + // value for REG. (This rule doesn't provide an address at which the + // register's value is saved.) + virtual bool ValExpressionRule(uint64 address, int reg, + const std::string &expression) = 0; + + // Indicate that the rules for the address range reported by the + // last call to Entry are complete. End should return true if + // everything is okay, or false if an error has occurred and parsing + // should stop. + virtual bool End() = 0; + + // Handler functions for Linux C++ exception handling data. These are + // only called if the data includes 'z' augmentation strings. + + // The Linux C++ ABI uses an extension of the DWARF CFI format to + // walk the stack to propagate exceptions from the throw to the + // appropriate catch, and do the appropriate cleanups along the way. + // CFI entries used for exception handling have two additional data + // associated with them: + // + // - The "language-specific data area" describes which exception + // types the function has 'catch' clauses for, and indicates how + // to go about re-entering the function at the appropriate catch + // clause. If the exception is not caught, it describes the + // destructors that must run before the frame is popped. + // + // - The "personality routine" is responsible for interpreting the + // language-specific data area's contents, and deciding whether + // the exception should continue to propagate down the stack, + // perhaps after doing some cleanup for this frame, or whether the + // exception will be caught here. + // + // In principle, the language-specific data area is opaque to + // everybody but the personality routine. In practice, these values + // may be useful or interesting to readers with extra context, and + // we have to at least skip them anyway, so we might as well report + // them to the handler. + + // This entry's exception handling personality routine's address is + // ADDRESS. If INDIRECT is true, then ADDRESS is the address at + // which the routine's address is stored. The default definition for + // this handler function simply returns true, allowing parsing of + // the entry to continue. + virtual bool PersonalityRoutine(uint64 address, bool indirect) { + return true; + } + + // This entry's language-specific data area (LSDA) is located at + // ADDRESS. If INDIRECT is true, then ADDRESS is the address at + // which the area's address is stored. The default definition for + // this handler function simply returns true, allowing parsing of + // the entry to continue. + virtual bool LanguageSpecificDataArea(uint64 address, bool indirect) { + return true; + } + + // This entry describes a signal trampoline --- this frame is the + // caller of a signal handler. The default definition for this + // handler function simply returns true, allowing parsing of the + // entry to continue. + // + // The best description of the rationale for and meaning of signal + // trampoline CFI entries seems to be in the GCC bug database: + // http://gcc.gnu.org/bugzilla/show_bug.cgi?id=26208 + virtual bool SignalHandler() { return true; } +}; + + +// The CallFrameInfo class makes calls on an instance of this class to +// report errors or warn about problems in the data it is parsing. +// These messages are sent to the message sink |aLog| provided to the +// constructor. +class CallFrameInfo::Reporter { + public: + // Create an error reporter which attributes troubles to the section + // named SECTION in FILENAME. + // + // Normally SECTION would be .debug_frame, but the Mac puts CFI data + // in a Mach-O section named __debug_frame. If we support + // Linux-style exception handling data, we could be reading an + // .eh_frame section. + Reporter(void (*aLog)(const char*), + const std::string &filename, + const std::string §ion = ".debug_frame") + : log_(aLog), filename_(filename), section_(section) { } + virtual ~Reporter() { } + + // The CFI entry at OFFSET ends too early to be well-formed. KIND + // indicates what kind of entry it is; KIND can be kUnknown if we + // haven't parsed enough of the entry to tell yet. + virtual void Incomplete(uint64 offset, CallFrameInfo::EntryKind kind); + + // The .eh_frame data has a four-byte zero at OFFSET where the next + // entry's length would be; this is a terminator. However, the buffer + // length as given to the CallFrameInfo constructor says there should be + // more data. + virtual void EarlyEHTerminator(uint64 offset); + + // The FDE at OFFSET refers to the CIE at CIE_OFFSET, but the + // section is not that large. + virtual void CIEPointerOutOfRange(uint64 offset, uint64 cie_offset); + + // The FDE at OFFSET refers to the CIE at CIE_OFFSET, but the entry + // there is not a CIE. + virtual void BadCIEId(uint64 offset, uint64 cie_offset); + + // The FDE at OFFSET refers to a CIE with version number VERSION, + // which we don't recognize. We cannot parse DWARF CFI if it uses + // a version number we don't recognize. + virtual void UnrecognizedVersion(uint64 offset, int version); + + // The FDE at OFFSET refers to a CIE with augmentation AUGMENTATION, + // which we don't recognize. We cannot parse DWARF CFI if it uses + // augmentations we don't recognize. + virtual void UnrecognizedAugmentation(uint64 offset, + const std::string &augmentation); + + // The pointer encoding ENCODING, specified by the CIE at OFFSET, is not + // a valid encoding. + virtual void InvalidPointerEncoding(uint64 offset, uint8 encoding); + + // The pointer encoding ENCODING, specified by the CIE at OFFSET, depends + // on a base address which has not been supplied. + virtual void UnusablePointerEncoding(uint64 offset, uint8 encoding); + + // The CIE at OFFSET contains a DW_CFA_restore instruction at + // INSN_OFFSET, which may not appear in a CIE. + virtual void RestoreInCIE(uint64 offset, uint64 insn_offset); + + // The entry at OFFSET, of kind KIND, has an unrecognized + // instruction at INSN_OFFSET. + virtual void BadInstruction(uint64 offset, CallFrameInfo::EntryKind kind, + uint64 insn_offset); + + // The instruction at INSN_OFFSET in the entry at OFFSET, of kind + // KIND, establishes a rule that cites the CFA, but we have not + // established a CFA rule yet. + virtual void NoCFARule(uint64 offset, CallFrameInfo::EntryKind kind, + uint64 insn_offset); + + // The instruction at INSN_OFFSET in the entry at OFFSET, of kind + // KIND, is a DW_CFA_restore_state instruction, but the stack of + // saved states is empty. + virtual void EmptyStateStack(uint64 offset, CallFrameInfo::EntryKind kind, + uint64 insn_offset); + + // The DW_CFA_remember_state instruction at INSN_OFFSET in the entry + // at OFFSET, of kind KIND, would restore a state that has no CFA + // rule, whereas the current state does have a CFA rule. This is + // bogus input, which the CallFrameInfo::Handler interface doesn't + // (and shouldn't) have any way to report. + virtual void ClearingCFARule(uint64 offset, CallFrameInfo::EntryKind kind, + uint64 insn_offset); + + private: + // A logging sink function, as supplied by LUL's user. + void (*log_)(const char*); + + protected: + // The name of the file whose CFI we're reading. + std::string filename_; + + // The name of the CFI section in that file. + std::string section_; +}; + + +using lul::CallFrameInfo; +using lul::Summariser; + +// A class that accepts parsed call frame information from the DWARF +// CFI parser and populates a google_breakpad::Module object with the +// contents. +class DwarfCFIToModule: public CallFrameInfo::Handler { + public: + + // DwarfCFIToModule uses an instance of this class to report errors + // detected while converting DWARF CFI to Breakpad STACK CFI records. + class Reporter { + public: + // Create a reporter that writes messages to the message sink + // |aLog|. FILE is the name of the file we're processing, and + // SECTION is the name of the section within that file that we're + // looking at (.debug_frame, .eh_frame, etc.). + Reporter(void (*aLog)(const char*), + const std::string &file, const std::string §ion) + : log_(aLog), file_(file), section_(section) { } + virtual ~Reporter() { } + + // The DWARF CFI entry at OFFSET says that REG is undefined, but the + // Breakpad symbol file format cannot express this. + virtual void UndefinedNotSupported(size_t offset, + const UniqueString* reg); + + // The DWARF CFI entry at OFFSET says that REG uses a DWARF + // expression to find its value, but parseDwarfExpr could not + // convert it to a sequence of PfxInstrs. + virtual void ExpressionCouldNotBeSummarised(size_t offset, + const UniqueString* reg); + + private: + // A logging sink function, as supplied by LUL's user. + void (*log_)(const char*); + protected: + std::string file_, section_; + }; + + // Register name tables. If TABLE is a vector returned by one of these + // functions, then TABLE[R] is the name of the register numbered R in + // DWARF call frame information. + class RegisterNames { + public: + // Intel's "x86" or IA-32. + static unsigned int I386(); + + // AMD x86_64, AMD64, Intel EM64T, or Intel 64 + static unsigned int X86_64(); + + // ARM. + static unsigned int ARM(); + }; + + // Create a handler for the dwarf2reader::CallFrameInfo parser that + // records the stack unwinding information it receives in SUMM. + // + // Use REGISTER_NAMES[I] as the name of register number I; *this + // keeps a reference to the vector, so the vector should remain + // alive for as long as the DwarfCFIToModule does. + // + // Use REPORTER for reporting problems encountered in the conversion + // process. + DwarfCFIToModule(const unsigned int num_dw_regs, + Reporter *reporter, + ByteReader* reader, + /*MOD*/UniqueStringUniverse* usu, + /*OUT*/Summariser* summ) + : summ_(summ), usu_(usu), num_dw_regs_(num_dw_regs), + reporter_(reporter), reader_(reader), return_address_(-1) { + } + virtual ~DwarfCFIToModule() {} + + virtual bool Entry(size_t offset, uint64 address, uint64 length, + uint8 version, const std::string &augmentation, + unsigned return_address); + virtual bool UndefinedRule(uint64 address, int reg); + virtual bool SameValueRule(uint64 address, int reg); + virtual bool OffsetRule(uint64 address, int reg, + int base_register, long offset); + virtual bool ValOffsetRule(uint64 address, int reg, + int base_register, long offset); + virtual bool RegisterRule(uint64 address, int reg, int base_register); + virtual bool ExpressionRule(uint64 address, int reg, + const std::string &expression); + virtual bool ValExpressionRule(uint64 address, int reg, + const std::string &expression); + virtual bool End(); + + private: + // Return the name to use for register I. + const UniqueString* RegisterName(int i); + + // The Summariser to which we should give entries + Summariser* summ_; + + // Universe for creating UniqueStrings in, should that be necessary. + UniqueStringUniverse* usu_; + + // The number of Dwarf-defined register names for this architecture. + const unsigned int num_dw_regs_; + + // The reporter to use to report problems. + Reporter *reporter_; + + // The ByteReader to use for parsing Dwarf expressions. + ByteReader* reader_; + + // The section offset of the current frame description entry, for + // use in error messages. + size_t entry_offset_; + + // The return address column for that entry. + unsigned return_address_; +}; + + +// Convert the Dwarf expression in |expr| into PfxInstrs stored in the +// SecMap referred to by |summ|, and return the index of the starting +// PfxInstr added, which must be >= 0. In case of failure return -1. +int32_t parseDwarfExpr(Summariser* summ, const ByteReader* reader, + string expr, bool debug, + bool pushCfaAtStart, bool derefAtEnd); + +} // namespace lul + +#endif // LulDwarfExt_h diff --git a/tools/profiler/lul/LulDwarfInt.h b/tools/profiler/lul/LulDwarfInt.h new file mode 100644 index 000000000..43126612f --- /dev/null +++ b/tools/profiler/lul/LulDwarfInt.h @@ -0,0 +1,193 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ + +// Copyright (c) 2008, 2010 Google Inc. All Rights Reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// CFI reader author: Jim Blandy <jimb@mozilla.com> <jimb@red-bean.com> + +// This file is derived from the following file in +// toolkit/crashreporter/google-breakpad: +// src/common/dwarf/dwarf2enums.h + +#ifndef LulDwarfInt_h +#define LulDwarfInt_h + +#include "LulCommonExt.h" +#include "LulDwarfExt.h" + +namespace lul { + +// These enums do not follow the google3 style only because they are +// known universally (specs, other implementations) by the names in +// exactly this capitalization. +// Tag names and codes. + +// Call Frame Info instructions. +enum DwarfCFI + { + DW_CFA_advance_loc = 0x40, + DW_CFA_offset = 0x80, + DW_CFA_restore = 0xc0, + DW_CFA_nop = 0x00, + DW_CFA_set_loc = 0x01, + DW_CFA_advance_loc1 = 0x02, + DW_CFA_advance_loc2 = 0x03, + DW_CFA_advance_loc4 = 0x04, + DW_CFA_offset_extended = 0x05, + DW_CFA_restore_extended = 0x06, + DW_CFA_undefined = 0x07, + DW_CFA_same_value = 0x08, + DW_CFA_register = 0x09, + DW_CFA_remember_state = 0x0a, + DW_CFA_restore_state = 0x0b, + DW_CFA_def_cfa = 0x0c, + DW_CFA_def_cfa_register = 0x0d, + DW_CFA_def_cfa_offset = 0x0e, + DW_CFA_def_cfa_expression = 0x0f, + DW_CFA_expression = 0x10, + DW_CFA_offset_extended_sf = 0x11, + DW_CFA_def_cfa_sf = 0x12, + DW_CFA_def_cfa_offset_sf = 0x13, + DW_CFA_val_offset = 0x14, + DW_CFA_val_offset_sf = 0x15, + DW_CFA_val_expression = 0x16, + + // Opcodes in this range are reserved for user extensions. + DW_CFA_lo_user = 0x1c, + DW_CFA_hi_user = 0x3f, + + // SGI/MIPS specific. + DW_CFA_MIPS_advance_loc8 = 0x1d, + + // GNU extensions. + DW_CFA_GNU_window_save = 0x2d, + DW_CFA_GNU_args_size = 0x2e, + DW_CFA_GNU_negative_offset_extended = 0x2f + }; + +// Exception handling 'z' augmentation letters. +enum DwarfZAugmentationCodes { + // If the CFI augmentation string begins with 'z', then the CIE and FDE + // have an augmentation data area just before the instructions, whose + // contents are determined by the subsequent augmentation letters. + DW_Z_augmentation_start = 'z', + + // If this letter is present in a 'z' augmentation string, the CIE + // augmentation data includes a pointer encoding, and the FDE + // augmentation data includes a language-specific data area pointer, + // represented using that encoding. + DW_Z_has_LSDA = 'L', + + // If this letter is present in a 'z' augmentation string, the CIE + // augmentation data includes a pointer encoding, followed by a pointer + // to a personality routine, represented using that encoding. + DW_Z_has_personality_routine = 'P', + + // If this letter is present in a 'z' augmentation string, the CIE + // augmentation data includes a pointer encoding describing how the FDE's + // initial location, address range, and DW_CFA_set_loc operands are + // encoded. + DW_Z_has_FDE_address_encoding = 'R', + + // If this letter is present in a 'z' augmentation string, then code + // addresses covered by FDEs that cite this CIE are signal delivery + // trampolines. Return addresses of frames in trampolines should not be + // adjusted as described in section 6.4.4 of the DWARF 3 spec. + DW_Z_is_signal_trampoline = 'S' +}; + +// Expression opcodes +enum DwarfExpressionOpcodes { + DW_OP_addr = 0x03, + DW_OP_deref = 0x06, + DW_OP_const1s = 0x09, + DW_OP_const2u = 0x0a, + DW_OP_const2s = 0x0b, + DW_OP_const4u = 0x0c, + DW_OP_const4s = 0x0d, + DW_OP_const8u = 0x0e, + DW_OP_const8s = 0x0f, + DW_OP_constu = 0x10, + DW_OP_consts = 0x11, + DW_OP_dup = 0x12, + DW_OP_drop = 0x13, + DW_OP_over = 0x14, + DW_OP_pick = 0x15, + DW_OP_swap = 0x16, + DW_OP_rot = 0x17, + DW_OP_xderef = 0x18, + DW_OP_abs = 0x19, + DW_OP_and = 0x1a, + DW_OP_div = 0x1b, + DW_OP_minus = 0x1c, + DW_OP_mod = 0x1d, + DW_OP_mul = 0x1e, + DW_OP_neg = 0x1f, + DW_OP_not = 0x20, + DW_OP_or = 0x21, + DW_OP_plus = 0x22, + DW_OP_plus_uconst = 0x23, + DW_OP_shl = 0x24, + DW_OP_shr = 0x25, + DW_OP_shra = 0x26, + DW_OP_xor = 0x27, + DW_OP_skip = 0x2f, + DW_OP_bra = 0x28, + DW_OP_eq = 0x29, + DW_OP_ge = 0x2a, + DW_OP_gt = 0x2b, + DW_OP_le = 0x2c, + DW_OP_lt = 0x2d, + DW_OP_ne = 0x2e, + DW_OP_lit0 = 0x30, + DW_OP_lit31 = 0x4f, + DW_OP_reg0 = 0x50, + DW_OP_reg31 = 0x6f, + DW_OP_breg0 = 0x70, + DW_OP_breg31 = 0x8f, + DW_OP_regx = 0x90, + DW_OP_fbreg = 0x91, + DW_OP_bregx = 0x92, + DW_OP_piece = 0x93, + DW_OP_deref_size = 0x94, + DW_OP_xderef_size = 0x95, + DW_OP_nop = 0x96, + DW_OP_push_object_address = 0x97, + DW_OP_call2 = 0x98, + DW_OP_call4 = 0x99, + DW_OP_call_ref = 0x9a, + DW_OP_form_tls_address = 0x9b, + DW_OP_call_frame_cfa = 0x9c, + DW_OP_bit_piece = 0x9d, + DW_OP_lo_user = 0xe0, + DW_OP_hi_user = 0xff +}; + +} // namespace lul + +#endif // LulDwarfInt_h diff --git a/tools/profiler/lul/LulElfExt.h b/tools/profiler/lul/LulElfExt.h new file mode 100644 index 000000000..7cb422ab7 --- /dev/null +++ b/tools/profiler/lul/LulElfExt.h @@ -0,0 +1,67 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ + +// Copyright (c) 2006, 2011, 2012 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file is derived from the following files in +// toolkit/crashreporter/google-breakpad: +// src/common/linux/dump_symbols.h + +#ifndef LulElfExt_h +#define LulElfExt_h + +// These two functions are the external interface to the +// ELF/Dwarf/EXIDX reader. + +#include "LulMainInt.h" + +using lul::SecMap; + +namespace lul { + +// Find all the unwind information in OBJ_FILE, an ELF executable +// or shared library, and add it to SMAP. +bool ReadSymbolData(const std::string& obj_file, + const std::vector<std::string>& debug_dirs, + SecMap* smap, + void* rx_avma, size_t rx_size, + void (*log)(const char*)); + +// The same as ReadSymbolData, except that OBJ_FILE is assumed to +// point to a mapped-in image of OBJ_FILENAME. +bool ReadSymbolDataInternal(const uint8_t* obj_file, + const std::string& obj_filename, + const std::vector<std::string>& debug_dirs, + SecMap* smap, + void* rx_avma, size_t rx_size, + void (*log)(const char*)); + +} // namespace lul + +#endif // LulElfExt_h diff --git a/tools/profiler/lul/LulElfInt.h b/tools/profiler/lul/LulElfInt.h new file mode 100644 index 000000000..98efc655c --- /dev/null +++ b/tools/profiler/lul/LulElfInt.h @@ -0,0 +1,201 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ + +// Copyright (c) 2006, 2012, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file is derived from the following files in +// toolkit/crashreporter/google-breakpad: +// src/common/linux/elfutils.h +// src/common/linux/file_id.h +// src/common/linux/elfutils-inl.h + +#ifndef LulElfInt_h +#define LulElfInt_h + +// This header defines functions etc internal to the ELF reader. It +// should not be included outside of LulElf.cpp. + +#include <elf.h> +#include <stdlib.h> + +#include "mozilla/Assertions.h" + +#include "LulPlatformMacros.h" + + +// (derived from) +// elfutils.h: Utilities for dealing with ELF files. +// + +# include <link.h> + + +namespace lul { + +// Traits classes so consumers can write templatized code to deal +// with specific ELF bits. +struct ElfClass32 { + typedef Elf32_Addr Addr; + typedef Elf32_Ehdr Ehdr; + typedef Elf32_Nhdr Nhdr; + typedef Elf32_Phdr Phdr; + typedef Elf32_Shdr Shdr; + typedef Elf32_Half Half; + typedef Elf32_Off Off; + typedef Elf32_Word Word; + static const int kClass = ELFCLASS32; + static const size_t kAddrSize = sizeof(Elf32_Addr); +}; + +struct ElfClass64 { + typedef Elf64_Addr Addr; + typedef Elf64_Ehdr Ehdr; + typedef Elf64_Nhdr Nhdr; + typedef Elf64_Phdr Phdr; + typedef Elf64_Shdr Shdr; + typedef Elf64_Half Half; + typedef Elf64_Off Off; + typedef Elf64_Word Word; + static const int kClass = ELFCLASS64; + static const size_t kAddrSize = sizeof(Elf64_Addr); +}; + +bool IsValidElf(const void* elf_header); +int ElfClass(const void* elf_base); + +// Attempt to find a section named |section_name| of type |section_type| +// in the ELF binary data at |elf_mapped_base|. On success, returns true +// and sets |*section_start| to point to the start of the section data, +// and |*section_size| to the size of the section's data. If |elfclass| +// is not NULL, set |*elfclass| to the ELF file class. +bool FindElfSection(const void *elf_mapped_base, + const char *section_name, + uint32_t section_type, + const void **section_start, + int *section_size, + int *elfclass); + +// Internal helper method, exposed for convenience for callers +// that already have more info. +template<typename ElfClass> +const typename ElfClass::Shdr* +FindElfSectionByName(const char* name, + typename ElfClass::Word section_type, + const typename ElfClass::Shdr* sections, + const char* section_names, + const char* names_end, + int nsection); + +// Attempt to find the first segment of type |segment_type| in the ELF +// binary data at |elf_mapped_base|. On success, returns true and sets +// |*segment_start| to point to the start of the segment data, and +// and |*segment_size| to the size of the segment's data. If |elfclass| +// is not NULL, set |*elfclass| to the ELF file class. +bool FindElfSegment(const void *elf_mapped_base, + uint32_t segment_type, + const void **segment_start, + int *segment_size, + int *elfclass); + +// Convert an offset from an Elf header into a pointer to the mapped +// address in the current process. Takes an extra template parameter +// to specify the return type to avoid having to dynamic_cast the +// result. +template<typename ElfClass, typename T> +const T* +GetOffset(const typename ElfClass::Ehdr* elf_header, + typename ElfClass::Off offset); + + +// (derived from) +// file_id.h: Return a unique identifier for a file +// + +static const size_t kMDGUIDSize = sizeof(MDGUID); + +class FileID { + public: + + // Load the identifier for the elf file mapped into memory at |base| into + // |identifier|. Return false if the identifier could not be created for the + // file. + static bool ElfFileIdentifierFromMappedFile(const void* base, + uint8_t identifier[kMDGUIDSize]); + + // Convert the |identifier| data to a NULL terminated string. The string will + // be formatted as a UUID (e.g., 22F065BB-FC9C-49F7-80FE-26A7CEBD7BCE). + // The |buffer| should be at least 37 bytes long to receive all of the data + // and termination. Shorter buffers will contain truncated data. + static void ConvertIdentifierToString(const uint8_t identifier[kMDGUIDSize], + char* buffer, int buffer_length); +}; + + + +template<typename ElfClass, typename T> +const T* GetOffset(const typename ElfClass::Ehdr* elf_header, + typename ElfClass::Off offset) { + return reinterpret_cast<const T*>(reinterpret_cast<uintptr_t>(elf_header) + + offset); +} + +template<typename ElfClass> +const typename ElfClass::Shdr* FindElfSectionByName( + const char* name, + typename ElfClass::Word section_type, + const typename ElfClass::Shdr* sections, + const char* section_names, + const char* names_end, + int nsection) { + MOZ_ASSERT(name != NULL); + MOZ_ASSERT(sections != NULL); + MOZ_ASSERT(nsection > 0); + + int name_len = strlen(name); + if (name_len == 0) + return NULL; + + for (int i = 0; i < nsection; ++i) { + const char* section_name = section_names + sections[i].sh_name; + if (sections[i].sh_type == section_type && + names_end - section_name >= name_len + 1 && + strcmp(name, section_name) == 0) { + return sections + i; + } + } + return NULL; +} + +} // namespace lul + + +// And finally, the external interface, offered to LulMain.cpp +#include "LulElfExt.h" + +#endif // LulElfInt_h diff --git a/tools/profiler/lul/LulMainInt.h b/tools/profiler/lul/LulMainInt.h new file mode 100644 index 000000000..a0dc918a7 --- /dev/null +++ b/tools/profiler/lul/LulMainInt.h @@ -0,0 +1,392 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef LulMainInt_h +#define LulMainInt_h + +#include "LulPlatformMacros.h" +#include "LulMain.h" // for TaggedUWord + +#include <vector> + +#include "mozilla/Assertions.h" + +// This file is provides internal interface inside LUL. If you are an +// end-user of LUL, do not include it in your code. The end-user +// interface is in LulMain.h. + + +namespace lul { + +using std::vector; + +//////////////////////////////////////////////////////////////// +// DW_REG_ constants // +//////////////////////////////////////////////////////////////// + +// These are the Dwarf CFI register numbers, as (presumably) defined +// in the ELF ABI supplements for each architecture. + +enum DW_REG_NUMBER { + // No real register has this number. It's convenient to be able to + // treat the CFA (Canonical Frame Address) as "just another + // register", though. + DW_REG_CFA = -1, +#if defined(LUL_ARCH_arm) + // ARM registers + DW_REG_ARM_R7 = 7, + DW_REG_ARM_R11 = 11, + DW_REG_ARM_R12 = 12, + DW_REG_ARM_R13 = 13, + DW_REG_ARM_R14 = 14, + DW_REG_ARM_R15 = 15, +#elif defined(LUL_ARCH_x64) + // Because the X86 (32 bit) and AMD64 (64 bit) summarisers are + // combined, a merged set of register constants is needed. + DW_REG_INTEL_XBP = 6, + DW_REG_INTEL_XSP = 7, + DW_REG_INTEL_XIP = 16, +#elif defined(LUL_ARCH_x86) + DW_REG_INTEL_XBP = 5, + DW_REG_INTEL_XSP = 4, + DW_REG_INTEL_XIP = 8, +#else +# error "Unknown arch" +#endif +}; + + +//////////////////////////////////////////////////////////////// +// PfxExpr // +//////////////////////////////////////////////////////////////// + +enum PfxExprOp { + // meaning of mOperand effect on stack + PX_Start, // bool start-with-CFA? start, with CFA on stack, or not + PX_End, // none stop; result is at top of stack + PX_SImm32, // int32 push signed int32 + PX_DwReg, // DW_REG_NUMBER push value of the specified reg + PX_Deref, // none pop X ; push *X + PX_Add, // none pop X ; pop Y ; push Y + X + PX_Sub, // none pop X ; pop Y ; push Y - X + PX_And, // none pop X ; pop Y ; push Y & X + PX_Or, // none pop X ; pop Y ; push Y | X + PX_CmpGES, // none pop X ; pop Y ; push (Y >=s X) ? 1 : 0 + PX_Shl // none pop X ; pop Y ; push Y << X +}; + +struct PfxInstr { + PfxInstr(PfxExprOp opcode, int32_t operand) + : mOpcode(opcode) + , mOperand(operand) + {} + explicit PfxInstr(PfxExprOp opcode) + : mOpcode(opcode) + , mOperand(0) + {} + bool operator==(const PfxInstr& other) { + return mOpcode == other.mOpcode && mOperand == other.mOperand; + } + PfxExprOp mOpcode; + int32_t mOperand; +}; + +static_assert(sizeof(PfxInstr) <= 8, "PfxInstr size changed unexpectedly"); + +// Evaluate the prefix expression whose PfxInstrs start at aPfxInstrs[start]. +// In the case of any mishap (stack over/underflow, running off the end of +// the instruction vector, obviously malformed sequences), +// return an invalid TaggedUWord. +// RUNS IN NO-MALLOC CONTEXT +TaggedUWord EvaluatePfxExpr(int32_t start, + const UnwindRegs* aOldRegs, + TaggedUWord aCFA, const StackImage* aStackImg, + const vector<PfxInstr>& aPfxInstrs); + + +//////////////////////////////////////////////////////////////// +// LExpr // +//////////////////////////////////////////////////////////////// + +// An expression -- very primitive. Denotes either "register + +// offset", a dereferenced version of the same, or a reference to a +// prefix expression stored elsewhere. So as to allow convenient +// handling of Dwarf-derived unwind info, the register may also denote +// the CFA. A large number of these need to be stored, so we ensure +// it fits into 8 bytes. See comment below on RuleSet to see how +// expressions fit into the bigger picture. + +enum LExprHow { + UNKNOWN=0, // This LExpr denotes no value. + NODEREF, // Value is (mReg + mOffset). + DEREF, // Value is *(mReg + mOffset). + PFXEXPR // Value is EvaluatePfxExpr(secMap->mPfxInstrs[mOffset]) +}; + +inline static const char* NameOf_LExprHow(LExprHow how) { + switch (how) { + case UNKNOWN: return "UNKNOWN"; + case NODEREF: return "NODEREF"; + case DEREF: return "DEREF"; + case PFXEXPR: return "PFXEXPR"; + default: return "LExpr-??"; + } +} + + +struct LExpr { + // Denotes an expression with no value. + LExpr() + : mHow(UNKNOWN) + , mReg(0) + , mOffset(0) + {} + + // Denotes any expressible expression. + LExpr(LExprHow how, int16_t reg, int32_t offset) + : mHow(how) + , mReg(reg) + , mOffset(offset) + { + switch (how) { + case UNKNOWN: MOZ_ASSERT(reg == 0 && offset == 0); break; + case NODEREF: break; + case DEREF: break; + case PFXEXPR: MOZ_ASSERT(reg == 0 && offset >= 0); break; + default: MOZ_ASSERT(0, "LExpr::LExpr: invalid how"); + } + } + + // Change the offset for an expression that references memory. + LExpr add_delta(long delta) + { + MOZ_ASSERT(mHow == NODEREF); + // If this is a non-debug build and the above assertion would have + // failed, at least return LExpr() so that the machinery that uses + // the resulting expression fails in a repeatable way. + return (mHow == NODEREF) ? LExpr(mHow, mReg, mOffset+delta) + : LExpr(); // Gone bad + } + + // Dereference an expression that denotes a memory address. + LExpr deref() + { + MOZ_ASSERT(mHow == NODEREF); + // Same rationale as for add_delta(). + return (mHow == NODEREF) ? LExpr(DEREF, mReg, mOffset) + : LExpr(); // Gone bad + } + + // Print a rule for recovery of |aNewReg| whose recovered value + // is this LExpr. + string ShowRule(const char* aNewReg) const; + + // Evaluate this expression, producing a TaggedUWord. |aOldRegs| + // holds register values that may be referred to by the expression. + // |aCFA| holds the CFA value, if any, that applies. |aStackImg| + // contains a chuck of stack that will be consulted if the expression + // references memory. |aPfxInstrs| holds the vector of PfxInstrs + // that will be consulted if this is a PFXEXPR. + // RUNS IN NO-MALLOC CONTEXT + TaggedUWord EvaluateExpr(const UnwindRegs* aOldRegs, + TaggedUWord aCFA, const StackImage* aStackImg, + const vector<PfxInstr>* aPfxInstrs) const; + + // Representation of expressions. If |mReg| is DW_REG_CFA (-1) then + // it denotes the CFA. All other allowed values for |mReg| are + // nonnegative and are DW_REG_ values. + LExprHow mHow:8; + int16_t mReg; // A DW_REG_ value + int32_t mOffset; // 32-bit signed offset should be more than enough. +}; + +static_assert(sizeof(LExpr) <= 8, "LExpr size changed unexpectedly"); + + +//////////////////////////////////////////////////////////////// +// RuleSet // +//////////////////////////////////////////////////////////////// + +// This is platform-dependent. For some address range, describes how +// to recover the CFA and then how to recover the registers for the +// previous frame. +// +// The set of LExprs contained in a given RuleSet describe a DAG which +// says how to compute the caller's registers ("new registers") from +// the callee's registers ("old registers"). The DAG can contain a +// single internal node, which is the value of the CFA for the callee. +// It would be possible to construct a DAG that omits the CFA, but +// including it makes the summarisers simpler, and the Dwarf CFI spec +// has the CFA as a central concept. +// +// For this to make sense, |mCfaExpr| can't have +// |mReg| == DW_REG_CFA since we have no previous value for the CFA. +// All of the other |Expr| fields can -- and usually do -- specify +// |mReg| == DW_REG_CFA. +// +// With that in place, the unwind algorithm proceeds as follows. +// +// (0) Initially: we have values for the old registers, and a memory +// image. +// +// (1) Compute the CFA by evaluating |mCfaExpr|. Add the computed +// value to the set of "old registers". +// +// (2) Compute values for the registers by evaluating all of the other +// |Expr| fields in the RuleSet. These can depend on both the old +// register values and the just-computed CFA. +// +// If we are unwinding without computing a CFA, perhaps because the +// RuleSets are derived from EXIDX instead of Dwarf, then +// |mCfaExpr.mHow| will be LExpr::UNKNOWN, so the computed value will +// be invalid -- that is, TaggedUWord() -- and so any attempt to use +// that will result in the same value. But that's OK because the +// RuleSet would make no sense if depended on the CFA but specified no +// way to compute it. +// +// A RuleSet is not allowed to cover zero address range. Having zero +// length would break binary searching in SecMaps and PriMaps. + +class RuleSet { +public: + RuleSet(); + void Print(void(*aLog)(const char*)) const; + + // Find the LExpr* for a given DW_REG_ value in this class. + LExpr* ExprForRegno(DW_REG_NUMBER aRegno); + + uintptr_t mAddr; + uintptr_t mLen; + // How to compute the CFA. + LExpr mCfaExpr; + // How to compute caller register values. These may reference the + // value defined by |mCfaExpr|. +#if defined(LUL_ARCH_x64) || defined(LUL_ARCH_x86) + LExpr mXipExpr; // return address + LExpr mXspExpr; + LExpr mXbpExpr; +#elif defined(LUL_ARCH_arm) + LExpr mR15expr; // return address + LExpr mR14expr; + LExpr mR13expr; + LExpr mR12expr; + LExpr mR11expr; + LExpr mR7expr; +#else +# error "Unknown arch" +#endif +}; + +// Returns |true| for Dwarf register numbers which are members +// of the set of registers that LUL unwinds on this target. +static inline bool registerIsTracked(DW_REG_NUMBER reg) { + switch (reg) { +# if defined(LUL_ARCH_x64) || defined(LUL_ARCH_x86) + case DW_REG_INTEL_XBP: case DW_REG_INTEL_XSP: case DW_REG_INTEL_XIP: + return true; +# elif defined(LUL_ARCH_arm) + case DW_REG_ARM_R7: case DW_REG_ARM_R11: case DW_REG_ARM_R12: + case DW_REG_ARM_R13: case DW_REG_ARM_R14: case DW_REG_ARM_R15: + return true; +# else +# error "Unknown arch" +# endif + default: + return false; + } +} + + +//////////////////////////////////////////////////////////////// +// SecMap // +//////////////////////////////////////////////////////////////// + +// A SecMap may have zero address range, temporarily, whilst RuleSets +// are being added to it. But adding a zero-range SecMap to a PriMap +// will make it impossible to maintain the total order of the PriMap +// entries, and so that can't be allowed to happen. + +class SecMap { +public: + // These summarise the contained mRuleSets, in that they give + // exactly the lowest and highest addresses that any of the entries + // in this SecMap cover. Hence invariants: + // + // mRuleSets is nonempty + // <=> mSummaryMinAddr <= mSummaryMaxAddr + // && mSummaryMinAddr == mRuleSets[0].mAddr + // && mSummaryMaxAddr == mRuleSets[#rulesets-1].mAddr + // + mRuleSets[#rulesets-1].mLen - 1; + // + // This requires that no RuleSet has zero length. + // + // mRuleSets is empty + // <=> mSummaryMinAddr > mSummaryMaxAddr + // + // This doesn't constrain mSummaryMinAddr and mSummaryMaxAddr uniquely, + // so let's use mSummaryMinAddr == 1 and mSummaryMaxAddr == 0 to denote + // this case. + + explicit SecMap(void(*aLog)(const char*)); + ~SecMap(); + + // Binary search mRuleSets to find one that brackets |ia|, or nullptr + // if none is found. It's not allowable to do this until PrepareRuleSets + // has been called first. + RuleSet* FindRuleSet(uintptr_t ia); + + // Add a RuleSet to the collection. The rule is copied in. Calling + // this makes the map non-searchable. + void AddRuleSet(const RuleSet* rs); + + // Add a PfxInstr to the vector of such instrs, and return the index + // in the vector. Calling this makes the map non-searchable. + uint32_t AddPfxInstr(PfxInstr pfxi); + + // Returns the entire vector of PfxInstrs. + const vector<PfxInstr>* GetPfxInstrs() { return &mPfxInstrs; } + + // Prepare the map for searching. Also, remove any rules for code + // address ranges which don't fall inside [start, +len). |len| may + // not be zero. + void PrepareRuleSets(uintptr_t start, size_t len); + + bool IsEmpty(); + + size_t Size() { return mRuleSets.size(); } + + // The min and max addresses of the addresses in the contained + // RuleSets. See comment above for invariants. + uintptr_t mSummaryMinAddr; + uintptr_t mSummaryMaxAddr; + +private: + // False whilst adding entries; true once it is safe to call FindRuleSet. + // Transition (false->true) is caused by calling PrepareRuleSets(). + bool mUsable; + + // A vector of RuleSets, sorted, nonoverlapping (post Prepare()). + vector<RuleSet> mRuleSets; + + // A vector of PfxInstrs, which are referred to by the RuleSets. + // These are provided as a representation of Dwarf expressions + // (DW_CFA_val_expression, DW_CFA_expression, DW_CFA_def_cfa_expression), + // are relatively expensive to evaluate, and and are therefore + // expected to be used only occasionally. + // + // The vector holds a bunch of separate PfxInstr programs, each one + // starting with a PX_Start and terminated by a PX_End, all + // concatenated together. When a RuleSet can't recover a value + // using a self-contained LExpr, it uses a PFXEXPR whose mOffset is + // the index in this vector of start of the necessary PfxInstr program. + vector<PfxInstr> mPfxInstrs; + + // A logging sink, for debugging. + void (*mLog)(const char*); +}; + +} // namespace lul + +#endif // ndef LulMainInt_h diff --git a/tools/profiler/lul/LulPlatformMacros.h b/tools/profiler/lul/LulPlatformMacros.h new file mode 100644 index 000000000..7004e594b --- /dev/null +++ b/tools/profiler/lul/LulPlatformMacros.h @@ -0,0 +1,39 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef LulPlatformMacros_h +#define LulPlatformMacros_h + +#include <stdint.h> +#include <stdlib.h> + +// Define platform selection macros in a consistent way. The primary +// factorisation is on (ARCH,OS) pairs ("PLATforms") but ARCH_ and OS_ +// macros are defined too, since they are sometimes convenient. + +#undef LUL_PLAT_x64_linux +#undef LUL_PLAT_x86_linux + +#undef LUL_ARCH_arm +#undef LUL_ARCH_x86 +#undef LUL_ARCH_x64 + +#undef LUL_OS_linux + +#if defined(__linux__) && defined(__x86_64__) +# define LUL_PLAT_x64_linux 1 +# define LUL_ARCH_x64 1 +# define LUL_OS_linux 1 + +#elif defined(__linux__) && defined(__i386__) +# define LUL_PLAT_x86_linux 1 +# define LUL_ARCH_x86 1 +# define LUL_OS_linux 1 + +#else +# error "Unsupported platform" +#endif + +#endif // LulPlatformMacros_h diff --git a/tools/profiler/merge-profiles.py b/tools/profiler/merge-profiles.py new file mode 100755 index 000000000..0c10c60e1 --- /dev/null +++ b/tools/profiler/merge-profiles.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# +# This script takes b2g process profiles and merged them into a single profile. +# The meta data is taken from the first profile. The startTime for each profile +# is used to syncronized the samples. Each thread is moved into the merged +# profile. +# +import json +import re +import sys + +def MergeProfiles(files): + threads = [] + fileData = [] + symTable = dict() + meta = None + libs = None + videoUrl = None + minStartTime = None + + for fname in files: + if fname.startswith("--video="): + videoUrl = fname[8:] + continue + + match = re.match('profile_([0-9]+)_(.+)\.sym', fname) + if match is None: + raise Exception("Filename '" + fname + "' doesn't match expected pattern") + pid = match.groups(0)[0] + pname = match.groups(0)[1] + + fp = open(fname, "r") + fileData = json.load(fp) + fp.close() + + if meta is None: + meta = fileData['profileJSON']['meta'].copy() + libs = fileData['profileJSON']['libs'] + minStartTime = meta['startTime'] + else: + minStartTime = min(minStartTime, fileData['profileJSON']['meta']['startTime']) + meta['startTime'] = minStartTime + + for thread in fileData['profileJSON']['threads']: + thread['name'] = thread['name'] + " (" + pname + ":" + pid + ")" + threads.append(thread) + + # Note that pid + sym, pid + location could be ambigious + # if we had pid=11 sym=1 && pid=1 sym=11. + pidStr = pid + ":" + + thread['startTime'] = fileData['profileJSON']['meta']['startTime'] + if meta['version'] >= 3: + stringTable = thread['stringTable'] + for i, str in enumerate(stringTable): + if str[:2] == '0x': + newLoc = pidStr + str + stringTable[i] = newLoc + symTable[newLoc] = str + else: + samples = thread['samples'] + for sample in thread['samples']: + for frame in sample['frames']: + if "location" in frame and frame['location'][0:2] == '0x': + oldLoc = frame['location'] + newLoc = pidStr + oldLoc + frame['location'] = newLoc + # Default to the unprefixed symbol if no translation is + symTable[newLoc] = oldLoc + + filesyms = fileData['symbolicationTable'] + for sym in filesyms.keys(): + symTable[pidStr + sym] = filesyms[sym] + + # For each thread, make the time offsets line up based on the + # earliest start + for thread in threads: + delta = thread['startTime'] - minStartTime + if meta['version'] >= 3: + idxTime = thread['samples']['schema']['time'] + for sample in thread['samples']['data']: + sample[idxTime] += delta + idxTime = thread['markers']['schema']['time'] + for marker in thread['markers']['data']: + marker[idxTime] += delta + else: + for sample in thread['samples']: + if "time" in sample: + sample['time'] += delta + for marker in thread['markers']: + marker['time'] += delta + + result = dict() + result['profileJSON'] = dict() + result['profileJSON']['meta'] = meta + result['profileJSON']['libs'] = libs + result['profileJSON']['threads'] = threads + result['symbolicationTable'] = symTable + result['format'] = "profileJSONWithSymbolicationTable,1" + if videoUrl: + result['profileJSON']['meta']['videoCapture'] = {"src": videoUrl} + + json.dump(result, sys.stdout) + + +if len(sys.argv) > 1: + MergeProfiles(sys.argv[1:]) + sys.exit(0) + +print "Usage: merge-profile.py profile_<pid1>_<pname1>.sym profile_<pid2>_<pname2>.sym > merged.sym" + + + diff --git a/tools/profiler/moz.build b/tools/profiler/moz.build new file mode 100644 index 000000000..5eb52fd1d --- /dev/null +++ b/tools/profiler/moz.build @@ -0,0 +1,36 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +IPDL_SOURCES += [ + 'gecko/ProfilerTypes.ipdlh', +] + +include('/ipc/chromium/chromium-config.mozbuild') + +EXPORTS += [ + 'public/GeckoProfiler.h', +] + +if CONFIG['MOZ_TASK_TRACER']: + EXPORTS += [ + 'tasktracer/GeckoTaskTracer.h', + 'tasktracer/GeckoTaskTracerImpl.h', + 'tasktracer/TracedTaskCommon.h', + ] + UNIFIED_SOURCES += [ + 'tasktracer/GeckoTaskTracer.cpp', + 'tasktracer/TracedTaskCommon.cpp', + ] + +XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell.ini'] + +if CONFIG['GNU_CXX']: + CXXFLAGS += [ + '-Wno-error=shadow', + '-Wno-ignored-qualifiers', # due to use of breakpad headers + ] + +with Files('**'):
+ BUG_COMPONENT = ('Core', 'Gecko Profiler')
diff --git a/tools/profiler/nm-symbolicate.py b/tools/profiler/nm-symbolicate.py new file mode 100755 index 000000000..f51d7f75f --- /dev/null +++ b/tools/profiler/nm-symbolicate.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys, subprocess, os + +def NMSymbolicate(library, addresses): + target_tools_prefix = os.environ.get("TARGET_TOOLS_PREFIX", "") + args = [ + target_tools_prefix + "nm", "-D", "-S", library + ] + nm_lines = subprocess.check_output(args).split("\n") + symbol_table = [] + for line in nm_lines: + pieces = line.split(" ", 4) + if len(pieces) != 4 or pieces[2] != "T": + continue + start = int(pieces[0], 16) + end = int(pieces[1], 16) + symbol = pieces[3] + symbol_table.append({ + "start": int(pieces[0], 16), + "end": int(pieces[0], 16) + int(pieces[1], 16), + "funcName": pieces[3] + }); + + for addressStr in addresses: + address = int(addressStr, 16) + symbolForAddress = None + for symbol in symbol_table: + if address >= symbol["start"] and address <= symbol["end"]: + symbolForAddress = symbol + break + if symbolForAddress: + print symbolForAddress["funcName"] + else: + print "??" # match addr2line + print ":0" # no line information from nm + +if len(sys.argv) > 1: + NMSymbolicate(sys.argv[1], sys.argv[2:]) + sys.exit(0) + +print "Usage: nm-symbolicate.py <library> <addresses> > merged.sym" + + diff --git a/tools/profiler/public/GeckoProfiler.h b/tools/profiler/public/GeckoProfiler.h new file mode 100644 index 000000000..92fc6e052 --- /dev/null +++ b/tools/profiler/public/GeckoProfiler.h @@ -0,0 +1,292 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* *************** SPS Sampler Information **************** + * + * SPS is an always on profiler that takes fast and low overheads samples + * of the program execution using only userspace functionity for portability. + * The goal of this module is to provide performance data in a generic + * cross platform way without requiring custom tools or kernel support. + * + * Non goals: Support features that are platform specific or replace + * platform specific profilers. + * + * Samples are collected to form a timeline with optional timeline event (markers) + * used for filtering. + * + * SPS collects samples in a platform independant way by using a speudo stack abstraction + * of the real program stack by using 'sample stack frames'. When a sample is collected + * all active sample stack frames and the program counter are recorded. + */ + +/* *************** SPS Sampler File Format **************** + * + * Simple new line seperated tag format: + * S -> BOF tags EOF + * tags -> tag tags + * tag -> CHAR - STRING + * + * Tags: + * 's' - Sample tag followed by the first stack frame followed by 0 or more 'c' tags. + * 'c' - Continue Sample tag gives remaining tag element. If a 'c' tag is seen without + * a preceding 's' tag it should be ignored. This is to support the behavior + * of circular buffers. + * If the 'stackwalk' feature is enabled this tag will have the format + * 'l-<library name>@<hex address>' and will expect an external tool to translate + * the tag into something readable through a symbolication processing step. + * 'm' - Timeline marker. Zero or more may appear before a 's' tag. + * 'l' - Information about the program counter library and address. Post processing + * can include function and source line. If built with leaf data enabled + * this tag will describe the last 'c' tag. + * 'r' - Responsiveness tag following an 's' tag. Gives an indication on how well the + * application is responding to the event loop. Lower is better. + * 't' - Elapse time since recording started. + * + */ + +#ifndef SAMPLER_H +#define SAMPLER_H + +#include "mozilla/Assertions.h" +#include "mozilla/Attributes.h" +#ifndef SPS_STANDALONE +#include "js/TypeDecls.h" +#endif +#include "mozilla/UniquePtr.h" +#include "mozilla/Vector.h" + +namespace mozilla { +class TimeStamp; + +namespace dom { +class Promise; +} // namespace dom + +} // namespace mozilla + +#ifndef SPS_STANDALONE +class nsIProfilerStartParams; +#endif + +enum TracingMetadata { + TRACING_DEFAULT, + TRACING_INTERVAL_START, + TRACING_INTERVAL_END, + TRACING_EVENT, + TRACING_EVENT_BACKTRACE, + TRACING_TIMESTAMP +}; + +#include <stdint.h> +#include <stdarg.h> + +// Insert a RAII in this scope to active a pseudo label. Any samples collected +// in this scope will contain this annotation. For dynamic strings use +// PROFILER_LABEL_PRINTF. Arguments must be string literals. +#define PROFILER_LABEL(name_space, info, category) do {} while (0) + +// Similar to PROFILER_LABEL, PROFILER_LABEL_FUNC will push/pop the enclosing +// functon name as the pseudostack label. +#define PROFILER_LABEL_FUNC(category) do {} while (0) + +// Format a dynamic string as a pseudo label. These labels will a considerable +// storage size in the circular buffer compared to regular labels. This function +// can be used to annotate custom information such as URL for the resource being +// decoded or the size of the paint. +#define PROFILER_LABEL_PRINTF(name_space, info, category, format, ...) do {} while (0) + +// Insert a marker in the profile timeline. This is useful to delimit something +// important happening such as the first paint. Unlike profiler_label that are +// only recorded if a sample is collected while it is active, marker will always +// be collected. +#define PROFILER_MARKER(info) do {} while (0) +#define PROFILER_MARKER_PAYLOAD(info, payload) do { mozilla::UniquePtr<ProfilerMarkerPayload> payloadDeletor(payload); } while (0) + +// Main thread specilization to avoid TLS lookup for performance critical use. +#define PROFILER_MAIN_THREAD_LABEL(name_space, info, category) do {} while (0) +#define PROFILER_MAIN_THREAD_LABEL_PRINTF(name_space, info, category, format, ...) do {} while (0) + +static inline void profiler_tracing(const char* aCategory, const char* aInfo, + TracingMetadata metaData = TRACING_DEFAULT) {} +class ProfilerBacktrace; + +static inline void profiler_tracing(const char* aCategory, const char* aInfo, + ProfilerBacktrace* aCause, + TracingMetadata metaData = TRACING_DEFAULT) {} + +// Initilize the profiler TLS, signal handlers on linux. If MOZ_PROFILER_STARTUP +// is set the profiler will be started. This call must happen before any other +// sampler calls. Particularly sampler_label/sampler_marker. +static inline void profiler_init(void* stackTop) {}; + +// Clean up the profiler module, stopping it if required. This function may +// also save a shutdown profile if requested. No profiler calls should happen +// after this point and all pseudo labels should have been popped. +static inline void profiler_shutdown() {}; + +// Start the profiler with the selected options. The samples will be +// recorded in a circular buffer. +// "aProfileEntries" is an abstract size indication of how big +// the profile's circular buffer should be. Multiply by 4 +// words to get the cost. +// "aInterval" the sampling interval. The profiler will do its +// best to sample at this interval. The profiler visualization +// should represent the actual sampling accuracy. +static inline void profiler_start(int aProfileEntries, double aInterval, + const char** aFeatures, uint32_t aFeatureCount, + const char** aThreadNameFilters, uint32_t aFilterCount) {} + +// Stop the profiler and discard the profile. Call 'profiler_save' before this +// to retrieve the profile. +static inline void profiler_stop() {} + +// These functions pause and resume the profiler. While paused the profile will not +// take any samples and will not record any data into its buffers. The profiler +// remains fully initialized in this state. Timeline markers will still be stored. +// This feature will keep javascript profiling enabled, thus allowing toggling the +// profiler without invalidating the JIT. +static inline bool profiler_is_paused() { return false; } +static inline void profiler_pause() {} +static inline void profiler_resume() {} + + +// Immediately capture the current thread's call stack and return it +static inline ProfilerBacktrace* profiler_get_backtrace() { return nullptr; } +static inline void profiler_get_backtrace_noalloc(char *output, size_t outputSize) { return; } + +// Free a ProfilerBacktrace returned by profiler_get_backtrace() +static inline void profiler_free_backtrace(ProfilerBacktrace* aBacktrace) {} + +static inline bool profiler_is_active() { return false; } + +// Check if an external profiler feature is active. +// Supported: +// * gpu +static inline bool profiler_feature_active(const char*) { return false; } + +// Internal-only. Used by the event tracer. +static inline void profiler_responsiveness(const mozilla::TimeStamp& aTime) {} + +// Internal-only. +static inline void profiler_set_frame_number(int frameNumber) {} + +// Get the profile encoded as a JSON string. +static inline mozilla::UniquePtr<char[]> profiler_get_profile(double aSinceTime = 0) { + return nullptr; +} + +// Get the profile encoded as a JSON object. +static inline JSObject* profiler_get_profile_jsobject(JSContext* aCx, + double aSinceTime = 0) { + return nullptr; +} + +#ifndef SPS_STANDALONE +// Get the profile encoded as a JSON object. +static inline void profiler_get_profile_jsobject_async(double aSinceTime = 0, + mozilla::dom::Promise* = 0) {} +static inline void profiler_get_start_params(int* aEntrySize, + double* aInterval, + mozilla::Vector<const char*>* aFilters, + mozilla::Vector<const char*>* aFeatures) {} +#endif + +// Get the profile and write it into a file +static inline void profiler_save_profile_to_file(char* aFilename) { } + +// Get the features supported by the profiler that are accepted by profiler_init. +// Returns a null terminated char* array. +static inline char** profiler_get_features() { return nullptr; } + +// Get information about the current buffer status. +// Retursn (using outparams) the current write position in the buffer, +// the total size of the buffer, and the generation of the buffer. +// This information may be useful to a user-interface displaying the +// current status of the profiler, allowing the user to get a sense +// for how fast the buffer is being written to, and how much +// data is visible. +static inline void profiler_get_buffer_info(uint32_t *aCurrentPosition, + uint32_t *aTotalSize, + uint32_t *aGeneration) +{ + *aCurrentPosition = 0; + *aTotalSize = 0; + *aGeneration = 0; +} + +// Discard the profile, throw away the profile and notify 'profiler-locked'. +// This function is to be used when entering private browsing to prevent +// the profiler from collecting sensitive data. +static inline void profiler_lock() {} + +// Re-enable the profiler and notify 'profiler-unlocked'. +static inline void profiler_unlock() {} + +static inline void profiler_register_thread(const char* name, void* guessStackTop) {} +static inline void profiler_unregister_thread() {} + +// These functions tell the profiler that a thread went to sleep so that we can avoid +// sampling it while it's sleeping. Calling profiler_sleep_start() twice without +// profiler_sleep_end() is an error. +static inline void profiler_sleep_start() {} +static inline void profiler_sleep_end() {} +static inline bool profiler_is_sleeping() { return false; } + +// Call by the JSRuntime's operation callback. This is used to enable +// profiling on auxilerary threads. +static inline void profiler_js_operation_callback() {} + +static inline double profiler_time() { return 0; } +static inline double profiler_time(const mozilla::TimeStamp& aTime) { return 0; } + +static inline bool profiler_in_privacy_mode() { return false; } + +static inline void profiler_log(const char *str) {} +static inline void profiler_log(const char *fmt, va_list args) {} + +class MOZ_RAII GeckoProfilerInitRAII { +public: + explicit GeckoProfilerInitRAII(void* stackTop) { + profiler_init(stackTop); + } + ~GeckoProfilerInitRAII() { + profiler_shutdown(); + } +}; + +class MOZ_RAII GeckoProfilerSleepRAII { +public: + GeckoProfilerSleepRAII() { + profiler_sleep_start(); + } + ~GeckoProfilerSleepRAII() { + profiler_sleep_end(); + } +}; + +/** + * Temporarily wake up the profiler while servicing events such as + * Asynchronous Procedure Calls (APCs). + */ +class MOZ_RAII GeckoProfilerWakeRAII { +public: + GeckoProfilerWakeRAII() + : mIssuedWake(profiler_is_sleeping()) + { + if (mIssuedWake) { + profiler_sleep_end(); + } + } + ~GeckoProfilerWakeRAII() { + if (mIssuedWake) { + MOZ_ASSERT(!profiler_is_sleeping()); + profiler_sleep_start(); + } + } +private: + bool mIssuedWake; +}; + +#endif // ifndef SAMPLER_H diff --git a/tools/profiler/tasktracer/GeckoTaskTracer.cpp b/tools/profiler/tasktracer/GeckoTaskTracer.cpp new file mode 100644 index 000000000..aefcb274e --- /dev/null +++ b/tools/profiler/tasktracer/GeckoTaskTracer.cpp @@ -0,0 +1,461 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "GeckoTaskTracer.h" +#include "GeckoTaskTracerImpl.h" + +#include "mozilla/MathAlgorithms.h" +#include "mozilla/StaticMutex.h" +#include "mozilla/ThreadLocal.h" +#include "mozilla/TimeStamp.h" +#include "mozilla/UniquePtr.h" +#include "mozilla/Unused.h" + +#include "nsString.h" +#include "nsThreadUtils.h" +#include "prtime.h" + +#include <stdarg.h> + +// We need a definition of gettid(), but older glibc versions don't provide a +// wrapper for it. +#if defined(__GLIBC__) +#include <unistd.h> +#include <sys/syscall.h> +#define gettid() static_cast<pid_t>(syscall(SYS_gettid)) +#elif defined(LINUX) +#include <sys/types.h> +pid_t gettid(); +#endif + +// NS_ENSURE_TRUE_VOID() without the warning on the debug build. +#define ENSURE_TRUE_VOID(x) \ + do { \ + if (MOZ_UNLIKELY(!(x))) { \ + return; \ + } \ + } while(0) + +// NS_ENSURE_TRUE() without the warning on the debug build. +#define ENSURE_TRUE(x, ret) \ + do { \ + if (MOZ_UNLIKELY(!(x))) { \ + return ret; \ + } \ + } while(0) + +namespace mozilla { +namespace tasktracer { + +static MOZ_THREAD_LOCAL(TraceInfo*) sTraceInfoTLS; +static mozilla::StaticMutex sMutex; + +// The generation of TraceInfo. It will be > 0 if the Task Tracer is started and +// <= 0 if stopped. +static mozilla::Atomic<bool> sStarted; +static nsTArray<UniquePtr<TraceInfo>>* sTraceInfos = nullptr; +static PRTime sStartTime; + +static const char sJSLabelPrefix[] = "#tt#"; + +namespace { + +static PRTime +GetTimestamp() +{ + return PR_Now() / 1000; +} + +static TraceInfo* +AllocTraceInfo(int aTid) +{ + StaticMutexAutoLock lock(sMutex); + + auto* info = sTraceInfos->AppendElement(MakeUnique<TraceInfo>(aTid)); + + return info->get(); +} + +static void +SaveCurTraceInfo() +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + info->mSavedCurTraceSourceId = info->mCurTraceSourceId; + info->mSavedCurTraceSourceType = info->mCurTraceSourceType; + info->mSavedCurTaskId = info->mCurTaskId; +} + +static void +RestoreCurTraceInfo() +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + info->mCurTraceSourceId = info->mSavedCurTraceSourceId; + info->mCurTraceSourceType = info->mSavedCurTraceSourceType; + info->mCurTaskId = info->mSavedCurTaskId; +} + +static void +CreateSourceEvent(SourceEventType aType) +{ + // Save the currently traced source event info. + SaveCurTraceInfo(); + + // Create a new unique task id. + uint64_t newId = GenNewUniqueTaskId(); + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + info->mCurTraceSourceId = newId; + info->mCurTraceSourceType = aType; + info->mCurTaskId = newId; + + uintptr_t* namePtr; +#define SOURCE_EVENT_NAME(type) \ + case SourceEventType::type: \ + { \ + static int CreateSourceEvent##type; \ + namePtr = (uintptr_t*)&CreateSourceEvent##type; \ + break; \ + } + + switch (aType) { +#include "SourceEventTypeMap.h" + default: + MOZ_CRASH("Unknown SourceEvent."); + } +#undef CREATE_SOURCE_EVENT_NAME + + // Log a fake dispatch and start for this source event. + LogDispatch(newId, newId, newId, aType); + LogVirtualTablePtr(newId, newId, namePtr); + LogBegin(newId, newId); +} + +static void +DestroySourceEvent() +{ + // Log a fake end for this source event. + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + LogEnd(info->mCurTraceSourceId, info->mCurTraceSourceId); + + // Restore the previously saved source event info. + RestoreCurTraceInfo(); +} + +inline static bool +IsStartLogging() +{ + return sStarted; +} + +static void +SetLogStarted(bool aIsStartLogging) +{ + MOZ_ASSERT(aIsStartLogging != IsStartLogging()); + sStarted = aIsStartLogging; + + StaticMutexAutoLock lock(sMutex); + if (!aIsStartLogging) { + for (uint32_t i = 0; i < sTraceInfos->Length(); ++i) { + (*sTraceInfos)[i]->mObsolete = true; + } + } +} + +static void +CleanUp() +{ + SetLogStarted(false); + StaticMutexAutoLock lock(sMutex); + + if (sTraceInfos) { + delete sTraceInfos; + sTraceInfos = nullptr; + } +} + +inline static void +ObsoleteCurrentTraceInfos() +{ + // Note that we can't and don't need to acquire sMutex here because this + // function is called before the other threads are recreated. + for (uint32_t i = 0; i < sTraceInfos->Length(); ++i) { + (*sTraceInfos)[i]->mObsolete = true; + } +} + +} // namespace anonymous + +nsCString* +TraceInfo::AppendLog() +{ + MutexAutoLock lock(mLogsMutex); + return mLogs.AppendElement(); +} + +void +TraceInfo::MoveLogsInto(TraceInfoLogsType& aResult) +{ + MutexAutoLock lock(mLogsMutex); + aResult.AppendElements(Move(mLogs)); +} + +void +InitTaskTracer(uint32_t aFlags) +{ + if (aFlags & FORKED_AFTER_NUWA) { + ObsoleteCurrentTraceInfos(); + return; + } + + MOZ_ASSERT(!sTraceInfos); + sTraceInfos = new nsTArray<UniquePtr<TraceInfo>>(); + + if (!sTraceInfoTLS.initialized()) { + Unused << sTraceInfoTLS.init(); + } +} + +void +ShutdownTaskTracer() +{ + CleanUp(); +} + +static void +FreeTraceInfo(TraceInfo* aTraceInfo) +{ + StaticMutexAutoLock lock(sMutex); + if (aTraceInfo) { + sTraceInfos->RemoveElement(aTraceInfo); + } +} + +void FreeTraceInfo() +{ + FreeTraceInfo(sTraceInfoTLS.get()); +} + +TraceInfo* +GetOrCreateTraceInfo() +{ + ENSURE_TRUE(sTraceInfoTLS.initialized(), nullptr); + ENSURE_TRUE(IsStartLogging(), nullptr); + + TraceInfo* info = sTraceInfoTLS.get(); + if (info && info->mObsolete) { + // TraceInfo is obsolete: remove it. + FreeTraceInfo(info); + info = nullptr; + } + + if (!info) { + info = AllocTraceInfo(gettid()); + sTraceInfoTLS.set(info); + } + + return info; +} + +uint64_t +GenNewUniqueTaskId() +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE(info, 0); + + pid_t tid = gettid(); + uint64_t taskid = ((uint64_t)tid << 32) | ++info->mLastUniqueTaskId; + return taskid; +} + +AutoSaveCurTraceInfo::AutoSaveCurTraceInfo() +{ + SaveCurTraceInfo(); +} + +AutoSaveCurTraceInfo::~AutoSaveCurTraceInfo() +{ + RestoreCurTraceInfo(); +} + +void +SetCurTraceInfo(uint64_t aSourceEventId, uint64_t aParentTaskId, + SourceEventType aSourceEventType) +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + info->mCurTraceSourceId = aSourceEventId; + info->mCurTaskId = aParentTaskId; + info->mCurTraceSourceType = aSourceEventType; +} + +void +GetCurTraceInfo(uint64_t* aOutSourceEventId, uint64_t* aOutParentTaskId, + SourceEventType* aOutSourceEventType) +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + *aOutSourceEventId = info->mCurTraceSourceId; + *aOutParentTaskId = info->mCurTaskId; + *aOutSourceEventType = info->mCurTraceSourceType; +} + +void +LogDispatch(uint64_t aTaskId, uint64_t aParentTaskId, uint64_t aSourceEventId, + SourceEventType aSourceEventType) +{ + LogDispatch(aTaskId, aParentTaskId, aSourceEventId, aSourceEventType, 0); +} + +void +LogDispatch(uint64_t aTaskId, uint64_t aParentTaskId, uint64_t aSourceEventId, + SourceEventType aSourceEventType, int aDelayTimeMs) +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + // aDelayTimeMs is the expected delay time in milliseconds, thus the dispatch + // time calculated of it might be slightly off in the real world. + uint64_t time = (aDelayTimeMs <= 0) ? GetTimestamp() : + GetTimestamp() + aDelayTimeMs; + + // Log format: + // [0 taskId dispatchTime sourceEventId sourceEventType parentTaskId] + nsCString* log = info->AppendLog(); + if (log) { + log->AppendPrintf("%d %lld %lld %lld %d %lld", + ACTION_DISPATCH, aTaskId, time, aSourceEventId, + aSourceEventType, aParentTaskId); + } +} + +void +LogBegin(uint64_t aTaskId, uint64_t aSourceEventId) +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + // Log format: + // [1 taskId beginTime processId threadId] + nsCString* log = info->AppendLog(); + if (log) { + log->AppendPrintf("%d %lld %lld %d %d", + ACTION_BEGIN, aTaskId, GetTimestamp(), getpid(), gettid()); + } +} + +void +LogEnd(uint64_t aTaskId, uint64_t aSourceEventId) +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + // Log format: + // [2 taskId endTime] + nsCString* log = info->AppendLog(); + if (log) { + log->AppendPrintf("%d %lld %lld", ACTION_END, aTaskId, GetTimestamp()); + } +} + +void +LogVirtualTablePtr(uint64_t aTaskId, uint64_t aSourceEventId, uintptr_t* aVptr) +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + // Log format: + // [4 taskId address] + nsCString* log = info->AppendLog(); + if (log) { + log->AppendPrintf("%d %lld %p", ACTION_GET_VTABLE, aTaskId, aVptr); + } +} + +AutoSourceEvent::AutoSourceEvent(SourceEventType aType) +{ + CreateSourceEvent(aType); +} + +AutoSourceEvent::~AutoSourceEvent() +{ + DestroySourceEvent(); +} + +void AddLabel(const char* aFormat, ...) +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + va_list args; + va_start(args, aFormat); + nsAutoCString buffer; + buffer.AppendPrintf(aFormat, args); + va_end(args); + + // Log format: + // [3 taskId "label"] + nsCString* log = info->AppendLog(); + if (log) { + log->AppendPrintf("%d %lld %lld \"%s\"", ACTION_ADD_LABEL, info->mCurTaskId, + GetTimestamp(), buffer.get()); + } +} + +// Functions used by GeckoProfiler. + +void +StartLogging() +{ + sStartTime = GetTimestamp(); + SetLogStarted(true); +} + +void +StopLogging() +{ + SetLogStarted(false); +} + +UniquePtr<TraceInfoLogsType> +GetLoggedData(TimeStamp aTimeStamp) +{ + auto result = MakeUnique<TraceInfoLogsType>(); + + // TODO: This is called from a signal handler. Use semaphore instead. + StaticMutexAutoLock lock(sMutex); + + for (uint32_t i = 0; i < sTraceInfos->Length(); ++i) { + (*sTraceInfos)[i]->MoveLogsInto(*result); + } + + return result; +} + +const PRTime +GetStartTime() +{ + return sStartTime; +} + +const char* +GetJSLabelPrefix() +{ + return sJSLabelPrefix; +} + +#undef ENSURE_TRUE_VOID +#undef ENSURE_TRUE + +} // namespace tasktracer +} // namespace mozilla diff --git a/tools/profiler/tasktracer/GeckoTaskTracer.h b/tools/profiler/tasktracer/GeckoTaskTracer.h new file mode 100644 index 000000000..9e36b3f0b --- /dev/null +++ b/tools/profiler/tasktracer/GeckoTaskTracer.h @@ -0,0 +1,92 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef GECKO_TASK_TRACER_H +#define GECKO_TASK_TRACER_H + +#include "mozilla/UniquePtr.h" +#include "nsCOMPtr.h" +#include "nsTArrayForwardDeclare.h" + +/** + * TaskTracer provides a way to trace the correlation between different tasks + * across threads and processes. Unlike sampling based profilers, TaskTracer can + * tell you where a task is dispatched from, what its original source was, how + * long it waited in the event queue, and how long it took to execute. + * + * Source Events are usually some kinds of I/O events we're interested in, such + * as touch events, timer events, network events, etc. When a source event is + * created, TaskTracer records the entire chain of Tasks and nsRunnables as they + * are dispatched to different threads and processes. It records latency, + * execution time, etc. for each Task and nsRunnable that chains back to the + * original source event. + */ + +class Task; +class nsIRunnable; +class nsCString; + +namespace mozilla { + +class TimeStamp; + +namespace tasktracer { + +enum { + FORKED_AFTER_NUWA = 1 << 0 +}; + +enum SourceEventType { + Unknown = 0, + Touch, + Mouse, + Key, + Bluetooth, + Unixsocket, + Wifi +}; + +class AutoSourceEvent +{ +public: + AutoSourceEvent(SourceEventType aType); + ~AutoSourceEvent(); +}; + +void InitTaskTracer(uint32_t aFlags = 0); +void ShutdownTaskTracer(); + +// Add a label to the currently running task, aFormat is the message to log, +// followed by corresponding parameters. +void AddLabel(const char* aFormat, ...); + +void StartLogging(); +void StopLogging(); +UniquePtr<nsTArray<nsCString>> GetLoggedData(TimeStamp aStartTime); + +// Returns the timestamp when Task Tracer is enabled in this process. +const PRTime GetStartTime(); + +/** + * Internal functions. + */ + +Task* CreateTracedTask(Task* aTask); + +already_AddRefed<nsIRunnable> +CreateTracedRunnable(already_AddRefed<nsIRunnable>&& aRunnable); + +// Free the TraceInfo allocated on a thread's TLS. Currently we are wrapping +// tasks running on nsThreads and base::thread, so FreeTraceInfo is called at +// where nsThread and base::thread release themselves. +void FreeTraceInfo(); + +const char* GetJSLabelPrefix(); + +} // namespace tasktracer +} // namespace mozilla. + +#endif diff --git a/tools/profiler/tasktracer/GeckoTaskTracerImpl.h b/tools/profiler/tasktracer/GeckoTaskTracerImpl.h new file mode 100644 index 000000000..5b748fb96 --- /dev/null +++ b/tools/profiler/tasktracer/GeckoTaskTracerImpl.h @@ -0,0 +1,102 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef GECKO_TASK_TRACER_IMPL_H +#define GECKO_TASK_TRACER_IMPL_H + +#include "GeckoTaskTracer.h" +#include "mozilla/Mutex.h" +#include "nsTArray.h" + +namespace mozilla { +namespace tasktracer { + +typedef nsTArray<nsCString> TraceInfoLogsType; + +struct TraceInfo +{ + TraceInfo(uint32_t aThreadId) + : mCurTraceSourceId(0) + , mCurTaskId(0) + , mSavedCurTraceSourceId(0) + , mSavedCurTaskId(0) + , mCurTraceSourceType(Unknown) + , mSavedCurTraceSourceType(Unknown) + , mThreadId(aThreadId) + , mLastUniqueTaskId(0) + , mObsolete(false) + , mLogsMutex("TraceInfoMutex") + { + MOZ_COUNT_CTOR(TraceInfo); + } + + ~TraceInfo() { MOZ_COUNT_DTOR(TraceInfo); } + + nsCString* AppendLog(); + void MoveLogsInto(TraceInfoLogsType& aResult); + + uint64_t mCurTraceSourceId; + uint64_t mCurTaskId; + uint64_t mSavedCurTraceSourceId; + uint64_t mSavedCurTaskId; + SourceEventType mCurTraceSourceType; + SourceEventType mSavedCurTraceSourceType; + uint32_t mThreadId; + uint32_t mLastUniqueTaskId; + mozilla::Atomic<bool> mObsolete; + + // This mutex protects the following log array because MoveLogsInto() might + // be called on another thread. + mozilla::Mutex mLogsMutex; + TraceInfoLogsType mLogs; +}; + +// Return the TraceInfo of current thread, allocate a new one if not exit. +TraceInfo* GetOrCreateTraceInfo(); + +uint64_t GenNewUniqueTaskId(); + +class AutoSaveCurTraceInfo +{ +public: + AutoSaveCurTraceInfo(); + ~AutoSaveCurTraceInfo(); +}; + +void SetCurTraceInfo(uint64_t aSourceEventId, uint64_t aParentTaskId, + SourceEventType aSourceEventType); + +void GetCurTraceInfo(uint64_t* aOutSourceEventId, uint64_t* aOutParentTaskId, + SourceEventType* aOutSourceEventType); + +/** + * Logging functions of different trace actions. + */ +enum ActionType { + ACTION_DISPATCH = 0, + ACTION_BEGIN, + ACTION_END, + ACTION_ADD_LABEL, + ACTION_GET_VTABLE +}; + +void LogDispatch(uint64_t aTaskId, uint64_t aParentTaskId, + uint64_t aSourceEventId, SourceEventType aSourceEventType); + +void LogDispatch(uint64_t aTaskId, uint64_t aParentTaskId, + uint64_t aSourceEventId, SourceEventType aSourceEventType, + int aDelayTimeMs); + +void LogBegin(uint64_t aTaskId, uint64_t aSourceEventId); + +void LogEnd(uint64_t aTaskId, uint64_t aSourceEventId); + +void LogVirtualTablePtr(uint64_t aTaskId, uint64_t aSourceEventId, uintptr_t* aVptr); + +} // namespace mozilla +} // namespace tasktracer + +#endif diff --git a/tools/profiler/tasktracer/SourceEventTypeMap.h b/tools/profiler/tasktracer/SourceEventTypeMap.h new file mode 100644 index 000000000..77dbc8330 --- /dev/null +++ b/tools/profiler/tasktracer/SourceEventTypeMap.h @@ -0,0 +1,11 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +SOURCE_EVENT_NAME(Unknown) +SOURCE_EVENT_NAME(Touch) +SOURCE_EVENT_NAME(Mouse) +SOURCE_EVENT_NAME(Key) +SOURCE_EVENT_NAME(Bluetooth) +SOURCE_EVENT_NAME(Unixsocket) +SOURCE_EVENT_NAME(Wifi) diff --git a/tools/profiler/tasktracer/TracedTaskCommon.cpp b/tools/profiler/tasktracer/TracedTaskCommon.cpp new file mode 100644 index 000000000..770eb202c --- /dev/null +++ b/tools/profiler/tasktracer/TracedTaskCommon.cpp @@ -0,0 +1,169 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "GeckoTaskTracerImpl.h" +#include "TracedTaskCommon.h" + +// NS_ENSURE_TRUE_VOID() without the warning on the debug build. +#define ENSURE_TRUE_VOID(x) \ + do { \ + if (MOZ_UNLIKELY(!(x))) { \ + return; \ + } \ + } while(0) + +namespace mozilla { +namespace tasktracer { + +TracedTaskCommon::TracedTaskCommon() + : mSourceEventType(SourceEventType::Unknown) + , mSourceEventId(0) + , mParentTaskId(0) + , mTaskId(0) + , mIsTraceInfoInit(false) +{ +} + +TracedTaskCommon::~TracedTaskCommon() +{ +} + +void +TracedTaskCommon::Init() +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + mTaskId = GenNewUniqueTaskId(); + mSourceEventId = info->mCurTraceSourceId; + mSourceEventType = info->mCurTraceSourceType; + mParentTaskId = info->mCurTaskId; + mIsTraceInfoInit = true; +} + +void +TracedTaskCommon::DispatchTask(int aDelayTimeMs) +{ + LogDispatch(mTaskId, mParentTaskId, mSourceEventId, mSourceEventType, + aDelayTimeMs); +} + +void +TracedTaskCommon::GetTLSTraceInfo() +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + mSourceEventType = info->mCurTraceSourceType; + mSourceEventId = info->mCurTraceSourceId; + mTaskId = info->mCurTaskId; + mIsTraceInfoInit = true; +} + +void +TracedTaskCommon::SetTLSTraceInfo() +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + if (mIsTraceInfoInit) { + info->mCurTraceSourceId = mSourceEventId; + info->mCurTraceSourceType = mSourceEventType; + info->mCurTaskId = mTaskId; + } +} + +void +TracedTaskCommon::ClearTLSTraceInfo() +{ + TraceInfo* info = GetOrCreateTraceInfo(); + ENSURE_TRUE_VOID(info); + + info->mCurTraceSourceId = 0; + info->mCurTraceSourceType = SourceEventType::Unknown; + info->mCurTaskId = 0; +} + +/** + * Implementation of class TracedRunnable. + */ +TracedRunnable::TracedRunnable(already_AddRefed<nsIRunnable>&& aOriginalObj) + : TracedTaskCommon() + , mOriginalObj(Move(aOriginalObj)) +{ + Init(); + LogVirtualTablePtr(mTaskId, mSourceEventId, reinterpret_cast<uintptr_t*>(mOriginalObj.get())); +} + +TracedRunnable::~TracedRunnable() +{ +} + +NS_IMETHODIMP +TracedRunnable::Run() +{ + SetTLSTraceInfo(); + LogBegin(mTaskId, mSourceEventId); + nsresult rv = mOriginalObj->Run(); + LogEnd(mTaskId, mSourceEventId); + ClearTLSTraceInfo(); + + return rv; +} + +/** + * Implementation of class TracedTask. + */ +TracedTask::TracedTask(Task* aOriginalObj) + : TracedTaskCommon() + , mOriginalObj(aOriginalObj) +{ + Init(); + LogVirtualTablePtr(mTaskId, mSourceEventId, reinterpret_cast<uintptr_t*>(aOriginalObj)); +} + +TracedTask::~TracedTask() +{ + if (mOriginalObj) { + delete mOriginalObj; + mOriginalObj = nullptr; + } +} + +void +TracedTask::Run() +{ + SetTLSTraceInfo(); + LogBegin(mTaskId, mSourceEventId); + mOriginalObj->Run(); + LogEnd(mTaskId, mSourceEventId); + ClearTLSTraceInfo(); +} + +/** + * CreateTracedRunnable() returns a TracedRunnable wrapping the original + * nsIRunnable object, aRunnable. + */ +already_AddRefed<nsIRunnable> +CreateTracedRunnable(already_AddRefed<nsIRunnable>&& aRunnable) +{ + nsCOMPtr<nsIRunnable> runnable = new TracedRunnable(Move(aRunnable)); + return runnable.forget(); +} + +/** + * CreateTracedTask() returns a TracedTask wrapping the original Task object, + * aTask. + */ +Task* +CreateTracedTask(Task* aTask) +{ + Task* task = new TracedTask(aTask); + return task; +} + +} // namespace tasktracer +} // namespace mozilla diff --git a/tools/profiler/tasktracer/TracedTaskCommon.h b/tools/profiler/tasktracer/TracedTaskCommon.h new file mode 100644 index 000000000..3594b8e9e --- /dev/null +++ b/tools/profiler/tasktracer/TracedTaskCommon.h @@ -0,0 +1,73 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef TRACED_TASK_COMMON_H +#define TRACED_TASK_COMMON_H + +#include "base/task.h" +#include "GeckoTaskTracer.h" +#include "nsCOMPtr.h" +#include "nsThreadUtils.h" + +namespace mozilla { +namespace tasktracer { + +class TracedTaskCommon +{ +public: + TracedTaskCommon(); + virtual ~TracedTaskCommon(); + + void DispatchTask(int aDelayTimeMs = 0); + + void SetTLSTraceInfo(); + void GetTLSTraceInfo(); + void ClearTLSTraceInfo(); + +protected: + void Init(); + + // TraceInfo of TLS will be set by the following parameters, including source + // event type, source event ID, parent task ID, and task ID of this traced + // task/runnable. + SourceEventType mSourceEventType; + uint64_t mSourceEventId; + uint64_t mParentTaskId; + uint64_t mTaskId; + bool mIsTraceInfoInit; +}; + +class TracedRunnable : public TracedTaskCommon + , public nsRunnable +{ +public: + NS_DECL_NSIRUNNABLE + + TracedRunnable(already_AddRefed<nsIRunnable>&& aOriginalObj); + +private: + virtual ~TracedRunnable(); + + nsCOMPtr<nsIRunnable> mOriginalObj; +}; + +class TracedTask : public TracedTaskCommon + , public Task +{ +public: + TracedTask(Task* aOriginalObj); + ~TracedTask(); + + virtual void Run(); + +private: + Task* mOriginalObj; +}; + +} // namespace tasktracer +} // namespace mozilla + +#endif diff --git a/tools/profiler/tests/head_profiler.js b/tools/profiler/tests/head_profiler.js new file mode 100644 index 000000000..a3821f51f --- /dev/null +++ b/tools/profiler/tests/head_profiler.js @@ -0,0 +1,31 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cu = Components.utils; + +function getInflatedStackLocations(thread, sample) { + let stackTable = thread.stackTable; + let frameTable = thread.frameTable; + let stringTable = thread.stringTable; + let SAMPLE_STACK_SLOT = thread.samples.schema.stack; + let STACK_PREFIX_SLOT = stackTable.schema.prefix; + let STACK_FRAME_SLOT = stackTable.schema.frame; + let FRAME_LOCATION_SLOT = frameTable.schema.location; + + // Build the stack from the raw data and accumulate the locations in + // an array. + let stackIndex = sample[SAMPLE_STACK_SLOT]; + let locations = []; + while (stackIndex !== null) { + let stackEntry = stackTable.data[stackIndex]; + let frame = frameTable.data[stackEntry[STACK_FRAME_SLOT]]; + locations.push(stringTable[frame[FRAME_LOCATION_SLOT]]); + stackIndex = stackEntry[STACK_PREFIX_SLOT]; + } + + // The profiler tree is inverted, so reverse the array. + return locations.reverse(); +} diff --git a/tools/profiler/tests/test_asm.js b/tools/profiler/tests/test_asm.js new file mode 100644 index 000000000..4d273a559 --- /dev/null +++ b/tools/profiler/tests/test_asm.js @@ -0,0 +1,79 @@ +// Check that asm.js code shows up on the stack. +function run_test() { + let p = Cc["@mozilla.org/tools/profiler;1"]; + + // Just skip the test if the profiler component isn't present. + if (!p) + return; + p = p.getService(Ci.nsIProfiler); + if (!p) + return; + + // This test assumes that it's starting on an empty SPS stack. + // (Note that the other profiler tests also assume the profiler + // isn't already started.) + do_check_true(!p.IsActive()); + + let jsFuns = Cu.getJSTestingFunctions(); + if (!jsFuns.isAsmJSCompilationAvailable()) + return; + + const ms = 10; + p.StartProfiler(10000, ms, ["js"], 1); + + let stack = null; + function ffi_function(){ + var delayMS = 5; + while (1) { + let then = Date.now(); + do {} while (Date.now() - then < delayMS); + + var thread0 = p.getProfileData().threads[0]; + + if (delayMS > 30000) + return; + + delayMS *= 2; + + if (thread0.samples.data.length == 0) + continue; + + var lastSample = thread0.samples.data[thread0.samples.data.length - 1]; + stack = String(getInflatedStackLocations(thread0, lastSample)); + if (stack.indexOf("trampoline") !== -1) + return; + } + } + + function asmjs_module(global, ffis) { + "use asm"; + var ffi = ffis.ffi; + function asmjs_function() { + ffi(); + } + return asmjs_function; + } + + do_check_true(jsFuns.isAsmJSModule(asmjs_module)); + + var asmjs_function = asmjs_module(null, {ffi:ffi_function}); + do_check_true(jsFuns.isAsmJSFunction(asmjs_function)); + + asmjs_function(); + + do_check_neq(stack, null); + + var i1 = stack.indexOf("entry trampoline"); + do_check_true(i1 !== -1); + var i2 = stack.indexOf("asmjs_function"); + do_check_true(i2 !== -1); + var i3 = stack.indexOf("FFI trampoline"); + do_check_true(i3 !== -1); + var i4 = stack.indexOf("ffi_function"); + do_check_true(i4 !== -1); + do_check_true(i1 < i2); + do_check_true(i2 < i3); + do_check_true(i3 < i4); + + p.StopProfiler(); +} diff --git a/tools/profiler/tests/test_enterjit_osr.js b/tools/profiler/tests/test_enterjit_osr.js new file mode 100644 index 000000000..a4bca590f --- /dev/null +++ b/tools/profiler/tests/test_enterjit_osr.js @@ -0,0 +1,59 @@ +// Check that the EnterJIT frame, added by the JIT trampoline and +// usable by a native unwinder to resume unwinding after encountering +// JIT code, is pushed as expected. +function run_test() { + let p = Cc["@mozilla.org/tools/profiler;1"]; + // Just skip the test if the profiler component isn't present. + if (!p) + return; + p = p.getService(Ci.nsIProfiler); + if (!p) + return; + + // This test assumes that it's starting on an empty SPS stack. + // (Note that the other profiler tests also assume the profiler + // isn't already started.) + do_check_true(!p.IsActive()); + + const ms = 5; + p.StartProfiler(100, ms, ["js"], 1); + + function arbitrary_name(){ + // A frame for |arbitrary_name| has been pushed. Do a sequence of + // increasingly long spins until we get a sample. + var delayMS = 5; + while (1) { + do_print("loop: ms = " + delayMS); + let then = Date.now(); + do { + let n = 10000; + while (--n); // OSR happens here + // Spin in the hope of getting a sample. + } while (Date.now() - then < delayMS); + let pr = p.getProfileData().threads[0]; + if (pr.samples.data.length > 0 || delayMS > 30000) + return pr; + delayMS *= 2; + } + }; + + var profile = arbitrary_name(); + + do_check_neq(profile.samples.data.length, 0); + var lastSample = profile.samples.data[profile.samples.data.length - 1]; + var stack = getInflatedStackLocations(profile, lastSample); + do_print(stack); + + // All we can really check here is ensure that there is exactly + // one arbitrary_name frame in the list. + var gotName = false; + for (var i = 0; i < stack.length; i++) { + if (stack[i].match(/arbitrary_name/)) { + do_check_eq(gotName, false); + gotName = true; + } + } + do_check_eq(gotName, true); + + p.StopProfiler(); +} diff --git a/tools/profiler/tests/test_enterjit_osr_disabling.js b/tools/profiler/tests/test_enterjit_osr_disabling.js new file mode 100644 index 000000000..dbf74c93a --- /dev/null +++ b/tools/profiler/tests/test_enterjit_osr_disabling.js @@ -0,0 +1,21 @@ +function run_test() { + let p = Cc["@mozilla.org/tools/profiler;1"]; + // Just skip the test if the profiler component isn't present. + if (!p) + return; + p = p.getService(Ci.nsIProfiler); + if (!p) + return; + + do_check_true(!p.IsActive()); + + p.StartProfiler(100, 10, ["js"], 1); + // The function is entered with the profiler enabled + (function (){ + p.StopProfiler(); + let n = 10000; + while (--n); // OSR happens here with the profiler disabled. + // An assertion will fail when this function returns, if the + // SPS stack was misbalanced. + })(); +} diff --git a/tools/profiler/tests/test_enterjit_osr_enabling.js b/tools/profiler/tests/test_enterjit_osr_enabling.js new file mode 100644 index 000000000..ae696057b --- /dev/null +++ b/tools/profiler/tests/test_enterjit_osr_enabling.js @@ -0,0 +1,21 @@ +function run_test() { + let p = Cc["@mozilla.org/tools/profiler;1"]; + // Just skip the test if the profiler component isn't present. + if (!p) + return; + p = p.getService(Ci.nsIProfiler); + if (!p) + return; + + do_check_true(!p.IsActive()); + + // The function is entered with the profiler disabled. + (function (){ + p.StartProfiler(100, 10, ["js"], 1); + let n = 10000; + while (--n); // OSR happens here with the profiler enabled. + // An assertion will fail when this function returns, if the + // SPS stack was misbalanced. + })(); + p.StopProfiler(); +} diff --git a/tools/profiler/tests/test_get_features.js b/tools/profiler/tests/test_get_features.js new file mode 100644 index 000000000..4fbd5891c --- /dev/null +++ b/tools/profiler/tests/test_get_features.js @@ -0,0 +1,18 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function run_test() { + // If we can't get the profiler component then assume gecko was + // built without it and pass all the tests + var profilerCc = Cc["@mozilla.org/tools/profiler;1"]; + if (!profilerCc) + return; + + var profiler = Cc["@mozilla.org/tools/profiler;1"].getService(Ci.nsIProfiler); + if (!profiler) + return; + + var profilerFeatures = profiler.GetFeatures([]); + do_check_true(profilerFeatures != null); +} diff --git a/tools/profiler/tests/test_pause.js b/tools/profiler/tests/test_pause.js new file mode 100644 index 000000000..fedff70c4 --- /dev/null +++ b/tools/profiler/tests/test_pause.js @@ -0,0 +1,35 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function run_test() { + // If we can't get the profiler component then assume gecko was + // built without it and pass all the tests + var profilerCc = Cc["@mozilla.org/tools/profiler;1"]; + if (!profilerCc) + return; + + var profiler = profilerCc.getService(Ci.nsIProfiler); + if (!profiler) + return; + + do_check_true(!profiler.IsActive()); + do_check_true(!profiler.IsPaused()); + + profiler.StartProfiler(1000, 10, [], 0); + + do_check_true(profiler.IsActive()); + + profiler.PauseSampling(); + + do_check_true(profiler.IsPaused()); + + profiler.ResumeSampling(); + + do_check_true(!profiler.IsPaused()); + + profiler.StopProfiler(); + do_check_true(!profiler.IsActive()); + do_check_true(!profiler.IsPaused()); + do_test_finished(); +} diff --git a/tools/profiler/tests/test_run.js b/tools/profiler/tests/test_run.js new file mode 100644 index 000000000..fef03a07d --- /dev/null +++ b/tools/profiler/tests/test_run.js @@ -0,0 +1,44 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function run_test() { + // If we can't get the profiler component then assume gecko was + // built without it and pass all the tests + var profilerCc = Cc["@mozilla.org/tools/profiler;1"]; + if (!profilerCc) + return; + + var profiler = Cc["@mozilla.org/tools/profiler;1"].getService(Ci.nsIProfiler); + if (!profiler) + return; + + do_check_true(!profiler.IsActive()); + + profiler.StartProfiler(1000, 10, [], 0); + + do_check_true(profiler.IsActive()); + + do_test_pending(); + + do_timeout(1000, function wait() { + // Check text profile format + var profileStr = profiler.GetProfile(); + do_check_true(profileStr.length > 10); + + // check json profile format + var profileObj = profiler.getProfileData(); + do_check_neq(profileObj, null); + do_check_neq(profileObj.threads, null); + do_check_true(profileObj.threads.length >= 1); + do_check_neq(profileObj.threads[0].samples, null); + // NOTE: The number of samples will be empty since we + // don't have any labels in the xpcshell code + + profiler.StopProfiler(); + do_check_true(!profiler.IsActive()); + do_test_finished(); + }); + + +} diff --git a/tools/profiler/tests/test_shared_library.js b/tools/profiler/tests/test_shared_library.js new file mode 100644 index 000000000..2bdbc0109 --- /dev/null +++ b/tools/profiler/tests/test_shared_library.js @@ -0,0 +1,23 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function run_test() { + // If we can't get the profiler component then assume gecko was + // built without it and pass all the tests + var profilerCc = Cc["@mozilla.org/tools/profiler;1"]; + if (!profilerCc) + return; + + var profiler = Cc["@mozilla.org/tools/profiler;1"].getService(Ci.nsIProfiler); + if (!profiler) + return; + + var sharedStr = profiler.getSharedLibraryInformation(); + sharedStr = sharedStr.toLowerCase(); + + // Let's not hardcode anything too specific + // just some sanity checks. + do_check_neq(sharedStr, null); + do_check_neq(sharedStr, ""); +} diff --git a/tools/profiler/tests/test_start.js b/tools/profiler/tests/test_start.js new file mode 100644 index 000000000..b04b130ff --- /dev/null +++ b/tools/profiler/tests/test_start.js @@ -0,0 +1,25 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +function run_test() { + // If we can't get the profiler component then assume gecko was + // built without it and pass all the tests + var profilerCc = Cc["@mozilla.org/tools/profiler;1"]; + if (!profilerCc) + return; + + var profiler = Cc["@mozilla.org/tools/profiler;1"].getService(Ci.nsIProfiler); + if (!profiler) + return; + + do_check_true(!profiler.IsActive()); + + profiler.StartProfiler(10, 100, [], 0); + + do_check_true(profiler.IsActive()); + + profiler.StopProfiler(); + + do_check_true(!profiler.IsActive()); +} diff --git a/tools/profiler/tests/xpcshell.ini b/tools/profiler/tests/xpcshell.ini new file mode 100644 index 000000000..997a7c142 --- /dev/null +++ b/tools/profiler/tests/xpcshell.ini @@ -0,0 +1,18 @@ +[DEFAULT] +head = head_profiler.js +tail = +skip-if = toolkit == 'android' + +[test_start.js] +skip-if = true +[test_get_features.js] +[test_shared_library.js] +[test_run.js] +skip-if = true +[test_pause.js] +[test_enterjit_osr.js] +[test_enterjit_osr_disabling.js] +skip-if = !debug +[test_enterjit_osr_enabling.js] +skip-if = !debug +[test_asm.js]
\ No newline at end of file diff --git a/tools/rb/README b/tools/rb/README new file mode 100644 index 000000000..c9b5c282c --- /dev/null +++ b/tools/rb/README @@ -0,0 +1,7 @@ +This is the Refcount Balancer. See +https://developer.mozilla.org/en-US/docs/Mozilla/Performance/Refcount_tracing_and_balancing +for documentation. + +Previous CVS history for the perl scripts is available at: +http://www.mozilla.org/webtools/bonsai/cvslog.cgi?file=mozilla-org/html/performance/find-leakers.pl&rev=&root=/cvsroot/ +http://www.mozilla.org/webtools/bonsai/cvslog.cgi?file=mozilla-org/html/performance/make-tree.pl&rev=&root=/cvsroot/ diff --git a/tools/rb/filter-log.pl b/tools/rb/filter-log.pl new file mode 100755 index 000000000..4a1f66741 --- /dev/null +++ b/tools/rb/filter-log.pl @@ -0,0 +1,44 @@ +#!/usr/bin/perl -w +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Filter a refcount log to show only the entries for a single object. +# Useful when manually examining refcount logs containing multiple +# objects. + +use 5.004; +use strict; +use Getopt::Long; + +GetOptions("object=s"); + +$::opt_object || + die qq{ +usage: filter-log-for.pl < logfile + --object <obj> The address of the object to examine (required) +}; + +warn "object $::opt_object\n"; + +LINE: while (<>) { + next LINE if (! /^</); + my $line = $_; + my @fields = split(/ /, $_); + + my $class = shift(@fields); + my $obj = shift(@fields); + next LINE unless ($obj eq $::opt_object); + my $sno = shift(@fields); + my $op = shift(@fields); + my $cnt = shift(@fields); + + print $line; + + # The lines in the stack trace + CALLSITE: while (<>) { + print; + last CALLSITE if (/^$/); + } +} diff --git a/tools/rb/find-comptr-leakers.pl b/tools/rb/find-comptr-leakers.pl new file mode 100755 index 000000000..925119935 --- /dev/null +++ b/tools/rb/find-comptr-leakers.pl @@ -0,0 +1,114 @@ +#!/usr/bin/perl -w +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Script loosely based on Chris Waterson's find-leakers.pl and make-tree.pl + +use 5.004; +use strict; +use Getopt::Long; + +# GetOption will create $opt_object, so ignore the +# warning that gets spit out about those vbls. +GetOptions("object=s", "list", "help"); + +# use $::opt_help twice to eliminate warning... +($::opt_help) && ($::opt_help) && die qq{ +usage: find-comptr-leakers.pl < logfile + --object <obj> Examine only object <obj> + --list Only list leaked objects + --help This message :-) +}; + +if ($::opt_object) { + warn "Examining only object $::opt_object (THIS IS BROKEN)\n"; +} else { + warn "Examining all objects\n"; +} + +my %allocs = ( ); +my %counter; +my $id = 0; + +my $accumulating = 0; +my $savedata = 0; +my $class; +my $obj; +my $sno; +my $op; +my $cnt; +my $ptr; +my $strace; + +sub save_data { + # save the data + if ($op eq 'nsCOMPtrAddRef') { + push @{ $allocs{$sno}->{$ptr} }, [ +1, $strace ]; + } + elsif ($op eq 'nsCOMPtrRelease') { + push @{ $allocs{$sno}->{$ptr} }, [ -1, $strace ]; + my $sum = 0; + my @ptrallocs = @{ $allocs{$sno}->{$ptr} }; + foreach my $alloc (@ptrallocs) { + $sum += @$alloc[0]; + } + if ( $sum == 0 ) { + delete($allocs{$sno}{$ptr}); + } + } +} + +LINE: while (<>) { + if (/^</) { + chop; # avoid \n in $ptr + my @fields = split(/ /, $_); + + ($class, $obj, $sno, $op, $cnt, $ptr) = @fields; + + $strace = ""; + + if ($::opt_list) { + save_data(); + } elsif (!($::opt_object) || ($::opt_object eq $obj)) { + $accumulating = 1; + } + } elsif ( $accumulating == 1 ) { + if ( /^$/ ) { + # if line is empty + $accumulating = 0; + save_data(); + } else { + $strace = $strace . $_; + } + } +} +if ( $accumulating == 1) { + save_data(); +} + +foreach my $serial (keys(%allocs)) { + foreach my $comptr (keys( %{$allocs{$serial}} )) { + my $sum = 0; + my @ptrallocs = @{ $allocs{$serial}->{$comptr} }; + foreach my $alloc (@ptrallocs) { + $sum += @$alloc[0]; + } + print "Object ", $serial, " held by ", $comptr, " is ", $sum, " out of balance.\n"; + unless ($::opt_list) { + print "\n"; + foreach my $alloc (@ptrallocs) { + if (@$alloc[0] == +1) { + print "Put into nsCOMPtr at:\n"; + } elsif (@$alloc[0] == -1) { + print "Released from nsCOMPtr at:\n"; + } + print @$alloc[1]; # the stack trace + print "\n"; + } + print "\n\n"; + } + } +} + diff --git a/tools/rb/find_leakers.py b/tools/rb/find_leakers.py new file mode 100755 index 000000000..4405d7a17 --- /dev/null +++ b/tools/rb/find_leakers.py @@ -0,0 +1,100 @@ +#!/usr/bin/python +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script processes a `refcount' log, and finds out if any object leaked. +# It simply goes through the log, finds `AddRef' or `Ctor' lines, and then +# sees if they `Release' or `Dtor'. If not, it reports them as leaks. +# Please see README file in the same directory. + + +import sys + +def print_output(allocation, obj_to_class): + '''Formats and prints output.''' + items = [] + for obj, count, in allocation.iteritems(): + # Adding items to a list, so we can sort them. + items.append((obj, count)) + # Sorting by count. + items.sort(key=lambda item: item[1]) + + for obj, count, in items: + print "{obj} ({count}) @ {class_name}".format(obj=obj, + count=count, + class_name=obj_to_class[obj]) + +def process_log(log_lines): + '''Process through the log lines, and print out the result. + + @param log_lines: List of strings. + ''' + allocation = {} + class_count = {} + obj_to_class = {} + + for log_line in log_lines: + if not log_line.startswith('<'): + continue + + (class_name, + obj, + ignore, + operation, + count,) = log_line.strip('\r\n').split(' ')[:5] + + # for AddRef/Release `count' is the refcount, + # for Ctor/Dtor it's the size. + + if ((operation == 'AddRef' and count == '1') or + operation == 'Ctor'): + # Examples: + # <nsStringBuffer> 0x01AFD3B8 1 AddRef 1 + # <PStreamNotifyParent> 0x08880BD0 8 Ctor (20) + class_count[class_name] = class_count.setdefault(class_name, 0) + 1 + allocation[obj] = class_count[class_name] + obj_to_class[obj] = class_name + + elif ((operation == 'Release' and count == '0') or + operation == 'Dtor'): + # Examples: + # <nsStringBuffer> 0x01AFD3B8 1 Release 0 + # <PStreamNotifyParent> 0x08880BD0 8 Dtor (20) + if obj not in allocation: + print "An object was released that wasn't allocated!", + print obj, "@", class_name + else: + allocation.pop(obj) + obj_to_class.pop(obj) + + # Printing out the result. + print_output(allocation, obj_to_class) + + +def print_usage(): + print + print "Usage: find-leakers.py [log-file]" + print + print "If `log-file' provided, it will read that as the input log." + print "Else, it will read the stdin as the input log." + print + +def main(): + '''Main method of the script.''' + if len(sys.argv) == 1: + # Reading log from stdin. + process_log(sys.stdin.readlines()) + elif len(sys.argv) == 2: + # Reading log from file. + with open(sys.argv[1], 'r') as log_file: + log_lines = log_file.readlines() + process_log(log_lines) + else: + print 'ERROR: Invalid number of arguments' + print_usage() + +if __name__ == '__main__': + main() + diff --git a/tools/rb/fix_linux_stack.py b/tools/rb/fix_linux_stack.py new file mode 100755 index 000000000..bdc8a15dc --- /dev/null +++ b/tools/rb/fix_linux_stack.py @@ -0,0 +1,317 @@ +#!/usr/bin/python +# vim:sw=4:ts=4:et: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script uses addr2line (part of binutils) to post-process the entries +# produced by NS_FormatCodeAddress(), which on Linux often lack a function +# name, a file name and a line number. + +import subprocess +import sys +import re +import os +import pty +import termios +from StringIO import StringIO + +class unbufferedLineConverter: + """ + Wrap a child process that responds to each line of input with one line of + output. Uses pty to trick the child into providing unbuffered output. + """ + def __init__(self, command, args = []): + pid, fd = pty.fork() + if pid == 0: + # We're the child. Transfer control to command. + os.execvp(command, [command] + args) + else: + # Disable echoing. + attr = termios.tcgetattr(fd) + attr[3] = attr[3] & ~termios.ECHO + termios.tcsetattr(fd, termios.TCSANOW, attr) + # Set up a file()-like interface to the child process + self.r = os.fdopen(fd, "r", 1) + self.w = os.fdopen(os.dup(fd), "w", 1) + def convert(self, line): + self.w.write(line + "\n") + return (self.r.readline().rstrip("\r\n"), self.r.readline().rstrip("\r\n")) + @staticmethod + def test(): + assert unbufferedLineConverter("rev").convert("123") == "321" + assert unbufferedLineConverter("cut", ["-c3"]).convert("abcde") == "c" + print "Pass" + +objdump_section_re = re.compile("^ [0-9a-f]* ([0-9a-f ]{8}) ([0-9a-f ]{8}) ([0-9a-f ]{8}) ([0-9a-f ]{8}).*") +def elf_section(file, section): + """ + Return the requested ELF section of the file as a str, representing + a sequence of bytes. + """ + # We can read the .gnu_debuglink section using either of: + # objdump -s --section=.gnu_debuglink $file + # readelf -x .gnu_debuglink $file + # Since readelf prints things backwards on little-endian platforms + # for some versions only (backwards on Fedora Core 6, forwards on + # Fedora 7), use objdump. + objdump = subprocess.Popen(['objdump', '-s', '--section=' + section, file], + stdout=subprocess.PIPE, + # redirect stderr so errors don't get printed + stderr=subprocess.PIPE) + (objdump_stdout, objdump_stderr) = objdump.communicate() + if objdump.returncode != 0: + return None + result = "" + # Turn hexadecimal dump into the bytes it represents + for line in StringIO(objdump_stdout).readlines(): + m = objdump_section_re.match(line) + if m: + for gnum in [0, 1, 2, 3]: + word = m.groups()[gnum] + if word != " ": + for idx in [0, 2, 4, 6]: + result += chr(int(word[idx:idx+2], 16)) + return result + +# FIXME: Hard-coded to gdb defaults (works on Fedora and Ubuntu). +global_debug_dir = '/usr/lib/debug'; + +endian_re = re.compile("\s*Data:\s+.*(little|big) endian.*$") + +# Table of 256 values, per documentation of .gnu_debuglink sections. +gnu_debuglink_crc32_table = [ + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, + 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, + 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, + 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, + 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, + 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, + 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, + 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, + 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, + 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, + 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, + 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, + 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, + 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, + 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, + 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, + 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, + 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, + 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, + 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, + 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, + 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, + 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, + 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, + 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, + 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, + 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, + 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, + 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, + 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, + 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, + 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, + 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, + 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, + 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, + 0x2d02ef8d +] + +def gnu_debuglink_crc32(stream): + # Note that python treats bitwise operators as though integers have + # an infinite number of bits (and thus such that negative integers + # 1-pad out to infinity). + crc = 0xffffffff + while True: + # Choose to read in 4096 byte chunks. + bytes = stream.read(4096) + if len(bytes) == 0: + break + for byte in bytes: + crc = gnu_debuglink_crc32_table[(crc ^ ord(byte)) & 0xff] ^ (crc >> 8) + return ~crc & 0xffffffff + +def separate_debug_file_for(file): + """ + Finds a separated file with the debug sections for a binary. Such + files are commonly installed by debug packages on linux distros. + Rules for finding them are documented in: + https://sourceware.org/gdb/current/onlinedocs/gdb/Separate-Debug-Files.html + """ + def have_debug_file(debugfile): + return os.path.isfile(debugfile) + + endian = None + readelf = subprocess.Popen(['readelf', '-h', file], + stdout=subprocess.PIPE) + for line in readelf.stdout.readlines(): + m = endian_re.match(line) + if m: + endian = m.groups()[0] + break + readelf.terminate() + if endian is None: + sys.stderr.write("Could not determine endianness of " + file + "\n") + return None + + def word32(s): + if type(s) != str or len(s) != 4: + raise StandardError("expected 4 byte string input") + s = list(s) + if endian == "big": + s.reverse() + return sum(map(lambda idx: ord(s[idx]) * (256 ** idx), range(0, 4))) + + buildid = elf_section(file, ".note.gnu.build-id"); + if buildid is not None: + # The build ID is an ELF note section, so it begins with a + # name size (4), a description size (size of contents), a + # type (3), and the name "GNU\0". + note_header = buildid[0:16] + buildid = buildid[16:] + if word32(note_header[0:4]) != 4 or \ + word32(note_header[4:8]) != len(buildid) or \ + word32(note_header[8:12]) != 3 or \ + note_header[12:16] != "GNU\0": + sys.stderr.write("malformed .note.gnu.build_id in " + file + "\n") + else: + buildid = "".join(map(lambda ch: "%02X" % ord(ch), buildid)).lower() + f = os.path.join(global_debug_dir, ".build-id", buildid[0:2], buildid[2:] + ".debug") + if have_debug_file(f): + return f + + debuglink = elf_section(file, ".gnu_debuglink"); + if debuglink is not None: + # The debuglink section contains a string, ending with a + # null-terminator and then 0 to three bytes of padding to fill the + # current 32-bit unit. (This padding is usually null bytes, but + # I've seen null-null-H, on Ubuntu x86_64.) This is followed by + # a 4-byte CRC. + debuglink_name = debuglink[:-4] + null_idx = debuglink_name.find("\0") + if null_idx == -1 or null_idx + 4 < len(debuglink_name): + sys.stderr.write("Malformed .gnu_debuglink in " + file + "\n") + return None + debuglink_name = debuglink_name[0:null_idx] + + debuglink_crc = word32(debuglink[-4:]) + + dirname = os.path.dirname(file) + possible_files = [ + os.path.join(dirname, debuglink_name), + os.path.join(dirname, ".debug", debuglink_name), + os.path.join(global_debug_dir, dirname.lstrip("/"), debuglink_name) + ] + for f in possible_files: + if have_debug_file(f): + fio = open(f, mode="r") + file_crc = gnu_debuglink_crc32(fio) + fio.close() + if file_crc == debuglink_crc: + return f + return None + +elf_type_re = re.compile("^\s*Type:\s+(\S+)") +elf_text_section_re = re.compile("^\s*\[\s*\d+\]\s+\.text\s+\w+\s+(\w+)\s+(\w+)\s+") + +def address_adjustment_for(file): + """ + Return the address adjustment to use for a file. + + addr2line wants offsets relative to the base address for shared + libraries, but it wants addresses including the base address offset + for executables. This returns the appropriate address adjustment to + add to an offset within file. See bug 230336. + """ + readelf = subprocess.Popen(['readelf', '-h', file], + stdout=subprocess.PIPE) + elftype = None + for line in readelf.stdout.readlines(): + m = elf_type_re.match(line) + if m: + elftype = m.groups()[0] + break + readelf.terminate() + + if elftype != "EXEC": + # If we're not dealing with an executable, return 0. + return 0 + + adjustment = 0 + readelf = subprocess.Popen(['readelf', '-S', file], + stdout=subprocess.PIPE) + for line in readelf.stdout.readlines(): + m = elf_text_section_re.match(line) + if m: + # Subtract the .text section's offset within the + # file from its base address. + adjustment = int(m.groups()[0], 16) - int(m.groups()[1], 16); + break + readelf.terminate() + return adjustment + +addr2lines = {} +def addressToSymbol(file, address): + converter = None + address_adjustment = None + cache = None + if not file in addr2lines: + debug_file = separate_debug_file_for(file) or file + converter = unbufferedLineConverter('/usr/bin/addr2line', ['-C', '-f', '-e', debug_file]) + address_adjustment = address_adjustment_for(file) + cache = {} + addr2lines[file] = (converter, address_adjustment, cache) + else: + (converter, address_adjustment, cache) = addr2lines[file] + if address in cache: + return cache[address] + result = converter.convert(hex(int(address, 16) + address_adjustment)) + cache[address] = result + return result + +# Matches lines produced by NS_FormatCodeAddress(). +line_re = re.compile("^(.*#\d+: )(.+)\[(.+) \+(0x[0-9A-Fa-f]+)\](.*)$") + +def fixSymbols(line): + result = line_re.match(line) + if result is not None: + (before, fn, file, address, after) = result.groups() + + if os.path.exists(file) and os.path.isfile(file): + (name, fileline) = addressToSymbol(file, address) + + # If addr2line gave us something useless, keep what we had before. + if name == "??": + name = fn + if fileline == "??:0" or fileline == "??:?": + fileline = file + + nl = '\n' if line[-1] == '\n' else '' + return "%s%s (%s)%s%s" % (before, name, fileline, after, nl) + else: + sys.stderr.write("Warning: File \"" + file + "\" does not exist.\n") + return line + else: + return line + +if __name__ == "__main__": + for line in sys.stdin: + sys.stdout.write(fixSymbols(line)) diff --git a/tools/rb/fix_macosx_stack.py b/tools/rb/fix_macosx_stack.py new file mode 100755 index 000000000..7d076d9b6 --- /dev/null +++ b/tools/rb/fix_macosx_stack.py @@ -0,0 +1,133 @@ +#!/usr/bin/python +# vim:sw=4:ts=4:et: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script uses |atos| to post-process the entries produced by +# NS_FormatCodeAddress(), which on Mac often lack a file name and a line +# number. + +import subprocess +import sys +import re +import os +import pty +import termios + +class unbufferedLineConverter: + """ + Wrap a child process that responds to each line of input with one line of + output. Uses pty to trick the child into providing unbuffered output. + """ + def __init__(self, command, args = []): + pid, fd = pty.fork() + if pid == 0: + # We're the child. Transfer control to command. + os.execvp(command, [command] + args) + else: + # Disable echoing. + attr = termios.tcgetattr(fd) + attr[3] = attr[3] & ~termios.ECHO + termios.tcsetattr(fd, termios.TCSANOW, attr) + # Set up a file()-like interface to the child process + self.r = os.fdopen(fd, "r", 1) + self.w = os.fdopen(os.dup(fd), "w", 1) + def convert(self, line): + self.w.write(line + "\n") + return self.r.readline().rstrip("\r\n") + @staticmethod + def test(): + assert unbufferedLineConverter("rev").convert("123") == "321" + assert unbufferedLineConverter("cut", ["-c3"]).convert("abcde") == "c" + print "Pass" + +def separate_debug_file_for(file): + return None + +address_adjustments = {} +def address_adjustment(file): + if not file in address_adjustments: + result = None + otool = subprocess.Popen(["otool", "-l", file], stdout=subprocess.PIPE) + while True: + line = otool.stdout.readline() + if line == "": + break + if line == " segname __TEXT\n": + line = otool.stdout.readline() + if not line.startswith(" vmaddr "): + raise StandardError("unexpected otool output") + result = int(line[10:], 16) + break + otool.stdout.close() + + if result is None: + raise StandardError("unexpected otool output") + + address_adjustments[file] = result + + return address_adjustments[file] + +atoses = {} +def addressToSymbol(file, address): + converter = None + if not file in atoses: + debug_file = separate_debug_file_for(file) or file + converter = unbufferedLineConverter('/usr/bin/xcrun', ['atos', '-arch', 'x86_64', '-o', debug_file]) + atoses[file] = converter + else: + converter = atoses[file] + return converter.convert("0x%X" % address) + +cxxfilt_proc = None +def cxxfilt(sym): + if cxxfilt_proc is None: + # --no-strip-underscores because atos already stripped the underscore + globals()["cxxfilt_proc"] = subprocess.Popen(['c++filt', + '--no-strip-underscores', + '--format', 'gnu-v3'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + cxxfilt_proc.stdin.write(sym + "\n") + return cxxfilt_proc.stdout.readline().rstrip("\n") + +# Matches lines produced by NS_FormatCodeAddress(). +line_re = re.compile("^(.*#\d+: )(.+)\[(.+) \+(0x[0-9A-Fa-f]+)\](.*)$") +atos_name_re = re.compile("^(.+) \(in ([^)]+)\) \((.+)\)$") + +def fixSymbols(line): + result = line_re.match(line) + if result is not None: + (before, fn, file, address, after) = result.groups() + address = int(address, 16) + + if os.path.exists(file) and os.path.isfile(file): + address += address_adjustment(file) + info = addressToSymbol(file, address) + + # atos output seems to have three forms: + # address + # address (in foo.dylib) + # symbol (in foo.dylib) (file:line) + name_result = atos_name_re.match(info) + if name_result is not None: + # Print the first two forms as-is, and transform the third + (name, library, fileline) = name_result.groups() + # atos demangles, but occasionally it fails. cxxfilt can mop + # up the remaining cases(!), which will begin with '_Z'. + if (name.startswith("_Z")): + name = cxxfilt(name) + info = "%s (%s, in %s)" % (name, fileline, library) + + nl = '\n' if line[-1] == '\n' else '' + return before + info + after + nl + else: + sys.stderr.write("Warning: File \"" + file + "\" does not exist.\n") + return line + else: + return line + +if __name__ == "__main__": + for line in sys.stdin: + sys.stdout.write(fixSymbols(line)) diff --git a/tools/rb/fix_stack_using_bpsyms.py b/tools/rb/fix_stack_using_bpsyms.py new file mode 100755 index 000000000..5d04cd02a --- /dev/null +++ b/tools/rb/fix_stack_using_bpsyms.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script uses breakpad symbols to post-process the entries produced by +# NS_FormatCodeAddress(), which on TBPL builds often lack a file name and a +# line number (and on Linux even the symbol is often bad). + +from __future__ import with_statement + +import sys +import os +import re +import subprocess +import bisect + +here = os.path.dirname(__file__) + +def prettyFileName(name): + if name.startswith("../") or name.startswith("..\\"): + # dom_quickstubs.cpp and many .h files show up with relative paths that are useless + # and/or don't correspond to the layout of the source tree. + return os.path.basename(name) + ":" + elif name.startswith("hg:"): + bits = name.split(":") + if len(bits) == 4: + (junk, repo, path, rev) = bits + # We could construct an hgweb URL with /file/ or /annotate/, like this: + # return "http://%s/annotate/%s/%s#l" % (repo, rev, path) + return path + ":" + return name + ":" + +class SymbolFile: + def __init__(self, fn): + addrs = [] # list of addresses, which will be sorted once we're done initializing + funcs = {} # hash: address --> (function name + possible file/line) + files = {} # hash: filenum (string) --> prettified filename ready to have a line number appended + with open(fn) as f: + for line in f: + line = line.rstrip() + # https://chromium.googlesource.com/breakpad/breakpad/+/master/docs/symbol_files.md + if line.startswith("FUNC "): + # FUNC <address> <size> <stack_param_size> <name> + bits = line.split(None, 4) + if len(bits) < 5: + bits.append('unnamed_function') + (junk, rva, size, ss, name) = bits + rva = int(rva,16) + funcs[rva] = name + addrs.append(rva) + lastFuncName = name + elif line.startswith("PUBLIC "): + # PUBLIC <address> <stack_param_size> <name> + (junk, rva, ss, name) = line.split(None, 3) + rva = int(rva,16) + funcs[rva] = name + addrs.append(rva) + elif line.startswith("FILE "): + # FILE <number> <name> + (junk, filenum, name) = line.split(None, 2) + files[filenum] = prettyFileName(name) + elif line[0] in "0123456789abcdef": + # This is one of the "line records" corresponding to the last FUNC record + # <address> <size> <line> <filenum> + (rva, size, line, filenum) = line.split(None) + rva = int(rva,16) + file = files[filenum] + name = lastFuncName + " [" + file + line + "]" + funcs[rva] = name + addrs.append(rva) + # skip everything else + #print "Loaded %d functions from symbol file %s" % (len(funcs), os.path.basename(fn)) + self.addrs = sorted(addrs) + self.funcs = funcs + + def addrToSymbol(self, address): + i = bisect.bisect(self.addrs, address) - 1 + if i > 0: + #offset = address - self.addrs[i] + return self.funcs[self.addrs[i]] + else: + return "" + +def findIdForPath(path): + """Finds the breakpad id for the object file at the given path.""" + # We should always be packaged with a "fileid" executable. + fileid_exe = os.path.join(here, 'fileid') + if not os.path.isfile(fileid_exe): + fileid_exe = fileid_exe + '.exe' + if not os.path.isfile(fileid_exe): + raise Exception("Could not find fileid executable in %s" % here) + + if not os.path.isfile(path): + for suffix in ('.exe', '.dll'): + if os.path.isfile(path + suffix): + path = path + suffix + try: + return subprocess.check_output([fileid_exe, path]).rstrip() + except subprocess.CalledProcessError as e: + raise Exception("Error getting fileid for %s: %s" % + (path, e.output)) + +def guessSymbolFile(full_path, symbolsDir): + """Guess a symbol file based on an object file's basename, ignoring the path and UUID.""" + fn = os.path.basename(full_path) + d1 = os.path.join(symbolsDir, fn) + root, _ = os.path.splitext(fn) + if os.path.exists(os.path.join(symbolsDir, root) + '.pdb'): + d1 = os.path.join(symbolsDir, root) + '.pdb' + fn = root + if not os.path.exists(d1): + return None + uuids = os.listdir(d1) + if len(uuids) == 0: + raise Exception("Missing symbol file for " + fn) + if len(uuids) > 1: + uuid = findIdForPath(full_path) + else: + uuid = uuids[0] + return os.path.join(d1, uuid, fn + ".sym") + +parsedSymbolFiles = {} +def getSymbolFile(file, symbolsDir): + p = None + if not file in parsedSymbolFiles: + symfile = guessSymbolFile(file, symbolsDir) + if symfile: + p = SymbolFile(symfile) + else: + p = None + parsedSymbolFiles[file] = p + else: + p = parsedSymbolFiles[file] + return p + +def addressToSymbol(file, address, symbolsDir): + p = getSymbolFile(file, symbolsDir) + if p: + return p.addrToSymbol(address) + else: + return "" + +# Matches lines produced by NS_FormatCodeAddress(). +line_re = re.compile("^(.*#\d+: )(.+)\[(.+) \+(0x[0-9A-Fa-f]+)\](.*)$") + +def fixSymbols(line, symbolsDir): + result = line_re.match(line) + if result is not None: + (before, fn, file, address, after) = result.groups() + address = int(address, 16) + symbol = addressToSymbol(file, address, symbolsDir) + if not symbol: + symbol = "%s + 0x%x" % (os.path.basename(file), address) + return before + symbol + after + "\n" + else: + return line + +if __name__ == "__main__": + symbolsDir = sys.argv[1] + for line in iter(sys.stdin.readline, ''): + print fixSymbols(line, symbolsDir), diff --git a/tools/rb/make-tree.pl b/tools/rb/make-tree.pl new file mode 100755 index 000000000..04f0d8534 --- /dev/null +++ b/tools/rb/make-tree.pl @@ -0,0 +1,303 @@ +#!/usr/bin/perl -w +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +use 5.004; +use strict; +use Getopt::Long; + +$::opt_prune_depth = 0; +$::opt_subtree_size = 0; +$::opt_reverse = 0; + +# GetOption will create $opt_object & $opt_exclude, so ignore the +# warning that gets spit out about those vbls. +GetOptions("object=s", "exclude=s", "comptrs=s", "ignore-balanced", "subtree-size=i", "prune-depth=i", + "collapse-to-method", "collapse-to-class", "old-style", "reverse"); + +$::opt_object || + die qq{ +usage: leak.pl < logfile + --object <obj> The address of the object to examine (required) + --exclude <file> Exclude routines listed in <file> + --comptrs <file> Subtract all the data in the balanced COMPtr log <file> + --ignore-balanced Ignore balanced subtrees + --subtree-size <n> Print subtrees with more than <n> nodes separately + --prune-depth <depth> Prune the tree to <depth> + --collapse-to-method Aggregate data by method + --collapse-to-class Aggregate data by class (subsumes --collapse-to-method) + --reverse Reverse call stacks, showing leaves first + --old-style Old-style formatting +}; + +$::opt_prune_depth = 0 if $::opt_prune_depth < 0; +$::opt_subtree_size = 0 if $::opt_subtree_size < 0; + +warn "object $::opt_object\n"; +warn "ignoring balanced subtrees\n" if $::opt_ignore_balanced; +warn "prune depth $::opt_prune_depth\n" if $::opt_prune_depth; +warn "collapsing to class\n" if $::opt_collapse_to_class; +warn "collapsing to method\n" if $::opt_collapse_to_method && !$::opt_collapse_to_class; +warn "reversing call stacks\n" if $::opt_reverse; + + +# The 'excludes' are functions that, if detected in a particular call +# stack, will cause the _entire_ call stack to be ignored. You might, +# for example, explicitly exclude two functions that have a matching +# AddRef/Release pair. + +my %excludes; + +if ($::opt_exclude) { + open(EXCLUDE, "<".$::opt_exclude) + || die "unable to open $::opt_exclude"; + + while (<EXCLUDE>) { + chomp $_; + warn "excluding $_\n"; + $excludes{$_} = 1; + } +} + +# Each entry in the tree rooted by callGraphRoot contains the following: +# #name# This call's name+offset string +# #refcount# The net reference count of this call +# #label# The label used for this subtree; only defined for labeled nodes +# #children# List of children in alphabetical order +# zero or more children indexed by method name+offset strings. + +my $callGraphRoot; +$callGraphRoot = { '#name#' => '.root', '#refcount#' => 'n/a' }; + +# The 'imbalance' is a gross count of how balanced a particular +# callsite is. It is used to prune away callsites that are detected to +# be balanced; that is, that have matching AddRef/Release() pairs. + +my %imbalance; +$imbalance{'.root'} = 'n/a'; + +# The main read loop. + +sub read_data($$$) { + my ($INFILE, $plus, $minus) = @_; + + LINE: while (<$INFILE>) { + next LINE if (! /^</); + my @fields = split(/ /, $_); + + my $class = shift(@fields); + my $obj = shift(@fields); + my $sno = shift(@fields); + next LINE unless ($obj eq $::opt_object); + + my $op = shift(@fields); + next LINE unless ($op eq $plus || $op eq $minus); + + my $cnt = shift(@fields); + + # Collect the remaining lines to create a stack trace. We need to + # filter out the frame numbers so that frames that differ only in + # their frame number are considered equivalent. However, we need to + # keep a frame number on each line so that the fix*.py scripts can + # parse the output. So we set the frame number to 0 for every frame. + my @stack; + CALLSITE: while (<$INFILE>) { + chomp; + last CALLSITE if (/^$/); + $_ =~ s/#\d+: /#00: /; # replace frame number with 0 + $stack[++$#stack] = $_; + } + + # Reverse the remaining fields to produce the call stack, with the + # oldest frame at the front of the array. + if (! $::opt_reverse) { + @stack = reverse(@stack); + } + + my $call; + + # If any of the functions in the stack are supposed to be excluded, + # march on to the next line. + foreach $call (@stack) { + next LINE if exists($excludes{$call}); + } + + + # Add the callstack as a path through the call graph, updating + # refcounts at each node. + + my $caller = $callGraphRoot; + + foreach $call (@stack) { + + # Chop the method offset if we're 'collapsing to method' or + # 'collapsing to class'. + $call =~ s/\+0x.*$//g if ($::opt_collapse_to_method || $::opt_collapse_to_class); + + # Chop the method name if we're 'collapsing to class'. + $call =~ s/::.*$//g if ($::opt_collapse_to_class); + + my $site = $caller->{$call}; + if (!$site) { + # This is the first time we've seen this callsite. Add a + # new entry to the call tree. + + $site = { '#name#' => $call, '#refcount#' => 0 }; + $caller->{$call} = $site; + } + + if ($op eq $plus) { + ++($site->{'#refcount#'}); + ++($imbalance{$call}); + } elsif ($op eq $minus) { + --($site->{'#refcount#'}); + --($imbalance{$call}); + } else { + die "Bad operation $op"; + } + + $caller = $site; + } + } +} + +read_data(*STDIN, "AddRef", "Release"); + +if ($::opt_comptrs) { + warn "Subtracting comptr log ". $::opt_comptrs . "\n"; + open(COMPTRS, "<".$::opt_comptrs) + || die "unable to open $::opt_comptrs"; + + # read backwards to subtract + read_data(*COMPTRS, "nsCOMPtrRelease", "nsCOMPtrAddRef"); +} + +sub num_alpha { + my ($aN, $aS, $bN, $bS); + ($aN, $aS) = ($1, $2) if $a =~ /^(\d+) (.+)$/; + ($bN, $bS) = ($1, $2) if $b =~ /^(\d+) (.+)$/; + return $a cmp $b unless defined $aN && defined $bN; + return $aN <=> $bN unless $aN == $bN; + return $aS cmp $bS; +} + +# Given a subtree and its nesting level, return true if that subtree should be pruned. +# If it shouldn't be pruned, destructively attempt to prune its children. +# Also compute the #children# properties of unpruned nodes. +sub prune($$) { + my ($site, $nest) = @_; + + # If they want us to prune the tree's depth, do so here. + return 1 if ($::opt_prune_depth && $nest >= $::opt_prune_depth); + + # If the subtree is balanced, ignore it. + return 1 if ($::opt_ignore_balanced && !$site->{'#refcount#'}); + + my $name = $site->{'#name#'}; + + # If the symbol isn't imbalanced, then prune here (and warn) + if ($::opt_ignore_balanced && !$imbalance{$name}) { + warn "discarding " . $name . "\n"; +# return 1; + } + + my @children; + foreach my $child (sort num_alpha keys(%$site)) { + if (substr($child, 0, 1) ne '#') { + if (prune($site->{$child}, $nest + 1)) { + delete $site->{$child}; + } else { + push @children, $site->{$child}; + } + } + } + $site->{'#children#'} = \@children; + return 0; +} + + +# Compute the #label# properties of this subtree. +# Return the subtree's number of nodes, not counting nodes reachable +# through a labeled node. +sub createLabels($) { + my ($site) = @_; + my @children = @{$site->{'#children#'}}; + my $nChildren = @children; + my $nDescendants = 0; + + foreach my $child (@children) { + my $childDescendants = createLabels($child); + if ($nChildren > 1 && $childDescendants > $::opt_subtree_size) { + die "Internal error" if defined($child->{'#label#'}); + $child->{'#label#'} = "__label__"; + $childDescendants = 1; + } + $nDescendants += $childDescendants; + } + return $nDescendants + 1; +} + + +my $nextLabel = 0; +my @labeledSubtrees; + +sub list($$$$$) { + my ($site, $nest, $nestStr, $childrenLeft, $root) = @_; + my $label = !$root && $site->{'#label#'}; + + # Assign a unique number to the label. + if ($label) { + die unless $label eq "__label__"; + $label = "__" . ++$nextLabel . "__"; + $site->{'#label#'} = $label; + push @labeledSubtrees, $site; + } + + print $nestStr; + if ($::opt_old_style) { + print $label, " " if $label; + print $site->{'#name#'}, ": bal=", $site->{'#refcount#'}, "\n"; + } else { + my $refcount = $site->{'#refcount#'}; + my $l = 8 - length $refcount; + $l = 1 if $l < 1; + print $refcount, " " x $l; + print $label, " " if $label; + print $site->{'#name#'}, "\n"; + } + + $nestStr .= $childrenLeft && !$::opt_old_style ? "| " : " "; + if (!$label) { + my @children = @{$site->{'#children#'}}; + $childrenLeft = @children; + foreach my $child (@children) { + $childrenLeft--; + list($child, $nest + 1, $nestStr, $childrenLeft); + } + } +} + + +if (!prune($callGraphRoot, 0)) { + createLabels $callGraphRoot if ($::opt_subtree_size); + list $callGraphRoot, 0, "", 0, 1; + while (@labeledSubtrees) { + my $labeledSubtree = shift @labeledSubtrees; + print "\n------------------------------\n", +$labeledSubtree->{'#label#'}, "\n"; + list $labeledSubtree, 0, "", 0, 1; + } + print "\n------------------------------\n" if @labeledSubtrees; +} + +print qq{ +Imbalance +--------- +}; + +foreach my $call (sort num_alpha keys(%imbalance)) { + print $call . " " . $imbalance{$call} . "\n"; +} + diff --git a/tools/rewriting/ThirdPartyPaths.txt b/tools/rewriting/ThirdPartyPaths.txt new file mode 100644 index 000000000..ab6c4b9cc --- /dev/null +++ b/tools/rewriting/ThirdPartyPaths.txt @@ -0,0 +1,62 @@ +browser/components/translation/cld2/ +build/stlport/ +db/sqlite3/src/ +dom/media/platforms/ffmpeg/libav +extensions/spellcheck/hunspell/src/ +gfx/2d/convolver +gfx/2d/image_operations +gfx/angle/ +gfx/cairo/ +gfx/graphite2/ +gfx/harfbuzz/ +gfx/ots/ +gfx/qcms/ +gfx/skia/ +gfx/ycbcr/ +intl/hyphenation/hyphen/ +intl/icu/ +ipc/chromium/ +js/src/ctypes/libffi/ +js/src/dtoa.c +js/src/jit/arm64/vixl/ +media/gmp-clearkey/0.1/openaes/ +media/kiss_fft/ +media/libav/ +media/libcubeb/ +media/libjpeg/ +media/libmkv/ +media/libnestegg/ +media/libogg/ +media/libopus/ +media/libpng/ +media/libsoundtouch/ +media/libspeex_resampler/ +media/libstagefright/ +media/libtheora/ +media/libtremor/ +media/libvorbis/ +media/libvpx/ +media/libyuv/ +media/mtransport/ +media/openmax_dl/ +media/pocketsphinx/ +media/sphinxbase/ +media/webrtc/trunk/ +mfbt/decimal/ +mfbt/double-conversion/ +mfbt/lz4 +modules/brotli/ +modules/freetype2/ +modules/libbz2/ +modules/libmar/ +modules/zlib/ +netwerk/sctp/src/ +netwerk/srtp/src/ +nsprpub/ +other-licenses/ +security/sandbox/chromium/ +third_party/aom/ +testing/gtest/gmock/ +testing/gtest/gtest/ +toolkit/components/protobuf/ +toolkit/crashreporter/google-breakpad/ |