From fc65e1f2fa4239c6271d666b3366cdf7cf440580 Mon Sep 17 00:00:00 2001 From: Moonchild Date: Sat, 13 Feb 2021 09:41:29 +0000 Subject: Issue #1739 - Implement numeric separators. Resolves #1739 --- js/src/frontend/TokenStream.cpp | 135 +++++++++++++++++++++++++++++++--------- 1 file changed, 105 insertions(+), 30 deletions(-) (limited to 'js/src/frontend/TokenStream.cpp') diff --git a/js/src/frontend/TokenStream.cpp b/js/src/frontend/TokenStream.cpp index 7a253cc0e9..083bcd5045 100644 --- a/js/src/frontend/TokenStream.cpp +++ b/js/src/frontend/TokenStream.cpp @@ -1476,21 +1476,45 @@ TokenStream::getTokenInternal(TokenKind* ttp, Modifier modifier) // Look for a decimal number. // if (c1kind == Dec) { + MOZ_ASSERT(JS7_ISDEC(c)); tp = newToken(-1); numStart = userbuf.addressOfNextRawChar() - 1; - - decimal: decimalPoint = NoDecimal; hasExp = false; - while (JS7_ISDEC(c)) + do { c = getCharIgnoreEOL(); + if (JS7_ISDEC(c)) + continue; + if (c != '_') + break; + c = getCharIgnoreEOL(); + if (!JS7_ISDEC(c)) { + ungetCharIgnoreEOL(c); + reportError(JSMSG_MISSING_DIGIT_AFTER_SEPARATOR); + goto error; + } + } while (true); + decimal_rest: if (c == '.') { decimalPoint = HasDecimal; - decimal_dot: - do { - c = getCharIgnoreEOL(); - } while (JS7_ISDEC(c)); + c = getCharIgnoreEOL(); + if (JS7_ISDEC(c)) { + decimal_dot: + do { + c = getCharIgnoreEOL(); + if (JS7_ISDEC(c)) + continue; + if (c != '_') + break; + c = getCharIgnoreEOL(); + if (!JS7_ISDEC(c)) { + ungetCharIgnoreEOL(c); + reportError(JSMSG_MISSING_DIGIT_AFTER_SEPARATOR); + goto error; + } + } while (true); + } } if (c == 'e' || c == 'E') { hasExp = true; @@ -1504,7 +1528,17 @@ TokenStream::getTokenInternal(TokenKind* ttp, Modifier modifier) } do { c = getCharIgnoreEOL(); - } while (JS7_ISDEC(c)); + if (JS7_ISDEC(c)) + continue; + if (c != '_') + break; + c = getCharIgnoreEOL(); + if (!JS7_ISDEC(c)) { + ungetCharIgnoreEOL(c); + reportError(JSMSG_MISSING_DIGIT_AFTER_SEPARATOR); + goto error; + } + } while (true); } ungetCharIgnoreEOL(c); @@ -1533,8 +1567,7 @@ TokenStream::getTokenInternal(TokenKind* ttp, Modifier modifier) if (!GetDecimalInteger(cx, numStart, userbuf.addressOfNextRawChar(), &dval)) goto error; } else { - const char16_t* dummy; - if (!js_strtod(cx, numStart, userbuf.addressOfNextRawChar(), &dummy, &dval)) + if (!GetDecimalNonInteger(cx, numStart, userbuf.addressOfNextRawChar(), &dval)) goto error; } tp->type = TOK_NUMBER; @@ -1576,8 +1609,19 @@ TokenStream::getTokenInternal(TokenKind* ttp, Modifier modifier) goto error; } numStart = userbuf.addressOfNextRawChar() - 1; // one past the '0x' - while (JS7_ISHEX(c)) + do { c = getCharIgnoreEOL(); + if (JS7_ISHEX(c)) + continue; + if (c != '_') + break; + c = getCharIgnoreEOL(); + if (!JS7_ISHEX(c)) { + ungetCharIgnoreEOL(c); + reportError(JSMSG_MISSING_DIGIT_AFTER_SEPARATOR); + goto error; + } + } while (true); } else if (c == 'b' || c == 'B') { radix = 2; c = getCharIgnoreEOL(); @@ -1587,8 +1631,19 @@ TokenStream::getTokenInternal(TokenKind* ttp, Modifier modifier) goto error; } numStart = userbuf.addressOfNextRawChar() - 1; // one past the '0b' - while (c == '0' || c == '1') + do { c = getCharIgnoreEOL(); + if (c == '0' || c == '1') + continue; + if (c != '_') + break; + c = getCharIgnoreEOL(); + if (c != '0' && c != '1') { + ungetCharIgnoreEOL(c); + reportError(JSMSG_MISSING_DIGIT_AFTER_SEPARATOR); + goto error; + } + } while (true); } else if (c == 'o' || c == 'O') { radix = 8; c = getCharIgnoreEOL(); @@ -1598,33 +1653,50 @@ TokenStream::getTokenInternal(TokenKind* ttp, Modifier modifier) goto error; } numStart = userbuf.addressOfNextRawChar() - 1; // one past the '0o' - while ('0' <= c && c <= '7') + do { + c = getCharIgnoreEOL(); + if ('0' <= c && c <= '7') + continue; + if (c != '_') + break; c = getCharIgnoreEOL(); + if (c < '0' || c > '7') { + ungetCharIgnoreEOL(c); + reportError(JSMSG_MISSING_DIGIT_AFTER_SEPARATOR); + goto error; + } + } while (true); } else if (JS7_ISDEC(c)) { + // Octal integer literals are not permitted in strict mode. + // Maybe one day we can get rid of this base-8 madness for good. + if (!reportStrictModeError(JSMSG_DEPRECATED_OCTAL)) + goto error; + radix = 8; numStart = userbuf.addressOfNextRawChar() - 1; // one past the '0' + bool nonOctalDecimalIntegerLiteral = false; while (JS7_ISDEC(c)) { - // Octal integer literals are not permitted in strict mode code. - if (!reportStrictModeError(JSMSG_DEPRECATED_OCTAL)) - goto error; - - // Outside strict mode, we permit 08 and 09 as decimal numbers, - // which makes our behaviour a superset of the ECMA numeric - // grammar. We might not always be so permissive, so we warn - // about it. - if (c >= '8') { - if (!warning(JSMSG_BAD_OCTAL, c == '8' ? "08" : "09")) - goto error; - - // Use the decimal scanner for the rest of the number. - goto decimal; - } + if (c >= '8') + nonOctalDecimalIntegerLiteral = true; c = getCharIgnoreEOL(); } + + if (c == '_') { + reportError(JSMSG_INVALID_NUMERIC_SEPARATOR); + goto error; + } + if (nonOctalDecimalIntegerLiteral) { + // Use the decimal scanner for the rest of the number. + decimalPoint = NoDecimal; + hasExp = false; + goto decimal_rest; + } } else { // '0' not followed by 'x', 'X' or a digit; scan as a decimal number. numStart = userbuf.addressOfNextRawChar() - 1; - goto decimal; + decimalPoint = NoDecimal; + hasExp = false; + goto decimal_rest; } ungetCharIgnoreEOL(c); @@ -1647,8 +1719,11 @@ TokenStream::getTokenInternal(TokenKind* ttp, Modifier modifier) double dval; const char16_t* dummy; - if (!GetPrefixInteger(cx, numStart, userbuf.addressOfNextRawChar(), radix, &dummy, &dval)) + if (!GetPrefixInteger(cx, numStart, userbuf.addressOfNextRawChar(), radix, + PrefixIntegerSeparatorHandling::SkipUnderscore, &dummy, &dval)) + { goto error; + } tp->type = TOK_NUMBER; tp->setNumber(dval, NoDecimal); goto out; -- cgit v1.2.3