summaryrefslogtreecommitdiff
path: root/services/sync/tests/unit/test_errorhandler_2.js
diff options
context:
space:
mode:
Diffstat (limited to 'services/sync/tests/unit/test_errorhandler_2.js')
-rw-r--r--services/sync/tests/unit/test_errorhandler_2.js1012
1 files changed, 1012 insertions, 0 deletions
diff --git a/services/sync/tests/unit/test_errorhandler_2.js b/services/sync/tests/unit/test_errorhandler_2.js
new file mode 100644
index 0000000000..41f8ee7276
--- /dev/null
+++ b/services/sync/tests/unit/test_errorhandler_2.js
@@ -0,0 +1,1012 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-sync/engines/clients.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/keys.js");
+Cu.import("resource://services-sync/policies.js");
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/status.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://testing-common/services/sync/utils.js");
+Cu.import("resource://gre/modules/FileUtils.jsm");
+
+var fakeServer = new SyncServer();
+fakeServer.start();
+
+do_register_cleanup(function() {
+ return new Promise(resolve => {
+ fakeServer.stop(resolve);
+ });
+});
+
+var fakeServerUrl = "http://localhost:" + fakeServer.port;
+
+const logsdir = FileUtils.getDir("ProfD", ["weave", "logs"], true);
+
+const PROLONGED_ERROR_DURATION =
+ (Svc.Prefs.get('errorhandler.networkFailureReportTimeout') * 2) * 1000;
+
+const NON_PROLONGED_ERROR_DURATION =
+ (Svc.Prefs.get('errorhandler.networkFailureReportTimeout') / 2) * 1000;
+
+Service.engineManager.clear();
+
+function setLastSync(lastSyncValue) {
+ Svc.Prefs.set("lastSync", (new Date(Date.now() - lastSyncValue)).toString());
+}
+
+var engineManager = Service.engineManager;
+engineManager.register(EHTestsCommon.CatapultEngine);
+
+// This relies on Service/ErrorHandler being a singleton. Fixing this will take
+// a lot of work.
+var errorHandler = Service.errorHandler;
+
+function run_test() {
+ initTestLogging("Trace");
+
+ Log.repository.getLogger("Sync.Service").level = Log.Level.Trace;
+ Log.repository.getLogger("Sync.SyncScheduler").level = Log.Level.Trace;
+ Log.repository.getLogger("Sync.ErrorHandler").level = Log.Level.Trace;
+
+ ensureLegacyIdentityManager();
+
+ run_next_test();
+}
+
+
+function clean() {
+ Service.startOver();
+ Status.resetSync();
+ Status.resetBackoff();
+ errorHandler.didReportProlongedError = false;
+}
+
+add_identity_test(this, function* test_crypto_keys_login_server_maintenance_error() {
+ Status.resetSync();
+ // Test crypto/keys server maintenance errors are not reported.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ yield configureIdentity({username: "broken.keys"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ // Force re-download of keys
+ Service.collectionKeys.clear();
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ function onUIUpdate() {
+ do_throw("Shouldn't get here!");
+ }
+ Svc.Obs.add("weave:ui:login:error", onUIUpdate);
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:clear-error", function onLoginFinish() {
+ Svc.Obs.remove("weave:ui:clear-error", onLoginFinish);
+
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, LOGIN_FAILED);
+ do_check_eq(Status.login, SERVER_MAINTENANCE);
+ do_check_false(errorHandler.didReportProlongedError);
+
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ setLastSync(NON_PROLONGED_ERROR_DURATION);
+ Service.sync();
+ yield deferred.promise;
+});
+
+add_task(function* test_sync_prolonged_server_maintenance_error() {
+ // Test prolonged server maintenance errors are reported.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ const BACKOFF = 42;
+ let engine = engineManager.get("catapult");
+ engine.enabled = true;
+ engine.exception = {status: 503,
+ headers: {"retry-after": BACKOFF}};
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
+ do_check_eq(Status.service, SYNC_FAILED);
+ do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
+ do_check_true(errorHandler.didReportProlongedError);
+
+ server.stop(() => {
+ clean();
+ deferred.resolve();
+ });
+ });
+
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(PROLONGED_ERROR_DURATION);
+ let ping = yield sync_and_validate_telem(true);
+ deepEqual(ping.status.sync, PROLONGED_SYNC_FAILURE);
+ deepEqual(ping.engines.find(e => e.failureReason).failureReason,
+ { name: "httperror", code: 503 });
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_info_collections_login_prolonged_server_maintenance_error(){
+ // Test info/collections prolonged server maintenance errors are reported.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ yield configureIdentity({username: "broken.info"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, SYNC_FAILED);
+ do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
+ do_check_true(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(PROLONGED_ERROR_DURATION);
+ Service.sync();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_meta_global_login_prolonged_server_maintenance_error(){
+ // Test meta/global prolonged server maintenance errors are reported.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ yield configureIdentity({username: "broken.meta"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, SYNC_FAILED);
+ do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
+ do_check_true(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(PROLONGED_ERROR_DURATION);
+ Service.sync();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_download_crypto_keys_login_prolonged_server_maintenance_error(){
+ // Test crypto/keys prolonged server maintenance errors are reported.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ yield configureIdentity({username: "broken.keys"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+ // Force re-download of keys
+ Service.collectionKeys.clear();
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, SYNC_FAILED);
+ do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
+ do_check_true(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(PROLONGED_ERROR_DURATION);
+ Service.sync();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_upload_crypto_keys_login_prolonged_server_maintenance_error(){
+ // Test crypto/keys prolonged server maintenance errors are reported.
+ let server = EHTestsCommon.sync_httpd_setup();
+
+ // Start off with an empty account, do not upload a key.
+ yield configureIdentity({username: "broken.keys"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, SYNC_FAILED);
+ do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
+ do_check_true(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(PROLONGED_ERROR_DURATION);
+ Service.sync();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_wipeServer_login_prolonged_server_maintenance_error(){
+ // Test that we report prolonged server maintenance errors that occur whilst
+ // wiping the server.
+ let server = EHTestsCommon.sync_httpd_setup();
+
+ // Start off with an empty account, do not upload a key.
+ yield configureIdentity({username: "broken.wipe"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, SYNC_FAILED);
+ do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
+ do_check_true(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(PROLONGED_ERROR_DURATION);
+ Service.sync();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_wipeRemote_prolonged_server_maintenance_error(){
+ // Test that we report prolonged server maintenance errors that occur whilst
+ // wiping all remote devices.
+ let server = EHTestsCommon.sync_httpd_setup();
+
+ server.registerPathHandler("/1.1/broken.wipe/storage/catapult", EHTestsCommon.service_unavailable);
+ yield configureIdentity({username: "broken.wipe"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+ EHTestsCommon.generateAndUploadKeys();
+
+ let engine = engineManager.get("catapult");
+ engine.exception = null;
+ engine.enabled = true;
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, SYNC_FAILED);
+ do_check_eq(Status.sync, PROLONGED_SYNC_FAILURE);
+ do_check_eq(Svc.Prefs.get("firstSync"), "wipeRemote");
+ do_check_true(errorHandler.didReportProlongedError);
+ server.stop(() => {
+ clean();
+ deferred.resolve();
+ });
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ Svc.Prefs.set("firstSync", "wipeRemote");
+ setLastSync(PROLONGED_ERROR_DURATION);
+ let ping = yield sync_and_validate_telem(true);
+ deepEqual(ping.failureReason, { name: "httperror", code: 503 });
+ yield deferred.promise;
+});
+
+add_task(function* test_sync_syncAndReportErrors_server_maintenance_error() {
+ // Test server maintenance errors are reported
+ // when calling syncAndReportErrors.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ const BACKOFF = 42;
+ let engine = engineManager.get("catapult");
+ engine.enabled = true;
+ engine.exception = {status: 503,
+ headers: {"retry-after": BACKOFF}};
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
+ do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
+ do_check_eq(Status.sync, SERVER_MAINTENANCE);
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(NON_PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_info_collections_login_syncAndReportErrors_server_maintenance_error() {
+ // Test info/collections server maintenance errors are reported
+ // when calling syncAndReportErrors.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ yield configureIdentity({username: "broken.info"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, LOGIN_FAILED);
+ do_check_eq(Status.login, SERVER_MAINTENANCE);
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(NON_PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_meta_global_login_syncAndReportErrors_server_maintenance_error() {
+ // Test meta/global server maintenance errors are reported
+ // when calling syncAndReportErrors.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ yield configureIdentity({username: "broken.meta"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, LOGIN_FAILED);
+ do_check_eq(Status.login, SERVER_MAINTENANCE);
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(NON_PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_download_crypto_keys_login_syncAndReportErrors_server_maintenance_error() {
+ // Test crypto/keys server maintenance errors are reported
+ // when calling syncAndReportErrors.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ yield configureIdentity({username: "broken.keys"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+ // Force re-download of keys
+ Service.collectionKeys.clear();
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, LOGIN_FAILED);
+ do_check_eq(Status.login, SERVER_MAINTENANCE);
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(NON_PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_upload_crypto_keys_login_syncAndReportErrors_server_maintenance_error() {
+ // Test crypto/keys server maintenance errors are reported
+ // when calling syncAndReportErrors.
+ let server = EHTestsCommon.sync_httpd_setup();
+
+ // Start off with an empty account, do not upload a key.
+ yield configureIdentity({username: "broken.keys"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, LOGIN_FAILED);
+ do_check_eq(Status.login, SERVER_MAINTENANCE);
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(NON_PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_wipeServer_login_syncAndReportErrors_server_maintenance_error() {
+ // Test crypto/keys server maintenance errors are reported
+ // when calling syncAndReportErrors.
+ let server = EHTestsCommon.sync_httpd_setup();
+
+ // Start off with an empty account, do not upload a key.
+ yield configureIdentity({username: "broken.wipe"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, LOGIN_FAILED);
+ do_check_eq(Status.login, SERVER_MAINTENANCE);
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(NON_PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_wipeRemote_syncAndReportErrors_server_maintenance_error(){
+ // Test that we report prolonged server maintenance errors that occur whilst
+ // wiping all remote devices.
+ let server = EHTestsCommon.sync_httpd_setup();
+
+ yield configureIdentity({username: "broken.wipe"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+ EHTestsCommon.generateAndUploadKeys();
+
+ let engine = engineManager.get("catapult");
+ engine.exception = null;
+ engine.enabled = true;
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, SYNC_FAILED);
+ do_check_eq(Status.sync, SERVER_MAINTENANCE);
+ do_check_eq(Svc.Prefs.get("firstSync"), "wipeRemote");
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ Svc.Prefs.set("firstSync", "wipeRemote");
+ setLastSync(NON_PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_task(function* test_sync_syncAndReportErrors_prolonged_server_maintenance_error() {
+ // Test prolonged server maintenance errors are
+ // reported when calling syncAndReportErrors.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ const BACKOFF = 42;
+ let engine = engineManager.get("catapult");
+ engine.enabled = true;
+ engine.exception = {status: 503,
+ headers: {"retry-after": BACKOFF}};
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:sync:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:sync:error", onUIUpdate);
+ do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
+ do_check_eq(Status.sync, SERVER_MAINTENANCE);
+ // syncAndReportErrors means dontIgnoreErrors, which means
+ // didReportProlongedError not touched.
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_info_collections_login_syncAndReportErrors_prolonged_server_maintenance_error() {
+ // Test info/collections server maintenance errors are reported
+ // when calling syncAndReportErrors.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ yield configureIdentity({username: "broken.info"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, LOGIN_FAILED);
+ do_check_eq(Status.login, SERVER_MAINTENANCE);
+ // syncAndReportErrors means dontIgnoreErrors, which means
+ // didReportProlongedError not touched.
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_meta_global_login_syncAndReportErrors_prolonged_server_maintenance_error() {
+ // Test meta/global server maintenance errors are reported
+ // when calling syncAndReportErrors.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ yield configureIdentity({username: "broken.meta"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, LOGIN_FAILED);
+ do_check_eq(Status.login, SERVER_MAINTENANCE);
+ // syncAndReportErrors means dontIgnoreErrors, which means
+ // didReportProlongedError not touched.
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_download_crypto_keys_login_syncAndReportErrors_prolonged_server_maintenance_error() {
+ // Test crypto/keys server maintenance errors are reported
+ // when calling syncAndReportErrors.
+ let server = EHTestsCommon.sync_httpd_setup();
+ yield EHTestsCommon.setUp(server);
+
+ yield configureIdentity({username: "broken.keys"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+ // Force re-download of keys
+ Service.collectionKeys.clear();
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, LOGIN_FAILED);
+ do_check_eq(Status.login, SERVER_MAINTENANCE);
+ // syncAndReportErrors means dontIgnoreErrors, which means
+ // didReportProlongedError not touched.
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_upload_crypto_keys_login_syncAndReportErrors_prolonged_server_maintenance_error() {
+ // Test crypto/keys server maintenance errors are reported
+ // when calling syncAndReportErrors.
+ let server = EHTestsCommon.sync_httpd_setup();
+
+ // Start off with an empty account, do not upload a key.
+ yield configureIdentity({username: "broken.keys"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, LOGIN_FAILED);
+ do_check_eq(Status.login, SERVER_MAINTENANCE);
+ // syncAndReportErrors means dontIgnoreErrors, which means
+ // didReportProlongedError not touched.
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_identity_test(this, function* test_wipeServer_login_syncAndReportErrors_prolonged_server_maintenance_error() {
+ // Test crypto/keys server maintenance errors are reported
+ // when calling syncAndReportErrors.
+ let server = EHTestsCommon.sync_httpd_setup();
+
+ // Start off with an empty account, do not upload a key.
+ yield configureIdentity({username: "broken.wipe"});
+ Service.serverURL = server.baseURI + "/maintenance/";
+ Service.clusterURL = server.baseURI + "/maintenance/";
+
+ let backoffInterval;
+ Svc.Obs.add("weave:service:backoff:interval", function observe(subject, data) {
+ Svc.Obs.remove("weave:service:backoff:interval", observe);
+ backoffInterval = subject;
+ });
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:ui:login:error", function onUIUpdate() {
+ Svc.Obs.remove("weave:ui:login:error", onUIUpdate);
+ do_check_true(Status.enforceBackoff);
+ do_check_eq(backoffInterval, 42);
+ do_check_eq(Status.service, LOGIN_FAILED);
+ do_check_eq(Status.login, SERVER_MAINTENANCE);
+ // syncAndReportErrors means dontIgnoreErrors, which means
+ // didReportProlongedError not touched.
+ do_check_false(errorHandler.didReportProlongedError);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_false(Status.enforceBackoff);
+ do_check_eq(Status.service, STATUS_OK);
+
+ setLastSync(PROLONGED_ERROR_DURATION);
+ errorHandler.syncAndReportErrors();
+ yield deferred.promise;
+});
+
+add_task(function* test_sync_engine_generic_fail() {
+ let server = EHTestsCommon.sync_httpd_setup();
+
+let engine = engineManager.get("catapult");
+ engine.enabled = true;
+ engine.sync = function sync() {
+ Svc.Obs.notify("weave:engine:sync:error", ENGINE_UNKNOWN_FAIL, "catapult");
+ };
+
+ let log = Log.repository.getLogger("Sync.ErrorHandler");
+ Svc.Prefs.set("log.appender.file.logOnError", true);
+
+ do_check_eq(Status.engines["catapult"], undefined);
+
+ let deferred = Promise.defer();
+ // Don't wait for reset-file-log until the sync is underway.
+ // This avoids us catching a delayed notification from an earlier test.
+ Svc.Obs.add("weave:engine:sync:finish", function onEngineFinish() {
+ Svc.Obs.remove("weave:engine:sync:finish", onEngineFinish);
+
+ log.info("Adding reset-file-log observer.");
+ Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+ Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+
+ // Put these checks here, not after sync(), so that we aren't racing the
+ // log handler... which resets everything just a few lines below!
+ _("Status.engines: " + JSON.stringify(Status.engines));
+ do_check_eq(Status.engines["catapult"], ENGINE_UNKNOWN_FAIL);
+ do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
+
+ // Test Error log was written on SYNC_FAILED_PARTIAL.
+ let entries = logsdir.directoryEntries;
+ do_check_true(entries.hasMoreElements());
+ let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
+ do_check_true(logfile.leafName.startsWith("error-sync-"), logfile.leafName);
+
+ clean();
+
+ let syncErrors = sumHistogram("WEAVE_ENGINE_SYNC_ERRORS", { key: "catapult" });
+ do_check_true(syncErrors, 1);
+
+ server.stop(() => {
+ clean();
+ deferred.resolve();
+ });
+ });
+ });
+
+ do_check_true(yield EHTestsCommon.setUp(server));
+ let ping = yield sync_and_validate_telem(true);
+ deepEqual(ping.status.service, SYNC_FAILED_PARTIAL);
+ deepEqual(ping.engines.find(e => e.status).status, ENGINE_UNKNOWN_FAIL);
+
+ yield deferred.promise;
+});
+
+add_test(function test_logs_on_sync_error_despite_shouldReportError() {
+ _("Ensure that an error is still logged when weave:service:sync:error " +
+ "is notified, despite shouldReportError returning false.");
+
+ let log = Log.repository.getLogger("Sync.ErrorHandler");
+ Svc.Prefs.set("log.appender.file.logOnError", true);
+ log.info("TESTING");
+
+ // Ensure that we report no error.
+ Status.login = MASTER_PASSWORD_LOCKED;
+ do_check_false(errorHandler.shouldReportError());
+
+ Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+ Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+
+ // Test that error log was written.
+ let entries = logsdir.directoryEntries;
+ do_check_true(entries.hasMoreElements());
+ let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
+ do_check_true(logfile.leafName.startsWith("error-sync-"), logfile.leafName);
+
+ clean();
+ run_next_test();
+ });
+ Svc.Obs.notify("weave:service:sync:error", {});
+});
+
+add_test(function test_logs_on_login_error_despite_shouldReportError() {
+ _("Ensure that an error is still logged when weave:service:login:error " +
+ "is notified, despite shouldReportError returning false.");
+
+ let log = Log.repository.getLogger("Sync.ErrorHandler");
+ Svc.Prefs.set("log.appender.file.logOnError", true);
+ log.info("TESTING");
+
+ // Ensure that we report no error.
+ Status.login = MASTER_PASSWORD_LOCKED;
+ do_check_false(errorHandler.shouldReportError());
+
+ Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+ Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+
+ // Test that error log was written.
+ let entries = logsdir.directoryEntries;
+ do_check_true(entries.hasMoreElements());
+ let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
+ do_check_true(logfile.leafName.startsWith("error-sync-"), logfile.leafName);
+
+ clean();
+ run_next_test();
+ });
+ Svc.Obs.notify("weave:service:login:error", {});
+});
+
+// This test should be the last one since it monkeypatches the engine object
+// and we should only have one engine object throughout the file (bug 629664).
+add_task(function* test_engine_applyFailed() {
+ let server = EHTestsCommon.sync_httpd_setup();
+
+ let engine = engineManager.get("catapult");
+ engine.enabled = true;
+ delete engine.exception;
+ engine.sync = function sync() {
+ Svc.Obs.notify("weave:engine:sync:applied", {newFailed:1}, "catapult");
+ };
+
+ let log = Log.repository.getLogger("Sync.ErrorHandler");
+ Svc.Prefs.set("log.appender.file.logOnError", true);
+
+ let deferred = Promise.defer();
+ Svc.Obs.add("weave:service:reset-file-log", function onResetFileLog() {
+ Svc.Obs.remove("weave:service:reset-file-log", onResetFileLog);
+
+ do_check_eq(Status.engines["catapult"], ENGINE_APPLY_FAIL);
+ do_check_eq(Status.service, SYNC_FAILED_PARTIAL);
+
+ // Test Error log was written on SYNC_FAILED_PARTIAL.
+ let entries = logsdir.directoryEntries;
+ do_check_true(entries.hasMoreElements());
+ let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
+ do_check_true(logfile.leafName.startsWith("error-sync-"), logfile.leafName);
+
+ clean();
+ server.stop(deferred.resolve);
+ });
+
+ do_check_eq(Status.engines["catapult"], undefined);
+ do_check_true(yield EHTestsCommon.setUp(server));
+ Service.sync();
+ yield deferred.promise;
+});