karma-sauce-launcher disconnects every test run resulting in failed runs with IE and Safari - karma-runner

Question

I have been having this exact issue for months now, through multiple versions of each component involved (karma, socket.io, sauce-connect). Currently I'm running on Karma 12.16.
Basically, when using https://github.com/karma-runner/karma-sauce-launcher, Safari and IE browsers launched disconnect every single test run resulting in only a partial test run. The relevant Karma log is
Driving the web on session: abf7c976abf1420a9c72b3e06c655040
DEBUG [wd]: > RESPONSE init({"base":"SauceLabs","browserName":"internet explorer","platform":"Windows 8.1","version":"11","tags":[],"name":"Karma and Sauce Labs demo","record-video":false,"record-screenshots":true,"build":null,"device-orientation":null,"disable-popup-handler":true}) "abf7c976abf1420a9c72b3e06c655040",null
INFO [launcher.sauce]: internet explorer 11 (Windows 8.1) session at https://saucelabs.com/tests/abf7c976abf1420a9c72b3e06c655040
DEBUG [launcher.sauce]: WebDriver channel for internet explorer 11 (Windows 8.1) instantiated, opening http://localhost:9876/?id=15483642
DEBUG [wd]: > CALL get("http://localhost:9876/?id=15483642")
DEBUG [wd]: > POST /session/:sessionID/url {"url":"http://localhost:9876/?id=15483642"}
DEBUG [web-server]: serving: /Volumes/OSXSTORAGE/Users/a/Dropbox/a/WebstormProjects/a-ui/node_modules/karma/static/client.html
DEBUG [web-server]: serving: /Volumes/OSXSTORAGE/Users/a/Dropbox/a/WebstormProjects/a-ui/node_modules/karma/static/karma.js
DEBUG [wd]: > RESPONSE get("http://localhost:9876/?id=15483642")
DEBUG [karma]: A browser has connected on socket xXySXoeppVtPhAr8FVR4
INFO [IE 11.0.0 (Windows)]: Connected on socket xXySXoeppVtPhAr8FVR4 with id 15483642
DEBUG [launcher]: internet explorer 11 (Windows 8.1) on SauceLabs (id 15483642) captured in 17.693 secs
DEBUG [web-server]: serving: /Volumes/OSXSTORAGE/Users/a/Dropbox/a/WebstormProjects/a-ui/node_modules/karma/static/context.html
DEBUG [web-server]: serving (cached): /Volumes/OSXSTORAGE/Users/a/Dropbox/a/WebstormProjects/a-ui/src/components/angular/angular.js
................................................................................
..............DEBUG [karma]: A browser has connected on socket mGf0VzQ8DKdfW_NSFVR5
DEBUG [IE 11.0.0 (Windows)]: New connection mGf0VzQ8DKdfW_NSFVR5 (already have xXySXoeppVtPhAr8FVR4)
WARN [IE 11.0.0 (Windows)]: Disconnected (1 times), because no message in 10000 ms.
IE 11.0.0 (Windows): Executed 94 of 94 DISCONNECTED (11.481 secs / 0.757 secs)
DEBUG [launcher.sauce]: Shutting down the internet explorer 11 (Windows 8.1) driver
DEBUG [wd]: > CALL get("about:blank")
DEBUG [wd]: > POST /session/:sessionID/url {"url":"about:blank"}
DEBUG [karma]: Run complete, exitting.
DEBUG [launcher]: Disconnecting all browsers
DEBUG [wd]: > RESPONSE get("about:blank")
DEBUG [wd]: > CALL quit()
DEBUG [wd]: > DELETE /session/:sessionID
DEBUG [wd]:
Ending your web drivage..
DEBUG [wd]: > RESPONSE quit()
I have tried increasing browserNoActivityTimeout to 60000, but that just results in the tests failing after 60s instead of 10s. Every one of these failed has the following:
DEBUG [karma]: A browser has connected on socket mGf0VzQ8DKdfW_NSFVR5
DEBUG [IE 11.0.0 (Windows)]: New connection mGf0VzQ8DKdfW_NSFVR5 (already have xXySXoeppVtPhAr8FVR4)
WARN [IE 11.0.0 (Windows)]: Disconnected (1 times), because no message in 10000 ms.
IE 11.0.0 (Windows): Executed 94 of 94 DISCONNECTED (11.481 secs / 0.757 secs)
Depending on the browser, sometimes it will only execute 50 tests (Executed 50 of 94 DISCONNECTED), and on others it will execute all tests, but still be considered a failed test run. I have never seen this occur with Chrome or Firefox.
The exact same karma.conf file will run every time locally with no issues. Has anyone encountered this? I have reached out to saucelabs support, but they are starting to insist that this is a karma-sauce-launcher issue.
Karma.conf:
var fs = require('fs');
module.exports = function(config) {
// Use ENV vars on Travis and sauce.json locally to get credentials
if (!process.env.SAUCE_USERNAME) {
if (!fs.existsSync('sauce.json')) {
console.log('Create a sauce.json with your credentials based on the sauce-sample.json file.');
process.exit(1);
} else {
process.env.SAUCE_USERNAME = require('./../sauce').username;
process.env.SAUCE_ACCESS_KEY = require('./../sauce').accessKey;
}
}
// Browsers to run on Sauce Labs
var customLaunchers = {
sl_safari_ml: {
base: 'SauceLabs',
browserName: 'safari',
platform: 'OS X 10.9'
}
};
config.set({
// base path that will be used to resolve all patterns (eg. files, exclude)
basePath: '../',
// frameworks to use
// available frameworks: https://npmjs.org/browse/keyword/karma-adapter
frameworks: ['jasmine'],
// list of files / patterns to load in the browser
files: [
'src/components/jquery/dist/jquery.js',
'src/components/angular/angular.js',
'src/components/angular-mocks/angular-mocks.js',
'src/components/angular-cookies/angular-cookies.js',
'src/components/angular-sanitize/angular-sanitize.js',
'src/components/angular-ui-router/release/angular-ui-router.js',
'src/components/angular-animate/angular-animate.js',
'src/components/angular-strap/dist/angular-strap.js',
'src/components/angular-strap/dist/angular-strap.tpl.js',
'src/components/speakingurl/lib/index.js',
'src/app/**/*.js',
'src/app/common/templates/*.html'
],
exclude: [
'src/app/**/*.e2e.spec.js'
],
ngHtml2JsPreprocessor: {
stripPrefix: 'src/app/common/',
moduleName: 'mywire.templates'
},
preprocessors: {
// load templates
'src/app/common/templates/*.html': ['ng-html2js']
},
// test results reporter to use
// possible values: 'dots', 'progress'
// available reporters: https://npmjs.org/browse/keyword/karma-reporter
reporters: ['dots'],
// web server port
port: 9876,
colors: true,
// level of logging
// possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG
logLevel: config.LOG_INFO,
sauceLabs: {
testName: 'Karma and Sauce Labs demo',
startConnect: false
},
browserNoActivityTimeout: 60000,
captureTimeout: 120000,
customLaunchers: customLaunchers,
// start these browsers
// available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
browsers: Object.keys(customLaunchers),
singleRun: true
});
};

Answer 1

Starting Karma version 0.12 there are some more timeout settings added.
Adding them to my karma config resolved this issue for me.
browserDisconnectTimeout
Type: Number
Default: 2000
Description: How long does Karma wait for
a browser to reconnect (in ms). With a flaky connection it is pretty
common that browser disconnects but the actual execution is running
without any problems. Karma does not treat a disconnection as
immediate failure and rather wait browserDisconnectTimeout ms. If the
browser reconnect during that time, everything is fine.
-
browserDisconnectTolerance
Type: Number
Default: 0
Description: The number of disconnections tolerated.
The disconnectTolerance value represents the maximum number of tries a browser will attempt in case of disconnection. Usually any disconnection is considered as a failure, but this option allows to define a tolerance level when there is a flaky network link between the karma server and the browsers.
-
browserNoActivityTimeout
Type: Number
Default: 10000
Description: How long does Karma wait for a message from a browser before disconnecting it (in ms).
If, during the execution, Karma does not receive any message from a browser within browserNoActivityTimeout ms, it will disconnect the browser.
source: http://karma-runner.github.io/0.12/config/configuration-file.html
more tips: http://oligofren.wordpress.com/2014/05/27/running-karma-tests-on-browserstack/