builder: mozilla-aurora-linux64
slave: bld-linux64-spot-266
starttime: 1457392972.99
results: exception (4)
buildid: 20160307152238
builduid: 2a9ccb62bcad4529a38346460a0cc034
revision: 9a446438d3a4e9279b776d3e296e7a7209acab77

========= Started set props: master (results: 0, elapsed: 0 secs) (at 2016-03-07 15:22:52.992640) =========
master: http://buildbot-master77.bb.releng.use1.mozilla.com:8001/
========= Finished set props: master (results: 0, elapsed: 0 secs) (at 2016-03-07 15:22:52.993067) =========

========= Started set props: basedir (results: 0, elapsed: 0 secs) (at 2016-03-07 15:22:52.993381) =========
bash -c pwd
 in dir /builds/slave/m-aurora-l64-00000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['bash', '-c', 'pwd']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-266.build.releng.use1.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/m-aurora-l64-00000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
/builds/slave/m-aurora-l64-00000000000000000
program finished with exit code 0
elapsedTime=0.004289
basedir: '/builds/slave/m-aurora-l64-00000000000000000'
========= master_lag: 0.18 =========
========= Finished set props: basedir (results: 0, elapsed: 0 secs) (at 2016-03-07 15:22:53.172980) =========

========= Started downloading to buildprops.json (results: 0, elapsed: 0 secs) (at 2016-03-07 15:22:53.173310) =========
========= Finished downloading to buildprops.json (results: 0, elapsed: 0 secs) (at 2016-03-07 15:22:53.202134) =========

========= Started 'rm -rf ...' (results: 0, elapsed: 0 secs) (at 2016-03-07 15:22:53.202527) =========
rm -rf properties
 in dir /builds/slave/m-aurora-l64-00000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['rm', '-rf', 'properties']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-266.build.releng.use1.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/m-aurora-l64-00000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.003476
========= master_lag: 0.04 =========
========= Finished 'rm -rf ...' (results: 0, elapsed: 0 secs) (at 2016-03-07 15:22:53.249953) =========

========= Started set props: script_repo_url (results: 0, elapsed: 0 secs) (at 2016-03-07 15:22:53.250285) =========
script_repo_url: https://hg.mozilla.org/build/mozharness
========= Finished set props: script_repo_url (results: 0, elapsed: 0 secs) (at 2016-03-07 15:22:53.250639) =========

========= Started 'rm -rf ...' (results: 0, elapsed: 0 secs) (at 2016-03-07 15:22:53.250914) =========
rm -rf scripts
 in dir /builds/slave/m-aurora-l64-00000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['rm', '-rf', 'scripts']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-266.build.releng.use1.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/m-aurora-l64-00000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.003381
========= master_lag: 0.04 =========
========= Finished 'rm -rf ...' (results: 0, elapsed: 0 secs) (at 2016-03-07 15:22:53.298807) =========

========= Started 'bash -c ...' exception (results: 4, elapsed: 12 mins, 42 secs) (at 2016-03-07 15:22:53.299155) =========
bash -c 'python /tools/checkouts/build-tools/buildfarm/utils/archiver_client.py mozharness --repo releases/mozilla-aurora --rev 9a446438d3a4e9279b776d3e296e7a7209acab77 --destination scripts --debug'
 in dir /builds/slave/m-aurora-l64-00000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['bash', '-c', u'python /tools/checkouts/build-tools/buildfarm/utils/archiver_client.py mozharness --repo releases/mozilla-aurora --rev 9a446438d3a4e9279b776d3e296e7a7209acab77 --destination scripts --debug']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-266.build.releng.use1.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/m-aurora-l64-00000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
2016-03-07 15:22:53,347 truncating revision to first 12 chars
2016-03-07 15:22:53,347 Setting DEBUG logging.
2016-03-07 15:22:53,347 attempt 1/10
2016-03-07 15:22:53,347 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/releases/mozilla-aurora/9a446438d3a4?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2016-03-07 15:22:53,821 attempt 1/10
2016-03-07 15:22:54,446 current task status: no status available at this point. state: PENDING
2016-03-07 15:22:54,446 sleeping for 9.00s (attempt 1/10)
2016-03-07 15:23:03,456 attempt 2/10
2016-03-07 15:23:04,039 current task status: no status available at this point. state: PENDING
2016-03-07 15:23:04,039 sleeping for 14.50s (attempt 2/10)
2016-03-07 15:23:18,554 attempt 3/10
2016-03-07 15:23:19,166 current task status: no status available at this point. state: PENDING
2016-03-07 15:23:19,166 sleeping for 22.75s (attempt 3/10)
2016-03-07 15:23:41,939 attempt 4/10
2016-03-07 15:23:42,512 current task status: no status available at this point. state: PENDING
2016-03-07 15:23:42,512 sleeping for 34.12s (attempt 4/10)
2016-03-07 15:24:16,672 attempt 5/10
2016-03-07 15:24:17,273 current task status: no status available at this point. state: PENDING
2016-03-07 15:24:17,273 sleeping for 51.19s (attempt 5/10)
2016-03-07 15:25:08,478 attempt 6/10
2016-03-07 15:25:09,073 current task status: Task has expired from pending for too long. Re-creating task. state: RETRY
2016-03-07 15:25:09,074 sleeping for 76.78s (attempt 6/10)
2016-03-07 15:26:25,932 attempt 7/10
2016-03-07 15:26:26,315 sleeping for 11.00s (attempt 1/10)
2016-03-07 15:26:37,326 attempt 2/10
2016-03-07 15:26:37,326 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/releases/mozilla-aurora/9a446438d3a4?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2016-03-07 15:26:37,706 sleeping for 17.50s (attempt 2/10)
2016-03-07 15:26:55,224 attempt 3/10
2016-03-07 15:26:55,225 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/releases/mozilla-aurora/9a446438d3a4?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2016-03-07 15:26:55,603 sleeping for 26.25s (attempt 3/10)
2016-03-07 15:27:21,880 attempt 4/10
2016-03-07 15:27:21,880 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/releases/mozilla-aurora/9a446438d3a4?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2016-03-07 15:27:22,259 sleeping for 39.38s (attempt 4/10)
2016-03-07 15:28:01,674 attempt 5/10
2016-03-07 15:28:01,674 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/releases/mozilla-aurora/9a446438d3a4?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2016-03-07 15:28:02,053 sleeping for 60.06s (attempt 5/10)
2016-03-07 15:29:02,176 attempt 6/10
2016-03-07 15:29:02,176 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/releases/mozilla-aurora/9a446438d3a4?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2016-03-07 15:29:02,557 sleeping for 90.09s (attempt 6/10)
2016-03-07 15:30:32,741 attempt 7/10
2016-03-07 15:30:32,742 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/releases/mozilla-aurora/9a446438d3a4?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2016-03-07 15:30:33,125 sleeping for 101.00s (attempt 7/10)
2016-03-07 15:32:14,225 attempt 8/10
2016-03-07 15:32:14,225 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/releases/mozilla-aurora/9a446438d3a4?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2016-03-07 15:32:14,602 sleeping for 99.00s (attempt 8/10)
2016-03-07 15:33:53,702 attempt 9/10
2016-03-07 15:33:53,702 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/releases/mozilla-aurora/9a446438d3a4?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2016-03-07 15:33:54,080 sleeping for 101.00s (attempt 9/10)
2016-03-07 15:35:35,180 attempt 10/10
2016-03-07 15:35:35,180 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/releases/mozilla-aurora/9a446438d3a4?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2016-03-07 15:35:35,563 Automation Error: could not determine a valid url response.
Traceback (most recent call last):
  File "/tools/checkouts/build-tools/buildfarm/utils/archiver_client.py", line 127, in get_url_response
    response = urllib2.urlopen(api_url, timeout=60)
  File "/usr/lib64/python2.6/urllib2.py", line 126, in urlopen
    return _opener.open(url, data, timeout)
  File "/usr/lib64/python2.6/urllib2.py", line 397, in open
    response = meth(req, response)
  File "/usr/lib64/python2.6/urllib2.py", line 510, in http_response
    'http', request, response, code, msg, hdrs)
  File "/usr/lib64/python2.6/urllib2.py", line 435, in error
    return self._call_chain(*args)
  File "/usr/lib64/python2.6/urllib2.py", line 369, in _call_chain
    result = func(*args)
  File "/usr/lib64/python2.6/urllib2.py", line 518, in http_error_default
    raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
HTTPError: HTTP Error 500: Internal Server Error
program finished with exit code 3
elapsedTime=762.267992
========= master_lag: 0.01 =========
========= Finished 'bash -c ...' exception (results: 4, elapsed: 12 mins, 42 secs) (at 2016-03-07 15:35:35.579203) =========

========= Skipped  (results: not started, elapsed: not started) =========
========= Skipped  (results: not started, elapsed: not started) =========
========= Skipped  (results: not started, elapsed: not started) =========
========= Skipped  (results: not started, elapsed: not started) =========
========= Skipped  (results: not started, elapsed: not started) =========
========= Skipped  (results: not started, elapsed: not started) =========
========= Skipped  (results: not started, elapsed: not started) =========
========= Started no change (results: 0, elapsed: 0 secs) (at 2016-03-07 15:35:35.580127) =========
bash -c 'for file in `ls -1`; do cat $file; done'
 in dir /builds/slave/m-aurora-l64-00000000000000000/properties (timeout 1200 secs)
 watching logfiles {}
 argv: ['bash', '-c', 'for file in `ls -1`; do cat $file; done']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-266.build.releng.use1.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/m-aurora-l64-00000000000000000/properties
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.005029

========= master_lag: 0.04 =========
========= Finished no change (results: 0, elapsed: 0 secs) (at 2016-03-07 15:35:35.627991) =========

========= Started 'rm -f ...' (results: 0, elapsed: 0 secs) (at 2016-03-07 15:35:35.628309) =========
rm -f oauth.txt
 in dir /builds/slave/m-aurora-l64-00000000000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['rm', '-f', 'oauth.txt']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-266.build.releng.use1.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/m-aurora-l64-00000000000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.002916
========= master_lag: 0.04 =========
========= Finished 'rm -f ...' (results: 0, elapsed: 0 secs) (at 2016-03-07 15:35:35.674556) =========

========= Started reboot skipped (results: 3, elapsed: 0 secs) (at 2016-03-07 15:35:35.674829) =========
========= Finished reboot skipped (results: 3, elapsed: 0 secs) (at 2016-03-07 15:35:35.675081) =========

========= Total master_lag: 0.36 =========