Reimport

tools/adb/systrace/catapult/common/lab/commits.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Print statistics about the rate of commits to a repository."""
+
+import datetime
+import itertools
+import json
+import math
+import urllib
+import urllib2
+
+
+_BASE_URL = 'https://chromium.googlesource.com'
+# Can be up to 10,000.
+_REVISION_COUNT = 10000
+
+_REPOSITORIES = [
+    'chromium/src',
+    'angle/angle',
+    'skia',
+    'v8/v8',
+]
+
+
+def Pairwise(iterable):
+  """s -> (s0,s1), (s1,s2), (s2, s3), ..."""
+  a, b = itertools.tee(iterable)
+  next(b, None)
+  return itertools.izip(a, b)
+
+
+def Percentile(data, percentile):
+  """Find a percentile of a list of values.
+
+  Parameters:
+    data: A sorted list of values.
+    percentile: The percentile to look up, from 0.0 to 1.0.
+
+  Returns:
+    The percentile.
+
+  Raises:
+    ValueError: If data is empty.
+  """
+  if not data:
+    raise ValueError()
+
+  k = (len(data) - 1) * percentile
+  f = math.floor(k)
+  c = math.ceil(k)
+
+  if f == c:
+    return data[int(k)]
+  return data[int(f)] * (c - k) + data[int(c)] * (k - f)
+
+
+def CommitTimes(repository, revision_count):
+  parameters = urllib.urlencode((('n', revision_count), ('format', 'JSON')))
+  url = '%s/%s/+log?%s' % (_BASE_URL, urllib.quote(repository), parameters)
+  data = json.loads(''.join(urllib2.urlopen(url).read().splitlines()[1:]))
+
+  commit_times = []
+  for revision in data['log']:
+    commit_time_string = revision['committer']['time']
+    commit_time = datetime.datetime.strptime(
+        commit_time_string, '%a %b %d %H:%M:%S %Y')
+    commit_times.append(commit_time - datetime.timedelta(hours=7))
+
+  return commit_times
+
+
+def IsWeekday(time):
+  return time.weekday() >= 0 and time.weekday() < 5
+
+
+def main():
+  for repository in _REPOSITORIES:
+    commit_times = CommitTimes(repository, _REVISION_COUNT)
+
+    commit_durations = []
+    for time1, time2 in Pairwise(commit_times):
+      #if not (IsWeekday(time1) and IsWeekday(time2)):
+      #  continue
+      commit_durations.append((time1 - time2).total_seconds() / 60.)
+    commit_durations.sort()
+
+    print 'REPOSITORY:', repository
+    print 'Start Date:', min(commit_times), 'PDT'
+    print '  End Date:', max(commit_times), 'PDT'
+    print '  Duration:', max(commit_times) - min(commit_times)
+    print '         n:', len(commit_times)
+
+    for p in (0.25, 0.50, 0.90):
+      percentile = Percentile(commit_durations, p)
+      print '%3d%% commit duration:' % (p * 100), '%6.1fm' % percentile
+    mean = math.fsum(commit_durations) / len(commit_durations)
+    print 'Mean commit duration:', '%6.1fm' % mean
+    print
+
+
+if __name__ == '__main__':
+  main()

tools/adb/systrace/catapult/common/lab/hardware.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Query build slave hardware info, and print it to stdout as csv."""
+
+import csv
+import json
+import logging
+import sys
+import urllib2
+
+
+_MASTERS = [
+    'chromium.perf',
+    'chromium.perf.fyi',
+    'client.catapult',
+    'tryserver.chromium.perf',
+    'tryserver.client.catapult',
+]
+
+
+_KEYS = [
+    'master', 'builder', 'hostname',
+
+    'os family', 'os version', 'bitness (userland)',
+
+    'product name', 'architecture', 'processor count', 'processor type',
+    'memory total',
+
+    'facter version', 'git version', 'puppet version', 'python version',
+    'ruby version',
+
+    'android device 1', 'android device 2', 'android device 3',
+    'android device 4', 'android device 5', 'android device 6',
+    'android device 7', 'android device 8',
+]
+_EXCLUDED_KEYS = frozenset([
+    'architecture (userland)',
+    'b directory',
+    'last puppet run',
+    'uptime',
+    'windows version',
+])
+
+
+def main():
+  writer = csv.DictWriter(sys.stdout, _KEYS)
+  writer.writeheader()
+
+  for master_name in _MASTERS:
+    master_data = json.load(urllib2.urlopen(
+        'http://build.chromium.org/p/%s/json/slaves' % master_name))
+
+    slaves = sorted(master_data.iteritems(),
+                    key=lambda x: (x[1]['builders'].keys(), x[0]))
+    for slave_name, slave_data in slaves:
+      for builder_name in slave_data['builders']:
+        row = {
+            'master': master_name,
+            'builder': builder_name,
+            'hostname': slave_name,
+        }
+
+        host_data = slave_data['host']
+        if host_data:
+          host_data = host_data.splitlines()
+          if len(host_data) > 1:
+            for line in host_data:
+              if not line:
+                continue
+              key, value = line.split(': ')
+              if key in _EXCLUDED_KEYS:
+                continue
+              row[key] = value
+
+        # Munge keys.
+        row = {key.replace('_', ' '): value for key, value in row.iteritems()}
+        if 'osfamily' in row:
+          row['os family'] = row.pop('osfamily')
+        if 'product name' not in row and slave_name.startswith('slave'):
+          row['product name'] = 'Google Compute Engine'
+
+        try:
+          writer.writerow(row)
+        except ValueError:
+          logging.error(row)
+          raise
+
+
+if __name__ == '__main__':
+  main()

tools/adb/systrace/catapult/common/lab/keychain_unlock.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Script to SSH into a list of bots and set up their keychains for Telemetry.
+# https://www.chromium.org/developers/telemetry/telemetry-mac-keychain-setup
+
+for hostname in "$@"
+do
+  ssh -t "$hostname" 'security unlock-keychain login.keychain
+security delete-generic-password -s "Chrome Safe Storage" login.keychain
+security add-generic-password -a Chrome -w "+NTclOvR4wLMgRlLIL9bHQ==" \
+  -s "Chrome Safe Storage" -A login.keychain'
+done