Make perf metrics collection & output via instrumentation simpler

- Added includeDetailedStats metadata to TimedTest for collecting additional
  metrics like memory usage and binder transactions
- Added PerformanceCollectorTestCase interface for hard-typing test class as
  performance test while still able to inherit from instrumenation test classes
- Reverted previous changes to PerformanceTestBase, will deprecate
  PerformanceTestBase/Case in a future CL
- Removed 'performance.' prefix from keys written to instrumentation output

test-runner/android/test/AndroidTestRunner.java

@@ -133,7 +133,7 @@ public class AndroidTestRunner extends BaseTestRunner {
         }
         return new TestResult();
     }
-    
+
     void setSkipExecution(boolean skip) {
         mSkipExecution = skip;
     }
@@ -165,7 +165,7 @@ public class AndroidTestRunner extends BaseTestRunner {
         for (TestCase testCase : mTestCases) {
             setContextIfAndroidTestCase(testCase, mContext, testContext);
             setInstrumentationIfInstrumentationTestCase(testCase, mInstrumentation);
-            setPerformanceWriterIfPerformanceTestCase(testCase, mPerfWriter);
+            setPerformanceWriterIfPerformanceCollectorTestCase(testCase, mPerfWriter);
             testCase.run(mTestResult);
         }
     }
@@ -188,10 +188,10 @@ public class AndroidTestRunner extends BaseTestRunner {
         }
     }
 
-    private void setPerformanceWriterIfPerformanceTestCase(
+    private void setPerformanceWriterIfPerformanceCollectorTestCase(
             Test test, PerformanceResultsWriter writer) {
-        if (PerformanceTestBase.class.isAssignableFrom(test.getClass())) {
-            ((PerformanceTestBase) test).setPerformanceResultsWriter(writer);
+        if (PerformanceCollectorTestCase.class.isAssignableFrom(test.getClass())) {
+            ((PerformanceCollectorTestCase) test).setPerformanceResultsWriter(writer);
         }
     }
 

test-runner/android/test/InstrumentationTestRunner.java

@@ -27,8 +27,6 @@ import android.os.Debug;
 import android.os.Looper;
 import android.os.Parcelable;
 import android.os.PerformanceCollector;
-import android.os.Process;
-import android.os.SystemClock;
 import android.os.PerformanceCollector.PerformanceResultsWriter;
 import android.test.suitebuilder.TestMethod;
 import android.test.suitebuilder.TestPredicates;
@@ -226,23 +224,6 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu
      * identifies the path to the generated code coverage file.
      */
     private static final String REPORT_KEY_COVERAGE_PATH = "coverageFilePath";
-    /**
-     * If included at the start of reporting keys, this prefix marks the key as a performance
-     * metric.
-     */
-    private static final String REPORT_KEY_PREFIX = "performance.";
-    /**
-     * If included in the status or final bundle sent to an IInstrumentationWatcher, this key
-     * reports the cpu time in milliseconds of the current test.
-     */
-    private static final String REPORT_KEY_PERF_CPU_TIME =
-        REPORT_KEY_PREFIX + PerformanceCollector.METRIC_KEY_CPU_TIME;
-    /**
-     * If included in the status or final bundle sent to an IInstrumentationWatcher, this key
-     * reports the run time in milliseconds of the current test.
-     */
-    private static final String REPORT_KEY_PERF_EXECUTION_TIME =
-        REPORT_KEY_PREFIX + PerformanceCollector.METRIC_KEY_EXECUTION_TIME;
 
     /**
      * The test is starting.
@@ -630,9 +611,9 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu
         int mTestNum = 0;
         int mTestResultCode = 0;
         String mTestClass = null;
+        PerformanceCollector mPerfCollector = new PerformanceCollector();
         boolean mIsTimedTest = false;
-        long mCpuTime = 0;
-        long mExecTime = 0;
+        boolean mIncludeDetailedStats = false;
 
         public WatcherResultPrinter(int numTests) {
             mResultTemplate = new Bundle();
@@ -675,20 +656,28 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu
             mTestResultCode = 0;
 
             mIsTimedTest = false;
+            mIncludeDetailedStats = false;
             try {
-                // Look for TimedTest annotation on both test class and test
-                // method
-                mIsTimedTest = test.getClass().isAnnotationPresent(TimedTest.class) ||
-                    test.getClass().getMethod(testName).isAnnotationPresent(TimedTest.class);
+                // Look for TimedTest annotation on both test class and test method
+                if (test.getClass().getMethod(testName).isAnnotationPresent(TimedTest.class)) {
+                    mIsTimedTest = true;
+                    mIncludeDetailedStats = test.getClass().getMethod(testName).getAnnotation(
+                            TimedTest.class).includeDetailedStats();
+                } else if (test.getClass().isAnnotationPresent(TimedTest.class)) {
+                    mIsTimedTest = true;
+                    mIncludeDetailedStats = test.getClass().getAnnotation(
+                            TimedTest.class).includeDetailedStats();
+                }
             } catch (SecurityException e) {
                 throw new IllegalStateException(e);
             } catch (NoSuchMethodException e) {
                 throw new IllegalStateException(e);
             }
 
-            if (mIsTimedTest) {
-                mExecTime = SystemClock.uptimeMillis();
-                mCpuTime = Process.getElapsedCpuTime();
+            if (mIsTimedTest && mIncludeDetailedStats) {
+                mPerfCollector.beginSnapshot("");
+            } else if (mIsTimedTest) {
+                mPerfCollector.startTiming("");
             }
         }
 
@@ -720,11 +709,10 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu
          * @see junit.framework.TestListener#endTest(Test)
          */
         public void endTest(Test test) {
-            if (mIsTimedTest) {
-                mCpuTime = Process.getElapsedCpuTime() - mCpuTime;
-                mExecTime = SystemClock.uptimeMillis() - mExecTime;
-                mTestResult.putLong(REPORT_KEY_PERF_CPU_TIME, mCpuTime);
-                mTestResult.putLong(REPORT_KEY_PERF_EXECUTION_TIME, mExecTime);
+            if (mIsTimedTest && mIncludeDetailedStats) {
+                mTestResult.putAll(mPerfCollector.endSnapshot());
+            } else if (mIsTimedTest) {
+                writeStopTiming(mPerfCollector.stopTiming(""));
             }
 
             if (mTestResultCode == 0) {
@@ -760,7 +748,7 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu
             for (Parcelable p :
                     results.getParcelableArrayList(PerformanceCollector.METRIC_KEY_ITERATIONS)) {
                 Bundle iteration = (Bundle)p;
-                String index = "performance.iteration" + i + ".";
+                String index = "iteration" + i + ".";
                 mTestResult.putString(index + PerformanceCollector.METRIC_KEY_LABEL,
                         iteration.getString(PerformanceCollector.METRIC_KEY_LABEL));
                 mTestResult.putLong(index + PerformanceCollector.METRIC_KEY_CPU_TIME,
@@ -772,15 +760,15 @@ public class InstrumentationTestRunner extends Instrumentation implements TestSu
         }
 
         public void writeMeasurement(String label, long value) {
-            mTestResult.putLong(REPORT_KEY_PREFIX + label, value);
+            mTestResult.putLong(label, value);
         }
 
         public void writeMeasurement(String label, float value) {
-            mTestResult.putFloat(REPORT_KEY_PREFIX + label, value);
+            mTestResult.putFloat(label, value);
         }
 
         public void writeMeasurement(String label, String value) {
-            mTestResult.putString(REPORT_KEY_PREFIX + label, value);
+            mTestResult.putString(label, value);
         }
 
         // TODO report the end of the cycle

test-runner/android/test/PerformanceCollectorTestCase.java

+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.test;
+
+import android.os.PerformanceCollector;
+import android.os.PerformanceCollector.PerformanceResultsWriter;
+
+/**
+ * A simple interface for passing in a PerformanceResultsWriter instance to be used with
+ * PerformanceCollector.
+ * <p/>
+ * A one line implementation of {@link #setPerformanceResultsWriter(PerformanceResultsWriter)}
+ * is sufficient in most cases:
+ * <p/>
+ * <code>mPerfCollector.setPerformanceResultsWriter(writer);</code>
+ *
+ * {@hide} Not needed for SDK.
+ */
+public interface PerformanceCollectorTestCase {
+    public PerformanceCollector mPerfCollector = new PerformanceCollector();
+
+    public void setPerformanceResultsWriter(PerformanceResultsWriter writer);
+}

test-runner/android/test/PerformanceTestBase.java

@@ -16,95 +16,12 @@
 
 package android.test;
 
-import android.os.Bundle;
-import android.os.PerformanceCollector;
-import android.os.PerformanceCollector.PerformanceResultsWriter;
-
-import java.lang.reflect.Method;
+import junit.framework.TestCase;
 
 /**
- * Provides hooks and wrappers to automatically and manually collect and report
- * performance data in tests.
- *
- * {@hide} Pending approval for public API.
+ * {@hide} Not needed for SDK.
  */
-public class PerformanceTestBase extends InstrumentationTestCase implements PerformanceTestCase {
-
-    private static PerformanceCollector sPerfCollector = new PerformanceCollector();
-    private static int sNumTestMethods = 0;
-    private static int sNumTestMethodsLeft = 0;
-
-    // Count number of tests, used to emulate beforeClass and afterClass from JUnit4
-    public PerformanceTestBase() {
-        if (sNumTestMethods == 0) {
-            Method methods[] = getClass().getMethods();
-            for (Method m : methods) {
-                if (m.getName().startsWith("test")) {
-                    sNumTestMethods ++;
-                    sNumTestMethodsLeft ++;
-                }
-            }
-        }
-    }
-
-    @Override
-    protected void setUp() throws Exception {
-        super.setUp();
-        // @beforeClass
-        // Will skew timing measured by TestRunner, but not by PerformanceCollector
-        if (sNumTestMethodsLeft == sNumTestMethods) {
-            sPerfCollector.beginSnapshot(this.getClass().getName());
-        }
-    }
-
-    @Override
-    protected void tearDown() throws Exception {
-        // @afterClass
-        // Will skew timing measured by TestRunner, but not by PerformanceCollector
-        if (--sNumTestMethodsLeft == 0) {
-            sPerfCollector.endSnapshot();
-        }
-        super.tearDown();
-    }
-
-    public void setPerformanceResultsWriter(PerformanceResultsWriter writer) {
-        sPerfCollector.setPerformanceResultsWriter(writer);
-    }
-
-    /**
-     * @see PerformanceCollector#beginSnapshot(String)
-     */
-    protected void beginSnapshot(String label) {
-        sPerfCollector.beginSnapshot(label);
-    }
-
-    /**
-     * @see PerformanceCollector#endSnapshot()
-     */
-    protected Bundle endSnapshot() {
-        return sPerfCollector.endSnapshot();
-    }
-
-    /**
-     * @see PerformanceCollector#startTiming(String)
-     */
-    protected void startTiming(String label) {
-        sPerfCollector.startTiming(label);
-    }
-
-    /**
-     * @see PerformanceCollector#addIteration(String)
-     */
-    protected Bundle addIteration(String label) {
-        return sPerfCollector.addIteration(label);
-    }
-
-    /**
-     * @see PerformanceCollector#stopTiming(String)
-     */
-    protected Bundle stopTiming(String label) {
-        return sPerfCollector.stopTiming(label);
-    }
+public class PerformanceTestBase extends TestCase implements PerformanceTestCase {
 
     public int startPerformance(PerformanceTestCase.Intermediates intermediates) {
         return 0;

core/java/android/test/TimedTest.java → test-runner/android/test/TimedTest.java

@@ -20,13 +20,17 @@ import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 
 /**
- * This annotation can be used on an {@link junit.framework.TestCase}'s test
- * methods. When the annotation is present, the test method is timed and the
- * results written through instrumentation output. It can also be used on the
- * class itself, which is equivalent to tagging all test methods with this
- * annotation.
+ * This annotation can be used on an {@link junit.framework.TestCase}'s test methods. When the
+ * annotation is present, the test method is timed and the results written through instrumentation
+ * output. It can also be used on the class itself, which is equivalent to tagging all test methods
+ * with this annotation.
+ * <p/>
+ * Setting {@link #includeDetailedStats()} to true causes additional metrics such as memory usage
+ * and binder transactions to be gathered and written through instrumentation output.
  *
  * {@hide} Pending approval for public API.
  */
 @Retention(RetentionPolicy.RUNTIME)
-public @interface TimedTest { }
\ No newline at end of file
+public @interface TimedTest {
+    boolean includeDetailedStats() default false;
+}