cts: add TestLog tags to XML report

b/16959454

- When configured properly, TF can store logs and report the URLs where
  they are stored back to the test runner. So capture those URLs and
  write them to the XML report for future processing.

- CtsXmlResultReporter implements ILogSaver, which has a testLogSaved callback
  that gives the URL of the test log file. The URL to the test log is included
  in the test result XML when the test fails and when the feature is enabled.

- Add a new tag <TestLog> to represent selected logs that testLogSaved
  tells us about. Put all logic regarding this tag into the TestLog
  class. TestLogType is an enum that defines what specific logs we are
  interested in and how to identify them by examining the data name
  passed by the testLogSaved callback.

- Finally, add a "include-test-log-tags" flag to control whether these
  tags are enabled in the XML. This has the possibility of doubling the
  size of reports, so we will only enable this feature in the lab
  setting.

Change-Id: Iecfa26abeee3a4d4d36dc95e5d0bfb5cac411236

tools/tradefed-host/src/com/android/cts/tradefed/result/CtsXmlResultReporter.java

@@ -27,10 +27,13 @@ import com.android.tradefed.build.IFolderBuildInfo;
 import com.android.tradefed.config.Option;
 import com.android.tradefed.config.Option.Importance;
 import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.ILogSaver;
+import com.android.tradefed.result.ILogSaverListener;
 import com.android.tradefed.result.ITestInvocationListener;
 import com.android.tradefed.result.ITestSummaryListener;
 import com.android.tradefed.result.InputStreamSource;
 import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.result.LogFile;
 import com.android.tradefed.result.LogFileSaver;
 import com.android.tradefed.result.TestSummary;
 import com.android.tradefed.util.FileUtil;
@@ -54,7 +57,9 @@ import java.util.Map;
  * <p/>
  * Outputs xml in format governed by the cts_result.xsd
  */
-public class CtsXmlResultReporter implements ITestInvocationListener, ITestSummaryListener {
+public class CtsXmlResultReporter
+        implements ITestInvocationListener, ITestSummaryListener, ILogSaverListener {
+
     private static final String LOG_TAG = "CtsXmlResultReporter";
 
     static final String TEST_RESULT_FILE_NAME = "testResult.xml";
@@ -89,11 +94,15 @@ public class CtsXmlResultReporter implements ITestInvocationListener, ITestSumma
     @Option(name = "result-server", description = "Server to publish test results.")
     private String mResultServer;
 
+    @Option(name = "include-test-log-tags", description = "Include test log tags in XML report.")
+    private boolean mIncludeTestLogTags = false;
+
     protected IBuildInfo mBuildInfo;
     private String mStartTime;
     private String mDeviceSerial;
     private TestResults mResults = new TestResults();
     private TestPackageResult mCurrentPkgResult = null;
+    private Test mCurrentTest = null;
     private boolean mIsDeviceInfoRun = false;
     private ResultReporter mReporter;
     private File mLogDir;
@@ -104,6 +113,11 @@ public class CtsXmlResultReporter implements ITestInvocationListener, ITestSumma
         mReportDir = reportDir;
     }
 
+    /** Set whether to include TestLog tags in the XML reports. */
+    public void setIncludeTestLogTags(boolean include) {
+        mIncludeTestLogTags = include;
+    }
+
     /**
      * {@inheritDoc}
      */
@@ -210,6 +224,20 @@ public class CtsXmlResultReporter implements ITestInvocationListener, ITestSumma
         }
     }
 
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public void testLogSaved(String dataName, LogDataType dataType, InputStreamSource dataStream,
+            LogFile logFile) {
+        if (mIncludeTestLogTags && mCurrentTest != null) {
+            TestLog log = TestLog.fromDataName(dataName, logFile.getUrl());
+            if (log != null) {
+                mCurrentTest.addTestLog(log);
+            }
+        }
+    }
+
     /**
      * Return the {@link LogFileSaver} to use.
      * <p/>
@@ -219,6 +247,10 @@ public class CtsXmlResultReporter implements ITestInvocationListener, ITestSumma
         return new LogFileSaver(mLogDir);
     }
 
+    @Override
+    public void setLogSaver(ILogSaver logSaver) {
+      // Don't need to keep a reference to logSaver, because we don't save extra logs in this class.
+    }
 
     @Override
     public void testRunStarted(String id, int numTests) {
@@ -235,7 +267,7 @@ public class CtsXmlResultReporter implements ITestInvocationListener, ITestSumma
     @Override
     public void testStarted(TestIdentifier test) {
         if (!mIsDeviceInfoRun) {
-            mCurrentPkgResult.insertTest(test);
+            mCurrentTest = mCurrentPkgResult.insertTest(test);
         }
     }
 

tools/tradefed-host/src/com/android/cts/tradefed/result/Test.java

@@ -16,12 +16,15 @@
 package com.android.cts.tradefed.result;
 
 import com.android.ddmlib.Log;
+import com.android.cts.tradefed.result.TestLog.TestLogType;
 
 import org.kxml2.io.KXmlSerializer;
 import org.xmlpull.v1.XmlPullParser;
 import org.xmlpull.v1.XmlPullParserException;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Data structure that represents a "Test" result XML element.
@@ -57,6 +60,12 @@ class Test extends AbstractXmlPullParser {
     private String mSummary;
     private String mDetails;
 
+    /**
+     * Log info for this test like a logcat dump or bugreport.
+     * Use *Locked methods instead of mutating this directly.
+     */
+    private List<TestLog> mTestLogs;
+
     /**
      * Create an empty {@link Test}
      */
@@ -75,6 +84,13 @@ class Test extends AbstractXmlPullParser {
         updateEndTime();
     }
 
+    /**
+     * Add a test log to this Test.
+     */
+    public void addTestLog(TestLog testLog) {
+        addTestLogLocked(testLog);
+    }
+
     /**
      * Set the name of this {@link Test}
      */
@@ -157,6 +173,8 @@ class Test extends AbstractXmlPullParser {
         serializer.attribute(CtsXmlResultReporter.ns, STARTTIME_ATTR, mStartTime);
         serializer.attribute(CtsXmlResultReporter.ns, ENDTIME_ATTR, mEndTime);
 
+        serializeTestLogsLocked(serializer);
+
         if (mMessage != null) {
             serializer.startTag(CtsXmlResultReporter.ns, SCENE_TAG);
             serializer.attribute(CtsXmlResultReporter.ns, MESSAGE_ATTR, mMessage);
@@ -328,10 +346,38 @@ class Test extends AbstractXmlPullParser {
                 mMessage = getAttribute(parser, MESSAGE_ATTR);
             } else if (eventType == XmlPullParser.START_TAG && parser.getName().equals(STACK_TAG)) {
                 mStackTrace = parser.nextText();
+            } else if (eventType == XmlPullParser.START_TAG && TestLog.isTag(parser.getName())) {
+                parseTestLog(parser);
             } else if (eventType == XmlPullParser.END_TAG && parser.getName().equals(TAG)) {
                 return;
             }
             eventType = parser.next();
         }
     }
+
+    /** Parse a TestLog entry from the parser positioned at a TestLog tag. */
+    private void parseTestLog(XmlPullParser parser) throws XmlPullParserException{
+        TestLog log = TestLog.fromXml(parser);
+        if (log == null) {
+            throw new XmlPullParserException("invalid XML: bad test log tag");
+        }
+        addTestLog(log);
+    }
+
+    /** Add a TestLog to the test in a thread safe manner. */
+    private synchronized void addTestLogLocked(TestLog testLog) {
+        if (mTestLogs == null) {
+            mTestLogs = new ArrayList<>(TestLogType.values().length);
+        }
+        mTestLogs.add(testLog);
+    }
+
+    /** Serialize the TestLogs of this test in a thread safe manner. */
+    private synchronized void serializeTestLogsLocked(KXmlSerializer serializer) throws IOException {
+        if (mTestLogs != null) {
+            for (TestLog log : mTestLogs) {
+                log.serialize(serializer);
+            }
+        }
+    }
 }

tools/tradefed-host/src/com/android/cts/tradefed/result/TestLog.java

+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.cts.tradefed.result;
+
+import org.kxml2.io.KXmlSerializer;
+import org.xmlpull.v1.XmlPullParser;
+
+import java.io.IOException;
+
+import javax.annotation.Nullable;
+
+/**
+ * TestLog describes a log for a test. It corresponds to the "TestLog" XML element.
+ */
+class TestLog {
+
+    private static final String TAG = "TestLog";
+    private static final String TYPE_ATTR = "type";
+    private static final String URL_ATTR = "url";
+
+    /** Type of log. */
+    public enum TestLogType {
+        LOGCAT("logcat-"),
+        BUGREPORT("bug-"),
+
+        ;
+
+        // This enum restricts the type of logs reported back to the server,
+        // because we do not intend to support them all. Using an enum somewhat
+        // assures that we will only see these expected types on the server side.
+
+        /**
+         * Returns the TestLogType from an ILogSaver data name or null
+         * if the data name is not supported.
+         */
+        @Nullable
+        static TestLogType fromDataName(String dataName) {
+            if (dataName == null) {
+                return null;
+            }
+
+            for (TestLogType type : values()) {
+                if (dataName.startsWith(type.dataNamePrefix)) {
+                    return type;
+                }
+            }
+            return null;
+        }
+
+        private final String dataNamePrefix;
+
+        private TestLogType(String dataNamePrefix) {
+            this.dataNamePrefix = dataNamePrefix;
+        }
+
+        String getAttrValue() {
+            return name().toLowerCase();
+        }
+    }
+
+    /** Type of the log like LOGCAT or BUGREPORT. */
+    private final TestLogType mLogType;
+
+    /** Url pointing to the log file. */
+    private final String mUrl;
+
+    /** Create a TestLog from an ILogSaver callback. */
+    @Nullable
+    static TestLog fromDataName(String dataName, String url) {
+        TestLogType logType = TestLogType.fromDataName(dataName);
+        if (logType == null) {
+            return null;
+        }
+
+        if (url == null) {
+            return null;
+        }
+
+        return new TestLog(logType, url);
+    }
+
+    /** Create a TestLog from XML given a XmlPullParser positioned at the TestLog tag. */
+    @Nullable
+    static TestLog fromXml(XmlPullParser parser) {
+        String type = parser.getAttributeValue(null, TYPE_ATTR);
+        if (type == null) {
+            return null;
+        }
+
+        String url = parser.getAttributeValue(null, URL_ATTR);
+        if (url == null) {
+            return null;
+        }
+
+        try {
+            TestLogType logType = TestLogType.valueOf(type.toUpperCase());
+            return new TestLog(logType, url);
+        } catch (IllegalArgumentException e) {
+            return null;
+        }
+    }
+
+    /** Create a TestLog directly given a log type and url. */
+    public static TestLog of(TestLogType logType, String url) {
+        return new TestLog(logType, url);
+    }
+
+    private TestLog(TestLogType logType, String url) {
+        this.mLogType = logType;
+        this.mUrl = url;
+    }
+
+    /** Returns this TestLog's log type. */
+    TestLogType getLogType() {
+        return mLogType;
+    }
+
+    /** Returns this TestLog's URL. */
+    String getUrl() {
+        return mUrl;
+    }
+
+    /** Serialize the TestLog to XML. */
+    public void serialize(KXmlSerializer serializer) throws IOException {
+        serializer.startTag(CtsXmlResultReporter.ns, TAG);
+        serializer.attribute(CtsXmlResultReporter.ns, TYPE_ATTR, mLogType.getAttrValue());
+        serializer.attribute(CtsXmlResultReporter.ns, URL_ATTR, mUrl);
+        serializer.endTag(CtsXmlResultReporter.ns, TAG);
+    }
+
+    /** Returns true if the tag is a TestLog tag. */
+    public static boolean isTag(String tagName) {
+        return TAG.equals(tagName);
+    }
+}

tools/tradefed-host/tests/src/com/android/cts/tradefed/UnitTests.java

@@ -21,6 +21,7 @@ import com.android.cts.tradefed.result.TestPackageResultTest;
 import com.android.cts.tradefed.result.TestResultsTest;
 import com.android.cts.tradefed.result.TestSummaryXmlTest;
 import com.android.cts.tradefed.result.TestTest;
+import com.android.cts.tradefed.result.TestLogTest;
 import com.android.cts.tradefed.testtype.Abi;
 import com.android.cts.tradefed.testtype.CtsTestTest;
 import com.android.cts.tradefed.testtype.DeqpTestRunnerTest;
@@ -55,6 +56,7 @@ public class UnitTests extends TestSuite {
         addTestSuite(TestResultsTest.class);
         addTestSuite(TestSummaryXmlTest.class);
         addTestSuite(TestTest.class);
+        addTestSuite(TestLogTest.class);
 
         // testtype package
         addTestSuite(CtsTestTest.class);

tools/tradefed-host/tests/src/com/android/cts/tradefed/result/CtsXmlResultReporterTest.java

@@ -22,6 +22,8 @@ import com.android.cts.util.AbiUtils;
 import com.android.ddmlib.testrunner.TestIdentifier;
 import com.android.tradefed.build.IFolderBuildInfo;
 import com.android.tradefed.log.LogUtil.CLog;
+import com.android.tradefed.result.LogDataType;
+import com.android.tradefed.result.LogFile;
 import com.android.tradefed.result.TestSummary;
 import com.android.tradefed.result.XmlResultReporter;
 import com.android.tradefed.util.FileUtil;
@@ -162,6 +164,8 @@ public class CtsXmlResultReporterTest extends TestCase {
         mResultReporter.testFailed(testId, trace);
         mResultReporter.testEnded(testId, emptyMap);
         mResultReporter.testRunEnded(3, emptyMap);
+        mResultReporter.testLogSaved("logcat-foo-bar", LogDataType.TEXT, null,
+                new LogFile("path", "url"));
         mResultReporter.invocationEnded(1);
         String output = getOutput();
         // TODO: consider doing xml based compare
@@ -171,6 +175,37 @@ public class CtsXmlResultReporterTest extends TestCase {
                 "<FailedScene message=\"this is a trace&#10;more trace\">     " +
                 "<StackTrace>this is a tracemore traceyet more trace</StackTrace>";
         assertTrue(output.contains(failureTag));
+
+        // Check that no TestLog tags were added, because the flag wasn't enabled.
+        final String testLogTag = String.format("<TestLog type=\"logcat\" url=\"url\" />");
+        assertFalse(output, output.contains(testLogTag));
+    }
+
+    /**
+     * Test that flips the include-test-log-tags flag and checks that logs are written to the XML.
+     */
+    public void testIncludeTestLogTags() {
+        Map<String, String> emptyMap = Collections.emptyMap();
+        final TestIdentifier testId = new TestIdentifier("FooTest", "testFoo");
+        final String trace = "this is a trace\nmore trace\nyet more trace";
+
+        // Include TestLogTags in the XML.
+        mResultReporter.setIncludeTestLogTags(true);
+
+        mResultReporter.invocationStarted(mMockBuild);
+        mResultReporter.testRunStarted(AbiUtils.createId(UnitTests.ABI.getName(), "run"), 1);
+        mResultReporter.testStarted(testId);
+        mResultReporter.testFailed(testId, trace);
+        mResultReporter.testEnded(testId, emptyMap);
+        mResultReporter.testRunEnded(3, emptyMap);
+        mResultReporter.testLogSaved("logcat-foo-bar", LogDataType.TEXT, null,
+                new LogFile("path", "url"));
+        mResultReporter.invocationEnded(1);
+
+        // Check for TestLog tags because the flag was enabled via setIncludeTestLogTags.
+        final String output = getOutput();
+        final String testLogTag = String.format("<TestLog type=\"logcat\" url=\"url\" />");
+        assertTrue(output, output.contains(testLogTag));
     }
 
     public void testDeviceSetup() {

tools/tradefed-host/tests/src/com/android/cts/tradefed/result/TestLogTest.java

+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.cts.tradefed.result;
+
+import com.android.cts.tradefed.result.TestLog.TestLogType;
+
+import org.kxml2.io.KXmlSerializer;
+import org.xmlpull.v1.XmlPullParser;
+import org.xmlpull.v1.XmlPullParserException;
+import org.xmlpull.v1.XmlPullParserFactory;
+
+import junit.framework.TestCase;
+
+import java.io.StringReader;
+import java.io.StringWriter;
+
+/** Tests for {@link TestLog}. */
+public class TestLogTest extends TestCase {
+
+    public void testTestLogType_fromDataName() {
+        assertNull(TestLogType.fromDataName(null));
+        assertNull(TestLogType.fromDataName(""));
+        assertNull(TestLogType.fromDataName("kmsg-foo_bar_test"));
+
+        assertEquals(TestLogType.LOGCAT,
+            TestLogType.fromDataName("logcat-foo_bar_test"));
+        assertEquals(TestLogType.BUGREPORT,
+            TestLogType.fromDataName("bug-foo_bar_test"));
+    }
+
+    public void testTestLogType_getAttrValue() {
+        assertEquals("logcat", TestLogType.LOGCAT.getAttrValue());
+        assertEquals("bugreport", TestLogType.BUGREPORT.getAttrValue());
+    }
+
+    public void testFromDataName() {
+        TestLog log = TestLog.fromDataName("logcat-baz_test", "http://logs/baz_test");
+        assertEquals(TestLogType.LOGCAT, log.getLogType());
+        assertEquals("http://logs/baz_test", log.getUrl());
+    }
+
+    public void testFromDataName_unrecognizedDataName() {
+        assertNull(TestLog.fromDataName("kmsg-baz_test", null));
+    }
+
+    public void testFromDataName_nullDataName() {
+        assertNull(TestLog.fromDataName(null, "http://logs/baz_test"));
+    }
+
+    public void testFromDataName_nullUrl() {
+        assertNull(TestLog.fromDataName("logcat-bar_test", null));
+    }
+
+    public void testFromDataName_allNull() {
+        assertNull(TestLog.fromDataName(null, null));
+    }
+
+    public void testFromXml() throws Exception {
+        TestLog log = TestLog.fromXml(newXml("<TestLog type=\"logcat\" url=\"http://logs/baz_test\">"));
+        assertEquals(TestLogType.LOGCAT, log.getLogType());
+        assertEquals("http://logs/baz_test", log.getUrl());
+    }
+
+    public void testFromXml_unrecognizedType() throws Exception {
+        assertNull(TestLog.fromXml(newXml("<TestLog type=\"kmsg\" url=\"http://logs/baz_test\">")));
+    }
+
+    public void testFromXml_noTypeAttribute() throws Exception {
+        assertNull(TestLog.fromXml(newXml("<TestLog url=\"http://logs/baz_test\">")));
+    }
+
+    public void testFromXml_noUrlAttribute() throws Exception {
+        assertNull(TestLog.fromXml(newXml("<TestLog type=\"bugreport\">")));
+    }
+
+    public void testFromXml_allNull() throws Exception {
+        assertNull(TestLog.fromXml(newXml("<TestLog>")));
+    }
+
+    public void testSerialize() throws Exception {
+        KXmlSerializer serializer = new KXmlSerializer();
+        StringWriter writer = new StringWriter();
+        serializer.setOutput(writer);
+
+        TestLog log = TestLog.of(TestLogType.LOGCAT, "http://logs/foo/bar");
+        log.serialize(serializer);
+        assertEquals("<TestLog type=\"logcat\" url=\"http://logs/foo/bar\" />", writer.toString());
+    }
+
+    public void testIsTag() {
+        assertTrue(TestLog.isTag("TestLog"));
+        assertFalse(TestLog.isTag("TestResult"));
+    }
+
+    private XmlPullParser newXml(String xml) throws Exception {
+        XmlPullParserFactory factory = org.xmlpull.v1.XmlPullParserFactory.newInstance();
+        XmlPullParser parser = factory.newPullParser();
+        parser.setInput(new StringReader(xml));
+
+        // Move the parser from the START_DOCUMENT stage to the START_TAG of the data.
+        parser.next();
+
+        return parser;
+    }
+}