Add a retry mechanism for the ADB command "root”. (#690)

Error "adb: unable to connect for root: closed" often occurs if `adb root` is called right after `adb reboot`. Add a retry for `adb root` to avoid this.
diff --git a/mobly/controllers/android_device_lib/adb.py b/mobly/controllers/android_device_lib/adb.py
index 05c2af0..fbcfd7d 100644
--- a/mobly/controllers/android_device_lib/adb.py
+++ b/mobly/controllers/android_device_lib/adb.py
@@ -28,6 +28,11 @@
 # do with port forwarding must happen under this lock.
 ADB_PORT_LOCK = threading.Lock()
 
+# Number of attempts to execute "adb root", and seconds for interval time of
+# this commands.
+ADB_ROOT_RETRY_ATTMEPTS = 3
+ADB_ROOT_RETRY_ATTEMPT_INTERVAL_SEC = 10
+
 # Qualified class name of the default instrumentation test runner.
 DEFAULT_INSTRUMENTATION_RUNNER = 'com.android.common.support.test.runner.AndroidJUnitRunner'
 
@@ -456,6 +461,38 @@
             return self._execute_adb_and_process_stdout(
                 'shell', instrumentation_command, shell=False, handler=handler)
 
+    def root(self):
+        """Enables ADB root mode on the device.
+
+        This method will retry to execute the command `adb root` when an
+        AdbError occurs, since sometimes the error `adb: unable to connect
+        for root: closed` is raised when executing `adb root` immediately after
+        the device is booted to OS.
+
+        Returns:
+            A string that is the stdout of root command.
+
+        Raises:
+            AdbError: If the command exit code is not 0.
+        """
+        for attempt in range(ADB_ROOT_RETRY_ATTMEPTS):
+            try:
+                return self._exec_adb_cmd('root',
+                                          args=None,
+                                          shell=False,
+                                          timeout=None,
+                                          stderr=None)
+            except AdbError as e:
+                if attempt + 1 < ADB_ROOT_RETRY_ATTMEPTS:
+                    logging.debug(
+                      'Retry the command "%s" since Error "%s" occurred.' %
+                      (utils.cli_cmd_to_string(e.cmd),
+                       e.stderr.decode('utf-8').strip()))
+                    # Buffer between "adb root" commands.
+                    time.sleep(ADB_ROOT_RETRY_ATTEMPT_INTERVAL_SEC)
+                else:
+                  raise e
+
     def __getattr__(self, name):
         def adb_call(args=None, shell=False, timeout=None, stderr=None):
             """Wrapper for an ADB command.
diff --git a/tests/mobly/controllers/android_device_lib/adb_test.py b/tests/mobly/controllers/android_device_lib/adb_test.py
index e4180cf..e920bbd 100755
--- a/tests/mobly/controllers/android_device_lib/adb_test.py
+++ b/tests/mobly/controllers/android_device_lib/adb_test.py
@@ -40,6 +40,12 @@
                                         '.instrumentation.tests/com.android'
                                         '.common.support.test.runner'
                                         '.AndroidJUnitRunner')
+
+# Mock root command outputs.
+MOCK_ROOT_SUCCESS_OUTPUT = 'adbd is already running as root'
+MOCK_ROOT_ERROR_OUTPUT = (
+    'adb: unable to connect for root: closed'.encode('utf-8'))
+
 # Mock Shell Command
 MOCK_SHELL_COMMAND = 'ls'
 MOCK_COMMAND_OUTPUT = '/system/bin/ls'.encode('utf-8')
@@ -712,6 +718,54 @@
             self.assertEqual(stderr,
                              mock_execute_and_process_stdout.return_value)
 
+    @mock.patch.object(adb.AdbProxy, '_exec_cmd')
+    def test_root_success(self, mock_exec_cmd):
+        mock_exec_cmd.return_value = MOCK_ROOT_SUCCESS_OUTPUT
+        output = adb.AdbProxy().root()
+        mock_exec_cmd.assert_called_once_with(
+            ['adb', 'root'],
+            shell=False,
+            timeout=None,
+            stderr=None)
+        self.assertEqual(output, MOCK_ROOT_SUCCESS_OUTPUT)
+
+    @mock.patch('time.sleep', return_value=mock.MagicMock())
+    @mock.patch.object(adb.AdbProxy, '_exec_cmd')
+    def test_root_success_with_retry(self, mock_exec_cmd, mock_sleep):
+        mock_exec_cmd.side_effect = [
+            adb.AdbError('adb root', '', MOCK_ROOT_ERROR_OUTPUT, 1),
+            MOCK_ROOT_SUCCESS_OUTPUT]
+        output = adb.AdbProxy().root()
+        mock_exec_cmd.assert_called_with(
+            ['adb', 'root'],
+            shell=False,
+            timeout=None,
+            stderr=None)
+        self.assertEqual(output, MOCK_ROOT_SUCCESS_OUTPUT)
+        self.assertEqual(mock_sleep.call_count, 1)
+        mock_sleep.assert_called_with(10)
+
+    @mock.patch('time.sleep', return_value=mock.MagicMock())
+    @mock.patch.object(adb.AdbProxy, '_exec_cmd')
+    def test_root_raises_adb_error_when_all_retries_failed(self, mock_exec_cmd,
+                                                           mock_sleep):
+        mock_exec_cmd.side_effect = adb.AdbError('adb root',
+                                                 '',
+                                                 MOCK_ROOT_ERROR_OUTPUT,
+                                                 1)
+        expected_msg = ('Error executing adb cmd "adb root". '
+                        'ret: 1, stdout: , stderr: %s' %
+                        MOCK_ROOT_ERROR_OUTPUT)
+        with self.assertRaisesRegex(adb.AdbError, expected_msg):
+            adb.AdbProxy().root()
+            mock_exec_cmd.assert_called_with(
+                ['adb', 'root'],
+                shell=False,
+                timeout=None,
+                stderr=None)
+        self.assertEqual(mock_sleep.call_count, 2)
+        mock_sleep.assert_called_with(10)
+
     def test_has_shell_command_called_correctly(self):
         with mock.patch.object(adb.AdbProxy, '_exec_cmd') as mock_exec_cmd:
             mock_exec_cmd.return_value = MOCK_DEFAULT_COMMAND_OUTPUT