From ededcf9b1a4edd78f41211ca22c32f0bbff445e8 Mon Sep 17 00:00:00 2001 From: Atsushi Mori Date: Sun, 7 Jul 2019 11:25:46 +0900 Subject: [PATCH 1/7] Fix android flaky tests --- ci-jobs/functional/run_appium.yml | 2 +- ci-jobs/functional_test.yml | 56 ++++++++++++++++++- .../android/find_by_accessibility_id_tests.py | 18 ++++-- .../android/helper/desired_capabilities.py | 4 +- test/functional/android/helper/test_helper.py | 2 +- test/functional/android/touch_action_tests.py | 21 ++++--- test/functional/android/webdriver_tests.py | 10 ++-- test/functional/ios/execute_driver_tests.py | 3 +- 8 files changed, 86 insertions(+), 30 deletions(-) diff --git a/ci-jobs/functional/run_appium.yml b/ci-jobs/functional/run_appium.yml index f1772710..7cdc7f18 100644 --- a/ci-jobs/functional/run_appium.yml +++ b/ci-jobs/functional/run_appium.yml @@ -3,7 +3,7 @@ steps: inputs: versionSpec: '11.x' displayName: Install Node 11.x -- script: npm install -g appium --chromedriver_version='2.44' +- script: npm install -g appium@beta --chromedriver_version='2.44' displayName: Install appium - task: UsePythonVersion@0 inputs: diff --git a/ci-jobs/functional_test.yml b/ci-jobs/functional_test.yml index 14584dfc..710cb91d 100644 --- a/ci-jobs/functional_test.yml +++ b/ci-jobs/functional_test.yml @@ -1,6 +1,7 @@ parameters: vmImage: 'macOS-10.14' pytestOpt: '--doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html' + androidSdkVer: 28 jobs: - template: ./functional/run_ios_test.yml @@ -8,7 +9,7 @@ jobs: name: 'func_test_ios1' vmImage: ${{ parameters.vmImage }} pytestOpt: ${{ parameters.pytestOpt }} - testFiles: 'find_*.py remote_fs_tests.py safari_tests.py' + testFiles: 'find_*.py remote_fs_tests.py safari_tests.py execute_driver_tests.py' - template: ./functional/run_ios_test.yml parameters: name: 'func_test_ios2' @@ -20,5 +21,54 @@ jobs: name: 'func_test_android1' vmImage: ${{ parameters.vmImage }} pytestOpt: ${{ parameters.pytestOpt }} - testFiles: 'location_tests.py' - sdkVer: 28 + testFiles: 'device_time_tests.py find_by_accessibility_id_tests.py find_by_image_tests.py find_by_uiautomator_tests.py' + sdkVer: ${{ parameters.androidSdkVer }} + - template: ./functional/run_android_test.yml + parameters: + name: 'func_test_android2' + vmImage: ${{ parameters.vmImage }} + pytestOpt: ${{ parameters.pytestOpt }} + testFiles: 'hw_actions_tests.py ime_tests.py keyboard_tests.py location_tests.py' + sdkVer: ${{ parameters.androidSdkVer }} + - template: ./functional/run_android_test.yml + parameters: + name: 'func_test_android3' + vmImage: ${{ parameters.vmImage }} + pytestOpt: ${{ parameters.pytestOpt }} + testFiles: 'activities_tests.py' + sdkVer: ${{ parameters.androidSdkVer }} + - template: ./functional/run_android_test.yml + parameters: + name: 'func_test_android4' + vmImage: ${{ parameters.vmImage }} + pytestOpt: ${{ parameters.pytestOpt }} + testFiles: 'finger_print_tests.py screen_record_tests.py settings_tests.py' + sdkVer: ${{ parameters.androidSdkVer }} + - template: ./functional/run_android_test.yml + parameters: + name: 'func_test_android5' + vmImage: ${{ parameters.vmImage }} + pytestOpt: ${{ parameters.pytestOpt }} + testFiles: 'context_switching_tests.py remote_fs_tests.py' + sdkVer: ${{ parameters.androidSdkVer }} + - template: ./functional/run_android_test.yml + parameters: + name: 'func_test_android6' + vmImage: ${{ parameters.vmImage }} + pytestOpt: ${{ parameters.pytestOpt }} + testFiles: 'webdriver_tests.py' + sdkVer: ${{ parameters.androidSdkVer }} + - template: ./functional/run_android_test.yml + parameters: + name: 'func_test_android7' + vmImage: ${{ parameters.vmImage }} + pytestOpt: ${{ parameters.pytestOpt }} + testFiles: 'applications_tests.py' + sdkVer: ${{ parameters.androidSdkVer }} + - template: ./functional/run_android_test.yml + parameters: + name: 'func_test_android8' + vmImage: ${{ parameters.vmImage }} + pytestOpt: ${{ parameters.pytestOpt }} + testFiles: 'network_connection_tests.py' + sdkVer: ${{ parameters.androidSdkVer }} diff --git a/test/functional/android/find_by_accessibility_id_tests.py b/test/functional/android/find_by_accessibility_id_tests.py index e66868e1..41ba2488 100644 --- a/test/functional/android/find_by_accessibility_id_tests.py +++ b/test/functional/android/find_by_accessibility_id_tests.py @@ -15,8 +15,10 @@ import unittest from appium import webdriver +from appium.webdriver.common.mobileby import MobileBy from .helper import desired_capabilities +from .helper.test_helper import wait_for_element class FindByAccessibilityIDTests(unittest.TestCase): @@ -28,9 +30,10 @@ def tearDown(self): self.driver.quit() def test_find_single_element(self): - self.driver.find_element_by_android_uiautomator('new UiSelector().text("Accessibility")').click() - self.driver.find_element_by_android_uiautomator('new UiSelector().text("Accessibility Node Querying")').click() - el = self.driver.find_element_by_accessibility_id('Task Take out Trash') + wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, 'new UiSelector().text("Accessibility")').click() + wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, + 'new UiSelector().text("Accessibility Node Querying")').click() + el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Task Take out Trash') self.assertIsNotNone(el) def test_find_multiple_elements(self): @@ -38,14 +41,17 @@ def test_find_multiple_elements(self): self.assertIsInstance(els, list) def test_element_find_single_element(self): - self.driver.find_element_by_android_uiautomator('new UiSelector().text("Accessibility")').click() - self.driver.find_element_by_android_uiautomator('new UiSelector().text("Accessibility Node Querying")').click() - el = self.driver.find_element_by_class_name('android.widget.ListView') + self.skipTest('Need to fix flaky test during running on CI') + wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, 'new UiSelector().text("Accessibility")').click() + wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, + 'new UiSelector().text("Accessibility Node Querying")').click() + el = wait_for_element(self.driver, MobileBy.CLASS_NAME, 'android.widget.ListView') sub_el = el.find_element_by_accessibility_id('Task Take out Trash') self.assertIsNotNone(sub_el) def test_element_find_multiple_elements(self): + wait_for_element(self.driver, MobileBy.CLASS_NAME, 'android.widget.ListView') el = self.driver.find_element_by_class_name('android.widget.ListView') sub_els = el.find_elements_by_accessibility_id('Animation') diff --git a/test/functional/android/helper/desired_capabilities.py b/test/functional/android/helper/desired_capabilities.py index 525753be..8048f74c 100644 --- a/test/functional/android/helper/desired_capabilities.py +++ b/test/functional/android/helper/desired_capabilities.py @@ -28,7 +28,9 @@ def get_desired_capabilities(app): 'deviceName': 'Android Emulator', 'app': PATH('../../apps/{}'.format(app)), 'newCommandTimeout': 240, - 'automationName': 'UIAutomator2' + 'automationName': 'UIAutomator2', + 'uiautomator2ServerInstallTimeout': 120000, + 'adbExecTimeout': 120000 } return desired_caps diff --git a/test/functional/android/helper/test_helper.py b/test/functional/android/helper/test_helper.py index 6237931e..0e1c43ca 100644 --- a/test/functional/android/helper/test_helper.py +++ b/test/functional/android/helper/test_helper.py @@ -16,7 +16,7 @@ from selenium.webdriver.support.ui import WebDriverWait # the emulator is sometimes slow and needs time to think -SLEEPY_TIME = 3 +SLEEPY_TIME = 10 def wait_for_element(driver, locator, value, timeout=SLEEPY_TIME): diff --git a/test/functional/android/touch_action_tests.py b/test/functional/android/touch_action_tests.py index 3fafb5bd..2c4b1647 100644 --- a/test/functional/android/touch_action_tests.py +++ b/test/functional/android/touch_action_tests.py @@ -13,7 +13,6 @@ # limitations under the License. import unittest -from time import sleep from selenium.common.exceptions import NoSuchElementException @@ -110,7 +109,7 @@ def test_press_and_moveto(self): action = TouchAction(self.driver) action.press(el1).move_to(el2).release().perform() - el = self.driver.find_element_by_accessibility_id('Views') + el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Views') self.assertIsNotNone(el) def test_press_and_moveto_x_y(self): @@ -120,7 +119,7 @@ def test_press_and_moveto_x_y(self): action = TouchAction(self.driver) action.press(el1).move_to(el2, 100, 100).release().perform() - el = self.driver.find_element_by_accessibility_id('Views') + el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Views') self.assertIsNotNone(el) def test_long_press(self): @@ -130,7 +129,7 @@ def test_long_press(self): action = TouchAction(self.driver) action.press(el1).move_to(el2).perform() - el = self.driver.find_element_by_accessibility_id('Views') + el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Views') action.tap(el).perform() el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Expandable Lists') @@ -155,13 +154,13 @@ def test_long_press_x_y(self): action = TouchAction(self.driver) action.press(el1).move_to(el2).perform() - el = self.driver.find_element_by_accessibility_id('Views') + el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Views') action.tap(el).perform() - el = self.driver.find_element_by_accessibility_id('Expandable Lists') + el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Expandable Lists') action.tap(el).perform() - el = self.driver.find_element_by_accessibility_id('1. Custom Adapter') + el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, '1. Custom Adapter') action.tap(el).perform() # the element "People Names" is located at 430:310 (top left corner) @@ -191,7 +190,7 @@ def test_drag_and_drop(self): # dnd is stimulated by longpress-move_to-release action.long_press(dd3).move_to(dd2).release().perform() - el = self.driver.find_element_by_id('com.example.android.apis:id/drag_text') + el = wait_for_element(self.driver, MobileBy.ID, 'com.example.android.apis:id/drag_text') self.assertTrue('drag_dot_3' in el.text) def test_driver_drag_and_drop(self): @@ -199,19 +198,19 @@ def test_driver_drag_and_drop(self): el2 = self.driver.find_element_by_accessibility_id('Animation') self.driver.scroll(el1, el2) - el = self.driver.find_element_by_accessibility_id('Views') + el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Views') action = TouchAction(self.driver) action.tap(el).perform() el = wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Drag and Drop') action.tap(el).perform() - dd3 = self.driver.find_element_by_id('com.example.android.apis:id/drag_dot_3') + dd3 = wait_for_element(self.driver, MobileBy.ID, 'com.example.android.apis:id/drag_dot_3') dd2 = self.driver.find_element_by_id('com.example.android.apis:id/drag_dot_2') self.driver.drag_and_drop(dd3, dd2) - el = self.driver.find_element_by_id('com.example.android.apis:id/drag_text') + el = wait_for_element(self.driver, MobileBy.ID, 'com.example.android.apis:id/drag_text') self.assertTrue('drag_dot_3' in el.text) def test_driver_swipe(self): diff --git a/test/functional/android/webdriver_tests.py b/test/functional/android/webdriver_tests.py index 55aac8dc..4c608888 100644 --- a/test/functional/android/webdriver_tests.py +++ b/test/functional/android/webdriver_tests.py @@ -47,11 +47,9 @@ def test_reset(self): self.assertTrue(self.driver.is_app_installed('com.example.android.apis')) def test_open_notifications(self): - self.driver.find_element_by_android_uiautomator('new UiSelector().text("App")').click() - self.driver.find_element_by_android_uiautomator('new UiSelector().text("Notification")').click() - self.driver.find_element_by_android_uiautomator('new UiSelector().text("Status Bar")').click() - - self.driver.find_element_by_android_uiautomator('new UiSelector().text(":-|")').click() + for word in ['App', 'Notification', 'Status Bar', ':-|']: + wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, + 'new UiSelector().text("{}")'.format(word)).click() self.driver.open_notifications() sleep(1) @@ -93,7 +91,7 @@ def test_send_keys(self): wait_for_element(self.driver, MobileBy.XPATH, "//android.widget.TextView[@text='{}']".format(text)).click() - el = self.driver.find_element(MobileBy.ID, 'com.example.android.apis:id/left_text_edit') + el = wait_for_element(self.driver, MobileBy.ID, 'com.example.android.apis:id/left_text_edit') el.send_keys(' text') self.assertEqual('Left is best text', el.text) diff --git a/test/functional/ios/execute_driver_tests.py b/test/functional/ios/execute_driver_tests.py index 1ebe338b..865cbb13 100644 --- a/test/functional/ios/execute_driver_tests.py +++ b/test/functional/ios/execute_driver_tests.py @@ -16,7 +16,8 @@ import unittest from appium import webdriver -from helper import desired_capabilities + +from .helper import desired_capabilities class ExecuteDriverTests(unittest.TestCase): From 8551f68301ad1dd0653343bdcfb7a930f9f36131 Mon Sep 17 00:00:00 2001 From: Atsushi Mori Date: Sun, 7 Jul 2019 14:29:47 +0900 Subject: [PATCH 2/7] Use androidSdkVer 27 for emulator --- ci-jobs/functional_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-jobs/functional_test.yml b/ci-jobs/functional_test.yml index 710cb91d..41b42606 100644 --- a/ci-jobs/functional_test.yml +++ b/ci-jobs/functional_test.yml @@ -1,7 +1,7 @@ parameters: vmImage: 'macOS-10.14' pytestOpt: '--doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html' - androidSdkVer: 28 + androidSdkVer: 27 jobs: - template: ./functional/run_ios_test.yml From 15b0cf8a44d3c4c5e4a0b57816fc01cf0c5b6e98 Mon Sep 17 00:00:00 2001 From: Atsushi Mori Date: Sun, 7 Jul 2019 18:08:57 +0900 Subject: [PATCH 3/7] Skip find_by_accessibility_id, find_by_uiautomator --- test/functional/android/find_by_accessibility_id_tests.py | 4 +++- test/functional/android/find_by_uiautomator_tests.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/functional/android/find_by_accessibility_id_tests.py b/test/functional/android/find_by_accessibility_id_tests.py index 41ba2488..0bd370de 100644 --- a/test/functional/android/find_by_accessibility_id_tests.py +++ b/test/functional/android/find_by_accessibility_id_tests.py @@ -14,6 +14,8 @@ import unittest +import pytest + from appium import webdriver from appium.webdriver.common.mobileby import MobileBy @@ -21,6 +23,7 @@ from .helper.test_helper import wait_for_element +@pytest.mark.skip(reason="Need to fix flaky test during running on CI") class FindByAccessibilityIDTests(unittest.TestCase): def setUp(self): desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') @@ -41,7 +44,6 @@ def test_find_multiple_elements(self): self.assertIsInstance(els, list) def test_element_find_single_element(self): - self.skipTest('Need to fix flaky test during running on CI') wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, 'new UiSelector().text("Accessibility")').click() wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, 'new UiSelector().text("Accessibility Node Querying")').click() diff --git a/test/functional/android/find_by_uiautomator_tests.py b/test/functional/android/find_by_uiautomator_tests.py index 2fa2c0ac..e3c52469 100644 --- a/test/functional/android/find_by_uiautomator_tests.py +++ b/test/functional/android/find_by_uiautomator_tests.py @@ -14,11 +14,14 @@ import unittest +import pytest + from appium import webdriver from .helper import desired_capabilities +@pytest.mark.skip(reason="Need to fix flaky test during running on CI") class FindByUIAutomatorTests(unittest.TestCase): def setUp(self): desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') From 64c56db3f2a337e88d8f77e979ed8539b43ea31a Mon Sep 17 00:00:00 2001 From: Atsushi Mori Date: Sat, 17 Aug 2019 18:29:49 +0900 Subject: [PATCH 4/7] Changed from https://github.com/ki4070ma/python-client/pull/5 --- ci-jobs/functional/run_android_test.yml | 4 ++ ci-jobs/functional/run_appium.yml | 2 +- ci-jobs/functional/run_ios_test.yml | 2 + ci-jobs/functional_test.yml | 17 +++++- test/functional/android/activities_tests.py | 6 +- test/functional/android/applications_tests.py | 11 +--- test/functional/android/chrome_tests.py | 4 +- test/functional/android/device_time_tests.py | 13 +---- .../android/find_by_accessibility_id_tests.py | 18 ++---- .../android/find_by_uiautomator_tests.py | 15 +---- test/functional/android/finger_print_tests.py | 13 +---- test/functional/android/helper/test_helper.py | 24 ++++++++ test/functional/android/hw_actions_tests.py | 15 ++--- test/functional/android/ime_tests.py | 13 +---- test/functional/android/keyboard_tests.py | 13 +---- test/functional/android/location_tests.py | 13 +---- test/functional/android/multi_action_tests.py | 13 +---- .../android/network_connection_tests.py | 14 ++--- test/functional/android/remote_fs_tests.py | 12 +--- .../functional/android/screen_record_tests.py | 13 +---- test/functional/android/settings_tests.py | 13 +---- test/functional/android/touch_action_tests.py | 11 +--- test/functional/android/webdriver_tests.py | 44 ++------------ test/functional/android/webelement_tests.py | 58 +++++++++++++++++++ 24 files changed, 153 insertions(+), 208 deletions(-) create mode 100644 test/functional/android/webelement_tests.py diff --git a/ci-jobs/functional/run_android_test.yml b/ci-jobs/functional/run_android_test.yml index ac6c6110..4a93844b 100644 --- a/ci-jobs/functional/run_android_test.yml +++ b/ci-jobs/functional/run_android_test.yml @@ -4,6 +4,7 @@ jobs: vmImage: ${{ parameters.vmImage }} variables: ANDROID_SDK_VERSION: ${{ parameters.sdkVer }} + CI: ${{ parameters.ci }} steps: - template: ./run_appium.yml - script: bash ci-jobs/functional/start-emulator.sh @@ -13,3 +14,6 @@ jobs: py.test ${{ parameters.testFiles}} ${{ parameters.pytestOpt }} displayName: Run Android functional tests - template: ./publish_test_result.yml + - template: ./save_appium_log.yml + parameters: + name: ${{ parameters.name }} diff --git a/ci-jobs/functional/run_appium.yml b/ci-jobs/functional/run_appium.yml index 7cdc7f18..42ebadd3 100644 --- a/ci-jobs/functional/run_appium.yml +++ b/ci-jobs/functional/run_appium.yml @@ -24,5 +24,5 @@ steps: appium --version node --version displayName: Check versions -- script: nohup appium --relaxed-security & +- script: nohup appium --relaxed-security > appium_log.txt & displayName: Run Appium in background diff --git a/ci-jobs/functional/run_ios_test.yml b/ci-jobs/functional/run_ios_test.yml index 1278c0e5..a0bd8d81 100644 --- a/ci-jobs/functional/run_ios_test.yml +++ b/ci-jobs/functional/run_ios_test.yml @@ -2,6 +2,8 @@ jobs: - job: ${{ parameters.name }} pool: vmImage: ${{ parameters.vmImage }} + variables: + CI: ${{ parameters.ci }} steps: - template: ./run_appium.yml - script: | diff --git a/ci-jobs/functional_test.yml b/ci-jobs/functional_test.yml index 41b42606..6ed303d0 100644 --- a/ci-jobs/functional_test.yml +++ b/ci-jobs/functional_test.yml @@ -1,7 +1,8 @@ parameters: vmImage: 'macOS-10.14' pytestOpt: '--doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html' - androidSdkVer: 27 + androidSdkVer: 28 + CI: true jobs: - template: ./functional/run_ios_test.yml @@ -10,12 +11,14 @@ jobs: vmImage: ${{ parameters.vmImage }} pytestOpt: ${{ parameters.pytestOpt }} testFiles: 'find_*.py remote_fs_tests.py safari_tests.py execute_driver_tests.py' + CI: ${{ parameters.ci }} - template: ./functional/run_ios_test.yml parameters: name: 'func_test_ios2' vmImage: ${{ parameters.vmImage }} pytestOpt: ${{ parameters.pytestOpt }} testFiles: 'applications_tests.py hw_actions_tests.py keyboard_tests.py screen_record_tests.py webdriver_tests.py' + CI: ${{ parameters.ci }} - template: ./functional/run_android_test.yml parameters: name: 'func_test_android1' @@ -23,20 +26,23 @@ jobs: pytestOpt: ${{ parameters.pytestOpt }} testFiles: 'device_time_tests.py find_by_accessibility_id_tests.py find_by_image_tests.py find_by_uiautomator_tests.py' sdkVer: ${{ parameters.androidSdkVer }} + CI: ${{ parameters.ci }} - template: ./functional/run_android_test.yml parameters: name: 'func_test_android2' vmImage: ${{ parameters.vmImage }} pytestOpt: ${{ parameters.pytestOpt }} - testFiles: 'hw_actions_tests.py ime_tests.py keyboard_tests.py location_tests.py' + testFiles: 'ime_tests.py keyboard_tests.py location_tests.py' sdkVer: ${{ parameters.androidSdkVer }} + CI: ${{ parameters.ci }} - template: ./functional/run_android_test.yml parameters: name: 'func_test_android3' vmImage: ${{ parameters.vmImage }} pytestOpt: ${{ parameters.pytestOpt }} - testFiles: 'activities_tests.py' + testFiles: 'activities_tests.py chrome_tests.py' sdkVer: ${{ parameters.androidSdkVer }} + CI: ${{ parameters.ci }} - template: ./functional/run_android_test.yml parameters: name: 'func_test_android4' @@ -44,6 +50,7 @@ jobs: pytestOpt: ${{ parameters.pytestOpt }} testFiles: 'finger_print_tests.py screen_record_tests.py settings_tests.py' sdkVer: ${{ parameters.androidSdkVer }} + CI: ${{ parameters.ci }} - template: ./functional/run_android_test.yml parameters: name: 'func_test_android5' @@ -51,6 +58,7 @@ jobs: pytestOpt: ${{ parameters.pytestOpt }} testFiles: 'context_switching_tests.py remote_fs_tests.py' sdkVer: ${{ parameters.androidSdkVer }} + CI: ${{ parameters.ci }} - template: ./functional/run_android_test.yml parameters: name: 'func_test_android6' @@ -58,6 +66,7 @@ jobs: pytestOpt: ${{ parameters.pytestOpt }} testFiles: 'webdriver_tests.py' sdkVer: ${{ parameters.androidSdkVer }} + CI: ${{ parameters.ci }} - template: ./functional/run_android_test.yml parameters: name: 'func_test_android7' @@ -65,6 +74,7 @@ jobs: pytestOpt: ${{ parameters.pytestOpt }} testFiles: 'applications_tests.py' sdkVer: ${{ parameters.androidSdkVer }} + CI: ${{ parameters.ci }} - template: ./functional/run_android_test.yml parameters: name: 'func_test_android8' @@ -72,3 +82,4 @@ jobs: pytestOpt: ${{ parameters.pytestOpt }} testFiles: 'network_connection_tests.py' sdkVer: ${{ parameters.androidSdkVer }} + CI: ${{ parameters.ci }} diff --git a/test/functional/android/activities_tests.py b/test/functional/android/activities_tests.py index bde1458b..71f70ad7 100644 --- a/test/functional/android/activities_tests.py +++ b/test/functional/android/activities_tests.py @@ -15,12 +15,16 @@ import unittest +import pytest + from appium import webdriver from .helper import desired_capabilities +from .helper.test_helper import BaseTest -class ActivitiesTests(unittest.TestCase): +@pytest.mark.skip(reason='Need to fix flaky test during running on CI') +class ActivitiesTests(BaseTest): def setUp(self): desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) diff --git a/test/functional/android/applications_tests.py b/test/functional/android/applications_tests.py index 13a98857..549aac87 100644 --- a/test/functional/android/applications_tests.py +++ b/test/functional/android/applications_tests.py @@ -16,19 +16,12 @@ import unittest from time import sleep -from appium import webdriver from appium.webdriver.applicationstate import ApplicationState -from .helper import desired_capabilities +from .helper.test_helper import BaseTest -class ApplicationsTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() +class ApplicationsTests(BaseTest): def test_background_app(self): self.driver.background_app(1) diff --git a/test/functional/android/chrome_tests.py b/test/functional/android/chrome_tests.py index 1d1216fe..a61a04dc 100644 --- a/test/functional/android/chrome_tests.py +++ b/test/functional/android/chrome_tests.py @@ -23,7 +23,9 @@ def setUp(self): 'platformName': 'Android', 'platformVersion': '9', 'deviceName': 'Android Emulator', - 'browserName': 'Chrome' + 'browserName': 'Chrome', + 'uiautomator2ServerInstallTimeout': 120000, + 'adbExecTimeout': 120000 } self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) diff --git a/test/functional/android/device_time_tests.py b/test/functional/android/device_time_tests.py index 42ca0129..4c708838 100644 --- a/test/functional/android/device_time_tests.py +++ b/test/functional/android/device_time_tests.py @@ -17,19 +17,10 @@ from dateutil.parser import parse -from appium import webdriver +from .helper.test_helper import BaseTest -from .helper import desired_capabilities - - -class DeviceTimeTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() +class DeviceTimeTests(BaseTest): def test_device_time(self): date_time = self.driver.device_time # convert to date ought to work diff --git a/test/functional/android/find_by_accessibility_id_tests.py b/test/functional/android/find_by_accessibility_id_tests.py index 0bd370de..b3577a18 100644 --- a/test/functional/android/find_by_accessibility_id_tests.py +++ b/test/functional/android/find_by_accessibility_id_tests.py @@ -14,24 +14,12 @@ import unittest -import pytest - -from appium import webdriver from appium.webdriver.common.mobileby import MobileBy -from .helper import desired_capabilities -from .helper.test_helper import wait_for_element - - -@pytest.mark.skip(reason="Need to fix flaky test during running on CI") -class FindByAccessibilityIDTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) +from .helper.test_helper import BaseTest, is_ci, wait_for_element - def tearDown(self): - self.driver.quit() +class FindByAccessibilityIDTests(BaseTest): def test_find_single_element(self): wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, 'new UiSelector().text("Accessibility")').click() wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, @@ -44,6 +32,8 @@ def test_find_multiple_elements(self): self.assertIsInstance(els, list) def test_element_find_single_element(self): + if is_ci(): + self.skipTest('Need to fix flaky test during running on CI') wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, 'new UiSelector().text("Accessibility")').click() wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, 'new UiSelector().text("Accessibility Node Querying")').click() diff --git a/test/functional/android/find_by_uiautomator_tests.py b/test/functional/android/find_by_uiautomator_tests.py index e3c52469..cde78b17 100644 --- a/test/functional/android/find_by_uiautomator_tests.py +++ b/test/functional/android/find_by_uiautomator_tests.py @@ -16,20 +16,11 @@ import pytest -from appium import webdriver +from .helper.test_helper import BaseTest -from .helper import desired_capabilities - - -@pytest.mark.skip(reason="Need to fix flaky test during running on CI") -class FindByUIAutomatorTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() +@pytest.mark.skip(reason='Need to fix flaky test during running on CI') # Due to "System UI isn't responding" dialog +class FindByUIAutomatorTests(BaseTest): def test_find_single_element(self): el = self.driver.find_element_by_android_uiautomator('new UiSelector().text("Animation")') self.assertIsNotNone(el) diff --git a/test/functional/android/finger_print_tests.py b/test/functional/android/finger_print_tests.py index 4f4e28e8..994bc0eb 100644 --- a/test/functional/android/finger_print_tests.py +++ b/test/functional/android/finger_print_tests.py @@ -15,19 +15,10 @@ import unittest -from appium import webdriver +from .helper.test_helper import BaseTest -from .helper import desired_capabilities - - -class FingerPrintTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() +class FingerPrintTests(BaseTest): def test_finger_print(self): result = self.driver.finger_print(1) self.assertEqual(None, result) diff --git a/test/functional/android/helper/test_helper.py b/test/functional/android/helper/test_helper.py index 0e1c43ca..60304042 100644 --- a/test/functional/android/helper/test_helper.py +++ b/test/functional/android/helper/test_helper.py @@ -12,9 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. + +import os +import unittest + from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait +from appium import webdriver + +from . import desired_capabilities + # the emulator is sometimes slow and needs time to think SLEEPY_TIME = 10 @@ -38,3 +46,19 @@ def wait_for_element(driver, locator, value, timeout=SLEEPY_TIME): return WebDriverWait(driver, timeout).until( EC.presence_of_element_located((locator, value)) ) + + +def is_ci(): + return os.getenv('CI', 'false') == 'true' + + +class BaseTest(unittest.TestCase): + + def setUp(self): + desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') + self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) + + def tearDown(self): + img_path = os.path.join(os.getcwd(), self._testMethodName + '.png') + self.driver.get_screenshot_as_file(img_path) + self.driver.quit() diff --git a/test/functional/android/hw_actions_tests.py b/test/functional/android/hw_actions_tests.py index bdbe406e..d67f902b 100644 --- a/test/functional/android/hw_actions_tests.py +++ b/test/functional/android/hw_actions_tests.py @@ -14,22 +14,15 @@ # limitations under the License. import unittest +from time import sleep -from appium import webdriver +from .helper.test_helper import BaseTest -from .helper import desired_capabilities - - -class HwActionsTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() +class HwActionsTests(BaseTest): def test_lock(self): self.driver.lock(-1) + sleep(10) try: self.assertTrue(self.driver.is_locked()) finally: diff --git a/test/functional/android/ime_tests.py b/test/functional/android/ime_tests.py index 4b75986f..df725e1e 100644 --- a/test/functional/android/ime_tests.py +++ b/test/functional/android/ime_tests.py @@ -16,22 +16,13 @@ import unittest from time import sleep -from appium import webdriver - -from .helper import desired_capabilities +from .helper.test_helper import BaseTest ANDROID_LATIN = 'com.android.inputmethod.latin/.LatinIME' # Android L/M/N GOOGLE_LATIN = 'com.google.android.inputmethod.latin/com.android.inputmethod.latin.LatinIME' # Android O/P -class IMETests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() - +class IMETests(BaseTest): def test_available_ime_engines(self): engines = self.driver.available_ime_engines self.assertIsInstance(engines, list) diff --git a/test/functional/android/keyboard_tests.py b/test/functional/android/keyboard_tests.py index acb4a56a..6c02cd4b 100644 --- a/test/functional/android/keyboard_tests.py +++ b/test/functional/android/keyboard_tests.py @@ -15,19 +15,10 @@ import unittest -from appium import webdriver +from .helper.test_helper import BaseTest -from .helper import desired_capabilities - - -class KeyboardTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() +class KeyboardTests(BaseTest): def test_press_keycode(self): # not sure how to test this. self.driver.press_keycode(176) diff --git a/test/functional/android/location_tests.py b/test/functional/android/location_tests.py index 5843358c..7085b92c 100644 --- a/test/functional/android/location_tests.py +++ b/test/functional/android/location_tests.py @@ -15,19 +15,10 @@ import unittest -from appium import webdriver +from .helper.test_helper import BaseTest -from .helper import desired_capabilities - - -class LocationTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() +class LocationTests(BaseTest): def test_toggle_location_services(self): self.driver.toggle_location_services() diff --git a/test/functional/android/multi_action_tests.py b/test/functional/android/multi_action_tests.py index a1825f41..55588c13 100644 --- a/test/functional/android/multi_action_tests.py +++ b/test/functional/android/multi_action_tests.py @@ -15,23 +15,14 @@ import unittest from time import sleep -from appium import webdriver from appium.webdriver.common.mobileby import MobileBy from appium.webdriver.common.multi_action import MultiAction from appium.webdriver.common.touch_action import TouchAction -from .helper import desired_capabilities -from .helper.test_helper import wait_for_element +from .helper.test_helper import BaseTest, wait_for_element -class MultiActionTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() - +class MultiActionTests(BaseTest): def test_parallel_actions(self): el1 = self.driver.find_element_by_accessibility_id('Content') el2 = self.driver.find_element_by_accessibility_id('Animation') diff --git a/test/functional/android/network_connection_tests.py b/test/functional/android/network_connection_tests.py index 2b0c96f2..ae675f31 100644 --- a/test/functional/android/network_connection_tests.py +++ b/test/functional/android/network_connection_tests.py @@ -15,25 +15,19 @@ import unittest -from appium import webdriver from appium.webdriver.connectiontype import ConnectionType -from .helper import desired_capabilities +from .helper.test_helper import BaseTest, is_ci -class NetworkConnectionTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() - +class NetworkConnectionTests(BaseTest): def test_get_network_connection(self): nc = self.driver.network_connection self.assertIsInstance(nc, int) def test_set_network_connection(self): + if is_ci(): + self.skipTest('Need to fix flaky test during running on CI') nc = self.driver.set_network_connection(ConnectionType.DATA_ONLY) self.assertIsInstance(nc, int) self.assertEqual(nc, ConnectionType.DATA_ONLY) diff --git a/test/functional/android/remote_fs_tests.py b/test/functional/android/remote_fs_tests.py index 2506f2f3..a185d458 100644 --- a/test/functional/android/remote_fs_tests.py +++ b/test/functional/android/remote_fs_tests.py @@ -19,20 +19,12 @@ from io import BytesIO from zipfile import ZipFile -from appium import webdriver from appium.common.helper import appium_bytes -from .helper import desired_capabilities +from .helper.test_helper import BaseTest -class RemoteFsTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() - +class RemoteFsTests(BaseTest): def test_push_pull_file(self): dest_path = '/data/local/tmp/test_push_file.txt' data = appium_bytes('This is the contents of the file to push to the device.', 'utf-8') diff --git a/test/functional/android/screen_record_tests.py b/test/functional/android/screen_record_tests.py index b856a269..fb1a0e4b 100644 --- a/test/functional/android/screen_record_tests.py +++ b/test/functional/android/screen_record_tests.py @@ -16,19 +16,10 @@ import unittest from time import sleep -from appium import webdriver +from .helper.test_helper import BaseTest -from .helper import desired_capabilities - - -class ScreenRecordTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() +class ScreenRecordTests(BaseTest): def test_screen_record(self): self.driver.start_recording_screen(timeLimit=10, forcedRestart=True) sleep(10) diff --git a/test/functional/android/settings_tests.py b/test/functional/android/settings_tests.py index d741248b..b974c028 100644 --- a/test/functional/android/settings_tests.py +++ b/test/functional/android/settings_tests.py @@ -15,19 +15,10 @@ import unittest -from appium import webdriver +from .helper.test_helper import BaseTest -from .helper import desired_capabilities - - -class SettingsTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() +class SettingsTests(BaseTest): def test_get_settings(self): settings = self.driver.get_settings() self.assertIsNotNone(settings) diff --git a/test/functional/android/touch_action_tests.py b/test/functional/android/touch_action_tests.py index 2c4b1647..318300f9 100644 --- a/test/functional/android/touch_action_tests.py +++ b/test/functional/android/touch_action_tests.py @@ -21,17 +21,10 @@ from appium.webdriver.common.touch_action import TouchAction from .helper import desired_capabilities -from .helper.test_helper import wait_for_element +from .helper.test_helper import BaseTest, wait_for_element -class TouchActionTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() - +class TouchActionTests(BaseTest): def test_tap(self): el = self.driver.find_element_by_accessibility_id('Animation') action = TouchAction(self.driver) diff --git a/test/functional/android/webdriver_tests.py b/test/functional/android/webdriver_tests.py index 4c608888..84f0a55d 100644 --- a/test/functional/android/webdriver_tests.py +++ b/test/functional/android/webdriver_tests.py @@ -18,20 +18,12 @@ from selenium.common.exceptions import NoSuchElementException -from appium import webdriver from appium.webdriver.common.mobileby import MobileBy -from .helper import desired_capabilities -from .helper.test_helper import wait_for_element +from .helper.test_helper import BaseTest, is_ci, wait_for_element -class WebdriverTests(unittest.TestCase): - def setUp(self): - desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') - self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) - - def tearDown(self): - self.driver.quit() +class WebdriverTests(BaseTest): def test_current_package(self): package = self.driver.current_package @@ -47,6 +39,9 @@ def test_reset(self): self.assertTrue(self.driver.is_app_installed('com.example.android.apis')) def test_open_notifications(self): + if is_ci(): + # TODO Due to unexpected dialog, "System UI isn't responding" + self.skipTest('Need to fix flaky test during running on CI.') for word in ['App', 'Notification', 'Status Bar', ':-|']: wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, 'new UiSelector().text("{}")'.format(word)).click() @@ -73,35 +68,6 @@ def test_open_notifications(self): sleep(1) self.driver.find_element_by_android_uiautomator('new UiSelector().text(":-|")') - def test_set_text(self): - self.driver.find_element_by_android_uiautomator( - 'new UiScrollable(new UiSelector().scrollable(true).instance(0)).scrollIntoView(new UiSelector().text("Views").instance(0));').click() - - wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Controls').click() - wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, '1. Light Theme').click() - - el = wait_for_element(self.driver, MobileBy.CLASS_NAME, 'android.widget.EditText') - el.send_keys('original text') - el.set_text('new text') - - self.assertEqual('new text', el.text) - - def test_send_keys(self): - for text in ['App', 'Activity', 'Custom Title']: - wait_for_element(self.driver, MobileBy.XPATH, - "//android.widget.TextView[@text='{}']".format(text)).click() - - el = wait_for_element(self.driver, MobileBy.ID, 'com.example.android.apis:id/left_text_edit') - el.send_keys(' text') - - self.assertEqual('Left is best text', el.text) - - def test_element_location_in_view(self): - el = self.driver.find_element_by_accessibility_id('Content') - loc = el.location_in_view - self.assertIsNotNone(loc['x']) - self.assertIsNotNone(loc['y']) - if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(WebdriverTests) diff --git a/test/functional/android/webelement_tests.py b/test/functional/android/webelement_tests.py new file mode 100644 index 00000000..12baaa57 --- /dev/null +++ b/test/functional/android/webelement_tests.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from appium import webdriver +from appium.webdriver.common.mobileby import MobileBy + +from .helper import desired_capabilities +from .helper.test_helper import BaseTest, wait_for_element + + +class WebelementTests(BaseTest): + def test_element_location_in_view(self): + el = self.driver.find_element_by_accessibility_id('Content') + loc = el.location_in_view + self.assertIsNotNone(loc['x']) + self.assertIsNotNone(loc['y']) + + def test_set_text(self): + self.driver.find_element_by_android_uiautomator( + 'new UiScrollable(new UiSelector().scrollable(true).instance(0)).scrollIntoView(new UiSelector().text("Views").instance(0));').click() + + wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, 'Controls').click() + wait_for_element(self.driver, MobileBy.ACCESSIBILITY_ID, '1. Light Theme').click() + + el = wait_for_element(self.driver, MobileBy.CLASS_NAME, 'android.widget.EditText') + el.send_keys('original text') + el.set_text('new text') + + self.assertEqual('new text', el.text) + + def test_send_keys(self): + for text in ['App', 'Activity', 'Custom Title']: + wait_for_element(self.driver, MobileBy.XPATH, + "//android.widget.TextView[@text='{}']".format(text)).click() + + el = wait_for_element(self.driver, MobileBy.ID, 'com.example.android.apis:id/left_text_edit') + el.send_keys(' text') + + self.assertEqual('Left is best text', el.text) + + +if __name__ == '__main__': + suite = unittest.TestLoader().loadTestsFromTestCase(WebelementTests) + unittest.TextTestRunner(verbosity=2).run(suite) From 748c116b8a274cdebaf28aa65e410eb2f11ae35e Mon Sep 17 00:00:00 2001 From: Atsushi Mori Date: Sat, 17 Aug 2019 18:32:28 +0900 Subject: [PATCH 5/7] Add save_appium_log.yml --- ci-jobs/functional/save_appium_log.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 ci-jobs/functional/save_appium_log.yml diff --git a/ci-jobs/functional/save_appium_log.yml b/ci-jobs/functional/save_appium_log.yml new file mode 100644 index 00000000..9819e9fb --- /dev/null +++ b/ci-jobs/functional/save_appium_log.yml @@ -0,0 +1,18 @@ +steps: +- task: CopyFiles@2 + condition: succeededOrFailed() + inputs: + contents: + '**/appium_log.txt' + targetFolder: $(Build.ArtifactStagingDirectory) +- task: CopyFiles@2 + condition: succeededOrFailed() + inputs: + contents: + '**/test_*.png' + targetFolder: $(Build.ArtifactStagingDirectory) +- task: PublishBuildArtifacts@1 + condition: succeededOrFailed() + inputs: + pathToPublish: $(Build.ArtifactStagingDirectory) + artifactName: ${{ parameters.name }} From 0b79f3ad43acdd86c40d5656d2aaeaa99520b022 Mon Sep 17 00:00:00 2001 From: Atsushi Mori Date: Sat, 17 Aug 2019 18:59:50 +0900 Subject: [PATCH 6/7] Don't run flaky tests on CI --- ci-jobs/functional_test.yml | 6 ++++-- test/functional/android/activities_tests.py | 3 --- test/functional/android/find_by_uiautomator_tests.py | 3 --- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ci-jobs/functional_test.yml b/ci-jobs/functional_test.yml index 6ed303d0..8fc79048 100644 --- a/ci-jobs/functional_test.yml +++ b/ci-jobs/functional_test.yml @@ -4,6 +4,8 @@ parameters: androidSdkVer: 28 CI: true +# [Android] Need to fix and add flaky tests for activities_tests, find_by_uiautomator_tests + jobs: - template: ./functional/run_ios_test.yml parameters: @@ -24,7 +26,7 @@ jobs: name: 'func_test_android1' vmImage: ${{ parameters.vmImage }} pytestOpt: ${{ parameters.pytestOpt }} - testFiles: 'device_time_tests.py find_by_accessibility_id_tests.py find_by_image_tests.py find_by_uiautomator_tests.py' + testFiles: 'device_time_tests.py find_by_accessibility_id_tests.py find_by_image_tests.py' sdkVer: ${{ parameters.androidSdkVer }} CI: ${{ parameters.ci }} - template: ./functional/run_android_test.yml @@ -40,7 +42,7 @@ jobs: name: 'func_test_android3' vmImage: ${{ parameters.vmImage }} pytestOpt: ${{ parameters.pytestOpt }} - testFiles: 'activities_tests.py chrome_tests.py' + testFiles: 'chrome_tests.py' sdkVer: ${{ parameters.androidSdkVer }} CI: ${{ parameters.ci }} - template: ./functional/run_android_test.yml diff --git a/test/functional/android/activities_tests.py b/test/functional/android/activities_tests.py index 71f70ad7..10da49ee 100644 --- a/test/functional/android/activities_tests.py +++ b/test/functional/android/activities_tests.py @@ -15,15 +15,12 @@ import unittest -import pytest - from appium import webdriver from .helper import desired_capabilities from .helper.test_helper import BaseTest -@pytest.mark.skip(reason='Need to fix flaky test during running on CI') class ActivitiesTests(BaseTest): def setUp(self): desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') diff --git a/test/functional/android/find_by_uiautomator_tests.py b/test/functional/android/find_by_uiautomator_tests.py index cde78b17..b202d1b3 100644 --- a/test/functional/android/find_by_uiautomator_tests.py +++ b/test/functional/android/find_by_uiautomator_tests.py @@ -14,12 +14,9 @@ import unittest -import pytest - from .helper.test_helper import BaseTest -@pytest.mark.skip(reason='Need to fix flaky test during running on CI') # Due to "System UI isn't responding" dialog class FindByUIAutomatorTests(BaseTest): def test_find_single_element(self): el = self.driver.find_element_by_android_uiautomator('new UiSelector().text("Animation")') From 605406a2e695d01e52dfbe3d4c3e8a2ecb031c4e Mon Sep 17 00:00:00 2001 From: Atsushi Mori Date: Sun, 18 Aug 2019 16:34:21 +0900 Subject: [PATCH 7/7] Rename class name --- test/functional/android/activities_tests.py | 4 ++-- test/functional/android/applications_tests.py | 4 ++-- test/functional/android/device_time_tests.py | 4 ++-- test/functional/android/find_by_accessibility_id_tests.py | 4 ++-- test/functional/android/find_by_uiautomator_tests.py | 4 ++-- test/functional/android/finger_print_tests.py | 4 ++-- test/functional/android/helper/test_helper.py | 7 ++++++- test/functional/android/hw_actions_tests.py | 4 ++-- test/functional/android/ime_tests.py | 4 ++-- test/functional/android/keyboard_tests.py | 4 ++-- test/functional/android/location_tests.py | 4 ++-- test/functional/android/multi_action_tests.py | 4 ++-- test/functional/android/network_connection_tests.py | 4 ++-- test/functional/android/remote_fs_tests.py | 4 ++-- test/functional/android/screen_record_tests.py | 4 ++-- test/functional/android/settings_tests.py | 4 ++-- test/functional/android/touch_action_tests.py | 4 ++-- test/functional/android/webdriver_tests.py | 4 ++-- test/functional/android/webelement_tests.py | 4 ++-- 19 files changed, 42 insertions(+), 37 deletions(-) diff --git a/test/functional/android/activities_tests.py b/test/functional/android/activities_tests.py index 10da49ee..63356fe8 100644 --- a/test/functional/android/activities_tests.py +++ b/test/functional/android/activities_tests.py @@ -18,10 +18,10 @@ from appium import webdriver from .helper import desired_capabilities -from .helper.test_helper import BaseTest +from .helper.test_helper import BaseTestCase -class ActivitiesTests(BaseTest): +class ActivitiesTests(BaseTestCase): def setUp(self): desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') self.driver = webdriver.Remote('http://localhost:4723/wd/hub', desired_caps) diff --git a/test/functional/android/applications_tests.py b/test/functional/android/applications_tests.py index 549aac87..41c40dad 100644 --- a/test/functional/android/applications_tests.py +++ b/test/functional/android/applications_tests.py @@ -18,10 +18,10 @@ from appium.webdriver.applicationstate import ApplicationState -from .helper.test_helper import BaseTest +from .helper.test_helper import BaseTestCase -class ApplicationsTests(BaseTest): +class ApplicationsTests(BaseTestCase): def test_background_app(self): self.driver.background_app(1) diff --git a/test/functional/android/device_time_tests.py b/test/functional/android/device_time_tests.py index 4c708838..deefdd36 100644 --- a/test/functional/android/device_time_tests.py +++ b/test/functional/android/device_time_tests.py @@ -17,10 +17,10 @@ from dateutil.parser import parse -from .helper.test_helper import BaseTest +from .helper.test_helper import BaseTestCase -class DeviceTimeTests(BaseTest): +class DeviceTimeTests(BaseTestCase): def test_device_time(self): date_time = self.driver.device_time # convert to date ought to work diff --git a/test/functional/android/find_by_accessibility_id_tests.py b/test/functional/android/find_by_accessibility_id_tests.py index b3577a18..9bb23e09 100644 --- a/test/functional/android/find_by_accessibility_id_tests.py +++ b/test/functional/android/find_by_accessibility_id_tests.py @@ -16,10 +16,10 @@ from appium.webdriver.common.mobileby import MobileBy -from .helper.test_helper import BaseTest, is_ci, wait_for_element +from .helper.test_helper import BaseTestCase, is_ci, wait_for_element -class FindByAccessibilityIDTests(BaseTest): +class FindByAccessibilityIDTests(BaseTestCase): def test_find_single_element(self): wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, 'new UiSelector().text("Accessibility")').click() wait_for_element(self.driver, MobileBy.ANDROID_UIAUTOMATOR, diff --git a/test/functional/android/find_by_uiautomator_tests.py b/test/functional/android/find_by_uiautomator_tests.py index b202d1b3..004db5c8 100644 --- a/test/functional/android/find_by_uiautomator_tests.py +++ b/test/functional/android/find_by_uiautomator_tests.py @@ -14,10 +14,10 @@ import unittest -from .helper.test_helper import BaseTest +from .helper.test_helper import BaseTestCase -class FindByUIAutomatorTests(BaseTest): +class FindByUIAutomatorTests(BaseTestCase): def test_find_single_element(self): el = self.driver.find_element_by_android_uiautomator('new UiSelector().text("Animation")') self.assertIsNotNone(el) diff --git a/test/functional/android/finger_print_tests.py b/test/functional/android/finger_print_tests.py index 994bc0eb..e77e72a4 100644 --- a/test/functional/android/finger_print_tests.py +++ b/test/functional/android/finger_print_tests.py @@ -15,10 +15,10 @@ import unittest -from .helper.test_helper import BaseTest +from .helper.test_helper import BaseTestCase -class FingerPrintTests(BaseTest): +class FingerPrintTests(BaseTestCase): def test_finger_print(self): result = self.driver.finger_print(1) self.assertEqual(None, result) diff --git a/test/functional/android/helper/test_helper.py b/test/functional/android/helper/test_helper.py index 60304042..9f3cbac7 100644 --- a/test/functional/android/helper/test_helper.py +++ b/test/functional/android/helper/test_helper.py @@ -49,10 +49,15 @@ def wait_for_element(driver, locator, value, timeout=SLEEPY_TIME): def is_ci(): + """Returns if current execution is running on CI + + Returns: + bool: `True` if current executions is on CI + """ return os.getenv('CI', 'false') == 'true' -class BaseTest(unittest.TestCase): +class BaseTestCase(unittest.TestCase): def setUp(self): desired_caps = desired_capabilities.get_desired_capabilities('ApiDemos-debug.apk') diff --git a/test/functional/android/hw_actions_tests.py b/test/functional/android/hw_actions_tests.py index d67f902b..3bb022f4 100644 --- a/test/functional/android/hw_actions_tests.py +++ b/test/functional/android/hw_actions_tests.py @@ -16,10 +16,10 @@ import unittest from time import sleep -from .helper.test_helper import BaseTest +from .helper.test_helper import BaseTestCase -class HwActionsTests(BaseTest): +class HwActionsTests(BaseTestCase): def test_lock(self): self.driver.lock(-1) sleep(10) diff --git a/test/functional/android/ime_tests.py b/test/functional/android/ime_tests.py index df725e1e..86bf7449 100644 --- a/test/functional/android/ime_tests.py +++ b/test/functional/android/ime_tests.py @@ -16,13 +16,13 @@ import unittest from time import sleep -from .helper.test_helper import BaseTest +from .helper.test_helper import BaseTestCase ANDROID_LATIN = 'com.android.inputmethod.latin/.LatinIME' # Android L/M/N GOOGLE_LATIN = 'com.google.android.inputmethod.latin/com.android.inputmethod.latin.LatinIME' # Android O/P -class IMETests(BaseTest): +class IMETests(BaseTestCase): def test_available_ime_engines(self): engines = self.driver.available_ime_engines self.assertIsInstance(engines, list) diff --git a/test/functional/android/keyboard_tests.py b/test/functional/android/keyboard_tests.py index 6c02cd4b..6c634da3 100644 --- a/test/functional/android/keyboard_tests.py +++ b/test/functional/android/keyboard_tests.py @@ -15,10 +15,10 @@ import unittest -from .helper.test_helper import BaseTest +from .helper.test_helper import BaseTestCase -class KeyboardTests(BaseTest): +class KeyboardTests(BaseTestCase): def test_press_keycode(self): # not sure how to test this. self.driver.press_keycode(176) diff --git a/test/functional/android/location_tests.py b/test/functional/android/location_tests.py index 7085b92c..1584fd5a 100644 --- a/test/functional/android/location_tests.py +++ b/test/functional/android/location_tests.py @@ -15,10 +15,10 @@ import unittest -from .helper.test_helper import BaseTest +from .helper.test_helper import BaseTestCase -class LocationTests(BaseTest): +class LocationTests(BaseTestCase): def test_toggle_location_services(self): self.driver.toggle_location_services() diff --git a/test/functional/android/multi_action_tests.py b/test/functional/android/multi_action_tests.py index 55588c13..3d0be2b0 100644 --- a/test/functional/android/multi_action_tests.py +++ b/test/functional/android/multi_action_tests.py @@ -19,10 +19,10 @@ from appium.webdriver.common.multi_action import MultiAction from appium.webdriver.common.touch_action import TouchAction -from .helper.test_helper import BaseTest, wait_for_element +from .helper.test_helper import BaseTestCase, wait_for_element -class MultiActionTests(BaseTest): +class MultiActionTests(BaseTestCase): def test_parallel_actions(self): el1 = self.driver.find_element_by_accessibility_id('Content') el2 = self.driver.find_element_by_accessibility_id('Animation') diff --git a/test/functional/android/network_connection_tests.py b/test/functional/android/network_connection_tests.py index ae675f31..d13937b7 100644 --- a/test/functional/android/network_connection_tests.py +++ b/test/functional/android/network_connection_tests.py @@ -17,10 +17,10 @@ from appium.webdriver.connectiontype import ConnectionType -from .helper.test_helper import BaseTest, is_ci +from .helper.test_helper import BaseTestCase, is_ci -class NetworkConnectionTests(BaseTest): +class NetworkConnectionTests(BaseTestCase): def test_get_network_connection(self): nc = self.driver.network_connection self.assertIsInstance(nc, int) diff --git a/test/functional/android/remote_fs_tests.py b/test/functional/android/remote_fs_tests.py index a185d458..fefe0839 100644 --- a/test/functional/android/remote_fs_tests.py +++ b/test/functional/android/remote_fs_tests.py @@ -21,10 +21,10 @@ from appium.common.helper import appium_bytes -from .helper.test_helper import BaseTest +from .helper.test_helper import BaseTestCase -class RemoteFsTests(BaseTest): +class RemoteFsTests(BaseTestCase): def test_push_pull_file(self): dest_path = '/data/local/tmp/test_push_file.txt' data = appium_bytes('This is the contents of the file to push to the device.', 'utf-8') diff --git a/test/functional/android/screen_record_tests.py b/test/functional/android/screen_record_tests.py index fb1a0e4b..26f5f764 100644 --- a/test/functional/android/screen_record_tests.py +++ b/test/functional/android/screen_record_tests.py @@ -16,10 +16,10 @@ import unittest from time import sleep -from .helper.test_helper import BaseTest +from .helper.test_helper import BaseTestCase -class ScreenRecordTests(BaseTest): +class ScreenRecordTests(BaseTestCase): def test_screen_record(self): self.driver.start_recording_screen(timeLimit=10, forcedRestart=True) sleep(10) diff --git a/test/functional/android/settings_tests.py b/test/functional/android/settings_tests.py index b974c028..a1ffe10b 100644 --- a/test/functional/android/settings_tests.py +++ b/test/functional/android/settings_tests.py @@ -15,10 +15,10 @@ import unittest -from .helper.test_helper import BaseTest +from .helper.test_helper import BaseTestCase -class SettingsTests(BaseTest): +class SettingsTests(BaseTestCase): def test_get_settings(self): settings = self.driver.get_settings() self.assertIsNotNone(settings) diff --git a/test/functional/android/touch_action_tests.py b/test/functional/android/touch_action_tests.py index 318300f9..8ebaf0d5 100644 --- a/test/functional/android/touch_action_tests.py +++ b/test/functional/android/touch_action_tests.py @@ -21,10 +21,10 @@ from appium.webdriver.common.touch_action import TouchAction from .helper import desired_capabilities -from .helper.test_helper import BaseTest, wait_for_element +from .helper.test_helper import BaseTestCase, wait_for_element -class TouchActionTests(BaseTest): +class TouchActionTests(BaseTestCase): def test_tap(self): el = self.driver.find_element_by_accessibility_id('Animation') action = TouchAction(self.driver) diff --git a/test/functional/android/webdriver_tests.py b/test/functional/android/webdriver_tests.py index 84f0a55d..3425e2c9 100644 --- a/test/functional/android/webdriver_tests.py +++ b/test/functional/android/webdriver_tests.py @@ -20,10 +20,10 @@ from appium.webdriver.common.mobileby import MobileBy -from .helper.test_helper import BaseTest, is_ci, wait_for_element +from .helper.test_helper import BaseTestCase, is_ci, wait_for_element -class WebdriverTests(BaseTest): +class WebdriverTests(BaseTestCase): def test_current_package(self): package = self.driver.current_package diff --git a/test/functional/android/webelement_tests.py b/test/functional/android/webelement_tests.py index 12baaa57..2b79824d 100644 --- a/test/functional/android/webelement_tests.py +++ b/test/functional/android/webelement_tests.py @@ -19,10 +19,10 @@ from appium.webdriver.common.mobileby import MobileBy from .helper import desired_capabilities -from .helper.test_helper import BaseTest, wait_for_element +from .helper.test_helper import BaseTestCase, wait_for_element -class WebelementTests(BaseTest): +class WebelementTests(BaseTestCase): def test_element_location_in_view(self): el = self.driver.find_element_by_accessibility_id('Content') loc = el.location_in_view