Skip to content

Commit 3d2cea3

Browse files
jonsimantovjonsimantov
authored
First pass at device randomization. (#924)
1 parent 4055173 commit 3d2cea3

File tree

2 files changed

+71
-20
lines changed

2 files changed

+71
-20
lines changed

‎.github/workflows/integration_tests.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -523,12 +523,13 @@ jobs:
523523
- name: Run Mobile integration tests on real device via FTL
524524
id: ftl_test
525525
if: ${{ matrix.device_type == 'real' }}
526-
uses: FirebaseExtended/github-actions/firebase-test-lab@v1.2
526+
uses: FirebaseExtended/github-actions/firebase-test-lab@v1.3
527527
with:
528528
credentials_json: ${{ secrets.FIREBASE_SERVICE_ACCOUNT_CREDENTIALS }}
529529
testapp_dir: testapps
530530
test_type: "game-loop"
531531
test_devices: ${{ matrix.device_detail }}
532+
test_device_selection: random
532533
max_attempts: 3
533534
validator: ${GITHUB_WORKSPACE}/scripts/gha/read_ftl_test_result.py
534535
- name: Read FTL Test Result

‎scripts/gha/print_matrix_configuration.py‎

Lines changed: 69 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -87,16 +87,63 @@
8787
BUILD_CONFIGS = ["Unity Version(s)", "Build OS(s)", "Platform(s)", "Test Device(s)"]
8888

8989
TEST_DEVICES = {
90-
"android_min": {"platform": ANDROID, "type": "real", "device": "model=Nexus10,version=19"},
91-
"android_target": {"platform": ANDROID, "type": "real", "device": "model=blueline,version=28"},
92-
"android_latest": {"platform": ANDROID, "type": "real", "device": "model=oriole,version=33"},
93-
"emulator_ftl_target": {"platform": ANDROID, "type": "real", "device": "model=Pixel2,version=28"},
90+
"android_min": {"platform": ANDROID, "type": "real",
91+
"device": [
92+
"model=1610,version=23", # vivo 1610
93+
"model=hammerhead,version=23", # Nexus 5
94+
"model=harpia,version=23", # Moto G Play
95+
]},
96+
"android_target": {"platform": ANDROID, "type": "real",
97+
"device": [
98+
"model=blueline,version=28", # Pixel 3
99+
"model=dreamlte,version=28", # Galaxy S8
100+
"model=gts3lltevzw,version=28", # Galaxy Tab S3
101+
"model=SH-01L,version=28", # AQUOS sense2 SH-01L
102+
]},
103+
"android_latest": {"platform": ANDROID, "type": "real",
104+
"device": [
105+
"model=oriole,version=33", # Pixel 6
106+
"model=panther,version=33", # Pixel 7
107+
"model=lynx,version=33", # Pixel 7a
108+
"model=cheetah,version=33", # Pixel 7 Pro
109+
"model=felix,version=33", # Pixel Fold
110+
"model=tangorpro,version=33", # Pixel Tablet
111+
"model=gts8uwifi,version=33", # Galaxy Tab S8 Ultra
112+
"model=b0q,version=33", # Galaxy S22 Ultra
113+
"model=b4q,version=33", # Galaxy Z Flip4
114+
]},
115+
"emulator_ftl_target": {"platform": ANDROID, "type": "real",
116+
"device": [
117+
"model=Pixel2,version=28",
118+
"model=Pixel2.arm,version=28",
119+
"model=MediumPhone.arm,version=28",
120+
"model=MediumTablet.arm,version=28",
121+
"model=SmallPhone.arm,version=28",
122+
]},
94123
"emulator_target": {"platform": ANDROID, "type": "virtual", "image": "system-images;android-28;google_apis;x86_64"},
95124
"emulator_latest": {"platform": ANDROID, "type": "virtual", "image": "system-images;android-30;google_apis;x86_64"},
96125
"emulator_32bit": {"platform": ANDROID, "type": "virtual", "image": "system-images;android-30;google_apis;x86"},
97-
"ios_min": {"platform": IOS, "type": "real", "device": "model=iphone8,version=14.7"},
98-
"ios_target": {"platform": IOS, "type": "real", "device": "model=iphone13pro,version=15.7"},
99-
"ios_latest": {"platform": IOS, "type": "real", "device": "model=iphone11pro,version=16.5"},
126+
"ios_min": {"platform": IOS, "type": "real",
127+
"device": [
128+
# Slightly different OS versions because of limited FTL selection.
129+
"model=iphone8,version=14.7",
130+
"model=iphone11pro,version=14.7",
131+
"model=iphone12pro,version=14.8",
132+
]},
133+
"ios_target": {"platform": IOS, "type": "real",
134+
"device": [
135+
# Slightly different OS versions because of limited FTL selection.
136+
"model=iphone13pro,version=15.7",
137+
"model=iphone8,version=15.7",
138+
"model=ipadmini4,version=15.4",
139+
]},
140+
"ios_latest": {"platform": IOS, "type": "real",
141+
"device": [
142+
"model=iphone14pro,version=16.6",
143+
"model=iphone11pro,version=16.6",
144+
"model=iphone8,version=16.6",
145+
"model=ipad10,version=16.6",
146+
]},
100147
"simulator_min": {"platform": IOS, "type": "virtual", "name": "iPhone 8", "version": "15.2"},
101148
"simulator_target": {"platform": IOS, "type": "virtual", "name": "iPhone 12", "version": "16.1"},
102149
"simulator_latest": {"platform": IOS, "type": "virtual", "name": "iPhone 12", "version": "16.2"},
@@ -152,10 +199,10 @@ def get_value(workflow, matrix_type, parm_key, config_parms_only=False):
152199
def filter_devices(devices, device_type, device_platform):
153200
""" Filter device by device_type
154201
"""
155-
filtered_value = filter(lambda device:
156-
TEST_DEVICES.get(device).get("type") in device_type
202+
filtered_value = filter(lambda device:
203+
TEST_DEVICES.get(device).get("type") in device_type
157204
and TEST_DEVICES.get(device).get("platform") in device_platform, devices)
158-
return list(filtered_value)
205+
return list(filtered_value)
159206

160207

161208
# TODO(sunmou): add auto_diff feature
@@ -166,7 +213,7 @@ def filter_values_on_diff(parm_key, value, auto_diff):
166213
def filter_non_desktop_platform(platform):
167214
mobile_platform = [ANDROID, IOS, TVOS]
168215
filtered_value = filter(lambda p: p in platform, mobile_platform)
169-
return list(filtered_value)
216+
return list(filtered_value)
170217

171218

172219
def filter_build_platforms(platforms):
@@ -220,13 +267,13 @@ def get_testapp_build_matrix(matrix_type, unity_versions, platforms, build_os, i
220267
unity_version = li[0]
221268
platform = li[1]
222269
os = li[2] if li[2] else (MACOS_RUNNER if (platform in [IOS, TVOS]) else WINDOWS_RUNNER)
223-
270+
224271
if platform in [IOS, TVOS]:
225-
# for iOS, tvOS platforms, exclude non macOS build_os
272+
# for iOS, tvOS platforms, exclude non macOS build_os
226273
if os==MACOS_RUNNER:
227274
for s in ios_sdk:
228275
# skip tvOS build for real devices
229-
if platform==TVOS and s=="real":
276+
if platform==TVOS and s=="real":
230277
continue
231278
matrix["include"].append({"unity_version": unity_version, "platform": platform, "os": os, "ios_sdk": s})
232279
else:
@@ -265,7 +312,7 @@ def get_testapp_test_matrix(matrix_type, unity_versions, platforms, build_os, mo
265312
# "test_device":"android_target",
266313
# "device_detail":"model=blueline,version=28", # secondary info
267314
# "device_type":"real", # secondary info
268-
# "ios_sdk": "NA"
315+
# "ios_sdk": "NA"
269316
# }
270317

271318
if matrix_type: unity_versions = get_value("integration_tests", matrix_type, "unity_versions")
@@ -294,7 +341,10 @@ def get_testapp_test_matrix(matrix_type, unity_versions, platforms, build_os, mo
294341
mobile_devices = get_value("integration_tests", matrix_type, "mobile_devices")
295342
for mobile_device in mobile_devices:
296343
device_detail = TEST_DEVICES.get(mobile_device).get("device")
297-
if not device_detail: device_detail = "NA"
344+
if device_detail:
345+
device_detail = ';'.join(device_detail)
346+
else:
347+
device_detail = "NA"
298348
device_type = TEST_DEVICES.get(mobile_device).get("type")
299349
device_platform = TEST_DEVICES.get(mobile_device).get("platform")
300350
# testapp & test device must match. e.g. iOS app only runs on iOS device, and cannot run on Android or tvOS devices
@@ -307,13 +357,13 @@ def get_testapp_test_matrix(matrix_type, unity_versions, platforms, build_os, mo
307357

308358

309359
def _get_test_os(platform, mobile_device_type=""):
310-
# Desktop platform test on their OS respectivly.
311-
# Mobile platform test on Linux machine if we run tests on FTL, else Mac machine if we run tests on simulators
360+
# Desktop platform test on their OS respectivly.
361+
# Mobile platform test on Linux machine if we run tests on FTL, else Mac machine if we run tests on simulators
312362
if platform == 'Windows':
313363
return WINDOWS_RUNNER
314364
elif platform == 'Linux' or (platform in [IOS, ANDROID] and mobile_device_type == 'real'):
315365
return LINUX_RUNNER
316-
else:
366+
else:
317367
return MACOS_RUNNER
318368

319369

0 commit comments

Comments
 (0)