Skip to content

Commit 7294675

Browse files
StypoxStypox
authored
Merge pull request #1093 from AudricV/yt_support-shorts-ui-playlists
[YouTube] Support Shorts UI in playlists
2 parents 93a90b8 + 698c710 commit 7294675

File tree

7 files changed

+700
-21
lines changed

7 files changed

+700
-21
lines changed

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java‎

Lines changed: 33 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
4141
// Names of some objects in JSON response frequently used in this class
4242
private static final String PLAYLIST_VIDEO_RENDERER = "playlistVideoRenderer";
4343
private static final String PLAYLIST_VIDEO_LIST_RENDERER = "playlistVideoListRenderer";
44+
private static final String RICH_GRID_RENDERER = "richGridRenderer";
45+
private static final String RICH_ITEM_RENDERER = "richItemRenderer";
46+
private static final String REEL_ITEM_RENDERER = "reelItemRenderer";
4447
private static final String SIDEBAR = "sidebar";
4548
private static final String VIDEO_OWNER_RENDERER = "videoOwnerRenderer";
4649

@@ -85,10 +88,6 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
8588
* browse response (the old returns instead a sidebar one).
8689
* </p>
8790
*
88-
* <p>
89-
* This new playlist UI is currently A/B tested.
90-
* </p>
91-
*
9291
* @return Whether the playlist response is using only the new playlist design
9392
*/
9493
private boolean checkIfResponseIsNewPlaylistInterface() {
@@ -327,17 +326,22 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, Extrac
327326
.map(content -> content.getObject("itemSectionRenderer")
328327
.getArray("contents")
329328
.getObject(0))
330-
.filter(contentItemSectionRendererContents ->
331-
contentItemSectionRendererContents.has(PLAYLIST_VIDEO_LIST_RENDERER)
332-
|| contentItemSectionRendererContents.has(
333-
"playlistSegmentRenderer"))
329+
.filter(content -> content.has(PLAYLIST_VIDEO_LIST_RENDERER)
330+
|| content.has(RICH_GRID_RENDERER))
334331
.findFirst()
335332
.orElse(null);
336333

337-
if (videoPlaylistObject != null && videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
338-
final JsonArray videosArray = videoPlaylistObject
339-
.getObject(PLAYLIST_VIDEO_LIST_RENDERER)
340-
.getArray("contents");
334+
if (videoPlaylistObject != null) {
335+
final JsonObject renderer;
336+
if (videoPlaylistObject.has(PLAYLIST_VIDEO_LIST_RENDERER)) {
337+
renderer = videoPlaylistObject.getObject(PLAYLIST_VIDEO_LIST_RENDERER);
338+
} else if (videoPlaylistObject.has(RICH_GRID_RENDERER)) {
339+
renderer = videoPlaylistObject.getObject(RICH_GRID_RENDERER);
340+
} else {
341+
return new InfoItemsPage<>(collector, null);
342+
}
343+
344+
final JsonArray videosArray = renderer.getArray("contents");
341345
collectStreamsFrom(collector, videosArray);
342346

343347
nextPage = getNextPageFrom(videosArray);
@@ -399,14 +403,26 @@ private Page getNextPageFrom(final JsonArray contents)
399403
private void collectStreamsFrom(@Nonnull final StreamInfoItemsCollector collector,
400404
@Nonnull final JsonArray videos) {
401405
final TimeAgoParser timeAgoParser = getTimeAgoParser();
402-
403406
videos.stream()
404407
.filter(JsonObject.class::isInstance)
405408
.map(JsonObject.class::cast)
406-
.filter(video -> video.has(PLAYLIST_VIDEO_RENDERER))
407-
.map(video -> new YoutubeStreamInfoItemExtractor(
408-
video.getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser))
409-
.forEachOrdered(collector::commit);
409+
.forEach(video -> {
410+
if (video.has(PLAYLIST_VIDEO_RENDERER)) {
411+
collector.commit(new YoutubeStreamInfoItemExtractor(
412+
video.getObject(PLAYLIST_VIDEO_RENDERER), timeAgoParser));
413+
} else if (video.has(RICH_ITEM_RENDERER)) {
414+
final JsonObject richItemRenderer = video.getObject(RICH_ITEM_RENDERER);
415+
if (richItemRenderer.has("content")) {
416+
final JsonObject richItemRendererContent =
417+
richItemRenderer.getObject("content");
418+
if (richItemRendererContent.has(REEL_ITEM_RENDERER)) {
419+
collector.commit(new YoutubeReelInfoItemExtractor(
420+
richItemRendererContent.getObject(REEL_ITEM_RENDERER),
421+
timeAgoParser));
422+
}
423+
}
424+
}
425+
});
410426
}
411427

412428
@Nonnull

‎extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public static void assertNotEmpty(String stringToCheck) {
5656
}
5757

5858
public static void assertNotEmpty(@Nullable String message, String stringToCheck) {
59-
assertNotNull(message, stringToCheck);
59+
assertNotNull(stringToCheck, message);
6060
assertFalse(stringToCheck.isEmpty(), message);
6161
}
6262

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/DefaultTests.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,6 @@ public static void defaultTestListOfItems(StreamingService expectedService, List
3838

3939
if (item instanceof StreamInfoItem) {
4040
StreamInfoItem streamInfoItem = (StreamInfoItem) item;
41-
assertNotEmpty("Uploader name not set: " + item, streamInfoItem.getUploaderName());
42-
43-
// assertNotEmpty("Uploader url not set: " + item, streamInfoItem.getUploaderUrl());
4441
final String uploaderUrl = streamInfoItem.getUploaderUrl();
4542
if (!isNullOrEmpty(uploaderUrl)) {
4643
assertIsSecureUrl(uploaderUrl);

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
2929
import org.schabi.newpipe.extractor.stream.Description;
3030
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
31+
import org.schabi.newpipe.extractor.utils.Utils;
3132

3233
import java.io.IOException;
3334

@@ -416,6 +417,120 @@ public void testDescription() throws ParsingException {
416417
}
417418
}
418419

420+
static class ShortsUI implements BasePlaylistExtractorTest {
421+
422+
private static PlaylistExtractor extractor;
423+
424+
@BeforeAll
425+
static void setUp() throws Exception {
426+
YoutubeTestsUtils.ensureStateless();
427+
NewPipe.init(DownloaderFactory.getDownloader(RESOURCE_PATH + "shortsUI"));
428+
extractor = YouTube.getPlaylistExtractor(
429+
"https://www.youtube.com/playlist?list=UUSHBR8-60-B28hp2BmDPdntcQ");
430+
extractor.fetchPage();
431+
}
432+
433+
@Test
434+
@Override
435+
public void testServiceId() throws Exception {
436+
assertEquals(YouTube.getServiceId(), extractor.getServiceId());
437+
}
438+
439+
@Test
440+
@Override
441+
public void testName() throws Exception {
442+
assertEquals("Short videos", extractor.getName());
443+
}
444+
445+
@Test
446+
@Override
447+
public void testId() throws Exception {
448+
assertEquals("UUSHBR8-60-B28hp2BmDPdntcQ", extractor.getId());
449+
}
450+
451+
@Test
452+
@Override
453+
public void testUrl() throws Exception {
454+
assertEquals("https://www.youtube.com/playlist?list=UUSHBR8-60-B28hp2BmDPdntcQ",
455+
extractor.getUrl());
456+
}
457+
458+
@Test
459+
@Override
460+
public void testOriginalUrl() throws Exception {
461+
assertEquals("https://www.youtube.com/playlist?list=UUSHBR8-60-B28hp2BmDPdntcQ",
462+
extractor.getOriginalUrl());
463+
}
464+
465+
@Test
466+
@Override
467+
public void testRelatedItems() throws Exception {
468+
defaultTestRelatedItems(extractor);
469+
}
470+
471+
// TODO: enable test when continuations are available
472+
@Disabled("Shorts UI doesn't return any continuation, even if when there are more than 100 "
473+
+ "items: this is a bug on YouTube's side, which is not related to the requirement "
474+
+ "of a valid visitorData like it is for Shorts channel tab")
475+
@Test
476+
@Override
477+
public void testMoreRelatedItems() throws Exception {
478+
defaultTestMoreItems(extractor);
479+
}
480+
481+
@Test
482+
@Override
483+
public void testThumbnailUrl() throws Exception {
484+
final String thumbnailUrl = extractor.getThumbnailUrl();
485+
assertIsSecureUrl(thumbnailUrl);
486+
ExtractorAsserts.assertContains("yt", thumbnailUrl);
487+
}
488+
489+
@Test
490+
@Override
491+
public void testBannerUrl() throws Exception {
492+
final String thumbnailUrl = extractor.getThumbnailUrl();
493+
assertIsSecureUrl(thumbnailUrl);
494+
ExtractorAsserts.assertContains("yt", thumbnailUrl);
495+
}
496+
497+
@Test
498+
@Override
499+
public void testUploaderName() throws Exception {
500+
assertEquals("YouTube", extractor.getUploaderName());
501+
}
502+
503+
@Test
504+
@Override
505+
public void testUploaderAvatarUrl() throws Exception {
506+
final String uploaderAvatarUrl = extractor.getUploaderAvatarUrl();
507+
ExtractorAsserts.assertContains("yt", uploaderAvatarUrl);
508+
}
509+
510+
@Test
511+
@Override
512+
public void testStreamCount() throws Exception {
513+
ExtractorAsserts.assertGreater(250, extractor.getStreamCount());
514+
}
515+
516+
@Test
517+
@Override
518+
public void testUploaderVerified() throws Exception {
519+
// YouTube doesn't provide this information for playlists
520+
assertFalse(extractor.isUploaderVerified());
521+
}
522+
523+
@Test
524+
void getPlaylistType() throws ParsingException {
525+
assertEquals(PlaylistInfo.PlaylistType.NORMAL, extractor.getPlaylistType());
526+
}
527+
528+
@Test
529+
void testDescription() throws ParsingException {
530+
assertTrue(Utils.isNullOrEmpty(extractor.getDescription().getContent()));
531+
}
532+
}
533+
419534
public static class ContinuationsTests {
420535

421536
@BeforeAll

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/shortsUI/generated_mock_0.json‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
{
2+
"request": {
3+
"httpMethod": "GET",
4+
"url": "https://www.youtube.com/sw.js",
5+
"headers": {
6+
"Referer": [
7+
"https://www.youtube.com"
8+
],
9+
"Origin": [
10+
"https://www.youtube.com"
11+
],
12+
"Accept-Language": [
13+
"en-GB, en;q\u003d0.9"
14+
]
15+
},
16+
"localization": {
17+
"languageCode": "en",
18+
"countryCode": "GB"
19+
}
20+
},
21+
"response": {
22+
"responseCode": 200,
23+
"responseMessage": "",
24+
"responseHeaders": {
25+
"access-control-allow-credentials": [
26+
"true"
27+
],
28+
"access-control-allow-origin": [
29+
"https://www.youtube.com"
30+
],
31+
"alt-svc": [
32+
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
33+
],
34+
"cache-control": [
35+
"private, max-age\u003d0"
36+
],
37+
"content-type": [
38+
"text/javascript; charset\u003dutf-8"
39+
],
40+
"cross-origin-opener-policy-report-only": [
41+
"same-origin; report-to\u003d\"youtube_main\""
42+
],
43+
"date": [
44+
"Mon, 07 Aug 2023 17:06:40 GMT"
45+
],
46+
"expires": [
47+
"Mon, 07 Aug 2023 17:06:40 GMT"
48+
],
49+
"origin-trial": [
50+
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
51+
],
52+
"p3p": [
53+
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
54+
],
55+
"permissions-policy": [
56+
"ch-ua-arch\u003d*, ch-ua-bitness\u003d*, ch-ua-full-version\u003d*, ch-ua-full-version-list\u003d*, ch-ua-model\u003d*, ch-ua-wow64\u003d*, ch-ua-form-factor\u003d*, ch-ua-platform\u003d*, ch-ua-platform-version\u003d*"
57+
],
58+
"report-to": [
59+
"{\"group\":\"youtube_main\",\"max_age\":2592000,\"endpoints\":[{\"url\":\"https://csp.withgoogle.com/csp/report-to/youtube_main\"}]}"
60+
],
61+
"server": [
62+
"ESF"
63+
],
64+
"set-cookie": [
65+
"YSC\u003d9y_BcrNyhW0; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
66+
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dTue, 10-Nov-2020 17:06:40 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
67+
"CONSENT\u003dPENDING+034; expires\u003dWed, 06-Aug-2025 17:06:40 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
68+
],
69+
"strict-transport-security": [
70+
"max-age\u003d31536000"
71+
],
72+
"x-content-type-options": [
73+
"nosniff"
74+
],
75+
"x-frame-options": [
76+
"SAMEORIGIN"
77+
],
78+
"x-xss-protection": [
79+
"0"
80+
]
81+
},
82+
"responseBody": "\n self.addEventListener(\u0027install\u0027, event \u003d\u003e {\n event.waitUntil(self.skipWaiting());\n });\n self.addEventListener(\u0027activate\u0027, event \u003d\u003e {\n event.waitUntil(\n self.clients.claim().then(() \u003d\u003e self.registration.unregister()));\n });\n ",
83+
"latestUrl": "https://www.youtube.com/sw.js"
84+
}
85+
}

‎extractor/src/test/resources/org/schabi/newpipe/extractor/services/youtube/extractor/playlist/shortsUI/generated_mock_1.json‎

Lines changed: 79 additions & 0 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)