@@ -58,7 +58,7 @@ const firstPage = await browser.newPage();
5858await firstPage .goto (REPOSITORIES_URL );
5959
6060const lastPageElement = firstPage .locator (' a[aria-label*="Page "]:nth-last-child(2)' );
61- const lastPageLabel = await lastPageElement .getAttribute (' aria-label' )
61+ const lastPageLabel = await lastPageElement .getAttribute (' aria-label' );
6262const lastPageNumber = Number (lastPageLabel .replace (/ \D / g , ' ' ));
6363console .log (lastPageNumber);
6464
@@ -81,7 +81,7 @@ await firstPage.goto(REPOSITORIES_URL);
8181
8282const lastPageLabel = await firstPage .$eval (
8383 ' a[aria-label*="Page "]:nth-last-child(2)' ,
84- (element ) => element .getAttribute (' aria-label' )
84+ (element ) => element .getAttribute (' aria-label' ),
8585);
8686const lastPageNumber = Number (lastPageLabel .replace (/ \D / g , ' ' ));
8787console .log (lastPageNumber);
@@ -132,7 +132,7 @@ const firstPage = await browser.newPage();
132132await firstPage .goto (REPOSITORIES_URL );
133133
134134const lastPageElement = firstPage .locator (' a[aria-label*="Page "]:nth-last-child(2)' );
135- const lastPageLabel = await lastPageElement .getAttribute (' aria-label' )
135+ const lastPageLabel = await lastPageElement .getAttribute (' aria-label' );
136136const lastPageNumber = Number (lastPageLabel .replace (/ \D / g , ' ' ));
137137
138138// Push all results from the first page to the repositories array
@@ -175,7 +175,7 @@ await firstPage.goto(REPOSITORIES_URL);
175175
176176const lastPageLabel = await firstPage .$eval (
177177 ' a[aria-label*="Page "]:nth-last-child(2)' ,
178- (element ) => element .getAttribute (' aria-label' )
178+ (element ) => element .getAttribute (' aria-label' ),
179179);
180180const lastPageNumber = Number (lastPageLabel .replace (/ \D / g , ' ' ));
181181
@@ -217,14 +217,14 @@ If we click around the pagination links, we can observe that all the URLs follow
217217That means we could construct URL for each page if we had an array of numbers with the same range as the pages. If ` lastPageNumber ` is ` 4 ` , the following code creates ` [0, 1, 2, 3, 4] ` :
218218
219219``` js
220- const array = Array (lastPageNumber + 1 ) // getting an array of certain size
221- const numbers = [... array .keys ()]; // getting the keys (the actual numbers) as another array
220+ const array = Array (lastPageNumber + 1 ); // getting an array of certain size
221+ const numbers = [... array .keys ()]; // getting the keys (the actual numbers) as another array
222222```
223223
224224Page ` 0 ` doesn't exist though and we've already scraped page ` 1 ` , so we need one more step to remove those:
225225
226226``` js
227- const pageNumbers = numbers .slice (2 ); // removes the first two numbers
227+ const pageNumbers = numbers .slice (2 ); // removes the first two numbers
228228```
229229
230230To have our code examples shorter, we'll squash the above to a single line of code:
@@ -237,23 +237,22 @@ Now let's scrape repositories for each of these numbers. We'll create promises f
237237
238238``` js
239239const pageNumbers = [... Array (lastPageNumber + 1 ).keys ()].slice (2 );
240- const promises = pageNumbers .map ((pageNumber ) =>
241- (async () => {
242- const paginatedPage = await browser .newPage ();
240+ const promises = pageNumbers .map ((pageNumber ) => (async () => {
241+ const paginatedPage = await browser .newPage ();
243242
244- // Construct the URL by setting the ?page=... parameter to value of pageNumber
245- const url = new URL (REPOSITORIES_URL );
246- url .searchParams .set (' page' , pageNumber);
243+ // Construct the URL by setting the ?page=... parameter to value of pageNumber
244+ const url = new URL (REPOSITORIES_URL );
245+ url .searchParams .set (' page' , pageNumber);
247246
248- // Scrape the page
249- await paginatedPage .goto (url .href );
250- const results = await scrapeRepos (paginatedPage)
247+ // Scrape the page
248+ await paginatedPage .goto (url .href );
249+ const results = await scrapeRepos (paginatedPage);
251250
252- // Push results to the repositories array
253- repositories .push (... results);
251+ // Push results to the repositories array
252+ repositories .push (... results);
254253
255- await paginatedPage .close ();
256- })()
254+ await paginatedPage .close ();
255+ })(),
257256);
258257await Promise .all (promises);
259258
@@ -302,7 +301,7 @@ const firstPage = await browser.newPage();
302301await firstPage .goto (REPOSITORIES_URL );
303302
304303const lastPageElement = firstPage .locator (' a[aria-label*="Page "]:nth-last-child(2)' );
305- const lastPageLabel = await lastPageElement .getAttribute (' aria-label' )
304+ const lastPageLabel = await lastPageElement .getAttribute (' aria-label' );
306305const lastPageNumber = Number (lastPageLabel .replace (/ \D / g , ' ' ));
307306
308307// Push all results from the first page to the repositories array
@@ -311,23 +310,22 @@ repositories.push(...(await scrapeRepos(firstPage)));
311310await firstPage .close ();
312311
313312const pageNumbers = [... Array (lastPageNumber + 1 ).keys ()].slice (2 );
314- const promises = pageNumbers .map ((pageNumber ) =>
315- (async () => {
316- const paginatedPage = await browser .newPage ();
313+ const promises = pageNumbers .map ((pageNumber ) => (async () => {
314+ const paginatedPage = await browser .newPage ();
317315
318- // Construct the URL by setting the ?page=... parameter to value of pageNumber
319- const url = new URL (REPOSITORIES_URL );
320- url .searchParams .set (' page' , pageNumber);
316+ // Construct the URL by setting the ?page=... parameter to value of pageNumber
317+ const url = new URL (REPOSITORIES_URL );
318+ url .searchParams .set (' page' , pageNumber);
321319
322- // Scrape the page
323- await paginatedPage .goto (url .href );
324- const results = await scrapeRepos (paginatedPage)
320+ // Scrape the page
321+ await paginatedPage .goto (url .href );
322+ const results = await scrapeRepos (paginatedPage);
325323
326- // Push results to the repositories array
327- repositories .push (... results);
324+ // Push results to the repositories array
325+ repositories .push (... results);
328326
329- await paginatedPage .close ();
330- })()
327+ await paginatedPage .close ();
328+ })(),
331329);
332330await Promise .all (promises);
333331
@@ -369,7 +367,7 @@ await firstPage.goto(REPOSITORIES_URL);
369367
370368const lastPageLabel = await firstPage .$eval (
371369 ' a[aria-label*="Page "]:nth-last-child(2)' ,
372- (element ) => element .getAttribute (' aria-label' )
370+ (element ) => element .getAttribute (' aria-label' ),
373371);
374372const lastPageNumber = Number (lastPageLabel .replace (/ \D / g , ' ' ));
375373
@@ -379,23 +377,22 @@ repositories.push(...(await scrapeRepos(page)));
379377await firstPage .close ();
380378
381379const pageNumbers = [... Array (lastPageNumber + 1 ).keys ()].slice (2 );
382- const promises = pageNumbers .map ((pageNumber ) =>
383- (async () => {
384- const paginatedPage = await browser .newPage ();
380+ const promises = pageNumbers .map ((pageNumber ) => (async () => {
381+ const paginatedPage = await browser .newPage ();
385382
386- // Construct the URL by setting the ?page=... parameter to value of pageNumber
387- const url = new URL (REPOSITORIES_URL );
388- url .searchParams .set (' page' , pageNumber);
383+ // Construct the URL by setting the ?page=... parameter to value of pageNumber
384+ const url = new URL (REPOSITORIES_URL );
385+ url .searchParams .set (' page' , pageNumber);
389386
390- // Scrape the page
391- await paginatedPage .goto (url .href );
392- const results = await scrapeRepos (paginatedPage)
387+ // Scrape the page
388+ await paginatedPage .goto (url .href );
389+ const results = await scrapeRepos (paginatedPage);
393390
394- // Push results to the repositories array
395- repositories .push (... results);
391+ // Push results to the repositories array
392+ repositories .push (... results);
396393
397- await paginatedPage .close ();
398- })()
394+ await paginatedPage .close ();
395+ })(),
399396);
400397await Promise .all (promises);
401398
0 commit comments