From f9e647113003078c995aa284f4f6568f107adbda Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Sun, 29 Dec 2024 16:13:44 -0800 Subject: [PATCH 01/25] Quick GetChild Implementation (git Fixed) --- src/tags/index.ts | 2 ++ src/tags/page-getters.ts | 12 ++++++++++++ types/entities.ts | 1 + 3 files changed, 15 insertions(+) diff --git a/src/tags/index.ts b/src/tags/index.ts index a959fa3..3304616 100644 --- a/src/tags/index.ts +++ b/src/tags/index.ts @@ -4,6 +4,7 @@ import { isCanonical, isCommon, getParentTags, + getChildTags, } from "./page-getters"; import { getTagId, getTagNameFromFeed } from "./works-feed-getters"; import { @@ -30,6 +31,7 @@ export const getTag = async ({ common: isCommon(tagPage), canonicalName: getCanonical(tagPage), parentTags: getParentTags(tagPage), + childTags: getChildTags(tagPage), }; }; diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts index 5d5d443..04366fe 100644 --- a/src/tags/page-getters.ts +++ b/src/tags/page-getters.ts @@ -52,3 +52,15 @@ export const getParentTags = ($tagPage: TagPage) => { }); return parentTags; }; +export const getChildTags = ($tagPage: TagPage) => { + const childTags: string[] = []; + $tagPage(".child > div > ul > li").each((i, element) => { + + //console.log(i, $tagPage(element).children().first().text()); + let childTag = $tagPage(element).children("a").first().text(); + if (childTag != '') { + childTags.push(childTag); + } + }) + return childTags; +} diff --git a/types/entities.ts b/types/entities.ts index 0cdad10..f7886c9 100644 --- a/types/entities.ts +++ b/types/entities.ts @@ -18,6 +18,7 @@ export interface Tag { // common and cannot be filtered on. canonicalName: string | null; parentTags: string[]; + childTags: string[]; } export interface User { From f9bb38eb8aae3b9a9f12124f8d2804174036c92e Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:27:39 -0800 Subject: [PATCH 02/25] working but slow implentation of tag categories --- src/tags/page-getters.ts | 10 ++++++---- types/entities.ts | 5 ++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts index 0009b2b..0d15e34 100644 --- a/src/tags/page-getters.ts +++ b/src/tags/page-getters.ts @@ -54,13 +54,15 @@ export const getParentTags = ($tagPage: TagPage) => { return parentTags; }; export const getChildTags = ($tagPage: TagPage) => { - const childTags: string[] = []; + const childTags: {tagName: string; Category: TagCategory }[] = []; $tagPage(".child > div > ul > li").each((i, element) => { - - //console.log(i, $tagPage(element).children().first().text()); let childTag = $tagPage(element).children("a").first().text(); if (childTag != '') { - childTags.push(childTag); + // parent and parent's parent is an element + let classcatagory = (element.parent?.parent as Element).attributes[0].value.split(' ')[0] + //console.log(classcatagory != 'freeforms' ? classcatagory : 'additional tags'); + let tc = (classcatagory != 'freeforms' ? classcatagory : 'additional tags') as TagCategory; + childTags.push({tagName: childTag, Category: tc}); } }) return childTags; diff --git a/types/entities.ts b/types/entities.ts index c5d2046..a1d5acf 100644 --- a/types/entities.ts +++ b/types/entities.ts @@ -18,7 +18,10 @@ export interface Tag { // common and cannot be filtered on. canonicalName: string | null; parentTags: string[]; - childTags: string[]; + childTags: Array<{ + tagName: string; + Category: TagCategory; + }>; subTags: Array<{ tagName: string; parentSubTag: string | null; From c77fccaeb3adfc10c3202898b9edc10d24b2f232 Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:59:05 -0800 Subject: [PATCH 03/25] iterate over each div instead of doing the backwards traversal --- src/tags/page-getters.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts index 0d15e34..6a327cc 100644 --- a/src/tags/page-getters.ts +++ b/src/tags/page-getters.ts @@ -54,7 +54,9 @@ export const getParentTags = ($tagPage: TagPage) => { return parentTags; }; export const getChildTags = ($tagPage: TagPage) => { + const childTags: {tagName: string; Category: TagCategory }[] = []; + /* $tagPage(".child > div > ul > li").each((i, element) => { let childTag = $tagPage(element).children("a").first().text(); if (childTag != '') { @@ -65,6 +67,18 @@ export const getChildTags = ($tagPage: TagPage) => { childTags.push({tagName: childTag, Category: tc}); } }) + */ + + const divCategory = $tagPage(".child > div"); + divCategory.each((i, divelement) => { + let classcatagory = divelement.attribs?.class?.split(' ')[0]; + let tagcatagory = (classcatagory != 'freeforms' ? classcatagory : 'additional tags') as TagCategory; + $tagPage(divelement).find('ul > li a').each((_, aElement) => { + let childTag = $tagPage(aElement).text(); + (childTag != '') && childTags.push({tagName: childTag, Category: tagcatagory}); + }) + }) + return childTags; } From 18eb2e2d2122c98fd97bf8e0d4bf700ce9d24818 Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:26:19 -0800 Subject: [PATCH 04/25] use cheerio map --- src/tags/page-getters.ts | 35 +++++++++-------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts index 6a327cc..730a7da 100644 --- a/src/tags/page-getters.ts +++ b/src/tags/page-getters.ts @@ -54,32 +54,15 @@ export const getParentTags = ($tagPage: TagPage) => { return parentTags; }; export const getChildTags = ($tagPage: TagPage) => { - - const childTags: {tagName: string; Category: TagCategory }[] = []; - /* - $tagPage(".child > div > ul > li").each((i, element) => { - let childTag = $tagPage(element).children("a").first().text(); - if (childTag != '') { - // parent and parent's parent is an element - let classcatagory = (element.parent?.parent as Element).attributes[0].value.split(' ')[0] - //console.log(classcatagory != 'freeforms' ? classcatagory : 'additional tags'); - let tc = (classcatagory != 'freeforms' ? classcatagory : 'additional tags') as TagCategory; - childTags.push({tagName: childTag, Category: tc}); - } - }) - */ - - const divCategory = $tagPage(".child > div"); - divCategory.each((i, divelement) => { - let classcatagory = divelement.attribs?.class?.split(' ')[0]; - let tagcatagory = (classcatagory != 'freeforms' ? classcatagory : 'additional tags') as TagCategory; - $tagPage(divelement).find('ul > li a').each((_, aElement) => { - let childTag = $tagPage(aElement).text(); - (childTag != '') && childTags.push({tagName: childTag, Category: tagcatagory}); - }) - }) - - return childTags; + return $tagPage(".child > div").map((_, divElement) => { + const $div = $tagPage(divElement); + const className = $div.attr("class")?.split(' ')[0] ?? ""; + const category = (className !== "freeforms" ? className : "additional tags") as TagCategory; + return $div.find("ul > li a").map((_, aElement) => { + const childTag = $tagPage(aElement).text(); + return childTag ? { tagName: childTag, Category: category} : null; + }).get(); + }).get(); } export const getSubTags = ($tagPage: TagPage) => { From 388238a7010e72056b25a194b4f469d3391424fd Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:33:41 -0800 Subject: [PATCH 05/25] fix capitlization --- src/tags/page-getters.ts | 2 +- types/entities.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts index 730a7da..00d204c 100644 --- a/src/tags/page-getters.ts +++ b/src/tags/page-getters.ts @@ -60,7 +60,7 @@ export const getChildTags = ($tagPage: TagPage) => { const category = (className !== "freeforms" ? className : "additional tags") as TagCategory; return $div.find("ul > li a").map((_, aElement) => { const childTag = $tagPage(aElement).text(); - return childTag ? { tagName: childTag, Category: category} : null; + return childTag ? { tagName: childTag, category: category} : null; }).get(); }).get(); } diff --git a/types/entities.ts b/types/entities.ts index a1d5acf..0b3ee43 100644 --- a/types/entities.ts +++ b/types/entities.ts @@ -20,7 +20,7 @@ export interface Tag { parentTags: string[]; childTags: Array<{ tagName: string; - Category: TagCategory; + category: TagCategory; }>; subTags: Array<{ tagName: string; From 04d4e64e23dcbd6cfac564439fabb28c9f2d05ea Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Thu, 2 Jan 2025 18:24:50 -0800 Subject: [PATCH 06/25] non-functional --- src/tags/page-getters.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts index 00d204c..8b5c62e 100644 --- a/src/tags/page-getters.ts +++ b/src/tags/page-getters.ts @@ -1,4 +1,4 @@ -import { TagCategory } from "types/entities"; +import { TagCategory, Tag } from "types/entities"; import { TagPage } from "../page-loaders"; import { Element } from "cheerio"; @@ -65,7 +65,8 @@ export const getChildTags = ($tagPage: TagPage) => { }).get(); } -export const getSubTags = ($tagPage: TagPage) => { +export const getSubTags = ($tagPage: TagPage): Tag["subTags"] => { + /* const subTags: { tagName: string; parentSubTag: string | null }[] = []; $tagPage(".sub > ul.tags > li").each((_, element) => { subTags.push({ tagName: $tagPage(element).children().first().text(), parentSubTag: null }); @@ -79,6 +80,17 @@ export const getSubTags = ($tagPage: TagPage) => { }); }); } - }); - return subTags; + });*/ + //return subTags; + return $tagPage(".sub > ul.tags > li").map((_, element) => { + if($tagPage($tagPage(element).has("ul.tags")).length) { + return $tagPage("ul.tags", element).children("li").map((_, child) => { + return { + tagName: $tagPage(child).children().first().text(), + parentSubTag: $tagPage($tagPage(child)).parents("li").children().first().text() + }; + }).get() + } + return { tagName: $tagPage(element).children().first().text(), parentSubTag: null } as Tag["subTags"][number]; + }).get(); }; From 97aa54abcb67e59c2260fcb2a0687c15e1bd61ed Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 13:45:58 -0800 Subject: [PATCH 07/25] Get Subtags works --- src/tags/page-getters.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts index 8b5c62e..856e0c5 100644 --- a/src/tags/page-getters.ts +++ b/src/tags/page-getters.ts @@ -84,12 +84,14 @@ export const getSubTags = ($tagPage: TagPage): Tag["subTags"] => { //return subTags; return $tagPage(".sub > ul.tags > li").map((_, element) => { if($tagPage($tagPage(element).has("ul.tags")).length) { - return $tagPage("ul.tags", element).children("li").map((_, child) => { + const topElement = { tagName: $tagPage(element).children().first().text(), parentSubTag: null }; + const subElements = $tagPage("ul.tags", element).children("li").map((_, child) => { return { tagName: $tagPage(child).children().first().text(), parentSubTag: $tagPage($tagPage(child)).parents("li").children().first().text() }; }).get() + return [topElement, ...subElements]; } return { tagName: $tagPage(element).children().first().text(), parentSubTag: null } as Tag["subTags"][number]; }).get(); From 2fd0ad9fd251c49414e40a839991f790cc609d9e Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 13:47:57 -0800 Subject: [PATCH 08/25] Explicit Type coersion no longer needed --- src/tags/page-getters.ts | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts index 856e0c5..2efa6ff 100644 --- a/src/tags/page-getters.ts +++ b/src/tags/page-getters.ts @@ -65,23 +65,7 @@ export const getChildTags = ($tagPage: TagPage) => { }).get(); } -export const getSubTags = ($tagPage: TagPage): Tag["subTags"] => { - /* - const subTags: { tagName: string; parentSubTag: string | null }[] = []; - $tagPage(".sub > ul.tags > li").each((_, element) => { - subTags.push({ tagName: $tagPage(element).children().first().text(), parentSubTag: null }); - if ($tagPage($tagPage(element)).has("ul.tags").length) { - $tagPage("ul.tags", element).children("li").each((_, child) => { - // each
  • element contains an element, - // which is why `.children().first()` is needed for both the `tagName` and `parentSubTag` - subTags.push({ - tagName: $tagPage(child).children().first().text(), - parentSubTag: $tagPage($tagPage(child)).parents("li").children().first().text() - }); - }); - } - });*/ - //return subTags; +export const getSubTags = ($tagPage: TagPage) => { return $tagPage(".sub > ul.tags > li").map((_, element) => { if($tagPage($tagPage(element).has("ul.tags")).length) { const topElement = { tagName: $tagPage(element).children().first().text(), parentSubTag: null }; @@ -93,6 +77,8 @@ export const getSubTags = ($tagPage: TagPage): Tag["subTags"] => { }).get() return [topElement, ...subElements]; } - return { tagName: $tagPage(element).children().first().text(), parentSubTag: null } as Tag["subTags"][number]; + return { tagName: $tagPage(element).children().first().text(), parentSubTag: null }; + //return { tagName: $tagPage(element).children().first().text(), parentSubTag: null } as Tag["subTags"][number]; + }).get(); }; From df74cf05c7e2a8cf2cc562f60d13f4b2f2769ff0 Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 13:55:53 -0800 Subject: [PATCH 09/25] Refactor as to not use extra arrays --- src/tags/page-getters.ts | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts index 2efa6ff..b5ecfdd 100644 --- a/src/tags/page-getters.ts +++ b/src/tags/page-getters.ts @@ -68,17 +68,19 @@ export const getChildTags = ($tagPage: TagPage) => { export const getSubTags = ($tagPage: TagPage) => { return $tagPage(".sub > ul.tags > li").map((_, element) => { if($tagPage($tagPage(element).has("ul.tags")).length) { - const topElement = { tagName: $tagPage(element).children().first().text(), parentSubTag: null }; - const subElements = $tagPage("ul.tags", element).children("li").map((_, child) => { - return { - tagName: $tagPage(child).children().first().text(), - parentSubTag: $tagPage($tagPage(child)).parents("li").children().first().text() - }; - }).get() - return [topElement, ...subElements]; + return [ + { + tagName: $tagPage(element).children().first().text(), + parentSubTag: null + }, + $tagPage("ul.tags", element).children("li").map((_, child) => { + return { + tagName: $tagPage(child).children().first().text(), + parentSubTag: $tagPage($tagPage(child)).parents("li").children().first().text() + }; + }).get() + ].flat(); } return { tagName: $tagPage(element).children().first().text(), parentSubTag: null }; - //return { tagName: $tagPage(element).children().first().text(), parentSubTag: null } as Tag["subTags"][number]; - }).get(); }; From d78dd309f4ef025276726451287e0807e5dfbb4b Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:05:20 -0800 Subject: [PATCH 10/25] Refactor getParentTags as well --- src/tags/page-getters.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts index b5ecfdd..2456257 100644 --- a/src/tags/page-getters.ts +++ b/src/tags/page-getters.ts @@ -47,11 +47,9 @@ export const getCanonical = ($tagPage: TagPage) => { }; export const getParentTags = ($tagPage: TagPage) => { - const parentTags: string[] = []; - $tagPage(".parent ul.tags li").each((_, element) => { - parentTags.push($tagPage(element).text()); - }); - return parentTags; + return $tagPage(".parent ul.tags li").map((_, element) => { + return $tagPage(element).text(); + }).get(); }; export const getChildTags = ($tagPage: TagPage) => { return $tagPage(".child > div").map((_, divElement) => { From e79039c57bb8ae3e5cd972caf20c7ce6e9d4a686 Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:15:24 -0800 Subject: [PATCH 11/25] refactor getSeriesWorks --- src/series/getters.ts | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 50f53b5..359f350 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -1,5 +1,5 @@ import { Author, Series, SeriesWorkSummary } from "types/entities"; -import { CheerioAPI, load } from "cheerio"; +import { CheerioAPI, Element, load } from "cheerio"; import { SeriesPage, WorkPage } from "../page-loaders"; import { getWorkBookmarkCount, @@ -114,18 +114,11 @@ export const getSeriesBookmarkCount = ($seriesPage: SeriesPage): number => { ); }; -export const getSeriesWorks = ( - $seriesPage: SeriesPage -): SeriesWorkSummary[] => { - const works: SeriesWorkSummary[] = []; - - $seriesPage("ul.index > li.work").each((index, element) => { - works[index] = getSeriesWork($seriesPage(element).html() as string); - }); - - return works; -}; - +export const getSeriesWorks = ($seriesPage: SeriesPage): SeriesWorkSummary[] => { + return $seriesPage("ul.index > li.work").map((index, element) => { + return getSeriesWork($seriesPage(element).html() as string); + }).get() +} // Helpers for series' works interface SeriesWork extends CheerioAPI { kind: "SeriesWork"; From f7c98f90c3c42943e753d1f3524132980aa13da2 Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:19:25 -0800 Subject: [PATCH 12/25] refactor getSeriesWork Fandoms --- src/series/getters.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 359f350..6b382f6 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -180,13 +180,10 @@ const getSeriesWorkSummary = ($work: SeriesWork) => { }; const getSeriesWorkFandoms = ($work: SeriesWork): string[] => { - const fandoms: string[] = []; - - $work("h5.fandoms a.tag").each(function (i, element) { - fandoms[i] = $work(element).text().trim(); - }); - return fandoms; -}; + return $work("h5.fandoms a.tag").map((i, element) => { + return $work(element).text().trim(); + }).get(); +} const getSeriesWorkCharacters = ($work: SeriesWork): string[] => { const characters: string[] = []; From aa1a73e3cba06c46e44dde574e5da6c4a733cc73 Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:21:12 -0800 Subject: [PATCH 13/25] refactor getSeriesWorkCharacters --- src/series/getters.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 6b382f6..b34b644 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -186,12 +186,9 @@ const getSeriesWorkFandoms = ($work: SeriesWork): string[] => { } const getSeriesWorkCharacters = ($work: SeriesWork): string[] => { - const characters: string[] = []; - - $work("li.characters a.tag").each(function (i, character) { - characters[i] = $work(character).text().trim(); - }); - return characters; + return $work("li.characters a.tag").map((i, character) => { + return $work(character).text().trim(); + }).get() }; const getSeriesWorkRelationships = ($work: SeriesWork): string[] => { From 6a987b48ee623837af255bcadede218bd1fc2166 Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:22:59 -0800 Subject: [PATCH 14/25] refactor getSeriesWorkRelationships --- src/series/getters.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index b34b644..65b5e75 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -192,12 +192,9 @@ const getSeriesWorkCharacters = ($work: SeriesWork): string[] => { }; const getSeriesWorkRelationships = ($work: SeriesWork): string[] => { - const ships: string[] = []; - - $work("li.relationships a.tag").each(function (i, ship) { - ships[i] = $work(ship).text().trim(); - }); - return ships; + return $work("li.relationships a.tag").map((i, ship) => { + return $work(ship).text().trim(); + }).get() }; const getSeriesWorkAdditionalTags = ($work: SeriesWork): string[] => { From 4a1738e10d87274ced472bda9c998691fe580452 Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:24:46 -0800 Subject: [PATCH 15/25] refactor getSeriesWorkAdditionalTags --- src/series/getters.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 65b5e75..3d29af2 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -198,12 +198,9 @@ const getSeriesWorkRelationships = ($work: SeriesWork): string[] => { }; const getSeriesWorkAdditionalTags = ($work: SeriesWork): string[] => { - const tags: string[] = []; - - $work("li.freeforms a.tag").each(function (i) { - tags[i] = $work(this).text().trim(); - }); - return tags; + return $work("li.freeforms a.tag").map((_, element) => { + return $work(element).text().trim(); + }).get(); }; const getSeriesWorkAuthors = ( From 9c661f330f9d548ea30fea6f3da47746e3f47294 Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:26:27 -0800 Subject: [PATCH 16/25] refactor getSeriesNotes --- src/series/getters.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 3d29af2..e1437bb 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -68,11 +68,7 @@ export const getSeriesDescription = ( export const getSeriesNotes = ($seriesPage: SeriesPage): string | null => { const notes = $seriesPage("dl.series dd:nth-of-type(5)"); - if (notes.prevAll().first().text().trim() === "Notes:") { - return notes.html()!.trim(); - } else { - return null; - } + return notes.prevAll().first().text().trim() === "Notes:" ? notes.html()!.trim() : null; }; export const getSeriesPublishDate = ($seriesPage: SeriesPage): string => { From 2ff33b3680b4a3d8eb27b3edb67d90cf24988e6c Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:54:31 -0800 Subject: [PATCH 17/25] $seriesPage("dl.series blockquote.userstuff").html() --- src/series/getters.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index e1437bb..6353e99 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -59,11 +59,8 @@ export const getSeriesAuthors = ( return authors; }; -export const getSeriesDescription = ( - $seriesPage: SeriesPage -): string | null => { - const description = $seriesPage("dl.series blockquote.userstuff").html(); - return description ? description.trim() : null; +export const getSeriesDescription = ($seriesPage: SeriesPage): string | null => { + return $seriesPage("dl.series blockquote.userstuff").html()?.trim() || null; }; export const getSeriesNotes = ($seriesPage: SeriesPage): string | null => { From b4e9f5f20f01b92612f85ef24d2c12a476d94487 Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 14:54:31 -0800 Subject: [PATCH 18/25] Refactor getSeriesDescription and make formatting consistant --- src/series/getters.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index e1437bb..6353e99 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -59,11 +59,8 @@ export const getSeriesAuthors = ( return authors; }; -export const getSeriesDescription = ( - $seriesPage: SeriesPage -): string | null => { - const description = $seriesPage("dl.series blockquote.userstuff").html(); - return description ? description.trim() : null; +export const getSeriesDescription = ($seriesPage: SeriesPage): string | null => { + return $seriesPage("dl.series blockquote.userstuff").html()?.trim() || null; }; export const getSeriesNotes = ($seriesPage: SeriesPage): string | null => { From b8d42f2dd81e5a263835bd115e4f39d1be2ffcb9 Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:08:48 -0800 Subject: [PATCH 19/25] refactor getSeriesAuthors --- src/series/getters.ts | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 6353e99..4bbc11a 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -31,32 +31,25 @@ export const getSeriesTitle = ($seriesPage: SeriesPage): string => { return $seriesPage("h2.heading").text().trim(); }; -export const getSeriesAuthors = ( - $seriesPage: SeriesPage -): Series["authors"] => { +export const getSeriesAuthors = ($seriesPage: SeriesPage): Series["authors"] => { const authorLinks = $seriesPage("dl.meta a[rel=author]"); - const authors: Author[] = []; - if ( - $seriesPage("dl.meta > dd:nth-of-type(1)").text().trim() === "Anonymous" - ) { + if ($seriesPage("dl.meta > dd:nth-of-type(1)").text().trim() === "Anonymous") { return [{ username: "Anonymous", pseud: "Anonymous", anonymous: true }]; } if (authorLinks.length !== 0) { - authorLinks.each((i, element) => { + return authorLinks.map((_, element) => { const url = element.attribs.href; const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!; - - authors.push({ + return { username: username, pseud: decodeURI(pseud), anonymous: false, - }); - }); + } as Author + }).get(); } - - return authors; + return [] as Author[]; }; export const getSeriesDescription = ($seriesPage: SeriesPage): string | null => { From 05f91a701dc9de8690a840fc785f279a2291b34b Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:09:56 -0800 Subject: [PATCH 20/25] forgot a couple semicolons --- src/series/getters.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 4bbc11a..9fa8255 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -103,7 +103,7 @@ export const getSeriesBookmarkCount = ($seriesPage: SeriesPage): number => { export const getSeriesWorks = ($seriesPage: SeriesPage): SeriesWorkSummary[] => { return $seriesPage("ul.index > li.work").map((index, element) => { return getSeriesWork($seriesPage(element).html() as string); - }).get() + }).get(); } // Helpers for series' works interface SeriesWork extends CheerioAPI { @@ -174,13 +174,13 @@ const getSeriesWorkFandoms = ($work: SeriesWork): string[] => { const getSeriesWorkCharacters = ($work: SeriesWork): string[] => { return $work("li.characters a.tag").map((i, character) => { return $work(character).text().trim(); - }).get() + }).get(); }; const getSeriesWorkRelationships = ($work: SeriesWork): string[] => { return $work("li.relationships a.tag").map((i, ship) => { return $work(ship).text().trim(); - }).get() + }).get(); }; const getSeriesWorkAdditionalTags = ($work: SeriesWork): string[] => { From f7668033cedcfaa69175c535650191ea9f59bdbe Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:14:59 -0800 Subject: [PATCH 21/25] refactor getSeriesWorkAuthors --- src/series/getters.ts | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/series/getters.ts b/src/series/getters.ts index 9fa8255..3aae6aa 100644 --- a/src/series/getters.ts +++ b/src/series/getters.ts @@ -189,28 +189,23 @@ const getSeriesWorkAdditionalTags = ($work: SeriesWork): string[] => { }).get(); }; -const getSeriesWorkAuthors = ( - $work: SeriesWork -): SeriesWorkSummary["authors"] => { +const getSeriesWorkAuthors = ($work: SeriesWork): SeriesWorkSummary["authors"] => { const authorLinks = $work("h4.heading a[rel='author']"); - const authors: Author[] = []; - if ($work("h4.heading").text().split("by")[1].trim() === "Anonymous") { return [{ username: "Anonymous", pseud: "Anonymous", anonymous: true }]; } if (authorLinks.length !== 0) { - authorLinks.each((i, element) => { + return authorLinks.map((_, element) => { const url = element.attribs.href; const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!; - - authors.push({ + return { username: username, pseud: decodeURI(pseud), anonymous: false, - }); - }); + } + }).get(); } - return authors; + return [] as Author[]; }; From a3723296183db7494741f7389fb958caa825689d Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:27:26 -0800 Subject: [PATCH 22/25] refactor getUserProfilePseuds --- src/users/getters.ts | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/users/getters.ts b/src/users/getters.ts index 1838f2b..5c83d78 100644 --- a/src/users/getters.ts +++ b/src/users/getters.ts @@ -1,3 +1,4 @@ +import { Element } from "cheerio"; import { UserProfile } from "../page-loaders"; import { getUserProfileUrl } from "../urls"; @@ -18,17 +19,11 @@ export const getUserProfileName = ($userProfile: UserProfile) => { const PSEUD_SUFFIX = ", "; export const getUserProfilePseuds = ($userProfile: UserProfile) => { const pseuds = $userProfile("dd.pseuds a"); - const pseudsArray: string[] = []; - - if (pseuds.length !== 0) { - pseuds.each((i, element) => { - const url = element.attribs.href; - const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!; - - pseudsArray.push(decodeURI(pseud)); - }); - } - return pseudsArray.join(PSEUD_SUFFIX); + return pseuds.length !== 0 + ? pseuds.map((_, element) => decodeURI(element.attribs.href.match(/users\/(.+)\/pseuds\/(.+)/)![2])) + .get() + .join(PSEUD_SUFFIX) + : ""; }; //Trim the results to only the date: From f9c5eeff49c306f5dbd5f301b852d55a48c0e48b Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:33:34 -0800 Subject: [PATCH 23/25] refactor getWorkAuthors --- src/works/chapter-getters.ts | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/works/chapter-getters.ts b/src/works/chapter-getters.ts index 8f3a3d3..b406773 100644 --- a/src/works/chapter-getters.ts +++ b/src/works/chapter-getters.ts @@ -37,26 +37,21 @@ export const getWorkTitle = ($chapterIndexPage: ChapterIndexPage) => { return $chapterIndexPage(".works-navigate h2 a[href^='/works/']").text(); }; -export const getWorkAuthors = ( - $chapterIndexPage: ChapterIndexPage -): Author[] => { +export const getWorkAuthors = ($chapterIndexPage: ChapterIndexPage): Author[] => { const authors: Author[] = []; const authorNode = $chapterIndexPage(".works-navigate h2 a[rel='author']"); if (authorNode.text().trim() === "Anonymous") { return [{ username: "Anonymous", pseud: "Anonymous", anonymous: true }]; } - - if (authorNode.length !== 0) { - authorNode.each((i, element) => { + return authorNode.length !== 0 + ? authorNode.map((_, element) => { const url = element.attribs.href; const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!; - - authors.push({ + return { username: username, pseud: decodeURI(pseud), anonymous: false, - }); - }); - } - return authors; + } + }).get() + : [] as Author[]; }; From 3ff1a6493860be5e03fdae97d053f576dc9dd498 Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:21:07 -0800 Subject: [PATCH 24/25] refactor getChaptersList --- src/works/chapter-getters.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/works/chapter-getters.ts b/src/works/chapter-getters.ts index b406773..17bc510 100644 --- a/src/works/chapter-getters.ts +++ b/src/works/chapter-getters.ts @@ -4,9 +4,9 @@ import { getWorkDetailsFromUrl, getWorkUrl } from "src/urls"; import { ChapterIndexPage } from "src/page-loaders"; const TITLE_SEPARATOR = ". "; -export const getChaptersList = ($chapterIndexPage: ChapterIndexPage) => { - const chapters: Chapter[] = []; - $chapterIndexPage("ol.index > li").each((index, li) => { +export const getChaptersList = ($chapterIndexPage: ChapterIndexPage): Chapter[] => { + //return chapters; + return $chapterIndexPage("ol.index > li").map((index, li) => { const link = $chapterIndexPage(li).find("a")[0]; const chapterText = $chapterIndexPage(link).text(); const { workId, chapterId } = getWorkDetailsFromUrl({ @@ -18,8 +18,7 @@ export const getChaptersList = ($chapterIndexPage: ChapterIndexPage) => { const dateNode = $chapterIndexPage( $chapterIndexPage(li).find(".datetime")[0] ); - - chapters.push({ + return { id: chapterId!, workId, index: index + 1, @@ -28,9 +27,8 @@ export const getChaptersList = ($chapterIndexPage: ChapterIndexPage) => { publishedAt: dateNode.text().replace(/[\(\)]/g, ""), // We rebuild the url so it gets the full path url: getWorkUrl({ workId, chapterId }), - }); - }); - return chapters; + } + }).get(); }; export const getWorkTitle = ($chapterIndexPage: ChapterIndexPage) => { From 7d4b8d6bf8b8eab69f0d7849fd65b28eb0451eab Mon Sep 17 00:00:00 2001 From: Nithin <10778861+darthnithin@users.noreply.github.com> Date: Tue, 7 Jan 2025 18:57:31 -0800 Subject: [PATCH 25/25] refactor getWorkAuthors --- src/works/work-getters.ts | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/works/work-getters.ts b/src/works/work-getters.ts index a56ce5d..36821f0 100644 --- a/src/works/work-getters.ts +++ b/src/works/work-getters.ts @@ -8,29 +8,25 @@ import { } from "types/entities"; import { WorkPage } from "../page-loaders"; +import { Element } from "cheerio"; export const getWorkAuthors = ($workPage: WorkPage): Author[] => { const authorLinks = $workPage("h3.byline a[rel='author']"); - const authors: Author[] = []; if ($workPage("h3.byline").text().trim() === "Anonymous") { return [{ username: "Anonymous", pseud: "Anonymous", anonymous: true }]; } - - if (authorLinks.length !== 0) { - authorLinks.each((i, element) => { - const url = element.attribs.href; - const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!; - - authors.push({ - username: username, - pseud: decodeURI(pseud), - anonymous: false, - }); - }); - } - - return authors; + return authorLinks.length !== 0 + ? authorLinks.map((_, element) => { + const url = element.attribs.href; + const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!; + return { + username: username, + pseud: decodeURI(pseud), + anonymous: false, + } as Author + }).get() + : [] as Author[]; }; export const getWorkTitle = ($workPage: WorkPage): string => {