From f9e647113003078c995aa284f4f6568f107adbda Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Sun, 29 Dec 2024 16:13:44 -0800
Subject: [PATCH 01/25] Quick GetChild Implementation (git Fixed)
---
src/tags/index.ts | 2 ++
src/tags/page-getters.ts | 12 ++++++++++++
types/entities.ts | 1 +
3 files changed, 15 insertions(+)
diff --git a/src/tags/index.ts b/src/tags/index.ts
index a959fa3..3304616 100644
--- a/src/tags/index.ts
+++ b/src/tags/index.ts
@@ -4,6 +4,7 @@ import {
isCanonical,
isCommon,
getParentTags,
+ getChildTags,
} from "./page-getters";
import { getTagId, getTagNameFromFeed } from "./works-feed-getters";
import {
@@ -30,6 +31,7 @@ export const getTag = async ({
common: isCommon(tagPage),
canonicalName: getCanonical(tagPage),
parentTags: getParentTags(tagPage),
+ childTags: getChildTags(tagPage),
};
};
diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts
index 5d5d443..04366fe 100644
--- a/src/tags/page-getters.ts
+++ b/src/tags/page-getters.ts
@@ -52,3 +52,15 @@ export const getParentTags = ($tagPage: TagPage) => {
});
return parentTags;
};
+export const getChildTags = ($tagPage: TagPage) => {
+ const childTags: string[] = [];
+ $tagPage(".child > div > ul > li").each((i, element) => {
+
+ //console.log(i, $tagPage(element).children().first().text());
+ let childTag = $tagPage(element).children("a").first().text();
+ if (childTag != '') {
+ childTags.push(childTag);
+ }
+ })
+ return childTags;
+}
diff --git a/types/entities.ts b/types/entities.ts
index 0cdad10..f7886c9 100644
--- a/types/entities.ts
+++ b/types/entities.ts
@@ -18,6 +18,7 @@ export interface Tag {
// common and cannot be filtered on.
canonicalName: string | null;
parentTags: string[];
+ childTags: string[];
}
export interface User {
From f9bb38eb8aae3b9a9f12124f8d2804174036c92e Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Thu, 2 Jan 2025 16:27:39 -0800
Subject: [PATCH 02/25] working but slow implentation of tag categories
---
src/tags/page-getters.ts | 10 ++++++----
types/entities.ts | 5 ++++-
2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts
index 0009b2b..0d15e34 100644
--- a/src/tags/page-getters.ts
+++ b/src/tags/page-getters.ts
@@ -54,13 +54,15 @@ export const getParentTags = ($tagPage: TagPage) => {
return parentTags;
};
export const getChildTags = ($tagPage: TagPage) => {
- const childTags: string[] = [];
+ const childTags: {tagName: string; Category: TagCategory }[] = [];
$tagPage(".child > div > ul > li").each((i, element) => {
-
- //console.log(i, $tagPage(element).children().first().text());
let childTag = $tagPage(element).children("a").first().text();
if (childTag != '') {
- childTags.push(childTag);
+ // parent and parent's parent is an element
+ let classcatagory = (element.parent?.parent as Element).attributes[0].value.split(' ')[0]
+ //console.log(classcatagory != 'freeforms' ? classcatagory : 'additional tags');
+ let tc = (classcatagory != 'freeforms' ? classcatagory : 'additional tags') as TagCategory;
+ childTags.push({tagName: childTag, Category: tc});
}
})
return childTags;
diff --git a/types/entities.ts b/types/entities.ts
index c5d2046..a1d5acf 100644
--- a/types/entities.ts
+++ b/types/entities.ts
@@ -18,7 +18,10 @@ export interface Tag {
// common and cannot be filtered on.
canonicalName: string | null;
parentTags: string[];
- childTags: string[];
+ childTags: Array<{
+ tagName: string;
+ Category: TagCategory;
+ }>;
subTags: Array<{
tagName: string;
parentSubTag: string | null;
From c77fccaeb3adfc10c3202898b9edc10d24b2f232 Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Thu, 2 Jan 2025 16:59:05 -0800
Subject: [PATCH 03/25] iterate over each div instead of doing the backwards
traversal
---
src/tags/page-getters.ts | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts
index 0d15e34..6a327cc 100644
--- a/src/tags/page-getters.ts
+++ b/src/tags/page-getters.ts
@@ -54,7 +54,9 @@ export const getParentTags = ($tagPage: TagPage) => {
return parentTags;
};
export const getChildTags = ($tagPage: TagPage) => {
+
const childTags: {tagName: string; Category: TagCategory }[] = [];
+ /*
$tagPage(".child > div > ul > li").each((i, element) => {
let childTag = $tagPage(element).children("a").first().text();
if (childTag != '') {
@@ -65,6 +67,18 @@ export const getChildTags = ($tagPage: TagPage) => {
childTags.push({tagName: childTag, Category: tc});
}
})
+ */
+
+ const divCategory = $tagPage(".child > div");
+ divCategory.each((i, divelement) => {
+ let classcatagory = divelement.attribs?.class?.split(' ')[0];
+ let tagcatagory = (classcatagory != 'freeforms' ? classcatagory : 'additional tags') as TagCategory;
+ $tagPage(divelement).find('ul > li a').each((_, aElement) => {
+ let childTag = $tagPage(aElement).text();
+ (childTag != '') && childTags.push({tagName: childTag, Category: tagcatagory});
+ })
+ })
+
return childTags;
}
From 18eb2e2d2122c98fd97bf8e0d4bf700ce9d24818 Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Thu, 2 Jan 2025 17:26:19 -0800
Subject: [PATCH 04/25] use cheerio map
---
src/tags/page-getters.ts | 35 +++++++++--------------------------
1 file changed, 9 insertions(+), 26 deletions(-)
diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts
index 6a327cc..730a7da 100644
--- a/src/tags/page-getters.ts
+++ b/src/tags/page-getters.ts
@@ -54,32 +54,15 @@ export const getParentTags = ($tagPage: TagPage) => {
return parentTags;
};
export const getChildTags = ($tagPage: TagPage) => {
-
- const childTags: {tagName: string; Category: TagCategory }[] = [];
- /*
- $tagPage(".child > div > ul > li").each((i, element) => {
- let childTag = $tagPage(element).children("a").first().text();
- if (childTag != '') {
- // parent and parent's parent is an element
- let classcatagory = (element.parent?.parent as Element).attributes[0].value.split(' ')[0]
- //console.log(classcatagory != 'freeforms' ? classcatagory : 'additional tags');
- let tc = (classcatagory != 'freeforms' ? classcatagory : 'additional tags') as TagCategory;
- childTags.push({tagName: childTag, Category: tc});
- }
- })
- */
-
- const divCategory = $tagPage(".child > div");
- divCategory.each((i, divelement) => {
- let classcatagory = divelement.attribs?.class?.split(' ')[0];
- let tagcatagory = (classcatagory != 'freeforms' ? classcatagory : 'additional tags') as TagCategory;
- $tagPage(divelement).find('ul > li a').each((_, aElement) => {
- let childTag = $tagPage(aElement).text();
- (childTag != '') && childTags.push({tagName: childTag, Category: tagcatagory});
- })
- })
-
- return childTags;
+ return $tagPage(".child > div").map((_, divElement) => {
+ const $div = $tagPage(divElement);
+ const className = $div.attr("class")?.split(' ')[0] ?? "";
+ const category = (className !== "freeforms" ? className : "additional tags") as TagCategory;
+ return $div.find("ul > li a").map((_, aElement) => {
+ const childTag = $tagPage(aElement).text();
+ return childTag ? { tagName: childTag, Category: category} : null;
+ }).get();
+ }).get();
}
export const getSubTags = ($tagPage: TagPage) => {
From 388238a7010e72056b25a194b4f469d3391424fd Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Thu, 2 Jan 2025 17:33:41 -0800
Subject: [PATCH 05/25] fix capitlization
---
src/tags/page-getters.ts | 2 +-
types/entities.ts | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts
index 730a7da..00d204c 100644
--- a/src/tags/page-getters.ts
+++ b/src/tags/page-getters.ts
@@ -60,7 +60,7 @@ export const getChildTags = ($tagPage: TagPage) => {
const category = (className !== "freeforms" ? className : "additional tags") as TagCategory;
return $div.find("ul > li a").map((_, aElement) => {
const childTag = $tagPage(aElement).text();
- return childTag ? { tagName: childTag, Category: category} : null;
+ return childTag ? { tagName: childTag, category: category} : null;
}).get();
}).get();
}
diff --git a/types/entities.ts b/types/entities.ts
index a1d5acf..0b3ee43 100644
--- a/types/entities.ts
+++ b/types/entities.ts
@@ -20,7 +20,7 @@ export interface Tag {
parentTags: string[];
childTags: Array<{
tagName: string;
- Category: TagCategory;
+ category: TagCategory;
}>;
subTags: Array<{
tagName: string;
From 04d4e64e23dcbd6cfac564439fabb28c9f2d05ea Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Thu, 2 Jan 2025 18:24:50 -0800
Subject: [PATCH 06/25] non-functional
---
src/tags/page-getters.ts | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts
index 00d204c..8b5c62e 100644
--- a/src/tags/page-getters.ts
+++ b/src/tags/page-getters.ts
@@ -1,4 +1,4 @@
-import { TagCategory } from "types/entities";
+import { TagCategory, Tag } from "types/entities";
import { TagPage } from "../page-loaders";
import { Element } from "cheerio";
@@ -65,7 +65,8 @@ export const getChildTags = ($tagPage: TagPage) => {
}).get();
}
-export const getSubTags = ($tagPage: TagPage) => {
+export const getSubTags = ($tagPage: TagPage): Tag["subTags"] => {
+ /*
const subTags: { tagName: string; parentSubTag: string | null }[] = [];
$tagPage(".sub > ul.tags > li").each((_, element) => {
subTags.push({ tagName: $tagPage(element).children().first().text(), parentSubTag: null });
@@ -79,6 +80,17 @@ export const getSubTags = ($tagPage: TagPage) => {
});
});
}
- });
- return subTags;
+ });*/
+ //return subTags;
+ return $tagPage(".sub > ul.tags > li").map((_, element) => {
+ if($tagPage($tagPage(element).has("ul.tags")).length) {
+ return $tagPage("ul.tags", element).children("li").map((_, child) => {
+ return {
+ tagName: $tagPage(child).children().first().text(),
+ parentSubTag: $tagPage($tagPage(child)).parents("li").children().first().text()
+ };
+ }).get()
+ }
+ return { tagName: $tagPage(element).children().first().text(), parentSubTag: null } as Tag["subTags"][number];
+ }).get();
};
From 97aa54abcb67e59c2260fcb2a0687c15e1bd61ed Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 13:45:58 -0800
Subject: [PATCH 07/25] Get Subtags works
---
src/tags/page-getters.ts | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts
index 8b5c62e..856e0c5 100644
--- a/src/tags/page-getters.ts
+++ b/src/tags/page-getters.ts
@@ -84,12 +84,14 @@ export const getSubTags = ($tagPage: TagPage): Tag["subTags"] => {
//return subTags;
return $tagPage(".sub > ul.tags > li").map((_, element) => {
if($tagPage($tagPage(element).has("ul.tags")).length) {
- return $tagPage("ul.tags", element).children("li").map((_, child) => {
+ const topElement = { tagName: $tagPage(element).children().first().text(), parentSubTag: null };
+ const subElements = $tagPage("ul.tags", element).children("li").map((_, child) => {
return {
tagName: $tagPage(child).children().first().text(),
parentSubTag: $tagPage($tagPage(child)).parents("li").children().first().text()
};
}).get()
+ return [topElement, ...subElements];
}
return { tagName: $tagPage(element).children().first().text(), parentSubTag: null } as Tag["subTags"][number];
}).get();
From 2fd0ad9fd251c49414e40a839991f790cc609d9e Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 13:47:57 -0800
Subject: [PATCH 08/25] Explicit Type coersion no longer needed
---
src/tags/page-getters.ts | 22 ++++------------------
1 file changed, 4 insertions(+), 18 deletions(-)
diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts
index 856e0c5..2efa6ff 100644
--- a/src/tags/page-getters.ts
+++ b/src/tags/page-getters.ts
@@ -65,23 +65,7 @@ export const getChildTags = ($tagPage: TagPage) => {
}).get();
}
-export const getSubTags = ($tagPage: TagPage): Tag["subTags"] => {
- /*
- const subTags: { tagName: string; parentSubTag: string | null }[] = [];
- $tagPage(".sub > ul.tags > li").each((_, element) => {
- subTags.push({ tagName: $tagPage(element).children().first().text(), parentSubTag: null });
- if ($tagPage($tagPage(element)).has("ul.tags").length) {
- $tagPage("ul.tags", element).children("li").each((_, child) => {
- // each
element contains an element,
- // which is why `.children().first()` is needed for both the `tagName` and `parentSubTag`
- subTags.push({
- tagName: $tagPage(child).children().first().text(),
- parentSubTag: $tagPage($tagPage(child)).parents("li").children().first().text()
- });
- });
- }
- });*/
- //return subTags;
+export const getSubTags = ($tagPage: TagPage) => {
return $tagPage(".sub > ul.tags > li").map((_, element) => {
if($tagPage($tagPage(element).has("ul.tags")).length) {
const topElement = { tagName: $tagPage(element).children().first().text(), parentSubTag: null };
@@ -93,6 +77,8 @@ export const getSubTags = ($tagPage: TagPage): Tag["subTags"] => {
}).get()
return [topElement, ...subElements];
}
- return { tagName: $tagPage(element).children().first().text(), parentSubTag: null } as Tag["subTags"][number];
+ return { tagName: $tagPage(element).children().first().text(), parentSubTag: null };
+ //return { tagName: $tagPage(element).children().first().text(), parentSubTag: null } as Tag["subTags"][number];
+
}).get();
};
From df74cf05c7e2a8cf2cc562f60d13f4b2f2769ff0 Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 13:55:53 -0800
Subject: [PATCH 09/25] Refactor as to not use extra arrays
---
src/tags/page-getters.ts | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts
index 2efa6ff..b5ecfdd 100644
--- a/src/tags/page-getters.ts
+++ b/src/tags/page-getters.ts
@@ -68,17 +68,19 @@ export const getChildTags = ($tagPage: TagPage) => {
export const getSubTags = ($tagPage: TagPage) => {
return $tagPage(".sub > ul.tags > li").map((_, element) => {
if($tagPage($tagPage(element).has("ul.tags")).length) {
- const topElement = { tagName: $tagPage(element).children().first().text(), parentSubTag: null };
- const subElements = $tagPage("ul.tags", element).children("li").map((_, child) => {
- return {
- tagName: $tagPage(child).children().first().text(),
- parentSubTag: $tagPage($tagPage(child)).parents("li").children().first().text()
- };
- }).get()
- return [topElement, ...subElements];
+ return [
+ {
+ tagName: $tagPage(element).children().first().text(),
+ parentSubTag: null
+ },
+ $tagPage("ul.tags", element).children("li").map((_, child) => {
+ return {
+ tagName: $tagPage(child).children().first().text(),
+ parentSubTag: $tagPage($tagPage(child)).parents("li").children().first().text()
+ };
+ }).get()
+ ].flat();
}
return { tagName: $tagPage(element).children().first().text(), parentSubTag: null };
- //return { tagName: $tagPage(element).children().first().text(), parentSubTag: null } as Tag["subTags"][number];
-
}).get();
};
From d78dd309f4ef025276726451287e0807e5dfbb4b Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 14:05:20 -0800
Subject: [PATCH 10/25] Refactor getParentTags as well
---
src/tags/page-getters.ts | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/src/tags/page-getters.ts b/src/tags/page-getters.ts
index b5ecfdd..2456257 100644
--- a/src/tags/page-getters.ts
+++ b/src/tags/page-getters.ts
@@ -47,11 +47,9 @@ export const getCanonical = ($tagPage: TagPage) => {
};
export const getParentTags = ($tagPage: TagPage) => {
- const parentTags: string[] = [];
- $tagPage(".parent ul.tags li").each((_, element) => {
- parentTags.push($tagPage(element).text());
- });
- return parentTags;
+ return $tagPage(".parent ul.tags li").map((_, element) => {
+ return $tagPage(element).text();
+ }).get();
};
export const getChildTags = ($tagPage: TagPage) => {
return $tagPage(".child > div").map((_, divElement) => {
From e79039c57bb8ae3e5cd972caf20c7ce6e9d4a686 Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 14:15:24 -0800
Subject: [PATCH 11/25] refactor getSeriesWorks
---
src/series/getters.ts | 19 ++++++-------------
1 file changed, 6 insertions(+), 13 deletions(-)
diff --git a/src/series/getters.ts b/src/series/getters.ts
index 50f53b5..359f350 100644
--- a/src/series/getters.ts
+++ b/src/series/getters.ts
@@ -1,5 +1,5 @@
import { Author, Series, SeriesWorkSummary } from "types/entities";
-import { CheerioAPI, load } from "cheerio";
+import { CheerioAPI, Element, load } from "cheerio";
import { SeriesPage, WorkPage } from "../page-loaders";
import {
getWorkBookmarkCount,
@@ -114,18 +114,11 @@ export const getSeriesBookmarkCount = ($seriesPage: SeriesPage): number => {
);
};
-export const getSeriesWorks = (
- $seriesPage: SeriesPage
-): SeriesWorkSummary[] => {
- const works: SeriesWorkSummary[] = [];
-
- $seriesPage("ul.index > li.work").each((index, element) => {
- works[index] = getSeriesWork($seriesPage(element).html() as string);
- });
-
- return works;
-};
-
+export const getSeriesWorks = ($seriesPage: SeriesPage): SeriesWorkSummary[] => {
+ return $seriesPage("ul.index > li.work").map((index, element) => {
+ return getSeriesWork($seriesPage(element).html() as string);
+ }).get()
+}
// Helpers for series' works
interface SeriesWork extends CheerioAPI {
kind: "SeriesWork";
From f7c98f90c3c42943e753d1f3524132980aa13da2 Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 14:19:25 -0800
Subject: [PATCH 12/25] refactor getSeriesWork Fandoms
---
src/series/getters.ts | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
diff --git a/src/series/getters.ts b/src/series/getters.ts
index 359f350..6b382f6 100644
--- a/src/series/getters.ts
+++ b/src/series/getters.ts
@@ -180,13 +180,10 @@ const getSeriesWorkSummary = ($work: SeriesWork) => {
};
const getSeriesWorkFandoms = ($work: SeriesWork): string[] => {
- const fandoms: string[] = [];
-
- $work("h5.fandoms a.tag").each(function (i, element) {
- fandoms[i] = $work(element).text().trim();
- });
- return fandoms;
-};
+ return $work("h5.fandoms a.tag").map((i, element) => {
+ return $work(element).text().trim();
+ }).get();
+}
const getSeriesWorkCharacters = ($work: SeriesWork): string[] => {
const characters: string[] = [];
From aa1a73e3cba06c46e44dde574e5da6c4a733cc73 Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 14:21:12 -0800
Subject: [PATCH 13/25] refactor getSeriesWorkCharacters
---
src/series/getters.ts | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/src/series/getters.ts b/src/series/getters.ts
index 6b382f6..b34b644 100644
--- a/src/series/getters.ts
+++ b/src/series/getters.ts
@@ -186,12 +186,9 @@ const getSeriesWorkFandoms = ($work: SeriesWork): string[] => {
}
const getSeriesWorkCharacters = ($work: SeriesWork): string[] => {
- const characters: string[] = [];
-
- $work("li.characters a.tag").each(function (i, character) {
- characters[i] = $work(character).text().trim();
- });
- return characters;
+ return $work("li.characters a.tag").map((i, character) => {
+ return $work(character).text().trim();
+ }).get()
};
const getSeriesWorkRelationships = ($work: SeriesWork): string[] => {
From 6a987b48ee623837af255bcadede218bd1fc2166 Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 14:22:59 -0800
Subject: [PATCH 14/25] refactor getSeriesWorkRelationships
---
src/series/getters.ts | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/src/series/getters.ts b/src/series/getters.ts
index b34b644..65b5e75 100644
--- a/src/series/getters.ts
+++ b/src/series/getters.ts
@@ -192,12 +192,9 @@ const getSeriesWorkCharacters = ($work: SeriesWork): string[] => {
};
const getSeriesWorkRelationships = ($work: SeriesWork): string[] => {
- const ships: string[] = [];
-
- $work("li.relationships a.tag").each(function (i, ship) {
- ships[i] = $work(ship).text().trim();
- });
- return ships;
+ return $work("li.relationships a.tag").map((i, ship) => {
+ return $work(ship).text().trim();
+ }).get()
};
const getSeriesWorkAdditionalTags = ($work: SeriesWork): string[] => {
From 4a1738e10d87274ced472bda9c998691fe580452 Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 14:24:46 -0800
Subject: [PATCH 15/25] refactor getSeriesWorkAdditionalTags
---
src/series/getters.ts | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/src/series/getters.ts b/src/series/getters.ts
index 65b5e75..3d29af2 100644
--- a/src/series/getters.ts
+++ b/src/series/getters.ts
@@ -198,12 +198,9 @@ const getSeriesWorkRelationships = ($work: SeriesWork): string[] => {
};
const getSeriesWorkAdditionalTags = ($work: SeriesWork): string[] => {
- const tags: string[] = [];
-
- $work("li.freeforms a.tag").each(function (i) {
- tags[i] = $work(this).text().trim();
- });
- return tags;
+ return $work("li.freeforms a.tag").map((_, element) => {
+ return $work(element).text().trim();
+ }).get();
};
const getSeriesWorkAuthors = (
From 9c661f330f9d548ea30fea6f3da47746e3f47294 Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 14:26:27 -0800
Subject: [PATCH 16/25] refactor getSeriesNotes
---
src/series/getters.ts | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/src/series/getters.ts b/src/series/getters.ts
index 3d29af2..e1437bb 100644
--- a/src/series/getters.ts
+++ b/src/series/getters.ts
@@ -68,11 +68,7 @@ export const getSeriesDescription = (
export const getSeriesNotes = ($seriesPage: SeriesPage): string | null => {
const notes = $seriesPage("dl.series dd:nth-of-type(5)");
- if (notes.prevAll().first().text().trim() === "Notes:") {
- return notes.html()!.trim();
- } else {
- return null;
- }
+ return notes.prevAll().first().text().trim() === "Notes:" ? notes.html()!.trim() : null;
};
export const getSeriesPublishDate = ($seriesPage: SeriesPage): string => {
From 2ff33b3680b4a3d8eb27b3edb67d90cf24988e6c Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 14:54:31 -0800
Subject: [PATCH 17/25] $seriesPage("dl.series blockquote.userstuff").html()
---
src/series/getters.ts | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/src/series/getters.ts b/src/series/getters.ts
index e1437bb..6353e99 100644
--- a/src/series/getters.ts
+++ b/src/series/getters.ts
@@ -59,11 +59,8 @@ export const getSeriesAuthors = (
return authors;
};
-export const getSeriesDescription = (
- $seriesPage: SeriesPage
-): string | null => {
- const description = $seriesPage("dl.series blockquote.userstuff").html();
- return description ? description.trim() : null;
+export const getSeriesDescription = ($seriesPage: SeriesPage): string | null => {
+ return $seriesPage("dl.series blockquote.userstuff").html()?.trim() || null;
};
export const getSeriesNotes = ($seriesPage: SeriesPage): string | null => {
From b4e9f5f20f01b92612f85ef24d2c12a476d94487 Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 14:54:31 -0800
Subject: [PATCH 18/25] Refactor getSeriesDescription and make formatting
consistant
---
src/series/getters.ts | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/src/series/getters.ts b/src/series/getters.ts
index e1437bb..6353e99 100644
--- a/src/series/getters.ts
+++ b/src/series/getters.ts
@@ -59,11 +59,8 @@ export const getSeriesAuthors = (
return authors;
};
-export const getSeriesDescription = (
- $seriesPage: SeriesPage
-): string | null => {
- const description = $seriesPage("dl.series blockquote.userstuff").html();
- return description ? description.trim() : null;
+export const getSeriesDescription = ($seriesPage: SeriesPage): string | null => {
+ return $seriesPage("dl.series blockquote.userstuff").html()?.trim() || null;
};
export const getSeriesNotes = ($seriesPage: SeriesPage): string | null => {
From b8d42f2dd81e5a263835bd115e4f39d1be2ffcb9 Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 15:08:48 -0800
Subject: [PATCH 19/25] refactor getSeriesAuthors
---
src/series/getters.ts | 21 +++++++--------------
1 file changed, 7 insertions(+), 14 deletions(-)
diff --git a/src/series/getters.ts b/src/series/getters.ts
index 6353e99..4bbc11a 100644
--- a/src/series/getters.ts
+++ b/src/series/getters.ts
@@ -31,32 +31,25 @@ export const getSeriesTitle = ($seriesPage: SeriesPage): string => {
return $seriesPage("h2.heading").text().trim();
};
-export const getSeriesAuthors = (
- $seriesPage: SeriesPage
-): Series["authors"] => {
+export const getSeriesAuthors = ($seriesPage: SeriesPage): Series["authors"] => {
const authorLinks = $seriesPage("dl.meta a[rel=author]");
- const authors: Author[] = [];
- if (
- $seriesPage("dl.meta > dd:nth-of-type(1)").text().trim() === "Anonymous"
- ) {
+ if ($seriesPage("dl.meta > dd:nth-of-type(1)").text().trim() === "Anonymous") {
return [{ username: "Anonymous", pseud: "Anonymous", anonymous: true }];
}
if (authorLinks.length !== 0) {
- authorLinks.each((i, element) => {
+ return authorLinks.map((_, element) => {
const url = element.attribs.href;
const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!;
-
- authors.push({
+ return {
username: username,
pseud: decodeURI(pseud),
anonymous: false,
- });
- });
+ } as Author
+ }).get();
}
-
- return authors;
+ return [] as Author[];
};
export const getSeriesDescription = ($seriesPage: SeriesPage): string | null => {
From 05f91a701dc9de8690a840fc785f279a2291b34b Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 15:09:56 -0800
Subject: [PATCH 20/25] forgot a couple semicolons
---
src/series/getters.ts | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/series/getters.ts b/src/series/getters.ts
index 4bbc11a..9fa8255 100644
--- a/src/series/getters.ts
+++ b/src/series/getters.ts
@@ -103,7 +103,7 @@ export const getSeriesBookmarkCount = ($seriesPage: SeriesPage): number => {
export const getSeriesWorks = ($seriesPage: SeriesPage): SeriesWorkSummary[] => {
return $seriesPage("ul.index > li.work").map((index, element) => {
return getSeriesWork($seriesPage(element).html() as string);
- }).get()
+ }).get();
}
// Helpers for series' works
interface SeriesWork extends CheerioAPI {
@@ -174,13 +174,13 @@ const getSeriesWorkFandoms = ($work: SeriesWork): string[] => {
const getSeriesWorkCharacters = ($work: SeriesWork): string[] => {
return $work("li.characters a.tag").map((i, character) => {
return $work(character).text().trim();
- }).get()
+ }).get();
};
const getSeriesWorkRelationships = ($work: SeriesWork): string[] => {
return $work("li.relationships a.tag").map((i, ship) => {
return $work(ship).text().trim();
- }).get()
+ }).get();
};
const getSeriesWorkAdditionalTags = ($work: SeriesWork): string[] => {
From f7668033cedcfaa69175c535650191ea9f59bdbe Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 15:14:59 -0800
Subject: [PATCH 21/25] refactor getSeriesWorkAuthors
---
src/series/getters.ts | 17 ++++++-----------
1 file changed, 6 insertions(+), 11 deletions(-)
diff --git a/src/series/getters.ts b/src/series/getters.ts
index 9fa8255..3aae6aa 100644
--- a/src/series/getters.ts
+++ b/src/series/getters.ts
@@ -189,28 +189,23 @@ const getSeriesWorkAdditionalTags = ($work: SeriesWork): string[] => {
}).get();
};
-const getSeriesWorkAuthors = (
- $work: SeriesWork
-): SeriesWorkSummary["authors"] => {
+const getSeriesWorkAuthors = ($work: SeriesWork): SeriesWorkSummary["authors"] => {
const authorLinks = $work("h4.heading a[rel='author']");
- const authors: Author[] = [];
-
if ($work("h4.heading").text().split("by")[1].trim() === "Anonymous") {
return [{ username: "Anonymous", pseud: "Anonymous", anonymous: true }];
}
if (authorLinks.length !== 0) {
- authorLinks.each((i, element) => {
+ return authorLinks.map((_, element) => {
const url = element.attribs.href;
const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!;
-
- authors.push({
+ return {
username: username,
pseud: decodeURI(pseud),
anonymous: false,
- });
- });
+ }
+ }).get();
}
- return authors;
+ return [] as Author[];
};
From a3723296183db7494741f7389fb958caa825689d Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 15:27:26 -0800
Subject: [PATCH 22/25] refactor getUserProfilePseuds
---
src/users/getters.ts | 17 ++++++-----------
1 file changed, 6 insertions(+), 11 deletions(-)
diff --git a/src/users/getters.ts b/src/users/getters.ts
index 1838f2b..5c83d78 100644
--- a/src/users/getters.ts
+++ b/src/users/getters.ts
@@ -1,3 +1,4 @@
+import { Element } from "cheerio";
import { UserProfile } from "../page-loaders";
import { getUserProfileUrl } from "../urls";
@@ -18,17 +19,11 @@ export const getUserProfileName = ($userProfile: UserProfile) => {
const PSEUD_SUFFIX = ", ";
export const getUserProfilePseuds = ($userProfile: UserProfile) => {
const pseuds = $userProfile("dd.pseuds a");
- const pseudsArray: string[] = [];
-
- if (pseuds.length !== 0) {
- pseuds.each((i, element) => {
- const url = element.attribs.href;
- const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!;
-
- pseudsArray.push(decodeURI(pseud));
- });
- }
- return pseudsArray.join(PSEUD_SUFFIX);
+ return pseuds.length !== 0
+ ? pseuds.map((_, element) => decodeURI(element.attribs.href.match(/users\/(.+)\/pseuds\/(.+)/)![2]))
+ .get()
+ .join(PSEUD_SUFFIX)
+ : "";
};
//Trim the results to only the date:
From f9c5eeff49c306f5dbd5f301b852d55a48c0e48b Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 15:33:34 -0800
Subject: [PATCH 23/25] refactor getWorkAuthors
---
src/works/chapter-getters.ts | 19 +++++++------------
1 file changed, 7 insertions(+), 12 deletions(-)
diff --git a/src/works/chapter-getters.ts b/src/works/chapter-getters.ts
index 8f3a3d3..b406773 100644
--- a/src/works/chapter-getters.ts
+++ b/src/works/chapter-getters.ts
@@ -37,26 +37,21 @@ export const getWorkTitle = ($chapterIndexPage: ChapterIndexPage) => {
return $chapterIndexPage(".works-navigate h2 a[href^='/works/']").text();
};
-export const getWorkAuthors = (
- $chapterIndexPage: ChapterIndexPage
-): Author[] => {
+export const getWorkAuthors = ($chapterIndexPage: ChapterIndexPage): Author[] => {
const authors: Author[] = [];
const authorNode = $chapterIndexPage(".works-navigate h2 a[rel='author']");
if (authorNode.text().trim() === "Anonymous") {
return [{ username: "Anonymous", pseud: "Anonymous", anonymous: true }];
}
-
- if (authorNode.length !== 0) {
- authorNode.each((i, element) => {
+ return authorNode.length !== 0
+ ? authorNode.map((_, element) => {
const url = element.attribs.href;
const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!;
-
- authors.push({
+ return {
username: username,
pseud: decodeURI(pseud),
anonymous: false,
- });
- });
- }
- return authors;
+ }
+ }).get()
+ : [] as Author[];
};
From 3ff1a6493860be5e03fdae97d053f576dc9dd498 Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Mon, 6 Jan 2025 16:21:07 -0800
Subject: [PATCH 24/25] refactor getChaptersList
---
src/works/chapter-getters.ts | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/src/works/chapter-getters.ts b/src/works/chapter-getters.ts
index b406773..17bc510 100644
--- a/src/works/chapter-getters.ts
+++ b/src/works/chapter-getters.ts
@@ -4,9 +4,9 @@ import { getWorkDetailsFromUrl, getWorkUrl } from "src/urls";
import { ChapterIndexPage } from "src/page-loaders";
const TITLE_SEPARATOR = ". ";
-export const getChaptersList = ($chapterIndexPage: ChapterIndexPage) => {
- const chapters: Chapter[] = [];
- $chapterIndexPage("ol.index > li").each((index, li) => {
+export const getChaptersList = ($chapterIndexPage: ChapterIndexPage): Chapter[] => {
+ //return chapters;
+ return $chapterIndexPage("ol.index > li").map((index, li) => {
const link = $chapterIndexPage(li).find("a")[0];
const chapterText = $chapterIndexPage(link).text();
const { workId, chapterId } = getWorkDetailsFromUrl({
@@ -18,8 +18,7 @@ export const getChaptersList = ($chapterIndexPage: ChapterIndexPage) => {
const dateNode = $chapterIndexPage(
$chapterIndexPage(li).find(".datetime")[0]
);
-
- chapters.push({
+ return {
id: chapterId!,
workId,
index: index + 1,
@@ -28,9 +27,8 @@ export const getChaptersList = ($chapterIndexPage: ChapterIndexPage) => {
publishedAt: dateNode.text().replace(/[\(\)]/g, ""),
// We rebuild the url so it gets the full path
url: getWorkUrl({ workId, chapterId }),
- });
- });
- return chapters;
+ }
+ }).get();
};
export const getWorkTitle = ($chapterIndexPage: ChapterIndexPage) => {
From 7d4b8d6bf8b8eab69f0d7849fd65b28eb0451eab Mon Sep 17 00:00:00 2001
From: Nithin <10778861+darthnithin@users.noreply.github.com>
Date: Tue, 7 Jan 2025 18:57:31 -0800
Subject: [PATCH 25/25] refactor getWorkAuthors
---
src/works/work-getters.ts | 28 ++++++++++++----------------
1 file changed, 12 insertions(+), 16 deletions(-)
diff --git a/src/works/work-getters.ts b/src/works/work-getters.ts
index a56ce5d..36821f0 100644
--- a/src/works/work-getters.ts
+++ b/src/works/work-getters.ts
@@ -8,29 +8,25 @@ import {
} from "types/entities";
import { WorkPage } from "../page-loaders";
+import { Element } from "cheerio";
export const getWorkAuthors = ($workPage: WorkPage): Author[] => {
const authorLinks = $workPage("h3.byline a[rel='author']");
- const authors: Author[] = [];
if ($workPage("h3.byline").text().trim() === "Anonymous") {
return [{ username: "Anonymous", pseud: "Anonymous", anonymous: true }];
}
-
- if (authorLinks.length !== 0) {
- authorLinks.each((i, element) => {
- const url = element.attribs.href;
- const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!;
-
- authors.push({
- username: username,
- pseud: decodeURI(pseud),
- anonymous: false,
- });
- });
- }
-
- return authors;
+ return authorLinks.length !== 0
+ ? authorLinks.map((_, element) => {
+ const url = element.attribs.href;
+ const [, username, pseud] = url.match(/users\/(.+)\/pseuds\/(.+)/)!;
+ return {
+ username: username,
+ pseud: decodeURI(pseud),
+ anonymous: false,
+ } as Author
+ }).get()
+ : [] as Author[];
};
export const getWorkTitle = ($workPage: WorkPage): string => {