From 8eb5a9e8f5c12bc35d8aca5a0157fc862477cc5c Mon Sep 17 00:00:00 2001 From: Reaper Gelera Date: Sun, 26 May 2024 15:16:41 +0530 Subject: [PATCH 1/5] initial sync job --- scripts/sync-lineage-os.js | 72 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/scripts/sync-lineage-os.js b/scripts/sync-lineage-os.js index 115d5ea..81ea761 100755 --- a/scripts/sync-lineage-os.js +++ b/scripts/sync-lineage-os.js @@ -1,20 +1,80 @@ #!/usr/bin/env node -const got = require('got'); +const _got = require('got'); const kluer = require('kleur'); const { logcons } = require('logcons'); const { STATUS_ENUM } = require('../db/status_enum'); const { upsertDevice } = require('../lib/sdk'); +const { parse } = require('yaml'); +const { dateStringToDate } = require('../lib/date-utils'); const success = kluer.green().bold; +const versionMap = { + '13.0': 6, + '14.1': 7.1, + '15.1': 8.1, + '16.0': 9, + '17.1': 10, + '18.1': 11, + '19.1': 12, + '20': 13, + '21': 14, +}; + const URL = 'https://raw.githubusercontent.com/LineageOS/hudson/main/updater/devices.json'; +const getDeviceDetailsUrl = deviceId => + `https://ungh.cc/repos/LineageOS/lineage_wiki/files/main/_data/devices/${deviceId}`; +const getAllDevices = async () => { + const data = await _got( + 'https://ungh.cc/repos/LineageOS/lineage_wiki/files/main' + ).json(); + + const devicePromises = data.files + .filter(d => { + return d.path.startsWith('_data/devices'); + }) + .map(async d => { + const [_, _a, name] = d.path.split('/'); + + const deviceDetails = (await got(getDeviceDetailsUrl(name)).json()).file + .contents; + const parsedDetails = parse(deviceDetails); + const codename = name.replace(/_variant(\d+)/, '').replace(/.yml$/, ''); + let releaseDate = ''; + if (typeof parsedDetails.release === 'object') { + // const [firstKey] = Object.keys(parsedDetails.release); + // releaseDate = dateStringToDate(parsedDetails.release[firstKey]); + } else { + releaseDate = dateStringToDate(parsedDetails.release); + } + + const versions = (parsedDetails.versions ?? []).map(d => { + return versionMap[d] ?? 'N/A'; + }); + return { + codename, + release: releaseDate, + versions, + }; + }); + + const result = await Promise.all(devicePromises); + return result.reduce((acc, item) => { + acc[item.codename] = item; + return acc; + }, {}); +}; + async function main() { + const deviceDetails = await getAllDevices(); + const response = await got(URL); const promises = (JSON.parse(response.body) || []).map(async deviceItem => { const codename = deviceItem.model; + const details = deviceDetails[codename]; const deviceName = `${deviceItem.oem} ${deviceItem.name}`; await upsertDevice({ @@ -22,7 +82,7 @@ async function main() { codename, rom: { status: STATUS_ENUM.unknown, - androidVersion: ['N/A'], + androidVersion: details?.versions || ['N/A'], links: [`https://download.lineageos.org/${codename}`], name: 'LineageOS', }, @@ -34,6 +94,14 @@ async function main() { console.log(success(`${logcons.tick()} Done, Syncing Lineage OS`)); } +function got(url) { + return _got(url, { + headers: { + Authorization: `Bearer ${process.env.GH_TOKEN}`, + }, + }); +} + exports.syncLineageOS = main; if (require.main === module) { From 27777b24e17b49fffd57fe5f556a8dd4794b1c83 Mon Sep 17 00:00:00 2001 From: Reaper Gelera Date: Mon, 27 May 2024 09:37:23 +0530 Subject: [PATCH 2/5] fix: table count for search --- lib/sdk.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/sdk.js b/lib/sdk.js index 2ea8225..0e051f2 100644 --- a/lib/sdk.js +++ b/lib/sdk.js @@ -158,9 +158,13 @@ const getDevices = async ({ }); } - const countData = await db('roms_devices_mapping').count( + const countData = await paginatedBaseQuery + .clone() + .clearOrder() + .clearCounters() + .count( 'roms_devices_mapping.id' - ); + ) const q = db .from(paginatedBaseQuery) From c08e16c9d05745d284471575a9bb4d78ede87c06 Mon Sep 17 00:00:00 2001 From: Reaper Gelera Date: Mon, 27 May 2024 09:47:07 +0530 Subject: [PATCH 3/5] fix: search term normalization --- lib/sdk.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/sdk.js b/lib/sdk.js index 0e051f2..eb2260a 100644 --- a/lib/sdk.js +++ b/lib/sdk.js @@ -142,6 +142,7 @@ const getDevices = async ({ paginatedBaseQuery.andWhere({ 'roms_devices_mapping.status': status }); if (searchTerm.length >= 3) { + const searchTermNormalized = `*${searchTerm.split(/[ ]/).join('*')}*`; paginatedBaseQuery.andWhere(function () { this.whereRaw( 'roms_devices_mapping.id in ?', @@ -152,19 +153,18 @@ const getDevices = async ({ FROM roms_search_index WHERE - keywords Match '${searchTerm}')` + keywords Match ?)`, + searchTermNormalized ) ); }); } const countData = await paginatedBaseQuery - .clone() - .clearOrder() - .clearCounters() - .count( - 'roms_devices_mapping.id' - ) + .clone() + .clearOrder() + .clearCounters() + .count('roms_devices_mapping.id'); const q = db .from(paginatedBaseQuery) From 18793f5bfb8d0212223cca932d551ab0e575ad91 Mon Sep 17 00:00:00 2001 From: Reaper Gelera Date: Mon, 27 May 2024 09:52:21 +0530 Subject: [PATCH 4/5] fix: secondary counts --- lib/sdk.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/sdk.js b/lib/sdk.js index eb2260a..798eb0d 100644 --- a/lib/sdk.js +++ b/lib/sdk.js @@ -162,9 +162,13 @@ const getDevices = async ({ const countData = await paginatedBaseQuery .clone() + .clearSelect() .clearOrder() .clearCounters() - .count('roms_devices_mapping.id'); + .clear('limit') + .clear('offset') + .count('roms_devices_mapping.id') + .first(); const q = db .from(paginatedBaseQuery) @@ -198,7 +202,7 @@ const getDevices = async ({ return { deviceList: [...normalized.values()], - count: countData[0]['count(`roms_devices_mapping`.`id`)'], + count: countData['count(`roms_devices_mapping`.`id`)'], }; }; From a3d62a7c0b77c45373009e94a6e40d698f95e478 Mon Sep 17 00:00:00 2001 From: Reaper Gelera Date: Mon, 27 May 2024 10:05:51 +0530 Subject: [PATCH 5/5] chore: cleanup --- scripts/sync-search-index.js | 40 +++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/scripts/sync-search-index.js b/scripts/sync-search-index.js index 9eee4d7..4d96559 100644 --- a/scripts/sync-search-index.js +++ b/scripts/sync-search-index.js @@ -1,26 +1,32 @@ -const {conch} = require("@barelyreaper/conch"); -const { db } = require("../db/db"); +const { conch } = require('@barelyreaper/conch'); +const { db } = require('../db/db'); async function main() { - await db("roms_search_index").del(); - const romData = await db("roms_devices_mapping") - .leftJoin("roms", "roms_devices_mapping.rom_id", "roms.id") - .leftJoin("devices", "roms_devices_mapping.device_id", "devices.id") + await db('roms_search_index').del(); + const romData = await db('roms_devices_mapping') + .leftJoin('roms', 'roms_devices_mapping.rom_id', 'roms.id') + .leftJoin('devices', 'roms_devices_mapping.device_id', 'devices.id') .select( - "roms_devices_mapping.*", - "roms.name as rom_name", - "devices.basename as device_name", - "devices.codename as device_codename", + 'roms_devices_mapping.*', + 'roms.name as rom_name', + 'devices.basename as device_name', + 'devices.codename as device_codename' ); - await conch(romData, async (row) => { - const execChain = await db("roms_search_index").insert({ - rom_mapping_id: row.id, - keywords: [row.rom_name, row.device_name, row.device_codename].join(", "), - }); - }, { limit: 20 }); + await conch( + romData, + async row => { + await db('roms_search_index').insert({ + rom_mapping_id: row.id, + keywords: [row.rom_name, row.device_name, row.device_codename].join( + ', ' + ), + }); + }, + { limit: 20 } + ); } -exports.syncSearchIndex = main +exports.syncSearchIndex = main; if (require.main === module) main().then(() => process.exit(0));