From 351d829d8cf76271e598d9d0054444559702928a Mon Sep 17 00:00:00 2001 From: "Jill \"oatmealine\" Monoids" Date: Fri, 16 Feb 2024 16:15:51 +0300 Subject: [PATCH] ids!!! yippeeeeeee --- .gitignore | 4 +- flake.nix | 2 +- index.js | 106 +++++++++++++++++++++++++++++++++++++++++++--- nlw.js | 2 + package-lock.json | 54 ++++++++++++++++++++++- package.json | 4 +- 6 files changed, 161 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 6a7d6d8..7ae2876 100644 --- a/.gitignore +++ b/.gitignore @@ -127,4 +127,6 @@ dist .yarn/unplugged .yarn/build-state.yml .yarn/install-state.gz -.pnp.* \ No newline at end of file +.pnp.* + +cache/ \ No newline at end of file diff --git a/flake.nix b/flake.nix index 7109e95..5a3e7c2 100644 --- a/flake.nix +++ b/flake.nix @@ -18,7 +18,7 @@ pname = "nlw-api"; inherit (package) version; - npmDepsHash = "sha256-JfY8PhfUfB4doZCxrRTTLCrkA6XZTgbBIRd0wolMA2Q="; + npmDepsHash = "sha256-doKVsiMdaDi8I8ivAnilSguv7xGcixAaSVh23YDoHE4="; doCheck = false; diff --git a/index.js b/index.js index 9aac9da..6f53c90 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,12 @@ import express from 'express'; import { fetchAllLevels as fetchNLWLevels } from './nlw.js'; import { fetchAllLevels as fetchIDSLevels } from './ids.js'; +import fs from 'fs/promises'; +import path from 'path'; +import { request } from 'undici'; +import PQueue from 'p-queue'; + +const cacheFolder = './cache/'; let levels = { nlw: { @@ -11,16 +17,99 @@ let levels = { ids: { regular: [], }, + metadata: [], }; +async function exists(f) { + try { + await fs.stat(f); + return true; + } catch { + return false; + } +} + +// a bit awful but oh well + +async function loadCache() { + if (await exists(path.join(cacheFolder, 'nlw-regular.json'))) levels.nlw.regular = JSON.parse(await fs.readFile(path.join(cacheFolder, 'nlw-regular.json'), 'utf8')); + if (await exists(path.join(cacheFolder, 'nlw-pending.json'))) levels.nlw.pending = JSON.parse(await fs.readFile(path.join(cacheFolder, 'nlw-pending.json'), 'utf8')); + if (await exists(path.join(cacheFolder, 'nlw-platformer.json'))) levels.nlw.platformer = JSON.parse(await fs.readFile(path.join(cacheFolder, 'nlw-platformer.json'), 'utf8')); + if (await exists(path.join(cacheFolder, 'ids-regular.json'))) levels.ids.regular = JSON.parse(await fs.readFile(path.join(cacheFolder, 'ids-regular.json'), 'utf8')); + if (await exists(path.join(cacheFolder, 'metadata.json'))) levels.metadata = JSON.parse(await fs.readFile(path.join(cacheFolder, 'metadata.json'), 'utf8')); +} + +async function saveCache() { + await fs.writeFile(path.join(cacheFolder, 'nlw-regular.json'), JSON.stringify(levels.nlw.regular)); + await fs.writeFile(path.join(cacheFolder, 'nlw-pending.json'), JSON.stringify(levels.nlw.pending)); + await fs.writeFile(path.join(cacheFolder, 'nlw-platformer.json'), JSON.stringify(levels.nlw.platformer)); + await fs.writeFile(path.join(cacheFolder, 'ids-regular.json'), JSON.stringify(levels.ids.regular)); + await fs.writeFile(path.join(cacheFolder, 'metadata.json'), JSON.stringify(levels.metadata)); +} + async function fetchSheets() { const nlw = await fetchNLWLevels(); const ids = await fetchIDSLevels(); - levels = { nlw, ids }; + levels = { nlw, ids, metadata: levels.metadata }; + await saveCache(); + await loadupMetadataQueue(); } -await fetchSheets(); -setInterval(fetchSheets, 1000 * 60 * 60); +async function fetchLevelData(name, creator) { + console.log('looking up metadata for', name, 'by', creator); + const { statusCode, headers, trailers, body } = await request(`https://history.geometrydash.eu/api/v1/search/level/advanced/?query=${name}&filter=cache_demon=true`); + const data = await body.json(); + + if (data.hits.length === 1) return data.hits[0]; + + const exact = data.hits.filter(h => h.cache_level_name.toLowerCase() === name.toLowerCase()); + if (exact.length === 1) return exact[0]; + + const creatorHits = data.hits.filter(h => creator.toLowerCase().toLowerCase().includes(h.cache_username)); + if (creatorHits.length === 1) return creatorHits[0]; + + return data.hits[0]; +} + +const metadataFetchQueue = new PQueue({ concurrency: 10, interval: 500, intervalCap: 2 }); +metadataFetchQueue.on('empty', () => { console.log('metadata fetch queue empty!'); }); + +async function loadupMetadataQueue() { + const list = [...levels.nlw.regular, ...levels.nlw.platformer, ...levels.nlw.pending, ...levels.ids.regular]; + + const noMetadata = list.filter(l => + levels.metadata.findIndex(m => m.name === l.name && m.creator === l.creator) === -1 + ); + if (noMetadata.length === 0) { + console.log('no metadata to fetch!'); + return + } + + console.log(noMetadata.length, 'levels with no metadata, starting fetch'); + + metadataFetchQueue.addAll( + noMetadata.map(level => (async () => { + const data = await fetchLevelData(level.name, level.creator); + if (!data) { + console.error('failed to find metadata!'); + return; + } + + console.log('id', data.online_id); + + levels.metadata.push({ + name: level.name, + creator: level.creator, + id: data.online_id, + }); + + await saveCache(); + })) + ); +} + +await loadCache(); +//await loadupMetadataQueue(); const app = express(); @@ -39,16 +128,16 @@ app.get('/list', (req, res) => { } else if (type === 'pending') { list = levels.nlw.pending; } else if (type === 'all') { - return res.json([ + list = [ ...levels.nlw.regular.map(l => ({ type: 'regular', ...l })), ...levels.nlw.platformer.map(l => ({ type: 'platformer', ...l })), ...levels.nlw.pending.map(l => ({ type: 'pending', ...l })), - ]); + ]; } else { return res.status(400); } - res.json(list); + res.json(list.map(l => ({ ...(levels.metadata.find(m => l.name === m.name && l.creator === m.creator) || {}), ...l }))); }); app.get('/ids', (req, res) => { @@ -57,4 +146,7 @@ app.get('/ids', (req, res) => { const port = process.env.PORT || 8080 app.listen(port); -console.log(`lisening on port ${port}`); \ No newline at end of file +console.log(`lisening on port ${port}`); + +await fetchSheets(); +setInterval(fetchSheets, 1000 * 60 * 60); \ No newline at end of file diff --git a/nlw.js b/nlw.js index 9ca3b35..75cddfd 100644 --- a/nlw.js +++ b/nlw.js @@ -17,6 +17,7 @@ const fruityLevels = { 'Missing Benefi s': 'Missing Benefits', 'troll levle': 'troll level', 'gardening map': 'gardening map ', + 'Flying Maze': 'Floating Maze', }; const brokenColors = { @@ -128,6 +129,7 @@ async function fetchLevels(sheet, platformer, pending) { const name = fruityLevels[level[0]] || level[0] || ""; if (name === 'miss you') broke = 'absolutely destroyed'; + if (name === 'None Yet!') continue; const obj = { sheetIndex: parseInt(i), diff --git a/package-lock.json b/package-lock.json index 32abc7d..a9b93fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,17 @@ "license": "ISC", "dependencies": { "express": "^4.18.2", - "google-spreadsheet": "^4.1.1" + "google-spreadsheet": "^4.1.1", + "p-queue": "^8.0.1", + "undici": "^6.6.2" + } + }, + "node_modules/@fastify/busboy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz", + "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==", + "engines": { + "node": ">=14" } }, "node_modules/accepts": { @@ -204,6 +214,11 @@ "node": ">= 0.6" } }, + "node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==" + }, "node_modules/express": { "version": "4.18.2", "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", @@ -531,6 +546,32 @@ "node": ">= 0.8" } }, + "node_modules/p-queue": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.0.1.tgz", + "integrity": "sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==", + "dependencies": { + "eventemitter3": "^5.0.1", + "p-timeout": "^6.1.2" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.2.tgz", + "integrity": "sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -724,6 +765,17 @@ "node": ">= 0.6" } }, + "node_modules/undici": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.6.2.tgz", + "integrity": "sha512-vSqvUE5skSxQJ5sztTZ/CdeJb1Wq0Hf44hlYMciqHghvz+K88U0l7D6u1VsndoFgskDcnU+nG3gYmMzJVzd9Qg==", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=18.0" + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/package.json b/package.json index e9ef3cc..f908c0c 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,8 @@ "type": "module", "dependencies": { "express": "^4.18.2", - "google-spreadsheet": "^4.1.1" + "google-spreadsheet": "^4.1.1", + "p-queue": "^8.0.1", + "undici": "^6.6.2" } }