oat
/
nlw-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ids!!! yippeeeeeee

This commit is contained in:
Jill 2024-02-16 16:15:51 +03:00
parent f8f89b670b
commit 351d829d8c
Signed by: oat
GPG Key ID: 33489AA58A955108
6 changed files with 161 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -127,4 +127,6 @@ dist
.yarn/unplugged
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.*
.pnp.*
cache/

2
flake.nix
View File

@ -18,7 +18,7 @@
pname = "nlw-api";
inherit (package) version;
npmDepsHash = "sha256-JfY8PhfUfB4doZCxrRTTLCrkA6XZTgbBIRd0wolMA2Q=";
npmDepsHash = "sha256-doKVsiMdaDi8I8ivAnilSguv7xGcixAaSVh23YDoHE4=";
doCheck = false;

106
index.js
View File

@ -1,6 +1,12 @@
import express from 'express';
import { fetchAllLevels as fetchNLWLevels } from './nlw.js';
import { fetchAllLevels as fetchIDSLevels } from './ids.js';
import fs from 'fs/promises';
import path from 'path';
import { request } from 'undici';
import PQueue from 'p-queue';
const cacheFolder = './cache/';
let levels = {
nlw: {
@ -11,16 +17,99 @@ let levels = {
ids: {
regular: [],
},
metadata: [],
};
async function exists(f) {
try {
await fs.stat(f);
return true;
} catch {
return false;
}
}
// a bit awful but oh well
async function loadCache() {
if (await exists(path.join(cacheFolder, 'nlw-regular.json'))) levels.nlw.regular = JSON.parse(await fs.readFile(path.join(cacheFolder, 'nlw-regular.json'), 'utf8'));
if (await exists(path.join(cacheFolder, 'nlw-pending.json'))) levels.nlw.pending = JSON.parse(await fs.readFile(path.join(cacheFolder, 'nlw-pending.json'), 'utf8'));
if (await exists(path.join(cacheFolder, 'nlw-platformer.json'))) levels.nlw.platformer = JSON.parse(await fs.readFile(path.join(cacheFolder, 'nlw-platformer.json'), 'utf8'));
if (await exists(path.join(cacheFolder, 'ids-regular.json'))) levels.ids.regular = JSON.parse(await fs.readFile(path.join(cacheFolder, 'ids-regular.json'), 'utf8'));
if (await exists(path.join(cacheFolder, 'metadata.json'))) levels.metadata = JSON.parse(await fs.readFile(path.join(cacheFolder, 'metadata.json'), 'utf8'));
}
async function saveCache() {
await fs.writeFile(path.join(cacheFolder, 'nlw-regular.json'), JSON.stringify(levels.nlw.regular));
await fs.writeFile(path.join(cacheFolder, 'nlw-pending.json'), JSON.stringify(levels.nlw.pending));
await fs.writeFile(path.join(cacheFolder, 'nlw-platformer.json'), JSON.stringify(levels.nlw.platformer));
await fs.writeFile(path.join(cacheFolder, 'ids-regular.json'), JSON.stringify(levels.ids.regular));
await fs.writeFile(path.join(cacheFolder, 'metadata.json'), JSON.stringify(levels.metadata));
}
async function fetchSheets() {
const nlw = await fetchNLWLevels();
const ids = await fetchIDSLevels();
levels = { nlw, ids };
levels = { nlw, ids, metadata: levels.metadata };
await saveCache();
await loadupMetadataQueue();
}
await fetchSheets();
setInterval(fetchSheets, 1000 * 60 * 60);
async function fetchLevelData(name, creator) {
console.log('looking up metadata for', name, 'by', creator);
const { statusCode, headers, trailers, body } = await request(`https://history.geometrydash.eu/api/v1/search/level/advanced/?query=${name}&filter=cache_demon=true`);
const data = await body.json();
if (data.hits.length === 1) return data.hits[0];
const exact = data.hits.filter(h => h.cache_level_name.toLowerCase() === name.toLowerCase());
if (exact.length === 1) return exact[0];
const creatorHits = data.hits.filter(h => creator.toLowerCase().toLowerCase().includes(h.cache_username));
if (creatorHits.length === 1) return creatorHits[0];
return data.hits[0];
}
const metadataFetchQueue = new PQueue({ concurrency: 10, interval: 500, intervalCap: 2 });
metadataFetchQueue.on('empty', () => { console.log('metadata fetch queue empty!'); });
async function loadupMetadataQueue() {
const list = [...levels.nlw.regular, ...levels.nlw.platformer, ...levels.nlw.pending, ...levels.ids.regular];
const noMetadata = list.filter(l =>
levels.metadata.findIndex(m => m.name === l.name && m.creator === l.creator) === -1
);
if (noMetadata.length === 0) {
console.log('no metadata to fetch!');
return
}
console.log(noMetadata.length, 'levels with no metadata, starting fetch');
metadataFetchQueue.addAll(
noMetadata.map(level => (async () => {
const data = await fetchLevelData(level.name, level.creator);
if (!data) {
console.error('failed to find metadata!');
return;
}
console.log('id', data.online_id);
levels.metadata.push({
name: level.name,
creator: level.creator,
id: data.online_id,
});
await saveCache();
}))
);
}
await loadCache();
//await loadupMetadataQueue();
const app = express();
@ -39,16 +128,16 @@ app.get('/list', (req, res) => {
} else if (type === 'pending') {
list = levels.nlw.pending;
} else if (type === 'all') {
return res.json([
list = [
...levels.nlw.regular.map(l => ({ type: 'regular', ...l })),
...levels.nlw.platformer.map(l => ({ type: 'platformer', ...l })),
...levels.nlw.pending.map(l => ({ type: 'pending', ...l })),
]);
];
} else {
return res.status(400);
}
res.json(list);
res.json(list.map(l => ({ ...(levels.metadata.find(m => l.name === m.name && l.creator === m.creator) || {}), ...l })));
});
app.get('/ids', (req, res) => {
@ -57,4 +146,7 @@ app.get('/ids', (req, res) => {
const port = process.env.PORT || 8080
app.listen(port);
console.log(`lisening on port ${port}`);
console.log(`lisening on port ${port}`);
await fetchSheets();
setInterval(fetchSheets, 1000 * 60 * 60);

2
nlw.js
View File

@ -17,6 +17,7 @@ const fruityLevels = {
'Missing Benefi s': 'Missing Benefits',
'troll levle': 'troll level',
'gardening map': 'gardening map ',
'Flying Maze': 'Floating Maze',
};
const brokenColors = {
@ -128,6 +129,7 @@ async function fetchLevels(sheet, platformer, pending) {
const name = fruityLevels[level[0]] || level[0] || "";
if (name === 'miss you') broke = 'absolutely destroyed';
if (name === 'None Yet!') continue;
const obj = {
sheetIndex: parseInt(i),

54
package-lock.json generated
View File

@ -10,7 +10,17 @@
"license": "ISC",
"dependencies": {
"express": "^4.18.2",
"google-spreadsheet": "^4.1.1"
"google-spreadsheet": "^4.1.1",
"p-queue": "^8.0.1",
"undici": "^6.6.2"
}
},
"node_modules/@fastify/busboy": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
"integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==",
"engines": {
"node": ">=14"
}
},
"node_modules/accepts": {
@ -204,6 +214,11 @@
"node": ">= 0.6"
}
},
"node_modules/eventemitter3": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz",
"integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="
},
"node_modules/express": {
"version": "4.18.2",
"resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
@ -531,6 +546,32 @@
"node": ">= 0.8"
}
},
"node_modules/p-queue": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.0.1.tgz",
"integrity": "sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==",
"dependencies": {
"eventemitter3": "^5.0.1",
"p-timeout": "^6.1.2"
},
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/p-timeout": {
"version": "6.1.2",
"resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.2.tgz",
"integrity": "sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==",
"engines": {
"node": ">=14.16"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/parseurl": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@ -724,6 +765,17 @@
"node": ">= 0.6"
}
},
"node_modules/undici": {
"version": "6.6.2",
"resolved": "https://registry.npmjs.org/undici/-/undici-6.6.2.tgz",
"integrity": "sha512-vSqvUE5skSxQJ5sztTZ/CdeJb1Wq0Hf44hlYMciqHghvz+K88U0l7D6u1VsndoFgskDcnU+nG3gYmMzJVzd9Qg==",
"dependencies": {
"@fastify/busboy": "^2.0.0"
},
"engines": {
"node": ">=18.0"
}
},
"node_modules/unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",

4
package.json
View File

@ -12,6 +12,8 @@
"type": "module",
"dependencies": {
"express": "^4.18.2",
"google-spreadsheet": "^4.1.1"
"google-spreadsheet": "^4.1.1",
"p-queue": "^8.0.1",
"undici": "^6.6.2"
}
}