nlw-api/index.js

import express from 'express';
import { fetchAllLevels as fetchNLWLevels } from './nlw.js';
import { fetchAllLevels as fetchIDSLevels } from './ids.js';
import fs from 'fs/promises';
import path from 'path';
import { request } from 'undici';
import PQueue from 'p-queue';

const cacheFolder = './cache/';

let levels = {
  nlw: {
    regular: [],
    pending: [],
    platformer: [],
  },
  ids: {
    regular: [],
  },
  metadata: [],
};

async function exists(f) {
  try {
    await fs.stat(f);
    return true;
  } catch {
    return false;
  }
}

// a bit awful but oh well

async function loadCache() {
  if (await exists(path.join(cacheFolder, 'nlw-regular.json'))) levels.nlw.regular = JSON.parse(await fs.readFile(path.join(cacheFolder, 'nlw-regular.json'), 'utf8'));
  if (await exists(path.join(cacheFolder, 'nlw-pending.json'))) levels.nlw.pending = JSON.parse(await fs.readFile(path.join(cacheFolder, 'nlw-pending.json'), 'utf8'));
  if (await exists(path.join(cacheFolder, 'nlw-platformer.json'))) levels.nlw.platformer = JSON.parse(await fs.readFile(path.join(cacheFolder, 'nlw-platformer.json'), 'utf8'));
  if (await exists(path.join(cacheFolder, 'ids-regular.json'))) levels.ids.regular = JSON.parse(await fs.readFile(path.join(cacheFolder, 'ids-regular.json'), 'utf8'));
  if (await exists(path.join(cacheFolder, 'metadata.json'))) levels.metadata = JSON.parse(await fs.readFile(path.join(cacheFolder, 'metadata.json'), 'utf8'));
}

async function saveCache() {
  await fs.writeFile(path.join(cacheFolder, 'nlw-regular.json'), JSON.stringify(levels.nlw.regular));
  await fs.writeFile(path.join(cacheFolder, 'nlw-pending.json'), JSON.stringify(levels.nlw.pending));
  await fs.writeFile(path.join(cacheFolder, 'nlw-platformer.json'), JSON.stringify(levels.nlw.platformer));
  await fs.writeFile(path.join(cacheFolder, 'ids-regular.json'), JSON.stringify(levels.ids.regular));
  await fs.writeFile(path.join(cacheFolder, 'metadata.json'), JSON.stringify(levels.metadata));
}

async function fetchSheets() {
  const nlw = await fetchNLWLevels();
  const ids = await fetchIDSLevels();
  levels = { nlw, ids, metadata: levels.metadata };
  await saveCache();
  await loadupMetadataQueue();
}

async function fetchLevelData(name, creator) {
  console.log('looking up metadata for', name, 'by', creator);
  const { statusCode, headers, trailers, body } = await request(`https://history.geometrydash.eu/api/v1/search/level/advanced/?query=${name}&filter=cache_demon=true`);
  const data = await body.json();

  if (data.hits.length === 1) return data.hits[0];

  const exact = data.hits.filter(h => h.cache_level_name.toLowerCase() === name.toLowerCase());
  if (exact.length === 1) return exact[0];

  const creatorHits = data.hits.filter(h => creator.toLowerCase().toLowerCase().includes(h.cache_username));
  if (creatorHits.length === 1) return creatorHits[0];

  return data.hits[0];
}

const metadataFetchQueue = new PQueue({ concurrency: 10, interval: 500, intervalCap: 2 });
metadataFetchQueue.on('empty', () => { console.log('metadata fetch queue empty!'); });

async function loadupMetadataQueue() {
  const list = [...levels.nlw.regular, ...levels.nlw.platformer, ...levels.nlw.pending, ...levels.ids.regular];

  const noMetadata = list.filter(l =>
    levels.metadata.findIndex(m => m.name === l.name && m.creator === l.creator) === -1
  );
  if (noMetadata.length === 0) {
    console.log('no metadata to fetch!');
    return
  }

  console.log(noMetadata.length, 'levels with no metadata, starting fetch');

  metadataFetchQueue.addAll(
    noMetadata.map(level => (async () => {
      const data = await fetchLevelData(level.name, level.creator);
      if (!data) {
        console.error('failed to find metadata!');
        return;
      }
    
      console.log('id', data.online_id);
    
      levels.metadata.push({
        name: level.name,
        creator: level.creator,
        id: data.online_id,
      });
    
      await saveCache();
    }))
  );
}

await loadCache();
//await loadupMetadataQueue();

const app = express();

app.get('/', (req, res) => {
  res.redirect('https://git.oat.zone/oat/nlw-api');
});

app.get('/list', (req, res) => {
  const type = req.query.type;

  let list;
  if (!type || type === '' || type === 'regular') {
    list = levels.nlw.regular;
  } else if (type === 'platformer') {
    list = levels.nlw.platformer;
  } else if (type === 'pending') {
    list = levels.nlw.pending;
  } else if (type === 'all') {
    list = [
      ...levels.nlw.regular.map(l => ({ type: 'regular', ...l })),
      ...levels.nlw.platformer.map(l => ({ type: 'platformer', ...l })),
      ...levels.nlw.pending.map(l => ({ type: 'pending', ...l })),
    ];
  } else {
    return res.status(400);
  }

  res.json(list.map(l => ({ ...(levels.metadata.find(m => l.name === m.name && l.creator === m.creator) || {}), ...l })));
});

app.get('/ids', (req, res) => {
  res.json(levels.ids.regular);
});

const port = process.env.PORT || 8080
app.listen(port);
console.log(`lisening on port ${port}`);

await fetchSheets();
setInterval(fetchSheets, 1000 * 60 * 60);