update flake to use CACHE_DIR

.gitignore

@@ -127,4 +127,6 @@ dist
 .yarn/unplugged
 .yarn/build-state.yml
 .yarn/install-state.gz
-.pnp.*
+.pnp.*
+
+cache/

README.md

@@ -25,6 +25,7 @@ Fetches the IDS list as a JSON. **Experimental.**
 Represents a generic level.
 
 - `sheetIndex`: The row index of the level on the associated spreadsheet. 0-indexed.
+- `id`: The ID of the level, manifested from the sheet using dark magic. Can be `undefined` very rarely.
 - `name`: The name of the level. Standardized to the in-game level name.
 - `creator`: The creator(s) of the level, as listed on the sheet.
 - `description`: Descriptions and notes as listed on the sheet.
@@ -44,4 +45,50 @@ Represents an IDS level.
 
 - `tier`: `"Fuck"`, `"Beginner"`, `"Easy"`, `"Medium"`, `"Hard"`, `"Very Hard"`, `"Insane"` or `"Extreme"`.
 - `skillset`: Level skillset, as listed on the sheet.
-- `broken`: If the level is broken in 2.2. `"no"`, `"yes"`, or rarely `null` if unknown.
+- `broken`: If the level is broken in 2.2. `"no"`, `"yes"`, or rarely `null` if unknown.
+
+### Self-hosting
+
+You can self-host the API yourself, if you so wish! Here's the rough steps:
+
+1. Grab yourself a Google Sheets API key. This may sound extremely intimidating, but you can create a read-only Sheets API key with not much hassle with [this guide](https://theoephraim.github.io/node-google-spreadsheet/#/guides/authentication?id=api-key).
+2. Install, either with NPM or Nix:
+    1. With NPM, run:
+        ```sh
+        npm install
+        ```
+        Then, you can start it with:
+        ```sh
+        API_KEY=... node index.js
+        ```
+        **However**, take note of the [environment variables](#environment-variables) available to use.
+    2. Use Nix to either just plainly run it:
+        ```sh
+        API_KEY=... nix run git+https://git.oat.zone/oat/nlw-api
+        ```
+        Or import it into your system flake like so:
+        ```nix
+        nlw-api.url = "git+https://git.oat.zone/oat/nlw-api";
+        # in your `nixosConfiguration`:
+        imports = [
+          inputs.nlw-api.nixosModules.nlw-api
+        ];
+        ```
+        Afterwards, you can use it as a regular NixOS service:
+        ```nix
+        services.nlw-api = {
+          enable = true;
+          domain = "nlw.oat.zone";
+          apiKey = builtins.readFile /etc/sheets-api-key;
+          port = 1995;
+        };
+        ```
+3. You're done! It will take a while to fetch every level and their IDs initially, but in time you will have yourself the caches set up and the server up and running.
+   
+### Environment variables
+
+You can pass these into the server as config (an `.env` file will **not** work):
+
+- `API_KEY`: A Google API key - see [this guide](https://theoephraim.github.io/node-google-spreadsheet/#/guides/authentication?id=api-key) to see how to get one hassle-free
+- `PORT`: The HTTP port to host the server on
+- `CACHE_DIR`: The directory to store cached data in, defaults to `./cache/`

flake.nix

@@ -18,7 +18,7 @@
             pname = "nlw-api";
             inherit (package) version;
 
-            npmDepsHash = "sha256-JfY8PhfUfB4doZCxrRTTLCrkA6XZTgbBIRd0wolMA2Q=";
+            npmDepsHash = "sha256-doKVsiMdaDi8I8ivAnilSguv7xGcixAaSVh23YDoHE4=";
 
             doCheck = false;
 
@@ -75,18 +75,15 @@
             environment = {
               PORT = toString cfg.port;
               API_KEY = cfg.apiKey;
+              CACHE_DIR = "/var/lib/nlw-api";
             };
 
             serviceConfig = {
               Restart = "on-failure";
               ExecStart = "${getExe cfg.package}";
               DynamicUser = "yes";
-              RuntimeDirectory = "nlw-api";
-              RuntimeDirectoryMode = "0755";
               StateDirectory = "nlw-api";
-              StateDirectoryMode = "0700";
-              CacheDirectory = "nlw-api";
-              CacheDirectoryMode = "0750";
+              StateDirectoryMode = "0755";
             };
           };
 

index.js

@@ -1,6 +1,12 @@
 import express from 'express';
 import { fetchAllLevels as fetchNLWLevels } from './nlw.js';
 import { fetchAllLevels as fetchIDSLevels } from './ids.js';
+import fs from 'fs/promises';
+import path from 'path';
+import { request } from 'undici';
+import PQueue from 'p-queue';
+
+const cacheFolder = process.env.CACHE_DIR || './cache/';
 
 let levels = {
   nlw: {
@@ -11,16 +17,99 @@ let levels = {
   ids: {
     regular: [],
   },
+  metadata: [],
 };
 
+async function exists(f) {
+  try {
+    await fs.stat(f);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// a bit awful but oh well
+
+async function loadCache() {
+  if (await exists(path.join(cacheFolder, 'nlw-regular.json'))) levels.nlw.regular = JSON.parse(await fs.readFile(path.join(cacheFolder, 'nlw-regular.json'), 'utf8'));
+  if (await exists(path.join(cacheFolder, 'nlw-pending.json'))) levels.nlw.pending = JSON.parse(await fs.readFile(path.join(cacheFolder, 'nlw-pending.json'), 'utf8'));
+  if (await exists(path.join(cacheFolder, 'nlw-platformer.json'))) levels.nlw.platformer = JSON.parse(await fs.readFile(path.join(cacheFolder, 'nlw-platformer.json'), 'utf8'));
+  if (await exists(path.join(cacheFolder, 'ids-regular.json'))) levels.ids.regular = JSON.parse(await fs.readFile(path.join(cacheFolder, 'ids-regular.json'), 'utf8'));
+  if (await exists(path.join(cacheFolder, 'metadata.json'))) levels.metadata = JSON.parse(await fs.readFile(path.join(cacheFolder, 'metadata.json'), 'utf8'));
+}
+
+async function saveCache() {
+  await fs.writeFile(path.join(cacheFolder, 'nlw-regular.json'), JSON.stringify(levels.nlw.regular));
+  await fs.writeFile(path.join(cacheFolder, 'nlw-pending.json'), JSON.stringify(levels.nlw.pending));
+  await fs.writeFile(path.join(cacheFolder, 'nlw-platformer.json'), JSON.stringify(levels.nlw.platformer));
+  await fs.writeFile(path.join(cacheFolder, 'ids-regular.json'), JSON.stringify(levels.ids.regular));
+  await fs.writeFile(path.join(cacheFolder, 'metadata.json'), JSON.stringify(levels.metadata));
+}
+
 async function fetchSheets() {
   const nlw = await fetchNLWLevels();
   const ids = await fetchIDSLevels();
-  levels = { nlw, ids };
+  levels = { nlw, ids, metadata: levels.metadata };
+  await saveCache();
+  await loadupMetadataQueue();
 }
 
-await fetchSheets();
-setInterval(fetchSheets, 1000 * 60 * 60);
+async function fetchLevelData(name, creator) {
+  console.log('looking up metadata for', name, 'by', creator);
+  const { statusCode, headers, trailers, body } = await request(`https://history.geometrydash.eu/api/v1/search/level/advanced/?query=${name}&filter=cache_demon=true`);
+  const data = await body.json();
+
+  if (data.hits.length === 1) return data.hits[0];
+
+  const exact = data.hits.filter(h => h.cache_level_name.toLowerCase() === name.toLowerCase());
+  if (exact.length === 1) return exact[0];
+
+  const creatorHits = data.hits.filter(h => creator.toLowerCase().toLowerCase().includes(h.cache_username));
+  if (creatorHits.length === 1) return creatorHits[0];
+
+  return data.hits[0];
+}
+
+const metadataFetchQueue = new PQueue({ concurrency: 10, interval: 500, intervalCap: 2 });
+metadataFetchQueue.on('empty', () => { console.log('metadata fetch queue empty!'); });
+
+async function loadupMetadataQueue() {
+  const list = [...levels.nlw.regular, ...levels.nlw.platformer, ...levels.nlw.pending, ...levels.ids.regular];
+
+  const noMetadata = list.filter(l =>
+    levels.metadata.findIndex(m => m.name === l.name && m.creator === l.creator) === -1
+  );
+  if (noMetadata.length === 0) {
+    console.log('no metadata to fetch!');
+    return
+  }
+
+  console.log(noMetadata.length, 'levels with no metadata, starting fetch');
+
+  metadataFetchQueue.addAll(
+    noMetadata.map(level => (async () => {
+      const data = await fetchLevelData(level.name, level.creator);
+      if (!data) {
+        console.error('failed to find metadata!');
+        return;
+      }
+    
+      console.log('id', data.online_id);
+    
+      levels.metadata.push({
+        name: level.name,
+        creator: level.creator,
+        id: data.online_id,
+      });
+    
+      await saveCache();
+    }))
+  );
+}
+
+await loadCache();
+//await loadupMetadataQueue();
 
 const app = express();
 
@@ -39,16 +128,16 @@ app.get('/list', (req, res) => {
   } else if (type === 'pending') {
     list = levels.nlw.pending;
   } else if (type === 'all') {
-    return res.json([
+    list = [
       ...levels.nlw.regular.map(l => ({ type: 'regular', ...l })),
       ...levels.nlw.platformer.map(l => ({ type: 'platformer', ...l })),
       ...levels.nlw.pending.map(l => ({ type: 'pending', ...l })),
-    ]);
+    ];
   } else {
     return res.status(400);
   }
 
-  res.json(list);
+  res.json(list.map(l => ({ ...(levels.metadata.find(m => l.name === m.name && l.creator === m.creator) || {}), ...l })));
 });
 
 app.get('/ids', (req, res) => {
@@ -57,4 +146,7 @@ app.get('/ids', (req, res) => {
 
 const port = process.env.PORT || 8080
 app.listen(port);
-console.log(`lisening on port ${port}`);
+console.log(`lisening on port ${port}`);
+
+await fetchSheets();
+setInterval(fetchSheets, 1000 * 60 * 60);

nlw.js

@@ -17,6 +17,7 @@ const fruityLevels = {
   'Missing Benefi s': 'Missing Benefits',
   'troll levle': 'troll level',
   'gardening map': 'gardening map ',
+  'Flying Maze': 'Floating Maze',
 };
 
 const brokenColors = {
@@ -128,6 +129,7 @@ async function fetchLevels(sheet, platformer, pending) {
 
     const name = fruityLevels[level[0]] || level[0] || "";
     if (name === 'miss you') broke = 'absolutely destroyed';
+    if (name === 'None Yet!') continue;
 
     const obj = {
       sheetIndex: parseInt(i),

package-lock.json

@@ -10,7 +10,17 @@
       "license": "ISC",
       "dependencies": {
         "express": "^4.18.2",
-        "google-spreadsheet": "^4.1.1"
+        "google-spreadsheet": "^4.1.1",
+        "p-queue": "^8.0.1",
+        "undici": "^6.6.2"
+      }
+    },
+    "node_modules/@fastify/busboy": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.0.tgz",
+      "integrity": "sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==",
+      "engines": {
+        "node": ">=14"
       }
     },
     "node_modules/accepts": {
@@ -204,6 +214,11 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/eventemitter3": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz",
+      "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="
+    },
     "node_modules/express": {
       "version": "4.18.2",
       "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
@@ -531,6 +546,32 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/p-queue": {
+      "version": "8.0.1",
+      "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.0.1.tgz",
+      "integrity": "sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==",
+      "dependencies": {
+        "eventemitter3": "^5.0.1",
+        "p-timeout": "^6.1.2"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
+    "node_modules/p-timeout": {
+      "version": "6.1.2",
+      "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.2.tgz",
+      "integrity": "sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==",
+      "engines": {
+        "node": ">=14.16"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/parseurl": {
       "version": "1.3.3",
       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -724,6 +765,17 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/undici": {
+      "version": "6.6.2",
+      "resolved": "https://registry.npmjs.org/undici/-/undici-6.6.2.tgz",
+      "integrity": "sha512-vSqvUE5skSxQJ5sztTZ/CdeJb1Wq0Hf44hlYMciqHghvz+K88U0l7D6u1VsndoFgskDcnU+nG3gYmMzJVzd9Qg==",
+      "dependencies": {
+        "@fastify/busboy": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=18.0"
+      }
+    },
     "node_modules/unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",

package.json

@@ -12,6 +12,8 @@
   "type": "module",
   "dependencies": {
     "express": "^4.18.2",
-    "google-spreadsheet": "^4.1.1"
+    "google-spreadsheet": "^4.1.1",
+    "p-queue": "^8.0.1",
+    "undici": "^6.6.2"
   }
 }