metadata garbage collection

ids.js

@@ -10,6 +10,8 @@ const fruityLevels = {
   'Dark matter (Real)': 'Dark matter',
   'Versus... MR. BEAST!!!': 'Vers', // ???
   'El Dorado': 'El Dorado ', // ????
+  'Photovoltaic ': 'Photovoltaic',
+  'Disconnect ': 'Disconnect',
 };
 
 // horrid. thank you ids

index.js

@@ -20,6 +20,10 @@ let levels = {
   metadata: [],
 };
 
+function getAllLevels() {
+  return [...levels.nlw.regular, ...levels.nlw.platformer, ...levels.nlw.pending, ...levels.ids.regular];
+}
+
 async function exists(f) {
   try {
     await fs.stat(f);
@@ -55,31 +59,61 @@ async function fetchSheets() {
   await loadupMetadataQueue();
 }
 
-async function fetchLevelData(name, creator) {
+async function fetchLevelData(name, creator, loose = false) {
   console.log('looking up metadata for', name, 'by', creator);
-  const { statusCode, headers, trailers, body } = await request(`https://history.geometrydash.eu/api/v1/search/level/advanced/?query=${name}&filter=cache_demon=true`);
+
+  const params = new URLSearchParams(loose ? {
+    'query': name,
+    'filter': 'cache_demon=true',
+  } : {
+    'filter': `cache_demon=true AND cache_level_name='${name}'`,
+  });
+
+  const { statusCode, headers, trailers, body } = await request(`https://history.geometrydash.eu/api/v1/search/level/advanced/?${params.toString()}`);
   const data = await body.json();
 
+  if (data.hits.length === 0 && !loose) {
+    return await fetchLevelData(name, creator, true);
+  } else if (data.hits.length === 0) {
+    return undefined;
+  }
+
   if (data.hits.length === 1) return data.hits[0];
 
   const exact = data.hits.filter(h => h.cache_level_name.toLowerCase() === name.toLowerCase());
   if (exact.length === 1) return exact[0];
 
-  const creatorHits = data.hits.filter(h => creator.toLowerCase().toLowerCase().includes(h.cache_username));
+  const creatorHits = data.hits.filter(h => creator.toLowerCase().includes(h.cache_username.toLowerCase()));
   if (creatorHits.length === 1) return creatorHits[0];
 
   return data.hits[0];
 }
 
 const metadataFetchQueue = new PQueue({ concurrency: 10, interval: 500, intervalCap: 2 });
-metadataFetchQueue.on('empty', () => { console.log('metadata fetch queue empty!'); });
+metadataFetchQueue.on('empty', async () => {
+  console.log('metadata fetch queue empty, idling');
+  await metadataSanityCheck();
+  await metadataGarbageCollect();
+});
+
+// hopefully will prevent cross-sheet duplicate fighting
+// eg. the same level in multiple sheets being marked as duplicate metadata
+function normalizeCreatorName(name) {
+  return name.replace('&', 'and').trim().toLowerCase();
+}
+
+function getMetadata(level) {
+  return levels.metadata.find(m =>
+    level.name === m.name &&
+    // normalized on the righthand side to prevent completely invalidating old caches
+    normalizeCreatorName(level.creator) === normalizeCreatorName(m.creator)
+  );
+}
 
 async function loadupMetadataQueue() {
-  const list = [...levels.nlw.regular, ...levels.nlw.platformer, ...levels.nlw.pending, ...levels.ids.regular];
+  const list = getAllLevels();
 
-  const noMetadata = list.filter(l =>
-    levels.metadata.findIndex(m => m.name === l.name && m.creator === l.creator) === -1
-  );
+  const noMetadata = list.filter(l => getMetadata(l) === undefined);
   if (noMetadata.length === 0) {
     console.log('no metadata to fetch!');
     return
@@ -89,6 +123,12 @@ async function loadupMetadataQueue() {
 
   metadataFetchQueue.addAll(
     noMetadata.map(level => (async () => {
+      // to prevent race conditions, let's check if the metadata exists already and cancel if so
+      if (getMetadata(level)) {
+        console.log(`metadata for ${level.name} by ${level.creator} already found, lol`);
+        return;
+      }
+
       const data = await fetchLevelData(level.name, level.creator);
       if (!data) {
         console.error('failed to find metadata!');
@@ -99,7 +139,7 @@ async function loadupMetadataQueue() {
     
       levels.metadata.push({
         name: level.name,
-        creator: level.creator,
+        creator: normalizeCreatorName(level.creator),
         id: data.online_id,
       });
     
@@ -108,6 +148,37 @@ async function loadupMetadataQueue() {
   );
 }
 
+async function metadataSanityCheck() {
+  const duplicates = levels.metadata.filter((l1, index) => levels.metadata.findIndex(l2 => l1.id === l2.id) !== index);
+
+  if (duplicates.length > 0) {
+    console.log('WARNING - duplicate IDs found in metadata table!:');
+    const ids = new Set(duplicates.map(l => l.id));
+    const badLevels = [...ids].flatMap(id => levels.metadata.filter(l => l.id === id));
+    for (const level of badLevels) {
+      console.log(`${level.id}  ${level.name} by ${level.creator}`);
+    }
+    console.log('clearing out, will be refreshed next metadata fetch cycle');
+    levels.metadata = levels.metadata.filter(l => !ids.has(l.id));
+    await saveCache();
+    //await loadupMetadataQueue();
+  }
+}
+
+async function metadataGarbageCollect() {
+  const metadata = [];
+  for (const level of getAllLevels()) {
+    const data = getMetadata(level);
+    if (data && metadata.findIndex(m => m.name === data.name && m.creator === data.creator) === -1) metadata.push(data);
+  }
+
+  if (metadata.length < levels.metadata.length) {
+    console.log(`garbage collecting metadata (${levels.metadata.length - metadata.length} levels)`);
+    levels.metadata = levels.metadata.filter(m => metadata.indexOf(m) !== -1);
+    await saveCache();
+  }
+}
+
 await loadCache();
 //await loadupMetadataQueue();
 
@@ -137,7 +208,7 @@ app.get('/list', (req, res) => {
     return res.status(400);
   }
 
-  res.json(list.map(l => ({ ...(levels.metadata.find(m => l.name === m.name && l.creator === m.creator) || {}), ...l })));
+  res.json(list.map(l => ({ ...(getMetadata(l) || {}), ...l })));
 });
 
 app.get('/ids', (req, res) => {

nlw.js

@@ -17,7 +17,7 @@ const fruityLevels = {
   'Missing Benefi s': 'Missing Benefits',
   'troll levle': 'troll level',
   'gardening map': 'gardening map ',
-  'Flying Maze': 'Floating Maze',
+  'Flying Maze': 'Floating Outskirts',
 };
 
 const brokenColors = {