move station post processor inline with api gets

.vscode/launch.json

@@ -26,6 +26,18 @@
 			"skipFiles": [
 				"<node_internals>/**"
 			],
+			"args": [
+				"--use-cache"
+			],
+			"type": "node"
+		},
+		{
+			"name": "Data:stations-api",
+			"program": "${workspaceFolder}/datagenerators/stations.mjs",
+			"request": "launch",
+			"skipFiles": [
+				"<node_internals>/**"
+			],
 			"type": "node"
 		},
 		{

datagenerators/stations-postprocessor.mjs

@@ -2,6 +2,8 @@
 
 import { readFileSync, writeFileSync } from 'fs';
 
+import * as url from 'node:url';
+
 // Load station data
 const stationInfo = JSON.parse(readFileSync('./datagenerators/output/stations-raw.json', 'utf8'));
 // const regionalCities = JSON.parse(readFileSync('./datagenerators/output/regionalcities.json', 'utf8'));
@@ -1109,14 +1111,31 @@ or where the fallback to the ICAO airport code occurred:
 	jq -c '.[] | select(.name | test("^[A-Z]{3}$")) | {state, city, simple, name}'
 */
 
-const diffMode = process.argv.includes('--diff');
-const onlyProblems = process.argv.includes('--only-problems');
-const noProblems = process.argv.includes('--no-problems');
-const onlyDuplicates = process.argv.includes('--only-dupes');
-const noPriority = process.argv.includes('--no-priority');
-const noSimple = process.argv.includes('--no-simple');
-const noCoordinates = process.argv.includes('--no-coords');
-const writeFile = process.argv.includes('--write');
+const readArguments = () => ({
+	diffMode: process.argv.includes('--diff'),
+	onlyProblems: process.argv.includes('--only-problems'),
+	noProblems: process.argv.includes('--no-problems'),
+	onlyDuplicates: process.argv.includes('--only-dupes'),
+	noPriority: process.argv.includes('--no-priority'),
+	noSimple: process.argv.includes('--no-simple'),
+	noCoordinates: process.argv.includes('--no-coords'),
+	writeFile: process.argv.includes('--write'),
+});
+
+const DEFAULT_OPTIONS = {
+	diffMode: false,
+	onlyProblems: false,
+	noProblems: false,
+	onlyDuplicates: false,
+	noPriority: false,
+	noSimple: false,
+	noCoordinates: false,
+	writeFile: false,
+};
+
+const postProcessor = (_options) => {
+	// combine default and provided options
+	const options = { ...DEFAULT_OPTIONS, ..._options };
 
 	// Process ALL stations at once to get the display name map
 	let displayNameMap = processAllStations(stationInfo);
@@ -1196,10 +1215,10 @@ cleanedMapByState.forEach((stateMap, _state) => {
 
 	// Filter results if requested
 	let finalResults = results;
-if (onlyProblems) {
+	if (options.onlyProblems) {
 		finalResults = results.filter((r) => r.potentialIssues.length > 0);
 	}
-if (onlyDuplicates) {
+	if (options.onlyDuplicates) {
 		finalResults = finalResults.filter((r) => r.potentialIssues.includes('duplicate'));
 	}
 
@@ -1207,7 +1226,7 @@ const outputResult = finalResults.map((result) => {
 		let outputItem = result;
 
 		// Don't include lat or long in diff mode
-	if (noCoordinates || diffMode) {
+		if (options.noCoordinates || options.diffMode) {
 			const {
 				lat: _lat, lon: _lon, ...resultWithoutLocation
 			} = result;
@@ -1215,19 +1234,19 @@ const outputResult = finalResults.map((result) => {
 		}
 
 		// Don't include potentialIssues when --no-problems is specified
-	if (noProblems || diffMode) {
+		if (options.noProblems || options.diffMode) {
 			const { potentialIssues: _potentialIssues, ...resultWithoutIssues } = outputItem;
 			outputItem = resultWithoutIssues;
 		}
 
 		// Remove type and priority if --no-priority is specified
-	if (noPriority || diffMode) {
+		if (options.noPriority || options.diffMode) {
 			const { type: _type, priority: _priority, ...resultWithoutPriority } = outputItem;
 			outputItem = resultWithoutPriority;
 		}
 
 		// remove simple field if --no-simple is specified
-	if (noSimple || diffMode) {
+		if (options.noSimple || options.diffMode) {
 			const { simple: _simple, ...resultWithoutSimple } = outputItem;
 			outputItem = resultWithoutSimple;
 		}
@@ -1235,13 +1254,41 @@ const outputResult = finalResults.map((result) => {
 		return outputItem;
 	});
 
-if (writeFile) {
 	const fileResults = results.map(({
-		simple: _simple, type: _type, potentialIssues: _potentialIssues, ...rest
+		simple: _simple, type: _type, potentialIssues: _potentialIssues, location: _location, ...rest
 	}) => rest);
 
+	if (options.writeFile) {
 		writeFileSync('./datagenerators/output/stations.json', compactStringifyToObject(fileResults));
 		console.log(`Wrote ${fileResults.length} processed stations to datagenerators/output/stations.json`);
 	} else {
 		console.log(compactStringifyToArray(outputResult));
 	}
+
+	// array to output object
+	const returnObject = {};
+	fileResults.forEach((item) => {
+		returnObject[item.id] = item;
+	});
+
+	return returnObject;
+};
+
+// determine if running from command line or module
+const commandLine = (() => {
+	if (import.meta.url.startsWith('file:')) { // (A)
+		const modulePath = url.fileURLToPath(import.meta.url);
+		if (process.argv[1] === modulePath) { // (B)
+			return true;
+		}
+	}
+	return false;
+}
+)();
+
+// run post processor if called from command line
+if (commandLine) {
+	postProcessor(readArguments());
+}
+
+export default postProcessor;

datagenerators/stations.mjs

@@ -1,3 +1,4 @@
+/* eslint-disable no-loop-func */
 // list all stations in a single file
 // only find stations with 4 letter codes
 
@@ -6,16 +7,23 @@ import https from './https.mjs';
 import states from './stations-states.mjs';
 import chunk from './chunk.mjs';
 import overrides from './stations-overrides.mjs';
+import postProcessor from './stations-postprocessor.mjs';
+
+// check for cached flag
+const USE_CACHE = process.argv.includes('--use-cache');
 
 // skip stations starting with these letters
 const skipStations = ['U', 'C', 'H', 'W', 'Y', 'T', 'S', 'M', 'O', 'L', 'A', 'F', 'B', 'N', 'V', 'R', 'D', 'E', 'I', 'G', 'J'];
 
 // chunk the list of states
-const chunkStates = chunk(states, 1);
+const chunkStates = chunk(states, 3);
 
 // store output
 const output = {};
+let completed = 0;
 
+// get data from api if desired
+if (!USE_CACHE) {
 	// process all chunks
 	for (let i = 0; i < chunkStates.length; i += 1) {
 		const stateChunk = chunkStates[i];
@@ -44,16 +52,12 @@ for (let i = 0; i < chunkStates.length; i += 1) {
 							console.log(`Duplicate station: ${state}-${id}`);
 							return;
 						}
-					// get any overrides if available
-					const override = overrides[id] ?? {};
 						output[id] = {
 							id,
 							city: station.properties.name,
 							state,
 							lat: station.geometry.coordinates[1],
 							lon: station.geometry.coordinates[0],
-						// finally add the overrides
-						...override,
 						};
 					});
 					next = stations?.pagination?.next;
@@ -62,7 +66,8 @@ for (let i = 0; i < chunkStates.length; i += 1) {
 					writeFileSync('./datagenerators/output/stations-raw.json', JSON.stringify(output, null, 2));
 				}
 				while (next && stations.features.length > 0);
-			console.log(`Complete: ${state}`);
+				completed += 1;
+				console.log(`Complete: ${state} ${completed}/${states.length}`);
 				return true;
 			} catch {
 				console.error(`Unable to get state: ${state}`);
@@ -70,3 +75,23 @@ for (let i = 0; i < chunkStates.length; i += 1) {
 			}
 		}));
 	}
+}
+
+// run the post processor
+// data is passed through the file stations-raw.json
+const postProcessed = postProcessor();
+
+// apply any overrides
+Object.entries(overrides).forEach(([id, values]) => {
+	// check for existing value
+	if (postProcessed[id]) {
+		// apply the overrides
+		postProcessed[id] = {
+			...postProcessed[id],
+			...values,
+		};
+	}
+});
+
+// write final file to disk
+writeFileSync('./datagenerators/output/stations.json', JSON.stringify(postProcessed, null, 2));