From 437100c433ead247504b313410da75d9c5f7a1e9 Mon Sep 17 00:00:00 2001 From: mrkmntal Date: Tue, 7 Apr 2026 22:11:03 -0400 Subject: [PATCH] cache ArcGIS basemap tiles on disk for faster map loads --- index.mjs | 12 ++++- proxy/cache.mjs | 50 +++++++++++++++++-- proxy/handlers.mjs | 16 ++++++ .../modules/utils/leaflet-weather-map.mjs | 12 +++-- server/scripts/modules/utils/url-rewrite.mjs | 8 +++ 5 files changed, 89 insertions(+), 9 deletions(-) diff --git a/index.mjs b/index.mjs index a74cd28..f8e090b 100644 --- a/index.mjs +++ b/index.mjs @@ -5,7 +5,15 @@ import { readFile } from 'fs/promises'; import { exec } from 'child_process'; import { promisify } from 'util'; import { - weatherProxy, radarProxy, outlookProxy, mesonetProxy, forecastProxy, openMeteoProxy, rainViewerProxy, + weatherProxy, + radarProxy, + outlookProxy, + mesonetProxy, + forecastProxy, + openMeteoProxy, + rainViewerProxy, + arcGisServerProxy, + arcGisServicesProxy, } from './proxy/handlers.mjs'; import playlist from './src/playlist.mjs'; import OVERRIDES from './src/overrides.mjs'; @@ -256,6 +264,8 @@ if (!process.env?.STATIC) { app.use('/forecast/', forecastProxy); app.use('/open-meteo/', openMeteoProxy); app.use('/rainviewer/', rainViewerProxy); + app.use('/arcgis-server/', arcGisServerProxy); + app.use('/arcgis-services/', arcGisServicesProxy); // Playlist route is available in server mode (not in static mode) app.get('/playlist.json', playlist); diff --git a/proxy/cache.mjs b/proxy/cache.mjs index 0dbfc11..549c616 100644 --- a/proxy/cache.mjs +++ b/proxy/cache.mjs @@ -32,10 +32,18 @@ const STALE_TIME_LIMIT_MS = 3 * 60 * 60 * 1000; const PERSISTED_HOSTS = new Set([ 'api.open-meteo.com', 'api.rainviewer.com', + 'server.arcgisonline.com', + 'services.arcgisonline.com', +]); +const BINARY_PERSISTED_HOSTS = new Set([ + 'server.arcgisonline.com', + 'services.arcgisonline.com', ]); const HOST_FALLBACK_TTLS = { 'api.open-meteo.com': 10 * 60, 'api.rainviewer.com': 2 * 60, + 'server.arcgisonline.com': 7 * 24 * 60 * 60, + 'services.arcgisonline.com': 7 * 24 * 60 * 60, }; class HttpCache { @@ -55,16 +63,30 @@ class HttpCache { return path.join(CACHE_DIR, `${HttpCache.hashKey(key)}.json`); } - static shouldPersist(url, options, response) { - if (options?.encoding === 'binary') return false; - if (typeof response?.data !== 'string') return false; + static getCacheBodyFilePath(key) { + return path.join(CACHE_DIR, `${HttpCache.hashKey(key)}.bin`); + } + static getPersistedHost(url) { try { const parsedUrl = new URL(url); - return PERSISTED_HOSTS.has(parsedUrl.hostname); + return parsedUrl.hostname; } catch { + return null; + } + } + + static shouldPersist(url, options, response) { + const host = HttpCache.getPersistedHost(url); + if (!host || !PERSISTED_HOSTS.has(host)) { return false; } + + if (options?.encoding === 'binary') { + return BINARY_PERSISTED_HOSTS.has(host) && Buffer.isBuffer(response?.data); + } + + return typeof response?.data === 'string'; } static getHostFallbackTtl(url) { @@ -96,10 +118,17 @@ class HttpCache { } if (now > parsed.entry.expiry + STALE_TIME_LIMIT_MS) { + if (parsed.entry.binaryBody === true) { + await unlink(HttpCache.getCacheBodyFilePath(parsed.key)).catch(() => null); + } await unlink(filePath); return; } + if (parsed.entry.binaryBody === true) { + parsed.entry.data = await readFile(HttpCache.getCacheBodyFilePath(parsed.key)); + } + this.cache.set(parsed.key, parsed.entry); })); } catch (error) { @@ -110,6 +139,18 @@ class HttpCache { static async persistEntry(key, entry) { try { await mkdir(CACHE_DIR, { recursive: true }); + + if (Buffer.isBuffer(entry.data)) { + const metadata = { + ...entry, + data: undefined, + binaryBody: true, + }; + await writeFile(HttpCache.getCacheBodyFilePath(key), entry.data); + await writeFile(HttpCache.getCacheFilePath(key), JSON.stringify({ key, entry: metadata })); + return; + } + await writeFile(HttpCache.getCacheFilePath(key), JSON.stringify({ key, entry })); } catch (error) { console.warn(`⚠️ Cache save | Failed to persist cache entry ${key}: ${error.message}`); @@ -118,6 +159,7 @@ class HttpCache { static async deletePersistedEntry(key) { try { + await unlink(HttpCache.getCacheBodyFilePath(key)).catch(() => null); await unlink(HttpCache.getCacheFilePath(key)); } catch (error) { if (error.code !== 'ENOENT') { diff --git a/proxy/handlers.mjs b/proxy/handlers.mjs index 4255d95..a845b61 100644 --- a/proxy/handlers.mjs +++ b/proxy/handlers.mjs @@ -64,3 +64,19 @@ export const rainViewerProxy = async (req, res) => { skipParams: ['u'], }); }; + +export const arcGisServerProxy = async (req, res) => { + await cache.handleRequest(req, res, 'https://server.arcgisonline.com', { + serviceName: 'ArcGIS Server', + skipParams: ['u'], + encoding: 'binary', + }); +}; + +export const arcGisServicesProxy = async (req, res) => { + await cache.handleRequest(req, res, 'https://services.arcgisonline.com', { + serviceName: 'ArcGIS Services', + skipParams: ['u'], + encoding: 'binary', + }); +}; diff --git a/server/scripts/modules/utils/leaflet-weather-map.mjs b/server/scripts/modules/utils/leaflet-weather-map.mjs index 73816ff..cd8ec39 100644 --- a/server/scripts/modules/utils/leaflet-weather-map.mjs +++ b/server/scripts/modules/utils/leaflet-weather-map.mjs @@ -4,8 +4,12 @@ import { getSmallIconFromWmoCode } from '../icons.mjs'; import { getOpenMeteoObservationSnapshot } from './weather.mjs'; import { temperature } from './units.mjs'; -const BASE_MAP_URL = 'https://server.arcgisonline.com/ArcGIS/rest/services/Canvas/World_Light_Gray_Base/MapServer/tile/{z}/{y}/{x}'; -const BOUNDARY_MAP_URL = 'https://services.arcgisonline.com/ArcGIS/rest/services/Reference/World_Boundaries_and_Places/MapServer/tile/{z}/{y}/{x}'; +const getBaseMapUrl = () => (window.WS4KP_SERVER_AVAILABLE + ? '/arcgis-server/ArcGIS/rest/services/Canvas/World_Light_Gray_Base/MapServer/tile/{z}/{y}/{x}' + : 'https://server.arcgisonline.com/ArcGIS/rest/services/Canvas/World_Light_Gray_Base/MapServer/tile/{z}/{y}/{x}'); +const getBoundaryMapUrl = () => (window.WS4KP_SERVER_AVAILABLE + ? '/arcgis-services/ArcGIS/rest/services/Reference/World_Boundaries_and_Places/MapServer/tile/{z}/{y}/{x}' + : 'https://services.arcgisonline.com/ArcGIS/rest/services/Reference/World_Boundaries_and_Places/MapServer/tile/{z}/{y}/{x}'); const DEFAULT_MAX_NEARBY_MARKERS = 7; const MIN_CITY_DISTANCE_METERS = 25000; const MIN_MARKER_PIXEL_DISTANCE = 85; @@ -26,14 +30,14 @@ const createMap = (mapElement) => window.L.map(mapElement, { }); const addBaseLayers = (map) => { - const baseLayer = window.L.tileLayer(BASE_MAP_URL, { + const baseLayer = window.L.tileLayer(getBaseMapUrl(), { maxZoom: 10, minZoom: 1, crossOrigin: true, className: 'radar-base-layer', }).addTo(map); - const boundaryLayer = window.L.tileLayer(BOUNDARY_MAP_URL, { + const boundaryLayer = window.L.tileLayer(getBoundaryMapUrl(), { maxZoom: 10, minZoom: 1, opacity: 0.6, diff --git a/server/scripts/modules/utils/url-rewrite.mjs b/server/scripts/modules/utils/url-rewrite.mjs index c815f0b..cc4267f 100644 --- a/server/scripts/modules/utils/url-rewrite.mjs +++ b/server/scripts/modules/utils/url-rewrite.mjs @@ -50,6 +50,14 @@ const rewriteUrl = (_url) => { url.protocol = window.location.protocol; url.host = window.location.host; url.pathname = `/rainviewer${url.pathname}`; + } else if (url.origin === 'https://server.arcgisonline.com') { + url.protocol = window.location.protocol; + url.host = window.location.host; + url.pathname = `/arcgis-server${url.pathname}`; + } else if (url.origin === 'https://services.arcgisonline.com') { + url.protocol = window.location.protocol; + url.host = window.location.host; + url.pathname = `/arcgis-services${url.pathname}`; } else if (typeof OVERRIDES !== 'undefined' && OVERRIDES?.RADAR_HOST && url.origin === `https://${OVERRIDES.RADAR_HOST}`) { // Handle override radar host url.protocol = window.location.protocol;