Use ETag middleware

Otherwise newer versions of npm will choke during `npm install` due
to this npm bug: https://github.com/npm/cli/issues/2610

lib/www/client/source/src/views/Plan.vue

@@ -119,7 +119,11 @@
         >
 
           <template v-slot:item.srss="{item}">
-            <v-icon small :title="srssInfo(item)">{{srssIcon(item)}}</v-icon>
+            <span style="white-space: nowrap;">
+              <v-icon small :title="srssInfo(item)">{{srssIcon(item)}}</v-icon>
+              /
+              <v-icon small :title="wxInfo(item)" v-if="item.meta.wx">{{wxIcon(item)}}</v-icon>
+            </span>
           </template>
 
           <template v-slot:item.sequence="{item, value}">
@@ -422,6 +426,123 @@ export default {
       plannerConfig: null,
       shiftAll: false, // Shift all sequences checkbox
 
+      // Weather API
+      wxData: null,
+      weathercode: {
+        0: {
+          description: "Clear sky",
+          icon: "mdi-weather-sunny"
+        },
+        1: {
+          description: "Mainly clear",
+          icon: "mdi-weather-sunny"
+        },
+        2: {
+          description: "Partly cloudy",
+          icon: "mdi-weather-partly-cloudy"
+        },
+        3: {
+          description: "Overcast",
+          icon: "mdi-weather-cloudy"
+        },
+        45: {
+          description: "Fog",
+          icon: "mde-weather-fog"
+        },
+        48: {
+          description: "Depositing rime fog",
+          icon: "mdi-weather-fog"
+        },
+        51: {
+          description: "Light drizzle",
+          icon: "mdi-weather-partly-rainy"
+        },
+        53: {
+          description: "Moderate drizzle",
+          icon: "mdi-weather-partly-rainy"
+        },
+        55: {
+          description: "Dense drizzle",
+          icon: "mdi-weather-rainy"
+        },
+        56: {
+          description: "Light freezing drizzle",
+          icon: "mdi-weather-partly-snowy-rainy"
+        },
+        57: {
+          description: "Freezing drizzle",
+          icon: "mdi-weather-partly-snowy-rainy"
+        },
+        61: {
+          description: "Light rain",
+          icon: "mdi-weather-rainy"
+        },
+        63: {
+          description: "Moderate rain",
+          icon: "mdi-weather-rainy"
+        },
+        65: {
+          description: "Heavy rain",
+          icon: "mdi-weather-pouring"
+        },
+        66: {
+          description: "Light freezing rain",
+          icon: "mdi-loading"
+        },
+        67: {
+          description: "Freezing rain",
+          icon: "mdi-loading"
+        },
+        71: {
+          description: "Light snow",
+          icon: "mdi-loading"
+        },
+        73: {
+          description: "Moderate snow",
+          icon: "mdi-loading"
+        },
+        75: {
+          description: "Heavy snow",
+          icon: "mdi-loading"
+        },
+        77: {
+          description: "Snow grains",
+          icon: "mdi-loading"
+        },
+        80: {
+          description: "Light rain showers",
+          icon: "mdi-loading"
+        },
+        81: {
+          description: "Moderate rain showers",
+          icon: "mdi-loading"
+        },
+        82: {
+          description: "Violent rain showers",
+          icon: "mdi-loading"
+        },
+        85: {
+          description: "Light snow showers",
+          icon: "mdi-loading"
+        },
+        86: {
+          description: "Snow showers",
+          icon: "mdi-loading"
+        },
+        95: {
+          description: "Thunderstorm",
+          icon: "mdi-loading"
+        },
+        96: {
+          description: "Hailstorm",
+          icon: "mdi-loading"
+        },
+        99: {
+          description: "Heavy hailstorm",
+          icon: "mdi-loading"
+        },
+      },
+
       // Context menu stuff
       contextMenuShow: false,
       contextMenuX: 0,
@@ -630,6 +751,113 @@ export default {
       return text.join("\n");
     },
 
+    wxInfo (line) {
+
+      function atm(key) {
+        return line.meta?.wx?.atmospheric?.hourly[key];
+      }
+
+      function mar(key) {
+        return line.meta?.wx?.marine?.hourly[key];
+      }
+
+      const code = atm("weathercode");
+
+      const description = this.weathercode[code]?.description ?? `WMO code ${code}`;
+      const wind_speed = Math.round(atm("windspeed_10m"));
+      const wind_direction = String(Math.round(atm("winddirection_10m"))).padStart(3, "0");
+      const pressure = Math.round(atm("surface_pressure"));
+      const temperature = Math.round(atm("temperature_2m"));
+      const humidity = atm("relativehumidity_2m");
+      const precipitation = atm("precipitation");
+      const precipitation_probability = atm("precipitation_probability");
+      const precipitation_str = precipitation_probability
+        ? `\nPrecipitation ${precipitation} mm (prob. ${precipitation_probability}%)`
+        : ""
+
+      const wave_height = mar("wave_height").toFixed(1);
+      const wave_direction = mar("wave_direction");
+      const wave_period = mar("wave_period");
+
+      return `${description}\n${temperature}° C\n${pressure} hPa\nWind ${wind_speed} kt ${wind_direction}°\nRelative humidity ${humidity}%${precipitation_str}\nWaves ${wave_height} m ${wave_direction}° @ ${wave_period} s`;
+    },
+
+    wxIcon (line) {
+      const code = line.meta?.wx?.atmospheric?.hourly?.weathercode;
+
+      return this.weathercode[code]?.icon ?? "mdi-help";
+
+    },
+
+    async wxQuery (line) {
+      function midpoint(line) {
+        // WARNING Fails if across the antimeridian
+        const longitude = (line.geometry.coordinates[0][0] + line.geometry.coordinates[1][0])/2;
+        const latitude = (line.geometry.coordinates[0][1] + line.geometry.coordinates[1][1])/2;
+        return [ longitude, latitude ];
+      }
+
+      function extract (fcst) {
+        const τ = (line.ts0.valueOf() + line.ts1.valueOf()) / 2000;
+        const [idx, ε] = fcst?.hourly?.time?.reduce( (acc, cur, idx) => {
+          const δ = Math.abs(cur - τ);
+          const retval = acc
+            ? acc[1] < δ
+              ? acc
+              : [ idx, δ ]
+            : [ idx, δ ];
+
+          return retval;
+        });
+
+        if (idx) {
+          const hourly = {};
+          for (let key in fcst?.hourly) {
+            fcst.hourly[key] = fcst.hourly[key][idx];
+          }
+        }
+
+        return fcst;
+      }
+
+      async function fetch_atmospheric (opts) {
+        const { longitude, latitude, dt0, dt1 } = opts;
+
+        const url = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&hourly=temperature_2m,relativehumidity_2m,precipitation_probability,precipitation,weathercode,pressure_msl,surface_pressure,windspeed_10m,winddirection_10m&daily=uv_index_max&windspeed_unit=kn&timeformat=unixtime&timezone=GMT&start_date=${dt0}&end_date=${dt1}&format=json`;
+        const init = {};
+        const res = await fetch (url, init);
+        if (res?.ok) {
+          const data = await res.json();
+
+          return extract(data);
+        }
+      }
+
+      async function fetch_marine (opts) {
+        const { longitude, latitude, dt0, dt1 } = opts;
+        const url = `https://marine-api.open-meteo.com/v1/marine?latitude=${latitude}&longitude=${longitude}&hourly=wave_height,wave_direction,wave_period&timeformat=unixtime&timezone=GMT&start_date=${dt0}&end_date=${dt1}&format=json`;
+
+        const init = {};
+        const res = await fetch (url, init);
+        if (res?.ok) {
+          const data = await res.json();
+
+          return extract(data);
+        }
+      }
+
+      if (line) {
+        const [ longitude, latitude ] = midpoint(line);
+        const dt0 = line.ts0.toISOString().substr(0, 10);
+        const dt1 = line.ts1.toISOString().substr(0, 10);
+
+        return {
+          atmospheric: await fetch_atmospheric({longitude, latitude, dt0, dt1}),
+          marine: await fetch_marine({longitude, latitude, dt0, dt1})
+        };
+      }
+    },
+
     lagAfter (item) {
       const pos = this.items.indexOf(item)+1;
       if (pos != 0) {
@@ -723,6 +951,9 @@ export default {
       for (const item of this.items) {
         item.ts0 = new Date(item.ts0);
         item.ts1 = new Date(item.ts1);
+        this.wxQuery(item).then( (wx) => {
+          item.meta = {...item.meta, wx};
+        });
       }
     },
 

lib/www/server/api/index.js

@@ -81,6 +81,9 @@ app.map({
 
 app.use(mw.auth.authentify);
 
+// Don't process the request if the data hasn't changed
+app.use(mw.etag.ifNoneMatch);
+
 // We must be authenticated before we can access these
 app.map({
 	'/project': {
@@ -260,12 +263,12 @@ app.map({
 	},
 	'/queue/outgoing/': {
 		'asaqc': {
-			get: [ mw.queue.asaqc.get ],
+			get: [ mw.etag.noSave, mw.queue.asaqc.get ],
 			post: [ mw.auth.access.write, mw.queue.asaqc.post ],
 			'/project/:project': {
-				get: [ mw.queue.asaqc.get ],
+				get: [ mw.etag.noSave, mw.queue.asaqc.get ],
 				'/sequence/:sequence': {
-					get: [ mw.queue.asaqc.get ],
+					get: [ mw.etag.noSave, mw.queue.asaqc.get ],
 				}
 			},
 			'/:id': {
@@ -289,6 +292,10 @@ app.map({
 //
 });
 
+app.use(mw.etag.save);
+// Invalidate cache on database events
+mw.etag.watch(app);
+
 // Generic error handler. Stops stack dumps
 // being sent to clients.
 app.use(function (err, req, res, next) {

lib/www/server/api/middleware/etag/cache.js

@@ -0,0 +1,45 @@
+
+const cache = {};
+
+function getCache (r) {
+	if (!r?.app?.locals) {
+		return cache;
+	}
+
+	if (!r.app.locals.etags) {
+		r.app.locals.etags = {};
+	}
+
+	return r.app.locals.etags;
+}
+
+function isCached (req) {
+	const cache = getCache(req);
+
+	if (req.url in cache) {
+		const cached = cache[req.url];
+		const etag = req.get("If-None-Match");
+		if (etag && etag == cached.etag) {
+			return cached;
+		}
+	}
+	return; // undefined
+}
+
+function saveResponse (res) {
+	if (res?.headersSent) {
+		const etag = res.get("ETag");
+		if (etag && res.locals.saveEtag !== false) {
+			const cache = getCache(res);
+			const req = res.req;
+			console.log(`Saving ETag: ${req.method} ${req.url} → ${etag}`);
+			cache[req.url] = {etag, headers: res.getHeaders()};
+		}
+	}
+};
+
+module.exports = {
+	getCache,
+	isCached,
+	saveResponse
+};

lib/www/server/api/middleware/etag/if-none-match.js

@@ -0,0 +1,39 @@
+const { isCached } = require('./cache');
+
+function isIdempotentMethod (method) {
+	const nonIdempotentMethods = [
+		"POST", "PUT", "PATH", "DELETE"
+	];
+
+	return !nonIdempotentMethods.includes(method.toUpperCase());
+};
+
+function setHeaders(res, headers) {
+	for (let [key, value] of Object.entries(headers)) {
+		res.set(key, value);
+	}
+	return res;
+}
+
+
+function ifNoneMatch (req, res, next) {
+
+	const cached = isCached(req);
+	if (cached) {
+		console.log("ETag match", req.url);
+		setHeaders(res, cached.headers);
+		if (req.method == "GET" || req.method == "HEAD") {
+			res.status(304).send();
+			// No next()
+		} else if (!isIdempotentMethod(req.method)) {
+			res.status(412).send();
+		}
+	} else {
+		// Either we didn't have this URL in the cache, or there was
+		// no If-None-Match header, or it didn't match the cached ETag.
+		// We let the request proceed normally.
+		next();
+	}
+}
+
+module.exports = ifNoneMatch;

lib/www/server/api/middleware/etag/index.js

@@ -0,0 +1,7 @@
+
+module.exports = {
+	ifNoneMatch: require('./if-none-match'),
+	save: require('./save'),
+	noSave: require('./no-save'),
+	watch: require('./watch')
+};

lib/www/server/api/middleware/etag/no-save.js

@@ -0,0 +1,7 @@
+
+function noSave (req, res, next) {
+	res.locals.saveEtag = false;
+	next();
+}
+
+module.exports = noSave;

lib/www/server/api/middleware/etag/save.js

@@ -0,0 +1,8 @@
+const { saveResponse } = require('./cache');
+
+function save (req, res, next) {
+	saveResponse(res);
+	next();
+}
+
+module.exports = save;

lib/www/server/api/middleware/etag/watch.js

@@ -0,0 +1,127 @@
+const { getCache } = require('./cache');
+const { listen } = require('../../../lib/db/notify');
+const channels = require('../../../lib/db/channels');
+
+const rels = [
+	{
+		channels: [ "realtime" ],
+		urls: [ ]
+	},
+	{
+		channels: [ "event" ],
+		urls: [ /^\/project\/([^\/]+)\/event/ ],
+		matches: [ "project" ]
+	},
+	{
+		channels: [ "project" ],
+		urls: [ /^\/project\/([^\/]+)\// ],
+		matches: [ "project" ]
+	},
+	{
+		channels: [ "preplot_lines", "preplot_points" ],
+		urls: [ /^\/project\/([^\/]+)\/line[\/?]?/ ],
+		matches: [ "project" ]
+	},
+	{
+		channels: [ "planned_lines" ],
+		urls: [ /^\/project\/([^\/]+)\/plan[\/?]?/ ],
+		matches: [ "project" ]
+	},
+	{
+		channels: [ "raw_lines", "raw_shots" ],
+		urls: [ /^\/project\/([^\/]+)\/sequence[\/?]?/ ],
+		matches: [ "project" ]
+	},
+	{
+		channels: [ "final_lines", "final_shots" ],
+		urls: [ /^\/project\/([^\/]+)\/sequence[\/?]?/ ],
+		matches: [ "project" ]
+	},
+	{
+		channels: [ "info" ],
+		urls: [ ],
+		matches: [ ],
+		callback (url, data) {
+			if (data.payload?.table == "info") {
+				const rx = /^\/project\/([^\/]+)\/info\/([^\/?]+)[\/?]?/;
+				const match = url.match(rx);
+				if (match) {
+					if (match[1] == data.payload.pid) {
+						if (match[2] == data.payload?.old?.key || match[2] == data.payload?.new?.key) {
+							return true;
+						}
+					}
+				}
+			}
+			return false;
+		}
+	},
+	{
+		channels: [ "queue_items" ],
+		urls: [ ]
+	},
+]
+
+function invalidateCache (data, cache) {
+	return new Promise((resolve, reject) => {
+		const channel = data.channel;
+		const project = data.payload.pid;
+		const operation = data.payload.operation;
+		const table = data.payload.table;
+		const fields = { channel, project, operation, table };
+
+		for (let rel of rels) {
+			if (rel.channels.includes(channel)) {
+				for (let url of rel.urls) {
+					for (let [key, data] of Object.entries(cache)) {
+						const match = key.match(url)?.slice(1);
+						if (match) {
+							if (rel.matches) {
+								if (rel.matches.every( (field, idx) => match[idx] == fields[field] )) {
+									console.log("DELETE ENTRY (MATCHES)", key);
+									delete cache[key];
+								}
+							} else {
+								// Delete unconditionally
+								console.log("DELETE ENTRY (UNCONDITIONAL)", key);
+								delete cache[key];
+							}
+						}
+
+					}
+				}
+
+				if (rel.callback) {
+					for (let key of Object.keys(cache)) {
+						if (rel.callback(key, data)) {
+							console.log("DELETE ENTRY (CALLBACK)", key);
+							delete cache[key];
+						}
+					}
+				}
+
+			}
+		}
+
+		resolve();
+	});
+}
+
+async function watch (app) {
+	if (!app.locals?.etags) {
+		app.locals.etags = {};
+	}
+
+	function etagWatch (data) {
+		invalidateCache(data, app.locals.etags);
+	}
+	try {
+		const client = await listen(channels, etagWatch);
+		console.log("ETag watch installed", client);
+	} catch (err) {
+		console.error(err);
+		console.log("ETag watch not installed");
+	}
+}
+
+module.exports = watch;

lib/www/server/api/middleware/index.js

@@ -15,5 +15,6 @@ module.exports = {
 	info: require('./info'),
 	meta: require('./meta'),
 	openapi: require('./openapi'),
-	rss: require('./rss')
+	rss: require('./rss'),
+	etag: require('./etag')
 };

lib/www/server/lib/db/connection.js

@@ -1,5 +1,5 @@
 const { Pool, Client, types } = require('pg');
-
+const createSubscriber = require('pg-listen');
 const cfg = require("../config");
 
 const pool = new Pool(cfg.db);
@@ -14,6 +14,12 @@ numericTypeOIDs.forEach(oid => {
 // 	types.setTypeParser(oid, function (v) { return JSON.parse(v); });
 // })
 
+function makeSubscriber (opts = cfg.db) {
+	const subscriber = createSubscriber(opts);
+	process.on("exit", () => subscriber.close());
+	return subscriber;
+}
+
 const transaction = {
 	async begin (client) {
 		return await client.query("BEGIN;");
@@ -72,6 +78,7 @@ async function fetchRow (cursor) {
 
 module.exports = {
 	pool,
+	makeSubscriber,
 	transaction,
 	setSurvey,
 	schema2pid,

lib/www/server/lib/db/notify.js

@@ -0,0 +1,36 @@
+const { makeSubscriber } = require('./connection');
+
+
+async function listen (addChannels, callback) {
+
+	const client = makeSubscriber();
+
+	client.events.on("error", (err) => {
+		console.error("Postgres LISTEN subscriber error", err);
+		setTimeout(() => client.connect(), 5000);
+	});
+
+	client.connect();
+
+	if (!Array.isArray(addChannels)) {
+		addChannels = [addChannels];
+	}
+
+	for (const channel of addChannels) {
+		await client.listenTo(channel);
+		client.notifications.on(channel, (payload) => {
+			const data = {
+				channel,
+				_received: new Date(),
+				payload
+			};
+			callback(data);
+		});
+	}
+
+	return client;
+}
+
+module.exports = {
+	listen
+};

lib/www/server/package.json

@@ -27,13 +27,14 @@
     "express-jwt": "^6.0.0",
     "json2csv": "^5.0.6",
     "jsonwebtoken": "^8.5.1",
-    "leaflet-headless": "gitlab:aaltronav/contrib/leaflet-headless#devel",
+    "leaflet-headless": "git+https://git@gitlab.com/aaltronav/contrib/leaflet-headless.git#devel",
     "marked": "^4.0.12",
     "netmask": "^2.0.2",
     "node-fetch": "^2.6.1",
     "nunjucks": "^3.2.3",
     "pg": "^8.3.3",
     "pg-cursor": "^2.7.3",
+    "pg-listen": "^1.7.0",
     "selenium-webdriver": "*",
     "ws": "^7.3.1",
     "yaml": "^2.0.0-0"