From 5a2af5c49ee9c3d6e95273799cb5a3816c6872fb Mon Sep 17 00:00:00 2001 From: "D. Berge" Date: Wed, 13 Sep 2023 21:58:06 +0200 Subject: [PATCH] Add CSV output option for events log --- .../api/middleware/event/sequence/get/csv.js | 85 +++++++++++++++++++ .../middleware/event/sequence/get/index.js | 2 + lib/www/server/package-lock.json | 66 ++++++++++++++ lib/www/server/package.json | 1 + 4 files changed, 154 insertions(+) create mode 100644 lib/www/server/api/middleware/event/sequence/get/csv.js diff --git a/lib/www/server/api/middleware/event/sequence/get/csv.js b/lib/www/server/api/middleware/event/sequence/get/csv.js new file mode 100644 index 0000000..c0538b2 --- /dev/null +++ b/lib/www/server/api/middleware/event/sequence/get/csv.js @@ -0,0 +1,85 @@ +const { stringify } = require('csv'); +const { transform, prepare } = require('../../../../../lib/sse'); + +const json = async function (req, res, next) { + try { + const query = req.query; + query.sequence = req.params.sequence; + const {events, sequences} = await prepare(req.params.project, query); + if ("download" in query || "d" in query) { + const extension = "csv"; + // Get the sequence number(s) (more than one sequence can be selected) + const seqNums = query.sequence.split(";"); + // If we've only been asked for a single sequence, get its line name + const lineName = (sequences.find(i => i.sequence == seqNums[0]) || {})?.meta?.lineName; + const filename = (seqNums.length == 1 && lineName) + ? `${lineName}-NavLog.${extension}` + : `${req.params.project}-${query.sequence}.${extension}`; + res.set("Content-Disposition", `attachment; filename="${filename}"`); + } + + const columns = { + id: "id", + unix_epoch: (row) => Math.floor(row.tstamp/1000), + timestamp: (row) => (new Date(row.tstamp)).toISOString(), + sequence: "sequence", + point: "point", + text: "remarks", + labels: (row) => row.labels.join(";"), + latitude: (row) => { + if (row.meta.geometry?.type == "Point" && row.meta.geometry?.coordinates) { + return row.meta.geometry.coordinates[1]; + } + }, + longitude: (row) => { + if (row.meta.geometry?.type == "Point" && row.meta.geometry?.coordinates) { + return row.meta.geometry.coordinates[0]; + } + } + }; + + let fields = [ "timestamp", "sequence", "point", "text", "labels", "latitude", "longitude", "id" ]; + + if (req.query.fields) { + fields = req.query.fields.split(/[,;:.\s+*|]+/); + } + + let delimiter = req.query.delimiter || ","; + + const stringifier = stringify({delimiter}); + stringifier.on('error', (err) => { + console.error(err.message); + }); + + stringifier.on('readable', () => { + while((row = stringifier.read()) !== null) { + res.write(row); + } + }); + + res.status(200); + + if (!req.query.header || req.query.header.toLowerCase() == "true" || req.query.header == "1") { + // Send header + stringifier.write(fields); + } + + events.forEach( event => { + stringifier.write(fields.map( field => { + if (typeof columns[field] === "function") { + return columns[field](event); + } else { + return event[columns[field]]; + } + })); + }); + + stringifier.end(); + res.end(); + next(); + } catch (err) { + next(err); + } +}; + +module.exports = json; diff --git a/lib/www/server/api/middleware/event/sequence/get/index.js b/lib/www/server/api/middleware/event/sequence/get/index.js index 7c0b149..e9d87da 100644 --- a/lib/www/server/api/middleware/event/sequence/get/index.js +++ b/lib/www/server/api/middleware/event/sequence/get/index.js @@ -2,6 +2,7 @@ const json = require('./json'); const geojson = require('./geojson'); const seis = require('./seis'); const html = require('./html'); +const csv = require('./csv'); const pdf = require('./pdf'); module.exports = async function (req, res, next) { @@ -11,6 +12,7 @@ module.exports = async function (req, res, next) { "application/geo+json": geojson, "application/vnd.seis+json": seis, "text/html": html, + "text/csv": csv, "application/pdf": pdf }; diff --git a/lib/www/server/package-lock.json b/lib/www/server/package-lock.json index 86341ef..d24ea71 100644 --- a/lib/www/server/package-lock.json +++ b/lib/www/server/package-lock.json @@ -15,6 +15,7 @@ "dependencies": { "body-parser": "gitlab:aaltronav/contrib/expressjs/body-parser", "cookie-parser": "^1.4.5", + "csv": "^6.3.3", "debug": "^4.3.4", "express": "^4.17.1", "express-jwt": "^8.4.1", @@ -575,6 +576,35 @@ "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" }, + "node_modules/csv": { + "version": "6.3.3", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.3.tgz", + "integrity": "sha512-TuOM1iZgdDiB6IuwJA8oqeu7g61d9CU9EQJGzCJ1AE03amPSh/UK5BMjAVx+qZUBb/1XEo133WHzWSwifa6Yqw==", + "dependencies": { + "csv-generate": "^4.2.8", + "csv-parse": "^5.5.0", + "csv-stringify": "^6.4.2", + "stream-transform": "^3.2.8" + }, + "engines": { + "node": ">= 0.1.90" + } + }, + "node_modules/csv-generate": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.2.8.tgz", + "integrity": "sha512-qQ5CUs4I58kfo90EDBKjdp0SpJ3xWnN1Xk1lZ1ITvfvMtNRf+jrEP8tNPeEPiI9xJJ6Bd/km/1hMjyYlTpY42g==" + }, + "node_modules/csv-parse": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.0.tgz", + "integrity": "sha512-RxruSK3M4XgzcD7Trm2wEN+SJ26ChIb903+IWxNOcB5q4jT2Cs+hFr6QP39J05EohshRFEvyzEBoZ/466S2sbw==" + }, + "node_modules/csv-stringify": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.2.tgz", + "integrity": "sha512-DXIdnnCUQYjDKTu6TgCSzRDiAuLxDjhl4ErFP9FGMF3wzBGOVMg9bZTLaUcYtuvhXgNbeXPKeaRfpgyqE4xySw==" + }, "node_modules/d3-queue": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/d3-queue/-/d3-queue-2.0.3.tgz", @@ -5713,6 +5743,11 @@ "node": ">= 0.6" } }, + "node_modules/stream-transform": { + "version": "3.2.8", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.2.8.tgz", + "integrity": "sha512-NUx0mBuI63KbNEEh9Yj0OzKB7iMOSTpkuODM2G7By+TTVihEIJ0cYp5X+pq/TdJRlsznt6CYR8HqxexyC6/bTw==" + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -6523,6 +6558,32 @@ } } }, + "csv": { + "version": "6.3.3", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.3.tgz", + "integrity": "sha512-TuOM1iZgdDiB6IuwJA8oqeu7g61d9CU9EQJGzCJ1AE03amPSh/UK5BMjAVx+qZUBb/1XEo133WHzWSwifa6Yqw==", + "requires": { + "csv-generate": "^4.2.8", + "csv-parse": "^5.5.0", + "csv-stringify": "^6.4.2", + "stream-transform": "^3.2.8" + } + }, + "csv-generate": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.2.8.tgz", + "integrity": "sha512-qQ5CUs4I58kfo90EDBKjdp0SpJ3xWnN1Xk1lZ1ITvfvMtNRf+jrEP8tNPeEPiI9xJJ6Bd/km/1hMjyYlTpY42g==" + }, + "csv-parse": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.0.tgz", + "integrity": "sha512-RxruSK3M4XgzcD7Trm2wEN+SJ26ChIb903+IWxNOcB5q4jT2Cs+hFr6QP39J05EohshRFEvyzEBoZ/466S2sbw==" + }, + "csv-stringify": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.2.tgz", + "integrity": "sha512-DXIdnnCUQYjDKTu6TgCSzRDiAuLxDjhl4ErFP9FGMF3wzBGOVMg9bZTLaUcYtuvhXgNbeXPKeaRfpgyqE4xySw==" + }, "d3-queue": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/d3-queue/-/d3-queue-2.0.3.tgz", @@ -10722,6 +10783,11 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, + "stream-transform": { + "version": "3.2.8", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.2.8.tgz", + "integrity": "sha512-NUx0mBuI63KbNEEh9Yj0OzKB7iMOSTpkuODM2G7By+TTVihEIJ0cYp5X+pq/TdJRlsznt6CYR8HqxexyC6/bTw==" + }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", diff --git a/lib/www/server/package.json b/lib/www/server/package.json index 614b63c..0ccd1f3 100644 --- a/lib/www/server/package.json +++ b/lib/www/server/package.json @@ -23,6 +23,7 @@ "dependencies": { "body-parser": "gitlab:aaltronav/contrib/expressjs/body-parser", "cookie-parser": "^1.4.5", + "csv": "^6.3.3", "debug": "^4.3.4", "express": "^4.17.1", "express-jwt": "^8.4.1",