diff --git a/makefile b/makefile index 883057a3544..da68484eff5 100644 --- a/makefile +++ b/makefile @@ -48,6 +48,9 @@ stop: monitor: @pm2 monit +logs: + @pm2 logs + # dev ######################################### dev: clear clean-dist install diff --git a/packages/rendering/lib/aws-metrics.js b/packages/rendering/lib/aws-metrics.js new file mode 100644 index 00000000000..a4cbf2be530 --- /dev/null +++ b/packages/rendering/lib/aws-metrics.js @@ -0,0 +1,124 @@ +// @flow + +type Metric = { + send: () => void, +}; + +process.env.AWS_PROFILE = 'frontend'; + +const AWS = require('aws-sdk'); + +AWS.config.update({ region: 'eu-west-1' }); + +// how frequently we send metrics to aws in ms +const METRICS_TIME_RESOLUTION = 60 * 1000; + +const sendMetric = (m: Array) => { + if (m.length === 0) { + return; + } + + const cloudWatchClient = new AWS.CloudWatch(); + + const params = { + MetricData: m, + Namespace: 'Application', + }; + + cloudWatchClient.putMetricData(params, err => { + if (err) { + console.error(err, err.stack); + } + }); +}; + +// handles sending matrics to AWS + +const collectAndSendAWSMetrics = function(...metrics: Metric) { + setInterval(() => { + console.log('Collecting metrics'); + metrics.forEach(m => m.send()); + }, METRICS_TIME_RESOLUTION); +}; + +// to record things like latency + +const TimingMetric = function TimingMetric( + app: String, + stage: String, + metricName: String, +) { + const values = []; + + return { + recordDuration: (n: Number) => { + values.push(n); + }, + + send: () => { + sendMetric( + values.map(v => ({ + Dimensions: [ + { + Name: 'ApplicationName', + Value: app, + }, + { + Name: 'Stage', + Value: stage, + }, + ], + MetricName: metricName, + Unit: 'Milliseconds', + Value: v, + })), + ); + + values.length = 0; + }, + }; +}; + +// to record memory or file sizes + +const BytesMetric = function BytesMetric( + app: String, + stage: String, + metricName: String, +) { + const values = []; + + return { + record: (n: Number) => { + values.push(n); + }, + + send: () => { + sendMetric( + values.map(v => ({ + Dimensions: [ + { + Name: 'ApplicationName', + Value: app, + }, + { + Name: 'Stage', + Value: stage, + }, + ], + MetricName: metricName, + Unit: 'Bytes', + Value: v, + })), + ); + + values.length = 0; + }, + }; +}; + +module.exports = { + collectAndSendAWSMetrics, + BytesMetric, + TimingMetric, +}; diff --git a/packages/rendering/lib/metrics-baseline.js b/packages/rendering/lib/metrics-baseline.js new file mode 100644 index 00000000000..7212b6db303 --- /dev/null +++ b/packages/rendering/lib/metrics-baseline.js @@ -0,0 +1,47 @@ +// @flow + +import os from 'os'; +import disk from 'diskusage'; +import { BytesMetric, collectAndSendAWSMetrics } from './aws-metrics'; + +const maxHeapMemory = BytesMetric('rendering', 'PROD', 'max-heap-memory'); +const freeDiskSpace = BytesMetric('rendering', 'PROD', 'free-disk-memory'); +const usedHeapMemory = BytesMetric('rendering', 'PROD', 'used-heap-memory'); +const freePhysicalMemory = BytesMetric( + 'rendering', + 'PROD', + 'free-physical-memory', +); +const totalPhysicalMemory = BytesMetric( + 'rendering', + 'PROD', + 'total-physical-memory', +); + +// transmits metrics to AWS + +collectAndSendAWSMetrics( + maxHeapMemory, + usedHeapMemory, + totalPhysicalMemory, + freePhysicalMemory, + freeDiskSpace, +); + +// records system metrics + +const recordBaselineCloudWatchMetrics = function recordBaselineCloudWatchMetrics() { + disk.check('/', (err, diskinfo) => { + if (err) { + console.error(err); + } else { + maxHeapMemory.record(process.memoryUsage().heapTotal); + usedHeapMemory.record(process.memoryUsage().heapUsed); + totalPhysicalMemory.record(os.totalmem()); + freePhysicalMemory.record(os.freemem()); + freeDiskSpace.record(diskinfo.free); + } + }); +}; + +export default recordBaselineCloudWatchMetrics; diff --git a/packages/rendering/package.json b/packages/rendering/package.json index 142718b3ec3..6d825627121 100644 --- a/packages/rendering/package.json +++ b/packages/rendering/package.json @@ -8,6 +8,8 @@ }, "license": "Apache-2.0", "dependencies": { + "aws-sdk": "^2.279.1", + "diskusage": "^0.2.4", "emotion": "^9.1.1", "express": "^4.16.3", "glob": "^7.1.2", diff --git a/packages/rendering/server.js b/packages/rendering/server.js index 019a4549425..afb5b1f0d3d 100644 --- a/packages/rendering/server.js +++ b/packages/rendering/server.js @@ -4,7 +4,7 @@ import path from 'path'; import express from 'express'; import type { $Request, $Response } from 'express'; - +import recordBaselineCloudWatchMetrics from './lib/metrics-baseline'; import document from '../../frontend/document'; import Article from '../../frontend/pages/Article'; import { dist, getPagesForSite, root } from '../../config'; @@ -80,5 +80,10 @@ if (process.env.NODE_ENV === 'production') { app.use((err, req, res, next) => { res.status(500).send(`
${err.stack}
`); }); + + setInterval(() => { + recordBaselineCloudWatchMetrics(); + }, 10 * 1000); + app.listen(9000); } diff --git a/yarn.lock b/yarn.lock index a3b1f5d86b2..0e70c04e68a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1233,6 +1233,20 @@ autoprefixer@^9.0.0: postcss "^7.0.2" postcss-value-parser "^3.2.3" +aws-sdk@^2.279.1: + version "2.300.0" + resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.300.0.tgz#862f993151b1d8c96f58a5c04aef562e04d56c6a" + dependencies: + buffer "4.9.1" + events "1.1.1" + ieee754 "1.1.8" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.1.0" + xml2js "0.4.19" + aws-sign2@~0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f" @@ -2253,7 +2267,7 @@ buffer-xor@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" -buffer@^4.3.0: +buffer@4.9.1, buffer@^4.3.0: version "4.9.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" dependencies: @@ -3294,6 +3308,12 @@ dir-glob@^2.0.0: arrify "^1.0.1" path-type "^3.0.0" +diskusage@^0.2.4: + version "0.2.4" + resolved "https://registry.yarnpkg.com/diskusage/-/diskusage-0.2.4.tgz#e956f7a1051e0c6a1af706154efe620a2ee432ec" + dependencies: + nan "^2.5.0" + doctrine@1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-1.5.0.tgz#379dce730f6166f76cefa4e6707a159b02c5a6fa" @@ -3760,7 +3780,7 @@ eventemitter2@~0.4.14: version "0.4.14" resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-0.4.14.tgz#8f61b75cde012b2e9eb284d4545583b5643b61ab" -events@^1.0.0: +events@1.1.1, events@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" @@ -4780,6 +4800,10 @@ icss-utils@^2.1.0: dependencies: postcss "^6.0.1" +ieee754@1.1.8: + version "1.1.8" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4" + ieee754@^1.1.4: version "1.1.11" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.11.tgz#c16384ffe00f5b7835824e67b6f2bd44a5229455" @@ -5276,6 +5300,10 @@ jest-docblock@^21.0.0: version "21.2.0" resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-21.2.0.tgz#51529c3b30d5fd159da60c27ceedc195faf8d414" +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + js-base64@^2.1.9: version "2.4.3" resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.4.3.tgz#2e545ec2b0f2957f41356510205214e98fad6582" @@ -6069,7 +6097,7 @@ mute-stream@0.0.7, mute-stream@~0.0.4: version "0.0.7" resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.7.tgz#3075ce93bc21b8fab43e1bc4da7e8115ed1e7bab" -nan@^2.3.0, nan@^2.6.2: +nan@^2.3.0, nan@^2.5.0, nan@^2.6.2: version "2.10.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f" @@ -8004,7 +8032,11 @@ safer-buffer@^2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" -sax@^1.2.4, sax@~1.2.1: +sax@1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + +sax@>=0.6.0, sax@^1.2.4, sax@~1.2.1: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -9129,6 +9161,13 @@ url-to-options@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/url-to-options/-/url-to-options-1.0.1.tgz#1505a03a289a48cbd7a434efbaeec5055f5633a9" +url@0.10.3: + version "0.10.3" + resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + dependencies: + punycode "1.3.2" + querystring "0.2.0" + url@^0.11.0: version "0.11.0" resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" @@ -9163,6 +9202,10 @@ utils-merge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" +uuid@3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" + uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0: version "3.2.1" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.2.1.tgz#12c528bb9d58d0b9265d9a2f6f0fe8be17ff1f14" @@ -9563,6 +9606,17 @@ xml-name-validator@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.0, xtend@~4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"