diff --git a/deploy/cloudwatch_api_logs.tf b/deploy/cloudwatch_api_logs.tf new file mode 100644 index 0000000000000000000000000000000000000000..2120e552377bca8055bd1762d3715e0865187b29 --- /dev/null +++ b/deploy/cloudwatch_api_logs.tf @@ -0,0 +1,75 @@ +locals { + API_GATEWAY_LOG_GROUP = "API-Gateway-Execution-Logs_${split("-", aws_api_gateway_stage.main.id)[1]}/${aws_api_gateway_stage.main.stage_name}" +} +output "log_groupe_name" { + value = local.API_GATEWAY_LOG_GROUP +} +resource "aws_lambda_function" "logs" { + filename = data.archive_file.lambda_logs_file.output_path + function_name = "${local.prefix}-logs-apigw" + role = aws_iam_role.iam_for_lambda_logs.arn + handler = "apigw_metrics.handler" + timeout = 10 + source_code_hash = data.archive_file.lambda_logs_file.output_base64sha256 + runtime = "nodejs12.x" + reserved_concurrent_executions = 2 + layers = [ + "arn:aws:lambda:eu-west-1:580247275435:layer:LambdaInsightsExtension:14" + ] + environment { + variables = { + API_GATEWAY_LOG_GROUP = local.API_GATEWAY_LOG_GROUP + } + } + + tags = local.common_tags +} + +resource "aws_lambda_function" "logs_events" { + filename = data.archive_file.lambda_logs_event_file.output_path + function_name = "${local.prefix}-logs-events-apigw" + role = aws_iam_role.iam_for_lambda_logs.arn + handler = "apigw_metrics_live.handler" + timeout = 10 + source_code_hash = data.archive_file.lambda_logs_event_file.output_base64sha256 + runtime = "nodejs12.x" + reserved_concurrent_executions = 200 + layers = [ + "arn:aws:lambda:eu-west-1:580247275435:layer:LambdaInsightsExtension:14" + ] + environment { + variables = { + API_GATEWAY_LOG_GROUP = local.API_GATEWAY_LOG_GROUP + } + } + + tags = local.common_tags +} + +data "archive_file" "lambda_logs_file" { + type = "zip" + output_path = "${local.lambda_loc}/zip/apigw_metrics.zip" + source_file = "${local.lambda_loc}/apigw_metrics/apigw_metrics.js" +} + +data "archive_file" "lambda_logs_event_file" { + type = "zip" + output_path = "${local.lambda_loc}/zip/apigw_metrics_events.zip" + source_file = "${local.lambda_loc}/apigw_metrics/apigw_metrics_live.js" +} + +resource "aws_iam_role" "iam_for_lambda_logs" { + name = "${local.prefix}-lambda-admin-cloudwatch-logs" + + assume_role_policy = file("./templates/lambda/assume-role-policy.json") +} + +resource "aws_iam_role_policy_attachment" "lambda_logs_FA" { + role = aws_iam_role.iam_for_lambda_logs.name + policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess" +} + +resource "aws_iam_role_policy_attachment" "lambda_cloudwatch_FA" { + role = aws_iam_role.iam_for_lambda_logs.name + policy_arn = "arn:aws:iam::aws:policy/CloudWatchFullAccess" +} \ No newline at end of file diff --git a/deploy/lambda/apigw_metrics/apigw_metrics.js b/deploy/lambda/apigw_metrics/apigw_metrics.js new file mode 100644 index 0000000000000000000000000000000000000000..2e33479d72728594f4ff3f2b07a12c930b9f5cc3 --- /dev/null +++ b/deploy/lambda/apigw_metrics/apigw_metrics.js @@ -0,0 +1,44 @@ +const AWS = require('aws-sdk') +const CW = new AWS.CloudWatch() +const CWL = new AWS.CloudWatchLogs() +const API_GATEWAY_LOG_GROUP = process.env.API_GATEWAY_LOG_GROUP +const now = Date.now()/1000 +const period = 30 * 60 // seconds +exports.handler = async function(event){ + + const params = { + logGroupName: API_GATEWAY_LOG_GROUP, + startTime: now - period, + endTime: now, + queryString: `fields @timestamp, @message + | parse @message \"(*) *\" as reqId, lMessage + | filter lMessage like /Method completed with status:/ + | parse lMessage \"Method completed with status: *\" as status + | display @timestamp, reqId, status` + }; + + // 1. Start the query. When we start a query, this returns a queryId for us to use on our next step. + const data = await CWL.startQuery(params).promise(); + const { queryId } = data; + console.debug('query id', queryId); + + while (true) { + + // 2. Send Insight query to CloudwatchLogs + const insightData = await CWL.getQueryResults({ queryId }).promise(); + console.log(JSON.stringify(insightData,null,4)) + // 3. Check if it is available + if (Array.isArray(insightData.results) && insightData.status === 'Complete') { + const insightResult = insightData.results; + + // Change this line to publish to SNS or send to Slack + console.log(JSON.stringify(insightResult, null, 4)) + break; + } + + // 4. Otherwise, Wait for 100 ms for insight api result + await new Promise((resolve, reject) => setTimeout(resolve, 100)); + } + + return 'ok'; +} \ No newline at end of file diff --git a/deploy/lambda/apigw_metrics/apigw_metrics_live.js b/deploy/lambda/apigw_metrics/apigw_metrics_live.js new file mode 100644 index 0000000000000000000000000000000000000000..bc448ca8c614e0547247e79006131c0d2dbd5b3b --- /dev/null +++ b/deploy/lambda/apigw_metrics/apigw_metrics_live.js @@ -0,0 +1,9 @@ +const AWS = require('aws-sdk') +const CW = new AWS.CloudWatch() +const CWL = new AWS.CloudWatchLogs() +const API_GATEWAY_LOG_GROUP = process.env.API_GATEWAY_LOG_GROUP +exports.handler = async function(event,context){ + + console.log(JSON.stringify(event)) + console.log(JSON.stringify(context)) +} \ No newline at end of file diff --git a/deploy/lambda/apigw_metrics/package-lock.json b/deploy/lambda/apigw_metrics/package-lock.json new file mode 100644 index 0000000000000000000000000000000000000000..9c3a1b414583486aca73bdf38d6b15d911e122df --- /dev/null +++ b/deploy/lambda/apigw_metrics/package-lock.json @@ -0,0 +1,234 @@ +{ + "name": "apigw_metrics", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "dependencies": { + "aws-sdk": "^2.864.0" + } + }, + "node_modules/aws-sdk": { + "version": "2.869.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.869.0.tgz", + "integrity": "sha512-Sj9H+OH1sizBJt6WyTFBvCthZ1hRNUi4qRFO922agf+cOfmq1r+PYLOcG/0qgLMe2aelRwfT2qE2AZ97mADiOw==", + "dependencies": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "node_modules/url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "node_modules/xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", + "engines": { + "node": ">=4.0" + } + } + }, + "dependencies": { + "aws-sdk": { + "version": "2.869.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.869.0.tgz", + "integrity": "sha512-Sj9H+OH1sizBJt6WyTFBvCthZ1hRNUi4qRFO922agf+cOfmq1r+PYLOcG/0qgLMe2aelRwfT2qE2AZ97mADiOw==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + } + }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + } + } +} diff --git a/deploy/lambda/apigw_metrics/package.json b/deploy/lambda/apigw_metrics/package.json new file mode 100644 index 0000000000000000000000000000000000000000..7e315ed09924196e440250f2c46eaf9b237e8c03 --- /dev/null +++ b/deploy/lambda/apigw_metrics/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "aws-sdk": "^2.864.0" + } +}