diff --git a/deploy/api_gateway.tf b/deploy/api_gateway.tf
index 7b31edb8c70cebd700d1a002dd897e0e23365bf6..599967028453ead1a93a96938790ef563e0468b1 100644
--- a/deploy/api_gateway.tf
+++ b/deploy/api_gateway.tf
@@ -83,15 +83,37 @@ resource "aws_api_gateway_deployment" "main" {
create_before_destroy = true
}
}
+locals {
+ stage_name = terraform.workspace
+}
+resource "aws_cloudwatch_log_group" "apigw_logs" {
+ name = "apigw_${aws_api_gateway_rest_api.main.id}/${local.stage_name}"
+ retention_in_days = 7
+ tags = local.common_tags
+}
resource "aws_api_gateway_stage" "main" {
deployment_id = aws_api_gateway_deployment.main.id
rest_api_id = aws_api_gateway_rest_api.main.id
- stage_name = terraform.workspace == "production" ? "api" : terraform.workspace
+ stage_name = local.stage_name
xray_tracing_enabled = true
tags = local.common_tags
+ access_log_settings {
+ format = jsonencode({
+ "stage" = "$context.stage",
+ "apiId" = "$context.apiId",
+ "requestId" = "$context.requestId",
+ "ip" = "$context.identity.sourceIp",
+ "requestTime" = "$context.requestTime",
+ "httpMethod" = "$context.httpMethod",
+ "resourcePath" = "$context.resourcePath",
+ "status" = "$context.status",
+ "protocol" = "$context.protocol",
+ "responseLength" = "$context.responseLength"
+ })
+ destination_arn = aws_cloudwatch_log_group.apigw_logs.arn
+ }
}
-
output "api_endpoint" {
value = aws_api_gateway_stage.main.invoke_url
}
diff --git a/deploy/cloudwatch_api_logs.tf b/deploy/cloudwatch_api_logs.tf
index a93dc1e985fd65ce41f63fc330dd013358de8d9b..64b01a7363ed1cde66ff5b22870dfd4686fe5076 100644
--- a/deploy/cloudwatch_api_logs.tf
+++ b/deploy/cloudwatch_api_logs.tf
@@ -1,76 +1,81 @@
-locals {
- API_GATEWAY_LOG_GROUP = "API-Gateway-Execution-Logs_${split("-", aws_api_gateway_stage.main.id)[1]}/${aws_api_gateway_stage.main.stage_name}"
-}
-
-output "log_groupe_name" {
- value = local.API_GATEWAY_LOG_GROUP
-}
-
-resource "aws_lambda_function" "logs" {
- filename = data.archive_file.lambda_logs_file.output_path
- function_name = "${local.prefix}-logs-apigw"
+resource "aws_lambda_function" "apigw_logs" {
+ filename = data.archive_file.lambda_logs.output_path
+ function_name = "${local.prefix}-analyse-logs-apigw"
role = aws_iam_role.iam_for_lambda_logs.arn
handler = "apigw_metrics.handler"
timeout = 10
- source_code_hash = data.archive_file.lambda_logs_file.output_base64sha256
+ source_code_hash = data.archive_file.lambda_logs.output_base64sha256
runtime = "nodejs12.x"
- reserved_concurrent_executions = 2
+ reserved_concurrent_executions = 5
layers = [
"arn:aws:lambda:eu-west-1:580247275435:layer:LambdaInsightsExtension:14"
]
environment {
variables = {
- API_GATEWAY_LOG_GROUP = local.API_GATEWAY_LOG_GROUP
+ API_GATEWAY_LOG_GROUP = aws_cloudwatch_log_group.apigw_logs.name
}
}
tags = local.common_tags
}
-resource "aws_lambda_function" "logs_events" {
- filename = data.archive_file.lambda_logs_event_file.output_path
- function_name = "${local.prefix}-logs-events-apigw"
- role = aws_iam_role.iam_for_lambda_logs.arn
- handler = "apigw_metrics_live.handler"
- timeout = 10
- source_code_hash = data.archive_file.lambda_logs_event_file.output_base64sha256
- runtime = "nodejs12.x"
- reserved_concurrent_executions = 200
- layers = [
- "arn:aws:lambda:eu-west-1:580247275435:layer:LambdaInsightsExtension:14"
- ]
- environment {
- variables = {
- API_GATEWAY_LOG_GROUP = local.API_GATEWAY_LOG_GROUP
- }
+resource "aws_cloudwatch_log_subscription_filter" "lambdafunction_logfilter" {
+ name = "${local.prefix}-lambdafunction-subscription"
+ log_group_name = aws_cloudwatch_log_group.apigw_logs.name
+ filter_pattern = " "
+ destination_arn = aws_lambda_function.apigw_logs.arn
+ lifecycle {
+ ignore_changes = [
+ filter_pattern
+ ]
}
+}
- tags = local.common_tags
+resource "aws_lambda_permission" "log_subscription" {
+ statement_id = "AllowClouWatchInvoke"
+ action = "lambda:InvokeFunction"
+ function_name = aws_lambda_function.apigw_logs.function_name
+ principal = "logs.${var.aws_region}.amazonaws.com"
+ source_arn = "${aws_cloudwatch_log_group.apigw_logs.arn}:*"
}
-data "archive_file" "lambda_logs_file" {
+data "archive_file" "lambda_logs" {
type = "zip"
output_path = "${local.lambda_loc}/zip/apigw_metrics.zip"
source_file = "${local.lambda_loc}/apigw_metrics/apigw_metrics.js"
}
-data "archive_file" "lambda_logs_event_file" {
- type = "zip"
- output_path = "${local.lambda_loc}/zip/apigw_metrics_events.zip"
- source_file = "${local.lambda_loc}/apigw_metrics/apigw_metrics_live.js"
-}
-
resource "aws_iam_role" "iam_for_lambda_logs" {
name = "${local.prefix}-lambda-admin-cloudwatch-logs"
assume_role_policy = file("./templates/lambda/assume-role-policy.json")
}
-resource "aws_iam_role_policy_attachment" "lambda_logs_FA" {
+resource "aws_iam_role_policy_attachment" "lambda_cloudwatch_FA" {
role = aws_iam_role.iam_for_lambda_logs.name
- policy_arn = "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
+ policy_arn = "arn:aws:iam::aws:policy/CloudWatchFullAccess"
}
-resource "aws_iam_role_policy_attachment" "lambda_cloudwatch_FA" {
+resource "aws_iam_role_policy_attachment" "lambda_apigw" {
role = aws_iam_role.iam_for_lambda_logs.name
- policy_arn = "arn:aws:iam::aws:policy/CloudWatchFullAccess"
+ policy_arn = aws_iam_policy.apigw_read_only.arn
+}
+
+resource "aws_iam_policy" "apigw_read_only" {
+ name = "ApiGatewayReadOnly"
+ path = "/"
+ description = "Read only for describing APIGateway services"
+ # Terraform's "jsonencode" function converts a
+ # Terraform expression result to valid JSON syntax.
+ policy = jsonencode({
+ Version = "2012-10-17"
+ Statement = [
+ {
+ Action = [
+ "apigateway:GET",
+ ]
+ Effect = "Allow"
+ Resource = "*"
+ }
+ ]
+ })
}
diff --git a/deploy/dynamodb.tf b/deploy/dynamodb.tf
index 66d4d049e66d14c35239cbc693ee5fcff90de329..f07f98d2bce4a6b5c3437e593cf5eed9e5678468 100644
--- a/deploy/dynamodb.tf
+++ b/deploy/dynamodb.tf
@@ -18,5 +18,57 @@ resource "aws_dynamodb_table" "main" {
lifecycle {
prevent_destroy = true
+ ignore_changes = [
+ write_capacity,
+ read_capacity
+ ]
}
}
+
+resource "aws_appautoscaling_target" "dynamodb_table_read_target" {
+ max_capacity = 10
+ min_capacity = 1
+ resource_id = "table/${aws_dynamodb_table.main.name}"
+ scalable_dimension = "dynamodb:table:ReadCapacityUnits"
+ service_namespace = "dynamodb"
+}
+
+resource "aws_appautoscaling_policy" "dynamodb_table_read_policy" {
+ name = "DynamoDBReadCapacityUtilization:${aws_appautoscaling_target.dynamodb_table_read_target.resource_id}"
+ policy_type = "TargetTrackingScaling"
+ resource_id = aws_appautoscaling_target.dynamodb_table_read_target.resource_id
+ scalable_dimension = aws_appautoscaling_target.dynamodb_table_read_target.scalable_dimension
+ service_namespace = aws_appautoscaling_target.dynamodb_table_read_target.service_namespace
+
+ target_tracking_scaling_policy_configuration {
+ predefined_metric_specification {
+ predefined_metric_type = "DynamoDBReadCapacityUtilization"
+ }
+
+ target_value = 50
+ }
+}
+
+resource "aws_appautoscaling_target" "dynamodb_table_write_target" {
+ max_capacity = 10
+ min_capacity = 1
+ resource_id = "table/${aws_dynamodb_table.main.name}"
+ scalable_dimension = "dynamodb:table:WriteCapacityUnits"
+ service_namespace = "dynamodb"
+}
+
+resource "aws_appautoscaling_policy" "dynamodb_table_write_policy" {
+ name = "DynamoDBWriteCapacityUtilization:${aws_appautoscaling_target.dynamodb_table_write_target.resource_id}"
+ policy_type = "TargetTrackingScaling"
+ resource_id = aws_appautoscaling_target.dynamodb_table_write_target.resource_id
+ scalable_dimension = aws_appautoscaling_target.dynamodb_table_write_target.scalable_dimension
+ service_namespace = aws_appautoscaling_target.dynamodb_table_write_target.service_namespace
+
+ target_tracking_scaling_policy_configuration {
+ predefined_metric_specification {
+ predefined_metric_type = "DynamoDBWriteCapacityUtilization"
+ }
+
+ target_value = 50
+ }
+}
\ No newline at end of file
diff --git a/deploy/lambda/apigw_metrics/apigw_metrics.js b/deploy/lambda/apigw_metrics/apigw_metrics.js
index 2e33479d72728594f4ff3f2b07a12c930b9f5cc3..4e6396025a50e64f04dd706eb65cc31b97abcfcf 100644
--- a/deploy/lambda/apigw_metrics/apigw_metrics.js
+++ b/deploy/lambda/apigw_metrics/apigw_metrics.js
@@ -1,44 +1,84 @@
const AWS = require('aws-sdk')
+const zlib = require('zlib');
const CW = new AWS.CloudWatch()
-const CWL = new AWS.CloudWatchLogs()
-const API_GATEWAY_LOG_GROUP = process.env.API_GATEWAY_LOG_GROUP
-const now = Date.now()/1000
-const period = 30 * 60 // seconds
-exports.handler = async function(event){
+const APIGW = new AWS.APIGateway()
- const params = {
- logGroupName: API_GATEWAY_LOG_GROUP,
- startTime: now - period,
- endTime: now,
- queryString: `fields @timestamp, @message
- | parse @message \"(*) *\" as reqId, lMessage
- | filter lMessage like /Method completed with status:/
- | parse lMessage \"Method completed with status: *\" as status
- | display @timestamp, reqId, status`
- };
-
- // 1. Start the query. When we start a query, this returns a queryId for us to use on our next step.
- const data = await CWL.startQuery(params).promise();
- const { queryId } = data;
- console.debug('query id', queryId);
+exports.handler = async function(event,context){
+ return new Promise((resolve,reject)=>{
- while (true) {
-
- // 2. Send Insight query to CloudwatchLogs
- const insightData = await CWL.getQueryResults({ queryId }).promise();
- console.log(JSON.stringify(insightData,null,4))
- // 3. Check if it is available
- if (Array.isArray(insightData.results) && insightData.status === 'Complete') {
- const insightResult = insightData.results;
-
- // Change this line to publish to SNS or send to Slack
- console.log(JSON.stringify(insightResult, null, 4))
- break;
- }
-
- // 4. Otherwise, Wait for 100 ms for insight api result
- await new Promise((resolve, reject) => setTimeout(resolve, 100));
- }
+ console.log("Event: "+JSON.stringify(event))
+ var payload = Buffer.from(event.awslogs.data, 'base64');
+ zlib.gunzip(payload, function(e, result) {
+ if (e) {
+ console.error(JSON.stringify(e, null, 2))
+ reject(e)
+ } else {
+ //début du traitement du log stream fourni
+ result = JSON.parse(result.toString('ascii'));
+ console.log("Event Data after decoding: ", JSON.stringify(result, null, 2));
+
+ var promises = []
- return 'ok';
-}
\ No newline at end of file
+ result.logEvents.forEach(log => {
+ //for each log, get api name and push a metric data object associated in the payload for PutMetricData
+ var message = JSON.parse(log.message)
+ var getRestApiPromise = APIGW.getRestApi({restApiId:message.apiId}).promise()
+ promises.push(new Promise((res,rej)=>{
+ getRestApiPromise.then(data=>{
+ console.log("getRestApi result: "+JSON.stringify(data))
+ res(
+ {
+ MetricName: '400 Errors',
+ Dimensions: [
+ {
+ Name: 'API/REST',
+ Value: data.name
+ },
+ {
+ Name: 'Stage',
+ Value: message.stage
+ },
+ {
+ Name: 'Method',
+ Value: message.httpMethod
+ },
+ {
+ Name: 'Resource',
+ Value: message.resourcePath
+ },
+ ],
+ Timestamp: Math.floor(log.timestamp/1000),
+ Unit: "Count",
+ Value: message.status === "400" ? "1":"0",
+ }
+ )
+ }).catch(err=>{
+ rej(err)
+ })
+ }))
+ });
+
+ //when all logs are processed, send the PutMetricData request
+ Promise.all(promises).then(metrics=>{ //if all the promises (cf line 26) succed (equivalent to all the getRestApiPromise)
+ const params = {
+ MetricData:metrics,
+ Namespace: "APIGW/custom"
+ };
+ console.log("PutMetricData parameters : "+JSON.stringify(params))
+ CW.putMetricData(params).promise()
+ .then(data=>{
+ console.log("PutMetricData succeeded :"+JSON.stringify(data))
+ resolve(data)
+ })
+ .catch(err=>{
+ console.error(err)
+ reject(err)
+ });
+ }).catch( err => { //if one of the promise (cf line 26) fails
+ console.error(err)
+ reject(err)
+ });
+ }
+ });
+ })
+};
\ No newline at end of file
diff --git a/deploy/lambda/apigw_metrics/apigw_metrics_live.js b/deploy/lambda/apigw_metrics/apigw_metrics_live.js
deleted file mode 100644
index bc448ca8c614e0547247e79006131c0d2dbd5b3b..0000000000000000000000000000000000000000
--- a/deploy/lambda/apigw_metrics/apigw_metrics_live.js
+++ /dev/null
@@ -1,9 +0,0 @@
-const AWS = require('aws-sdk')
-const CW = new AWS.CloudWatch()
-const CWL = new AWS.CloudWatchLogs()
-const API_GATEWAY_LOG_GROUP = process.env.API_GATEWAY_LOG_GROUP
-exports.handler = async function(event,context){
-
- console.log(JSON.stringify(event))
- console.log(JSON.stringify(context))
-}
\ No newline at end of file
diff --git a/deploy/lambda/apigw_metrics/package-lock.json b/deploy/lambda/apigw_metrics/package-lock.json
index 9c3a1b414583486aca73bdf38d6b15d911e122df..8b913e17125c5e264a1eb192f04948a4204f08a4 100644
--- a/deploy/lambda/apigw_metrics/package-lock.json
+++ b/deploy/lambda/apigw_metrics/package-lock.json
@@ -5,7 +5,8 @@
"packages": {
"": {
"dependencies": {
- "aws-sdk": "^2.864.0"
+ "aws-sdk": "^2.864.0",
+ "zlib": "^1.0.5"
}
},
"node_modules/aws-sdk": {
@@ -133,6 +134,15 @@
"engines": {
"node": ">=4.0"
}
+ },
+ "node_modules/zlib": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/zlib/-/zlib-1.0.5.tgz",
+ "integrity": "sha1-bnyXL8NxxkWmr7A6sUdp3vEU/MA=",
+ "hasInstallScript": true,
+ "engines": {
+ "node": ">=0.2.0"
+ }
}
},
"dependencies": {
@@ -229,6 +239,11 @@
"version": "9.0.7",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz",
"integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0="
+ },
+ "zlib": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/zlib/-/zlib-1.0.5.tgz",
+ "integrity": "sha1-bnyXL8NxxkWmr7A6sUdp3vEU/MA="
}
}
}
diff --git a/deploy/lambda/apigw_metrics/package.json b/deploy/lambda/apigw_metrics/package.json
index 7e315ed09924196e440250f2c46eaf9b237e8c03..43875eae23bd34a8e7fdb50e69d99be0cb6d9a6c 100644
--- a/deploy/lambda/apigw_metrics/package.json
+++ b/deploy/lambda/apigw_metrics/package.json
@@ -1,5 +1,6 @@
{
"dependencies": {
- "aws-sdk": "^2.864.0"
+ "aws-sdk": "^2.864.0",
+ "zlib": "^1.0.5"
}
}
diff --git a/deploy/lambda/src/create-one.js b/deploy/lambda/src/create-one.js
index 4f12e2847724c7eb5df55bc747aceea805b499a0..94950b45ae88624d20feeab6cc1d98bcc05d27cb 100644
--- a/deploy/lambda/src/create-one.js
+++ b/deploy/lambda/src/create-one.js
@@ -8,8 +8,9 @@ const RESERVED_RESPONSE = `Error: You're using AWS reserved keywords as attribut
exports.handler = async (event = {}) => {
console.log(event)
if (!event.body) {
- console.error("400 invalid request, you are missing the parameter body")
- return { statusCode: 400, body: 'invalid request, you are missing the parameter body' };
+ var err = { statusCode: 400, body: JSON.stringify({status:32001,msg:'invalid request, you are missing the parameter body'})};
+ console.error(err)
+ return err;
}
const item = typeof event.body == 'object' ? event.body : JSON.parse(event.body);
item[PRIMARY_KEY] = uuidv4();
diff --git a/deploy/lambda/src/delete-one.js b/deploy/lambda/src/delete-one.js
index 6ed81e10a0892318fbd3cfa5ded543f00d42b903..637264d37d9081ddb4cd06552a79dc528f1c5996 100644
--- a/deploy/lambda/src/delete-one.js
+++ b/deploy/lambda/src/delete-one.js
@@ -4,15 +4,12 @@ const db = new AWS.DynamoDB.DocumentClient();
const TABLE_NAME = process.env.TABLE_NAME || '';
const PRIMARY_KEY = process.env.TABLE_KEY || '';
exports.handler = async (event = {}) => {
- if (!event.queryStringParameters) {
- console.error("400 invalid request, you are missing the path parameter id")
- return { statusCode: 400, body: `Error: You are missing the path parameter id` };
+ if (!event.queryStringParameters || !event.queryStringParameters.id) {
+ var err = { statusCode: 400, body: JSON.stringify({status:32002,msg:'invalid request, you are missing the path parameter id'})};
+ console.error(err)
+ return err;
}
const requestedItemId = event.queryStringParameters.id;
- if (!requestedItemId) {
- console.error("400 invalid request, you are missing the path parameter id")
- return { statusCode: 400, body: `Error: You are missing the path parameter id` };
- }
const params = {
TableName: TABLE_NAME,
Key: {
diff --git a/deploy/lambda/src/get-one.js b/deploy/lambda/src/get-one.js
index a3f22ae96aab5569e1694208cb3692f50da75f8b..f55214599c9a4e661559f0372e8295254870d129 100644
--- a/deploy/lambda/src/get-one.js
+++ b/deploy/lambda/src/get-one.js
@@ -4,15 +4,12 @@ const db = new AWS.DynamoDB.DocumentClient();
const TABLE_NAME = process.env.TABLE_NAME || '';
const PRIMARY_KEY = process.env.TABLE_KEY || '';
exports.handler = async (event = {}) => {
- if (!event.queryStringParameters) {
- console.error("400 invalid request, you are missing the path parameter id")
- return { statusCode: 400, body: `Error: You are missing the path parameter id` };
+ if (!event.queryStringParameters || !event.queryStringParameters.id) {
+ var err = { statusCode: 400, body: JSON.stringify({status:32002,msg:'invalid request, you are missing the path parameter id'})};
+ console.error(err)
+ return err;
}
const requestedItemId = event.queryStringParameters.id;
- if (!requestedItemId) {
- console.error("400 invalid request, you are missing the path parameter id")
- return { statusCode: 400, body: `Error: You are missing the path parameter id` };
- }
const params = {
TableName: TABLE_NAME,
Key: {
diff --git a/deploy/lambda/src/update-one.js b/deploy/lambda/src/update-one.js
index 12a48747e464efa644f697aa464377fe00653a9a..21f2c55a83247d693e64cf7a83cee03e3d438622 100644
--- a/deploy/lambda/src/update-one.js
+++ b/deploy/lambda/src/update-one.js
@@ -6,18 +6,16 @@ const PRIMARY_KEY = process.env.TABLE_KEY || '';
const RESERVED_RESPONSE = `Error: You're using AWS reserved keywords as attributes`, DYNAMODB_EXECUTION_ERROR = `Error: Execution update, caused a Dynamodb error, please take a look at your CloudWatch Logs.`;
exports.handler = async (event = {}) => {
if (!event.body) {
- console.error("400 invalid request, you are missing the parameter body")
- return { statusCode: 400, body: 'invalid request, you are missing the parameter body' };
+ var err = { statusCode: 400, body: JSON.stringify({status:32001,msg:'invalid request, you are missing the parameter body'})};
+ console.error(err)
+ return err;
}
- if (!event.queryStringParameters) {
- console.error("400 invalid request, you are missing the path parameter id")
- return { statusCode: 400, body: `Error: You are missing the path parameter id` };
+ if (!event.queryStringParameters || !event.queryStringParameters.id) {
+ var err = { statusCode: 400, body: JSON.stringify({status:32002,msg:'invalid request, you are missing the path parameter id'})};
+ console.error(err)
+ return err;
}
const editedItemId = event.queryStringParameters.id;
- if (!editedItemId) {
- console.error("400 invalid request, you are missing the path parameter id")
- return { statusCode: 400, body: 'invalid request, you are missing the path parameter id' };
- }
const editedItem = typeof event.body == 'object' ? event.body : JSON.parse(event.body);
const editedItemProperties = Object.keys(editedItem);
if (!editedItem || editedItemProperties.length < 1) {
diff --git a/deploy/vars/dev.tfvars b/deploy/vars/dev.tfvars
index 35e13d2bd25c800b85a1e028c7e8d840e1fc9c87..4988439cf49a532170d92319edde3f91a11a8416 100644
--- a/deploy/vars/dev.tfvars
+++ b/deploy/vars/dev.tfvars
@@ -1,3 +1,3 @@
-dynamodb_read_capacity = 1
-dynamodb_write_capacity = 1
+dynamodb_read_capacity = 2
+dynamodb_write_capacity = 2
dynamodb_enable_pitr = false