Commit b8b14843 authored by Bertrand PINEL's avatar Bertrand PINEL
Browse files

Add provision of Cloud directory in project file

parent da64fe8c
......@@ -8,6 +8,11 @@ It could also be seen as a starter for setting up Mirage, allowing the developpe
Complete code is available at https://gitlab.ippon.fr/bpinel/ember-aws-ehipster
A full tutorial is available on the Ippon Blog :
* in english: https://blog.ippon.tech/a-guinea-pig-in-the-cloud/
* in french: https://blog.ippon.fr/2019/03/19/un-cochon-dinde-dans-le-cloud/
Installation
------------------------------------------------------------------------------
......
'use strict';
const AWS = require('aws-sdk');
const dynamo = new AWS.DynamoDB.DocumentClient();
const tableName = 'JsonApiTable';
const EPOCH = 1545907609050;
const MAX_OBJECTS=1000; // Max number of objects returned by finder
// Instagram inspired --> https://instagram-engineering.com/sharding-ids-at-instagram-1cf5a71e5a5c
function generateRowId(subid) {
var ts = new Date().getTime() - EPOCH; // limit to recent
// 41 bits for time in milliseconds (gives us 41 years of IDs with a custom epoch
var randid = Math.floor(Math.random() * 512);
ts = (ts * 64); // bit-shift << 6
// Given shard (if any...)
ts = ts + subid;
// random value
return (ts * 512) + (randid % 512);
}
const createObject = (obj) => {
let objout = { "type": obj.ObjectType,
"id": obj.Id,
"attributes": {},
"relationships": {}
};
for (var attr in obj) {
if (attr !== 'ObjectType' && attr !== 'Id') {
if(!attr.endsWith('_id')) {
objout.attributes[attr] = obj[attr];
} else {
let relationDetails = attr.split('_');
if (obj[attr].startsWith('[')) {
// hasMany relationship
let idsArray = obj[attr].substring(1, obj[attr].length-1).split(',');
let relData = [];
for (let l=0;l<idsArray.length;l++) {
let relObject = {"type": relationDetails[0], "id": Number(idsArray[l]) };
relData.push(relObject); }
objout.relationships[relationDetails[0]] = {
"links": {
"self": "/"+obj.ObjectType+"/"+obj.Id+"/relationships/"+relationDetails[0],
"related": "/"+obj.ObjectType+"/"+obj.Id+"/"+relationDetails[0]
},
"data": relData
};
} else {
// belongsTo relationship
objout.relationships[relationDetails[0]] = {
"links": {
"self": "/"+obj.ObjectType+"/"+obj.Id+"/relationships/"+relationDetails[0],
"related": "/"+obj.ObjectType+"/"+obj.Id+"/"+relationDetails[0]
},
"data": {"type": relationDetails[1], "id": Number(obj[attr])}
};
}
}
} else {
objout[(attr==='ObjectType')?'type':'id'] = obj[attr];
}
}
console.log('Return object is '+JSON.stringify(objout));
return objout;
}
const handlingData = (data) => {
if (Array.isArray(data)) {
let outdata = [];
for (let i=0;i<data.length;i++){
outdata.push(createObject(data[i]));
}
return outdata;
} else {
return createObject(data);
}
}
const handlingRelationships = (data) => {
if (Array.isArray(data)) {
console.warn("Not handling correctly relationship when retrieving list of objects");
} else {
}
return {};
}
const createResponse = (statusCode, body) => {
console.log("Body is "+JSON.stringify(body));
return {
'statusCode': statusCode,
'data': handlingData(body),
'relationships': handlingRelationships(body)
}
};
const buildQuery = (queryParams, type) => {
let filterExpression = [];
let params = {
TableName: tableName,
KeyConditionExpression: 'ObjectType = :objectType',
FilterExpression: "",
ExpressionAttributeValues: {':objectType': type}
}
for (var filter in queryParams) {
//"filter[email]": "klaroutte@yopmail.com",
if (filter.startsWith('filter')){
let attr = filter.substring(7, filter.length-1);
let value = queryParams[filter];
filterExpression.push(attr+" = :"+attr);
params["ExpressionAttributeValues"][":"+attr] = value;
}
params["FilterExpression"] = filterExpression.join(' and ');
}
console.log('Query is '+JSON.stringify(params));
return params;
}
const getMethod = (event, context, callback) => {
let params = {
TableName: tableName,
},
type = event.params.path.type,
query = event.params.querystring,
id = Number(event.params.path.id),
dbGet = {};
if (id) {
params.Key = {
'ObjectType': type,
'Id': id
};
dbGet = (params) => { return dynamo.get(params).promise() };
console.log('Ehipster lambda GET single value with params: ', params);
} else if (Object.keys(query).length === 0) {
// No filter expression
params.KeyConditionExpression = 'ObjectType = :objectType';
params.ExpressionAttributeValues = { ':objectType': type };
dbGet = (params) => { return dynamo.query(params).promise() };
console.log('Ehipster lambda GET multiple values with params: ', params);
} else {
// Use filter expression
params = buildQuery(query, type);
dbGet = (params) => { return dynamo.query(params).promise() };
console.log('Ehipster lambda GET query values with params: ', params);
}
dbGet(params).then( (data) => {
//console.log('Ehipster lambda GET data received: ', data);
if (id && !data.Item) {
callback(null, createResponse(404, "ITEM NOT FOUND"));
return;
} else if (id && data.Item) {
console.log(`RETRIEVED ITEM SUCCESSFULLY WITH doc = ${data.Item}`);
callback(null, createResponse(200, data.Item));
} else {
console.log('SCANNING TABLE');
console.log(`RETRIEVED ITEMS SUCCESSFULLY WITH doc = ${data.Items}`);
callback(null, createResponse(200, data.Items));
}
}).catch( (err) => {
console.log(`GET ITEM FAILED FOR Entry = ${params}, WITH ERROR: ${err}`);
callback(null, createResponse(500, err));
});
};
const putMethod = (event, context, callback) => {
const body=event['body-json'];
const attrs = body.data.attributes;
const relations = body.data.relationships;
// Without any body content, there is nothing to put...
if (!body || !attrs) {
callback(null, createResponse(500, 'No content found in body'));
return;
}
// Retrieving the type and generating a new id for created item
let type = event.params.path.type,
id = generateRowId(1);
// Final content contains at least these two fields + the atributes
let content = {
"ObjectType": type,
"Id": id
};
// Adding attributes as column in dynamoDb
for (var prop in attrs) {
if (attrs[prop] != null && attrs[prop].length>0)
// Empty field cannot be persisted
content[prop] = attrs[prop];
}
// Dealing with relationships if any
if (relations){
for (var relName in relations) {
let newCol = relName+'_'+type+'_id';
let relData = relations[relName]["data"];
if (!Array.isArray(relData)) {
content[newCol] = relData["id"];
} else {
let ids = [];
for (var i=0; i<relData.length;i++){
let currentData = relData[i];
ids.push(currentData["id"]);
}
content[newCol] = '['+ids.join(',')+']';
}
}
}
const entry = {
TableName: tableName,
Item: content
};
console.log('Try saving entity of type '+type+' and content '+JSON.stringify(entry));
//let dbPut = (entry) => { return dynamo.put(entry).promise() };
dynamo.put(entry, function(err, data) {
if (err) {
console.log("Error", err);
callback(null, createResponse(500, 'Error '+err));
} else {
body.data.id = id;
body['statusCode'] = 200;
console.log(`PUT ITEM SUCCEEDED WITH data=`+JSON.stringify(body));
callback(null, body);
}
});
};
const patchMethod = (event, context, callback) => {
const body=event['body-json'];
const attrs = body.data.attributes;
let updateExpressionList = [];
let expressionAttributeList = [];
for (var attr in attrs) {
updateExpressionList.push(attr+' = :'+attr);
expressionAttributeList[':'+attr] = attrs[attr];
}
let updateExpression = 'set '+updateExpressionList.join(',');
let type = event.params.path.type,
id = Number(event.params.path.id),
params = {
'TableName': tableName,
'Key': {
'ObjectType': type,
'Id': id
},
'UpdateExpression': updateExpression,
'ExpressionAttributeValues': expressionAttributeList,
'ReturnValues': 'UPDATED_NEW'
};
let dbUpdate = (params) => { return dynamo.update(params).promise() };
dbUpdate(params).then( (data) => {
if (!data.Attributes) {
callback(null, createResponse(404, "ITEM NOT FOUND FOR UPDATE"));
return;
}
console.log(`UPDATE ITEM OF TYPE ${type} SUCCESSFULLY WITH id = ${id}`);
callback(null, body);
}).catch( (err) => {
console.log(`UPDATE ITEM OF TYPE ${type} FAILED FOR id = ${id}, WITH ERROR: ${err}`);
callback(null, createResponse(500, err));
});
};
const deleteMethod = (event, context, callback) => {
let type = event.params.path.type,
id = Number(event.params.path.id),
params = {
'TableName': tableName,
'Key': {
'ObjectType': type,
'Id': id
},
'ReturnValues': 'ALL_OLD'
};
let dbDelete = (params) => { return dynamo.delete(params).promise() };
dbDelete(params).then( (data) => {
if (!data.Attributes) {
callback(null, createResponse(404, "ITEM NOT FOUND FOR DELETION"));
return;
}
console.log(`DELETED ITEM OF TYPE ${type} SUCCESSFULLY WITH id = ${id}`);
const empty = {'data': []};
callback(null, empty);
}).catch( (err) => {
console.log(`DELETE ITEM OF TYPE ${type} FAILED FOR id = ${id}, WITH ERROR: ${err}`);
callback(null, createResponse(500, err));
});
};
exports.handler = (event, context, callback) => {
console.log("JSONAPILambda ********************** Received Event *******************\n"+ JSON.stringify(event));
console.log("JSONAPILambda httpMethod="+event.context.httpMethod);
switch ( event.context.httpMethod ) {
case 'GET':
getMethod(event,context,callback);
break;
case 'PUT':
case 'POST':
putMethod(event,context,callback);
break;
case 'PATCH':
patchMethod(event,context,callback);
break;
case 'DELETE':
deleteMethod(event,context,callback);
break;
default:
callback(null, createResponse(500, 'Unsupported Method: ' + context.httpMethod));
break;
}
};
Installing AWS infrastructure through Terraform scripts
==============================================================================
This file explains how to set up the expected infrastructure awaited to implement the JSON API.
The installation is based on Terraform (https://www.terraform.io/)
It expects that both AWS CLI and Terraform have been previously installed and set up correctly.
The AWS Credentials should also have been set up
How it works
------------------------------------------------------------------------------
AWS JSON API server relies on 3 main components, plus the needed IAM roles :
- A DynamoDB table manipuling all serialized objects
- A lambda function in charge of the translation from JSON API format to dynamoDB objects and managing the relationships and the optional parameters provided in the request
- An API Gateway configured to received the REST HTTP requests and proxying the Lambda
It also create three S3 buckets, one for storing the lambda code (nammed lambda-jsonapi-code-bucket) and the two other for receiving the ‘staging’ and ‘production’ version of the application (respectively nammed ember-aws-ehipster-staging and ember-aws-ehipster-production)
Several Terraform files are available, each one creating the needed component and their relationships with each other.
Installation
------------------------------------------------------------------------------
One this is done, simply run the terraform script :
```
terraform init
terraform apply -var bucket_name=<bucket name for static web site>
```
\ No newline at end of file
resource "aws_api_gateway_rest_api" "jsonapi" {
name = "JsonApiRestGateway"
description = "HTTP request front API for JSON API rest persistance"
minimum_compression_size = 4096
}
# Path /{type}
resource "aws_api_gateway_resource" "typePath" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
parent_id = "${aws_api_gateway_rest_api.jsonapi.root_resource_id}"
path_part = "{type}"
}
# Path /{type}/{id}
resource "aws_api_gateway_resource" "typeIdPath" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
parent_id = "${aws_api_gateway_resource.typePath.id}"
path_part = "{id}"
}
# GET on path /{type}
resource "aws_api_gateway_method" "typePathGet" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typePath.id}"
http_method = "GET"
authorization = "NONE"
}
# OPTIONS on path /{type} for supporting CORS
resource "aws_api_gateway_method" "typePathOptions" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typePath.id}"
http_method = "OPTIONS"
authorization = "NONE"
}
resource "aws_api_gateway_method_response" "typeOptions200" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typePath.id}"
http_method = "${aws_api_gateway_method.typePathOptions.http_method}"
status_code = "200"
response_models {
"application/json" = "Empty"
}
response_parameters {
"method.response.header.Access-Control-Allow-Headers" = true,
"method.response.header.Access-Control-Allow-Methods" = true,
"method.response.header.Access-Control-Allow-Origin" = true
}
}
resource "aws_api_gateway_integration" "typeOptionsIntegration" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typePath.id}"
http_method = "${aws_api_gateway_method.typePathOptions.http_method}"
type = "MOCK"
passthrough_behavior = "WHEN_NO_TEMPLATES"
request_templates {
"application/json" = "{\"statusCode\": 200}"
}
}
resource "aws_api_gateway_integration_response" "typeOptionsIntegrationResponse" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typePath.id}"
http_method = "${aws_api_gateway_method.typePathOptions.http_method}"
status_code = "${aws_api_gateway_method_response.typeOptions200.status_code}"
response_parameters = {
"method.response.header.Access-Control-Allow-Headers" = "'Content-Type,Access-Control-Allow-Origin,X-Amz-Date,Authorization,X-Requested-With,X-Requested-By,X-Api-Key,X-Amz-Security-Token'",
"method.response.header.Access-Control-Allow-Methods" = "'GET,OPTIONS,POST'",
"method.response.header.Access-Control-Allow-Origin" = "'*'"
}
}
# POST on path /{type}
resource "aws_api_gateway_method" "typePathPost" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typePath.id}"
http_method = "POST"
authorization = "NONE"
}
# GET on path /{type}/{id}
resource "aws_api_gateway_method" "typeIdPathGet" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typeIdPath.id}"
http_method = "GET"
authorization = "NONE"
}
# OPTIONS on path /{type}/{id}
resource "aws_api_gateway_method" "typeIdPathOptions" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typeIdPath.id}"
http_method = "OPTIONS"
authorization = "NONE"
}
resource "aws_api_gateway_method_response" "typeIdOptions200" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typeIdPath.id}"
http_method = "${aws_api_gateway_method.typeIdPathOptions.http_method}"
status_code = "200"
response_models {
"application/json" = "Empty"
}
response_parameters {
"method.response.header.Access-Control-Allow-Headers" = true,
"method.response.header.Access-Control-Allow-Methods" = true,
"method.response.header.Access-Control-Allow-Origin" = true
}
}
resource "aws_api_gateway_integration" "typeIdOptionsIntegration" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typeIdPath.id}"
http_method = "${aws_api_gateway_method.typeIdPathOptions.http_method}"
type = "MOCK"
passthrough_behavior = "WHEN_NO_TEMPLATES"
request_templates {
"application/json" = "{\"statusCode\": 200}"
}
}
resource "aws_api_gateway_integration_response" "typeIdOptionsIntegrationResponse" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typeIdPath.id}"
http_method = "${aws_api_gateway_method.typeIdPathOptions.http_method}"
status_code = "${aws_api_gateway_method_response.typeIdOptions200.status_code}"
response_parameters = {
"method.response.header.Access-Control-Allow-Headers" = "'Content-Type,Access-Control-Allow-Origin,X-Amz-Date,Authorization,X-Requested-With,X-Requested-By,X-Api-Key,X-Amz-Security-Token'",
"method.response.header.Access-Control-Allow-Methods" = "'GET,OPTIONS,DELETE,PATCH'",
"method.response.header.Access-Control-Allow-Origin" = "'*'"
}
}
# DELETE on path /{type}/{id}
resource "aws_api_gateway_method" "typeIdPathDelete" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typeIdPath.id}"
http_method = "DELETE"
authorization = "NONE"
}
# PATCH on path /{type}/{id}
resource "aws_api_gateway_method" "typeIdPathPatch" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typeIdPath.id}"
http_method = "PATCH"
authorization = "NONE"
}
##############################################
# GET on {type}
# Setup Integration Request for GET on {type}
resource "aws_api_gateway_integration" "lambdaJsonTypeGet" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_method.typePathGet.resource_id}"
http_method = "${aws_api_gateway_method.typePathGet.http_method}"
integration_http_method = "POST"
type = "AWS"
uri = "${aws_lambda_function.lambda-jsonapi.invoke_arn}"
# Transforms the incoming XML request to JSON
passthrough_behavior = "WHEN_NO_TEMPLATES"
request_templates {
"application/json" = "${file("template.vm")}"
}
}
# Setup Integration Response for status code 200 and GET on {type}
resource "aws_api_gateway_method_response" "200TypeGet" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_method.typePathGet.resource_id}"
http_method = "${aws_api_gateway_method.typePathGet.http_method}"
status_code = "200"
response_parameters {
"method.response.header.Access-Control-Allow-Origin" = true
}
}
resource "aws_api_gateway_integration_response" "200TypeGetIntegrationResponse" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_resource.typePath.id}"
http_method = "${aws_api_gateway_method.typePathGet.http_method}"
status_code = "${aws_api_gateway_method_response.200TypeGet.status_code}"
response_templates = {
"application/json" = ""
}
response_parameters {
"method.response.header.Access-Control-Allow-Origin" = "'*'"
}
depends_on = ["aws_api_gateway_integration.lambdaJsonTypeGet"]
}
##############################################
# POST on {type}
# Setup Integration Request for POST on {type}
resource "aws_api_gateway_integration" "lambdaJsonTypePost" {
rest_api_id = "${aws_api_gateway_rest_api.jsonapi.id}"
resource_id = "${aws_api_gateway_method.typePathPost.resource_id}"
http_method = "${aws_api_gateway_method.typePathPost.http_method}"
integration_http_method = "POST"
type = "AWS"
uri = "${aws_lambda_function.lambda-jsonapi.invoke_arn}"
# Transforms the incoming XML request to JSON
passthrough_behavior = "WHEN_NO_TEMPLATES"
request_templates {
"application/json" = "${file("template.vm")}"
}
}
# Setup Integration Response for status code 200 and POST on {type}