Node.js streaming csv downloads proxy
-
Upload
ismael-celis -
Category
Software
-
view
1.814 -
download
1
description
Transcript of Node.js streaming csv downloads proxy
Streaming downloads proxy service with
Node.js
ismael celis @ismasan
bootic.net - Hosted e-commerce in South America
background job Email
attachment
Previous setup
Previous setup
Previous setup
• Memory limitations • Email deliverability • Code bloat • inflexible
New setup
• Monolithic micro • Leverage existing API
New setup
• Monolithic micro • Leverage existing API
curl -H “Authorization: Bearer xxx” \https://api.bootic.net/v1/orders.json?created_at:gte=2014-02-01&page=2
New setup
API -> CSV Stream
// pipe generated CSV onto the HTTP responsevar writer = csv.createCsvStreamWriter(response)// Turn a series of paginated requests // to the backend API into a stream of datavar stream = apistream.instance(uri, token)// Pipe data stream into CSV writerstream.pipe(writer)
API -> CSV Stream
response.setHeader('Content-Type', ‘text/csv'); response.setHeader('Content-disposition', 'attachment;filename=' + name + '.csv');
API -> mappers -> CSV Stream
{ "code": "123EFCD", "total": 80000, "status": "shipped", "date": "2014-02-03", "items": [ {"product_title": "iPhone 5", "units": 2, "unit_price": 30000}, {"product_title": "Samsung Galaxy S4", "units": 1, "unit_price": 20000} ]}
code, total, date, status, product, units, unit_price, total2 123EFCD, 80000, 2014-02-03, shipped, iPhone 5, 2, 30000, 800003 123EFCD, 80000, 2014-02-03, shipped, Samsung Galaxy S4, 1, 20000, 80000
API -> mappers -> CSV Stream
var OrderMapper = csvmapper.define(function () { this.scope('items', function () { this .map('id', '/id') .map('order', '/code') .map('status', '/status') .map('discount', '/discount_total') .map('shipping price', '/shipping_total') .map('total', '/total') .map('year', '/updated_on', year) .map('month', '/updated_on', month) .map('day', '/updated_on', day) .map('payment method', '/payment_method_type') .map('name', '/contact/name') .map('email', '/contact/email') .map('address', '/address', address) .map('product', 'product_title') .map('variant', 'variant_title') .map('sku', 'variant_sku') .map('unit price', 'unit_price') .map('quantity', 'units')
API -> mappers -> CSV Stream
var writer = csv.createCsvStreamWriter(res);var stream = apistream.instance(uri, token)var mapper = new OrdersMapper()// First line in CSV is the headerswriter.writeRecord(mapper.headers())// mapper.eachRow() turns a single API resource into 1 or more CSV rowsstream.on('item', function (item) { mapper.eachRow(item, function (row) { writer.writeRecord(row) })})
API -> mappers -> CSV Stream
stream.on('item', function (item) { mapper.eachRow(item, function (row) { writer.writeRecord(row) })})
Paremeter definitions
https://api.bootic.net/v1/orders.json? created_at:gte=2014-02-01 & page=2
Paremeter definitions
var OrdersParams = params.define(function () { this .param('sort', 'updated_on:desc') .param('per_page', 20) .param('status', 'closed,pending,invalid,shipped') })
Paremeter definitions
var params = new OrdersParams(request.query)// Compose API url using sanitized / defaulted paramsvar uri = "https://api.com/orders?" + params.query;var stream = apistream.instance(uri, token)
Secure CSV downloads
JSON Web Tokens
headers . claims . signature
http://self-issued.info/docs/draft-ietf-oauth-json-web-token.html
eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMDA4MTkzODAsDQogImh0dHA6Ly9leGFtcGxlLmNvbS9pc19yb290Ijp0cnVlfQ.dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk
JSON Web Tokens
headers {“typ":"JWT", "alg":"HS256"}
claims{ “shop_id":"acme", "iat":1300819380, "aud":"orders", "filters": {"status": "shipped"} }
signature
+ Base64
+ Base64
HMAC SHA-256 (headers + claims, secret) + Base64
Rails: Generate token (Ruby)
# controllers/downloads_controller.rbdef create url = Rails.application.config.downloads_host claims = params[:download_options] # Add an issued_at timestamp claims[:iat] = (Time.now.getutc.to_f * 1000).to_i # Scope data on current account claims[“shop_id"] = current_shop.id # generate JWT token = JWT.encode(claims, Rails.application.config.downloads_secret) # Redirect to download URL. Browser will trigger download dialog redirect_to “#{url}?jwt=#{token}" end
Rails: Generate token (Ruby)
claims[:iat] = (Time.now.getutc.to_f * 1000).to_iclaims[“shop_id"] = current_shop.idtoken = JWT.encode(claims, secret)redirect_to "#{url}?jwt=#{token}"
Node: validate JWT
var TTL = 60000;var tokenMiddleware = function(req, res, next){ try{ var decoded = jwt.decode(req.query.jwt, secret); if(decoded.shop_id != req.param(‘shop_id') { res.send(400, ‘JWT and query shop ids do not match'); return } var now = new Date(), utc = getUtcCurrentDate(); if(utc - Number(decoded.iat) > TTL) { res.send(401, "Web token has expired") return } req.query = decoded // all good, carry on next() } catch(e) { res.send(401, 'Unauthorized or invalid web token'); }}
Node: validate JWT
var decoded = jwt.decode(req.query.jwt, secret);
?jwt=eyJ0eXAiOiJKV1QiLA0KICJhbGciOiJIUzI1NiJ.eyJpc3MiOiJqb2UiLA0KICJleHAiOjEzMD.dBjftJeZ4CVP-mB92K27uhbUJU1p1r_wW1gFWFOEjXk
Node: validate JWT
if(decoded.shop_id != req.param(‘shop_id') { res.send(400, ‘JWT and query shop ids do not match'); return}
Node: validate JWT
var now = new Date(), utc = getUtcCurrentDate();if(utc - Number(decoded.iat) > TTL) { res.send(401, "Web token has expired") return}
Node: validate JWT
req.query = decoded // all good, carry onnext()
Node: HTTP handlers
app.get('/:shop_id/orders.csv', tokenMiddleware, handler.create('orders', ...));
app.get('/:shop_id/contacts.csv', tokenMiddleware, handler.create('contacts', ...));
app.get('/:shop_id/products.csv', tokenMiddleware, handler.create('products', ...));
}
goo.gl/nolmRK
ismael celis @ismasan