Post on 17-Feb-2017
@theburningmonk
tame |> (cloud <| complexity) |> with |> (fsharp >> powered >> DSLs)
image by nerovivo license : https://creativecommons.org/licenses/by-sa/2.0/
@theburningmonk
Under-Abstraction
@theburningmonk
Oversimplification
@theburningmonk
Impedance Mismatch
@theburningmonk
Amazon DynamoDB
Amazon SimpleWorkflow
Amazon CloudWatch
CASE STUDY
@theburningmonk
F# DSLs. Awesome!
@theburningmonk
Amazon DynamoDB
Amazon SimpleWorkflow
Amazon CloudWatch
CASE STUDY
@theburningmonk
@theburningmonk
managedkey-value store
@theburningmonk
redundancy9-9s guarantee
@theburningmonk
great performance
@theburningmonk
name your throughput
@theburningmonk
@theburningmonk
can be changed on-the-fly
@theburningmonk
@theburningmonk
infinitely scalable(but you still have to pay for it)
@theburningmonk
Hash Key
Range Key
@theburningmonk
Query:given a hash keyfilter on
range key, orlocal secondary index
@theburningmonk
Hash Key Range Key
Local Secondary Index
Global Secondary Index
@theburningmonk
Scan:FULL TABLE search(performance + cost concern)
@theburningmonk
Hash Key Range Key
Local Secondary Index
Global Secondary Index
Hash Key Range Key
Local Secondary Index
Who are the TOP 3 players in “Starship X” with a score of at least 1000?
Global Secondary Index
@theburningmonk
@theburningmonk
select GameTitle, UserId, TopScore from GameScores where GameTitle = “Starship X” and TopScore >= 1000 order desc limit 3 with (NoConsistentRead, Index(GameTitleIndex, true))
@theburningmonk
GOAL
Disguise complexity
@theburningmonk
GOAL
SELECT UserId, TopScore FROM GameScore WHERE GameTitle CONTAINS “Zelda” ORDER DESCLIMIT 3 WITH (NoConsistentRead)
@theburningmonk
Query
AST
Execution
@theburningmonk
@theburningmonk
SELECT * FROM GameScore
Abstract Syntax Tree (AST)
FParsec
@theburningmonk
SELECT * FROM GameScore
keyword keyword
* | attribute, attribute, …
table name
@theburningmonk
SELECT * FROM GameScore
type Attributes = | Asterisk | Attributes of string[]
@theburningmonk
SELECT * FROM GameScore
type Query = { Attributes : Attributes Table : string }
@theburningmonk
SELECT * FROM GameScore
Parser for “SELECT” keyword
pSelect
@theburningmonk
SELECT * FROM GameScorepSelect
let pSelect = skipStringCI "select"
@theburningmonk
SELECT * FROM GameScorepSelect
let pSelect = skipStringCI "select"matches the string “select” (Case
Insensitive) and ignores it
@theburningmonk
SELECT * FROM GameScorepFrom
let pFrom = skipStringCI "from"
@theburningmonk
SELECT * FROM GameScore
Parser for a string that represents the table name
pTableName
@theburningmonk
SELECT * FROM GameScorepTableName
let isTableName = isLetter <||> isDigit let pTableName = many1Satisfy isTableName
@theburningmonk
SELECT * FROM GameScorepTableName
let isTableName = isLetter <||> isDigit let pTableName = many1Satisfy isTableName
@theburningmonk
SELECT * FROM GameScorepTableName
let isTableName = isLetter <||> isDigit let pTableName = many1Satisfy isTableName
@theburningmonk
SELECT * FROM GameScorepTableName
let isTableName = isLetter <||> isDigit let pTableName = many1Satisfy isTableName
@theburningmonk
SELECT * FROM GameScorepTableName
let isTableName = isLetter <||> isDigit let pTableName = many1Satisfy isTableName
parses a sequence of one or more chars that satisfies the
predicate function
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeName
UserId, GameTitle, TopScore, …
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeName
UserId, GameTitle, TopScore, …
let pAsterisk = stringCIReturn "*" Asterisk
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeName
UserId, GameTitle, TopScore, …
let pAsterisk = stringCIReturn "*" Asteriskmatches the specified string and return the given value
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeName
UserId, GameTitle, TopScore, …
let isAttributeName = isLetter <||> isDigit let pAttributeName = many1Satisfy isAttributeName
@theburningmonk
SELECT * FROM GameScore
UserId, GameTitle, TopScore, …
pAttributeName pCommapAsterisk
*
let pComma = skipStringCI ","
@theburningmonk
SELECT * FROM GameScore
UserId, GameTitle, TopScore, …
pAttributeName pCommapAsterisk
*
let pAttributeNames = sepBy1 pAttributeName pComma
@theburningmonk
SELECT * FROM GameScore
UserId, GameTitle, TopScore, …
pAttributeName pCommapAsterisk
*
let pAttributeNames = sepBy1 pAttributeName pComma
parses one or more occurrences of pAttributeName separated by pComma
@theburningmonk
SELECT * FROM GameScore
UserId, GameTitle, TopScore, …
pAttributeName pComma
sepBy1
pAttributeNames
pAsterisk
*
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeNames
UserId, GameTitle, TopScore, …
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeNames
UserId, GameTitle, TopScore, …
let pAttribute = pAsterisk <|> pAttributeNames
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeNames
UserId, GameTitle, TopScore, …
let pAttribute = pAsterisk <|> pAttributeNames
@theburningmonk
SELECT * FROM GameScore
pAsterisk
*
pAttributeNames
UserId, GameTitle, TopScore, …
choice
pAttribute
@theburningmonk
SELECT * FROM GameScorepAttribute
@theburningmonk
SELECT * FROM GameScorepAttribute pTableNamepFrompSelect
@theburningmonk
SELECT * FROM GameScorepAttribute pTableNamepFrompSelect
let pQuery = tuple4 pSelect pAttribute pFrom pTableName |>> (fun (_, attributes, _, table) -> { Attributes = attributes Table = table })
@theburningmonk
SELECT * FROM GameScorepAttribute pTableNamepFrompSelect
let pQuery = tuple4 pSelect pAttribute pFrom pTableName |>> (fun (_, attributes, _, table) -> { Attributes = attributes Table = table })
@theburningmonk
SELECT * FROM GameScorepAttribute pTableNamepFrompSelect
let pQuery = tuple4 pSelect pAttribute pFrom pTableName |>> (fun (_, attributes, _, table) -> { Attributes = attributes Table = table })
@theburningmonk
SELECT * FROM GameScorepAttribute pTableNamepFrompSelect
let pQuery = tuple4 pSelect pAttribute pFrom pTableName |>> (fun (_, attributes, _, table) -> { Attributes = attributes Table = table })
Query
@theburningmonk
SELECT * FROM GameScorepAttribute pTableNamepFrompSelect
tuple4
pQuery
@theburningmonk
@theburningmonk
< 50 lines of code
@theburningmonk
Amazing
F# + FParsec =
@theburningmonk
Recap
@theburningmonk
@theburningmonk
select GameTitle, UserId, TopScore from GameScores where GameTitle = “Starship X” and TopScore >= 1000 order desc limit 3 with (NoConsistentRead, Index(GameTitleIndex, true))
@theburningmonk
Amazon DynamoDB
Amazon SimpleWorkflow
Amazon CloudWatch
CASE STUDY
@theburningmonk
@theburningmonk
Decision Worker
@theburningmonk
Decision WorkerPoll
@theburningmonk
Decision WorkerDecision
Task
@theburningmonk
Decision WorkerDecide
@theburningmonk
Activity Worker
Decision Worker
@theburningmonk
Activity Worker
Decision Worker
Poll
@theburningmonk
Activity Worker
Decision Worker
Activity Task
@theburningmonk
Activity Worker
Decision Worker
Complete
@theburningmonk
Workers can run from anywhere
@theburningmonk
@theburningmonk
input = “Yan”
result = “Hello Yan!”
Start
Finish
Activity
@theburningmonk
image by Ryan Hageman license : https://creativecommons.org/licenses/by-sa/2.0/
SWF-based Application
API
Heartbeats
Error Handling
Polling
API
Activity Worker
Decision Worker
Heartbeats
Error Handling
Polling
API
Activity Worker
Decision Worker
Heartbeats
Error Handling
Polling
API
Boilerplate
– Kris Jordan
“Good simplicity is less with leverage, not less with less. Good simplicity is complexity
disguised, not complexity denied.”
http://bit.ly/1pOLeKl
Start
Finish
Activity?
@theburningmonk
the workflow is implied by decision
worker logic…
@theburningmonk
instead..
@theburningmonk
the workflow should drive decision worker logic
@theburningmonk
the workflow should driveautomate
decision worker logic
Amazon.SimpleWorkflow.Extensions
github.com/fsprojects/Amazon.SimpleWorkflow.Extensions
@theburningmonk
GOAL
Remove boilerplates
@theburningmonk
GOAL
Code that matches the way you think
@theburningmonk
Start
Finish
Activity
Start
Finish
Activity
@theburningmonk
Workflows can be nested
input
result
@theburningmonk
Recap
Activity Worker
Decision Worker
Heartbeats
Error Handling
Polling
API
@theburningmonk
Amazon DynamoDB
Amazon SimpleWorkflow
Amazon CloudWatch
CASE STUDY
@theburningmonk
@theburningmonk
@theburningmonk
@theburningmonk
wanna find correlations?
@theburningmonk
wanna find correlations?
you can DIY it!
;-)
@theburningmonk
“what latencies spiked at the same time as payment service?”
Amazon.CloudWatch.Selector
github.com/fsprojects/Amazon.CloudWatch.Selector
@theburningmonk
Find metrics whose 5 min average exceeded
1 second during last 12 hours
@theburningmonk
cloudWatch.Select( unitIs “milliseconds” + average (>) 1000.0 @ last 12 hours |> intervalOf 5 minutes)
@theburningmonk
cloudWatch.Select(“ unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes”)
@theburningmonk
“did any cache nodes’ CPU spike
yesterday?”
@theburningmonk
cloudWatch.Select( namespaceLike “elasticache” + nameLike “cpu” + max (>) 80.0 @ last 24 hours |> intervalOf 15 minutes)
@theburningmonk
cloudWatch.Select( namespaceLike “elasticache” + nameLike “cpu” + max (>) 80.0 @ last 24 hours |> intervalOf 15 minutes)
Regex
@theburningmonk
@theburningmonk
@theburningmonk
@theburningmonk
namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes
@theburningmonk
namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes
Filters
@theburningmonk
namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes
TimeFrame
@theburningmonk
namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes
Period
@theburningmonk
type Query = { Filter : Filter TimeFrame : TimeFrame Period : Period option }
@theburningmonk
Query
Internal DSL
External DSL
@theburningmonk
namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes
@theburningmonk
type MetricTerm = Namespace | Name
type Filter = | MetricFilter of MetricTerm * (string -> bool)
@theburningmonk
namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes
@theburningmonk
type MetricTerm = Namespace | Name
type Unit = | Unit
type Filter = | MetricFilter of MetricTerm * (string -> bool) | UnitFilter of Unit * (string -> bool)
@theburningmonk
namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes
@theburningmonk
type MetricTerm = Namespace | Name
type Unit = | Unit
type StatsTerm = | Average | Min | Max | Sum | SampleCount
type Filter = | MetricFilter of MetricTerm * (string -> bool) | UnitFilter of Unit * (string -> bool) | StatsFilter of StatsTerm * (float -> bool)
@theburningmonk
namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes
@theburningmonk
type MetricTerm = Namespace | Name
type Unit = | Unit
type StatsTerm = | Average | Min | Max | Sum | SampleCount
type Filter = | MetricFilter of MetricTerm * (string -> bool) | UnitFilter of Unit * (string -> bool) | StatsFilter of StatsTerm * (float -> bool) | CompositeFilter of Filter * Filter
@theburningmonk
namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes
@theburningmonk
type TimeFrame = | Last of TimeSpan | Since of DateTime | Between of DateTime * DateTime
@theburningmonk
namespaceIs ‘JustEat’ and nameLike ‘cpu’ and unitIs ‘milliseconds’ and average > 1000.0 duringLast 12 hours at intervalOf 5 minutes
@theburningmonk
type Period = | Period of TimeSpan
@theburningmonk
type Query = { Filter : Filter TimeFrame : TimeFrame Period : Period option }
@theburningmonk
Active Patternsa primer on
@theburningmonk
allow patterns to be abstracted away into
named functions
@theburningmonk
Single-Case Patterns
@theburningmonk
let (|Float|) input = match Double.TryParse input with | true, n -> n | _ -> failwithf “not a float [%s]” input
@theburningmonk
let (|Float|) input = match Double.TryParse input with | true, n -> n | _ -> failwithf “not a float [%s]” input
@theburningmonk
let (|Float|) input = match Double.TryParse input with | true, n -> n | _ -> failwithf “not a float [%s]” input
Float : string -> float
@theburningmonk
match someString with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x
@theburningmonk
let (|Float|) input = match Double.TryParse input with | true, n -> n
match someString with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x
@theburningmonk
let (|Float|) input = match Double.TryParse input with | true, n -> n
match someString with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x
@theburningmonk
match someString with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x
let (|Float|) input = match Double.TryParse input with | true, n -> n
@theburningmonk
match someString with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x
let (|Float|) input = match Double.TryParse input with | true, n -> n
@theburningmonk
match “42” with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x
@theburningmonk
match “boo” with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x
Error!!!
@theburningmonk
Partial Patterns
@theburningmonk
let (|Float|_|) input = match Double.TryParse input with | true, n -> Some n | _ -> None
@theburningmonk
let (|Float|_|) input = match Double.TryParse input with | true, n -> Some n | _ -> None
Float : string -> float option
@theburningmonk
match “boo” with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x | _ -> “not a float”
@theburningmonk
match “boo” with | Float 42.0 -> “ftw” | Float 11.0 -> “palprime” | Float x -> sprintf “just %f” x | _ -> “not a float”
@theburningmonk
Multi-Case Patterns
@theburningmonk
let (|Prime|NotPrime|NaN|) input = match Double.TryParse input with | true, n when isPrime n -> Prime n | true, n -> NotPrime n | _ -> NaN
@theburningmonk
let (|Prime|NotPrime|NaN|) input = match Double.TryParse input with | true, n when isPrime n -> Prime n | true, n -> NotPrime n | _ -> NaN
@theburningmonk
let (|Prime|NotPrime|NaN|) input = match Double.TryParse input with | true, n when isPrime n -> Prime n | true, n -> NotPrime n | _ -> NaN
@theburningmonk
let (|Prime|NotPrime|NaN|) input = match Double.TryParse input with | true, n when isPrime n -> Prime n | true, n -> NotPrime n | _ -> NaN
@theburningmonk
let (|Prime|NotPrime|NaN|) input = match Double.TryParse input with | true, n when isPrime n -> Prime n | true, n -> NotPrime n | _ -> NaN
@theburningmonk
match someString with | Prime n -> … | NotPrime n -> … | NaN -> …
@theburningmonk
match someString with | Prime n & Float 11.0 -> … | Prime n -> … | Float 42.0 | Float 8.0 -> … | NotPrime n -> … | NaN -> …
@theburningmonk
match someString with | Prime n & Float 11.0 -> … | Prime n -> … | Float 42.0 | Float 8.0 -> … | NotPrime n -> … | NaN -> …
@theburningmonk
match someString with | Prime (IsPalindrome) -> … | Prime _ -> … | _ -> …
@theburningmonk
Tokenise(string -> string list)
@theburningmonk
[ “namespaceIs”; “‘JustEat’”; “and”; “nameLike”; “‘cpu’”; “and”; … ]
@theburningmonk
“namespaceIs” “and” …
F# List = LinkedList
“‘JustEat’”
@theburningmonk
“namespaceIs” “and” …
F# List = LinkedList
“namespaceIs” “‘JustEat’” “and”:::: :: tail
cons (::) operator to prepend
“‘JustEat’”
@theburningmonk
“namespaceIs” “and” …
F# List = LinkedList
“namespaceIs” “‘JustEat’” “and”:::: :: tailmatch aList with |
“‘JustEat’”
works as a pattern too!
@theburningmonk
let (|StringCI|_|) (str : string) (input : string) = if str.ToLower() = input.ToLower() then Some () else None
let (|Float|_|) input = match System.Double.TryParse input with | true, n -> Some n | _ -> None
@theburningmonk
let (|EmptyString|_|) input = if String.IsNullOrWhiteSpace input then Some () else None
let (|StartsWith|_|) char (input : string) = if input.[0] = char then Some () else None
let (|EndsWith|_|) char (input : string) = if input.[input.Length-1] = char then Some () else None
@theburningmonk
let (|QuotedString|_|) (input : string) = match input with | EmptyString -> None | str when str.Length < 2 -> None | StartsWith ''' & EndsWith ''' -> Some <| input.Substring(1, input.Length - 2) | _ -> None
@theburningmonk
let (|NamespaceIs|_|) = function | StringCI “NamespaceIs" :: QuotedString ns :: tl -> Some (eqFilter MetricFilter Namespace ns, tl) | _ -> None
@theburningmonk
let (|NamespaceIs|_|) = function | StringCI “NamespaceIs" :: QuotedString ns :: tl -> Some (eqFilter MetricFilter Namespace ns, tl) | _ -> None
namespaceIs ‘JustEat’ and nameLike ‘cpu’ and …
@theburningmonk
let (|NamespaceIs|_|) = function | StringCI “NamespaceIs" :: QuotedString ns :: tl -> Some (eqFilter MetricFilter Namespace ns, tl) | _ -> None
“namespaceIs” “‘JustEat’” :::: …
@theburningmonk
let (|NamespaceIs|_|) = function | StringCI “NamespaceIs" :: QuotedString ns :: tl -> Some (eqFilter MetricFilter Namespace ns, tl) | _ -> None
type MetricTerm = Namespace | Name
type Filter = | MetricFilter of MetricTerm * (string -> bool)
@theburningmonk
let (|NamespaceIs|_|) = function | StringCI “NamespaceIs" :: QuotedString ns :: tl -> Some (eqFilter MetricFilter Namespace ns, tl) | _ -> None
“and” :: “nameLike” :: “‘cpu’” :: “and” :: …
@theburningmonk
let rec loop acc = function | NamespaceIs (filter, tl) | NamespaceLike (filter, tl) | NameIs (filter, tl) | NameLike (filter, tl) | UnitIs (filter, tl) | Average (filter, tl) | Sum (filter, tl) | Min (filter, tl) | Max (filter, tl) | SampleCount (filter, tl) -> match tl with | And tl -> loop (filter::acc) tl | _ -> flatten (filter::acc), tl | _ -> failwith “No filters?!?!?”
@theburningmonk
let rec loop acc = function | NamespaceIs (filter, tl) | NamespaceLike (filter, tl) | NameIs (filter, tl) | NameLike (filter, tl) | UnitIs (filter, tl) | Average (filter, tl) | Sum (filter, tl) | Min (filter, tl) | Max (filter, tl) | SampleCount (filter, tl) -> match tl with | And tl -> loop (filter::acc) tl | _ -> flatten (filter::acc), tl | _ -> failwith “No filters?!?!?”
@theburningmonk
let rec loop acc = function | NamespaceIs (filter, tl) | NamespaceLike (filter, tl) | NameIs (filter, tl) | NameLike (filter, tl) | UnitIs (filter, tl) | Average (filter, tl) | Sum (filter, tl) | Min (filter, tl) | Max (filter, tl) | SampleCount (filter, tl) -> match tl with | And tl -> loop (filter::acc) tl | _ -> flatten (filter::acc), tl | _ -> failwith “No filters?!?!?”
@theburningmonk
let rec loop acc = function | NamespaceIs (filter, tl) | NamespaceLike (filter, tl) | NameIs (filter, tl) | NameLike (filter, tl) | UnitIs (filter, tl) | Average (filter, tl) | Sum (filter, tl) | Min (filter, tl) | Max (filter, tl) | SampleCount (filter, tl) -> match tl with | And tl -> loop (filter::acc) tl | _ -> flatten (filter::acc), tl | _ -> failwith “No filters?!?!?”
@theburningmonk
let rec loop acc = function | NamespaceIs (filter, tl) | NamespaceLike (filter, tl) | NameIs (filter, tl) | NameLike (filter, tl) | UnitIs (filter, tl) | Average (filter, tl) | Sum (filter, tl) | Min (filter, tl) | Max (filter, tl) | SampleCount (filter, tl) -> match tl with | And tl -> loop (filter::acc) tl | _ -> flatten (filter::acc), tl | _ -> failwith “No filters?!?!?”
@theburningmonk
let parse (input : string) = input |> tokenize |> parseFilter |> parseTimeFrame |> parsePeriod
@theburningmonk
Amazing
F# =
@theburningmonk
usable from anywhere you can run F# code
e.g. F# REPL, executable, ..
Internal DSL
@theburningmonk
useful for building tools e.g. CLI, …
External DSL
@theburningmonk
@theburningmonk
@theburningmonk
@theburningmonk
Recap
@theburningmonk
@theburningmonk
@theburningmonk
Amazon DynamoDB
Amazon SimpleWorkflow
Amazon CloudWatch
CASE STUDY
@theburningmonktheburningmonk.comgithub.com/theburningmonk