mongodb training | mongodb online training | mongodb training and certification | mongodb course
Building Real Time Systems on MongoDB Using the Oplog at Stripe
description
Transcript of Building Real Time Systems on MongoDB Using the Oplog at Stripe
MongoDB and the OplogEVAN BRODER @ebroder
AGENDAINTRO TO THE OPLOGEXAMPLE APPLICATIONS
INTROTO THE OPLOG
PRIMARY
SECONDARIES
APPLICATION
APPLICATION
save{_id: 1, a: 2}
THINGS I’VE DONE:
- save {_id: 1, a: 2}
APPLICATION
update where{a: 2},{$set: {a: 3}}
THINGS I’VE DONE:
- save {_id: 1, a: 2}- update {_id: 1}, {$set: {a: 3}}
THINGS I’VE DONE:
- save {_id: 1, a: 2}- update {_id: 1}, {$set: {a: 3}}- insert…- delete…- delete…- save…- update…
THINGS I’VE DONE:
save…
THINGS I’VE DONE:
THINGS I’VE DONE:
save…
THINGS I’VE DONE:
save…
TRIGGERS
GOAL:EVENT PROCESSING
GOAL:DETECT INSERTIONS
WARNINGTHIS CODE IS NOT PRODUCTION-READY
oplog = mongo_connection['local']['oplog.rs']
ns = 'eventdb.events'
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.each do |op|
puts op['o']['_id']
end
end
oplog = mongo_connection['local']['oplog.rs']
ns = 'eventdb.events'
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.each do |op|
puts op['o']['_id']
end
end
oplog = mongo_connection['local']['oplog.rs']
ns = 'eventdb.events'
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.each do |op|
puts op['o']['_id']
end
end
oplog = mongo_connection['local']['oplog.rs']
ns = 'eventdb.events'
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.each do |op|
puts op['o']['_id']
end
end
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
oplog.find({'op' => 'i', 'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
DATA TRANSFORMATIONS
GOAL:MONGODB TO POSTGRESQL
start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start},
'op' => 'i',
'ns' => ns}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
start_entry = oplog.find_one({},
{:sort => {'$natural' => -1}})
start = start_entry['ts']
oplog.find({'ts' => {'$gt' => start}}) do |cursor|
cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY)
cursor.add_option(Mongo::Constants::OP_QUERY_TAILABLE)
cursor.add_option(Mongo::Constants::OP_QUERY_AWAIT_DATA)
loop do
cursor.each do |op|
puts op['o']['_id']
end
end
end
cursor.each do |op|
puts op['o']['_id']
end
cursor.each do |op|
case op['op']
when 'i'
puts op['o']['_id']
else
# ¯\_(ツ)_/¯
end
end
cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') +
') VALUES (' +
op['o'].values.map(&:inspect).join(', ') + ')'
else
# ¯\_(ツ)_/¯
end
end
cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') +
') VALUES (' +
op['o'].values.map(&:inspect).join(', ') + ')'
when 'd'
query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
else
# ¯\_(ツ)_/¯
end
end
query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
when 'u'
query = "UPDATE #{op['ns']} SET"
updates = op['o']['$set'] ? op['o']['$set'] : op['o']
updates.each do |k, v|
query += " #{k}=#{v.inspect}"
end
query += " WHERE _id="
query += op['o2']['_id'].inspect
else
# ¯\_(ツ)_/¯
end
end
cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') + ') VALUES (' +
op['o'].values.map(&:inspect).join(', ') + ')'
when 'd'
query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
when 'u'
query = "UPDATE #{op['ns']} SET"
updates = op['o']['$set'] ? op['o']['$set'] : op['o']
updates.each do |k, v|
query += " #{k}=#{v.inspect}"
end
query += " WHERE _id=" + op['o2']['_id'].inspect
else
# ¯\_(ツ)_/¯
end
end
github.com/stripe/mosql
github.com/stripe/zerowing
cursor.each do |op|
case op['op']
when 'i'
query = "INSERT INTO #{op['ns']} (" +
op['o'].keys.join(', ') + ') VALUES (' +
op['o'].values.map(&:inspect).join(', ') + ')'
when 'd'
query = "DELETE FROM #{op['ns']} WHERE _id=" +
op['o']['_id'].inspect
when 'u'
query = "UPDATE #{op['ns']} SET"
updates = op['o']['$set'] ? op['o']['$set'] : op['o']
updates.each do |k, v|
query += " #{k}=#{v.inspect}"
end
query += " WHERE _id=" + op['o2']['_id'].inspect
else
# ¯\_(ツ)_/¯
end
end
DISASTER RECOVERY
task = collection.find_one({'finished' => nil}
# do something with task…
collection.update({'_id' => task.id},
{'$set' => {'finished' => Time.now.to_i}})
loop do
collection.remove(
{'finished' => {'$lt' => Time.now.to_i - 30}})
sleep(10)
end
evan@caron:~$ mongo
MongoDB shell version: 2.4.10
connecting to: test
normal:PRIMARY> null < (Date.now() / 1000) - 30
true
THINGS I’VE DONE:
insertdelete…
THINGS I’VE DONE:
> db.getReplicationInfo()
{
"logSizeMB" : 48964.3541015625,
"usedMB" : 46116.4,
"timeDiff" : 316550,
"timeDiffHours" : 87.93,
"tFirst" : "Thu Apr 11 2013 07:24:29 GMT+0000 (UTC)",
"tLast" : "Sun Apr 14 2013 23:20:19 GMT+0000 (UTC)",
"now" : "Sat May 24 2014 07:52:35 GMT+0000 (UTC)"
}
> db.getReplicationInfo()
{
"logSizeMB" : 48964.3541015625,
"usedMB" : 46116.4,
"timeDiff" : 316550,
"timeDiffHours" : 87.93,
"tFirst" : "Thu Apr 11 2013 07:24:29 GMT+0000 (UTC)",
"tLast" : "Sun Apr 14 2013 23:20:19 GMT+0000 (UTC)",
"now" : "Sat May 24 2014 07:52:35 GMT+0000 (UTC)"
}
new_oplog.find({'ts' => {'$gt' => start}}) do |cursor| cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY) cursor.each do |op| if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks' old_task = old_tasks.find_one({'_id' => op['o']['_id']}) if old_task['finished'] == nil # found one! # save old_task to a file, and we'll re-queue it later end end
old_connection['admin'].command({'applyOps' => [op]}) endend
new_oplog.find({'ts' => {'$gt' => start}}) do |cursor| cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY) cursor.each do |op| if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks' old_task = old_tasks.find_one({'_id' => op['o']['_id']}) if old_task['finished'] == false # found one! # save old_task to a file, and we'll re-queue it later end end
old_connection['admin'].command({'applyOps' => [op]}) endend
new_oplog.find({'ts' => {'$gt' => start}}) do |cursor| cursor.add_option(Mongo::Constants::OP_QUERY_OPLOG_REPLAY) cursor.each do |op| if op['op'] == 'd' && op['ns'] == 'monsterdb.tasks' old_task = old_tasks.find_one({'_id' => op['o']['_id']}) if old_task['finished'] == false # found one! # save old_task to a file, and we'll re-queue it later end end
old_connection['admin'].command({'applyOps' => [op]}) endend
THINGS I’VE DONE:
save…
THINGS I’VE DONE:
save…
QUESTIONS?