AnyEventInternals(samples of async programming)

Mons Anderson

AnyEvent::Internals

AE::*

use AE 5;

my $w = AE::io $fh, 0, sub { warn '$fh is readable'; };my $w = AE::io $fh, 1, sub { warn '$fh is writable'; };

my $w = AE::timer 1,0, sub { warn '1 second passed'; };my $w = AE::timer 0,1, sub { warn 'every 1 second'; };

my $w = AE::signal TERM => sub { warn 'TERM received'; };

my $w = AE::idle { warn 'Event loop is idle';};

I/OWaitSigMisc

AnyEvent::Internals

AE::cv

use AE 5;

my $cv = AE::cv;

any sub { async sub { calls sub { $cv->send; }; };};

$cv->recv; # Run loop until get send

AnyEvent::Internals

AE::cv

use AE 5;

my $cv = AE::cv { warn "All done" };

for (1..10) { $cv->begin; async_call sub { may_be_nested sub { $cv->end; } }}

AnyEvent::Internals

AE::HTTP

use AnyEvent::HTTP;my $cv = AE::cv;http_request GET => 'http://www.google.ru', sub { my ($body,$hdr) = @_; warn "$hdr->{Status}: $hdr->{Reason}\n"; warn "Body: ".length($body)." bytes\n"; $cv->send; },;$cv->recv;

$ perl http_request.pl200: OKBody: 10875 bytes

AnyEvent::Internals

AE::HTTP

sub http_request($$@) { my $cb = pop; my ($method,$url) = (uc shift, shift); my (%arg) = ( timeout => 10, recurse => 10, @_); my %state; my %hdr = (connection => 'close'); my $err = sub { %state = (); $cb->(undef, { Status => $_[1], Reason => $_[0] || 599, URL => $_[2] || $url } ); }; return $err->("Too many redirections") if $arg{recurse} < 0;

while (my ($k, $v) = each %{$arg{headers}}) { $hdr{lc $k} = $v };

$hdr{"user-agent"} //= "AnyEvent/$AnyEvent::VERSION";

$hdr{"content-length"} = length $arg{body} if length $arg{body} or $method ne "GET";

# ...}

AnyEvent::Internals

AE::HTTP

use AnyEvent::Socket;sub http_request($$@) { # ... my ($rscheme, $uauthority, $rpath, $query, $fragment) = $url =~ m{(?:([^:/?#]+):)?(?://([^/?#]*))? ([^?#]*)(?:\?([^#]*))?(?:#(.*))?}x; $rscheme = lc $rscheme;

my $rport = $rscheme eq "http" ? 80 : $rscheme eq "https" ? 443 : return $err->("Only http/https are supported");

$uauthority =~ /^(?: .*\@ )? ([^\@:]+) (?: : (\d+) )?$/x or return $err->("Unparsable URL"); my $rhost = $1; $rport = $2 if defined $2; $hdr{host} //= defined $2 ? "$uhost:$2" : "$uhost"; $rhost =~ s/^\[(.*)\]$/$1/; $rpath .= "?$query" if length $query; $rpath =~ s%^/?%/%;

$state{connect_guard} = tcp_connect $rhost, $rport, sub { # ... }, sub { $arg{timeout} }; defined wantarray && AnyEvent::Util::guard { %state = () };}

AnyEvent::Internals

AE::HTTP

use AnyEvent::Socket;

tcp_connect $host, $port, sub { # Connect sub my ($fh, $addr, $port) = @_; # ... }, sub { # Prepare sub my ($socket) = @_; return $timeout; };

AnyEvent::Internals

AE::HTTP

use AnyEvent::Handle;sub http_request($$@) { # ... $state{connect_guard} = tcp_connect $rhost, $rport, sub { $state{fh} = shift or return $err->("$!"); return unless delete $state{connect_guard};

$state{handle} = new AnyEvent::Handle fh => $state{fh}, timeout => $arg{timeout}, on_error => sub { $err->($_[2]) }, on_eof => sub { $err->("Unexpected end-of-file") }; $state{handle}->starttls ("connect") if $rscheme eq "https";

$state{handle}->push_write ( "$method $rpath HTTP/1.0\015\012". join( "", map { defined $hdr{$_} ? "\u$_: ".delete($hdr{$_})."\015\012" : delete $hdr{$_}||() } keys %hdr )."\015\012". delete($arg{body}) ); $state{handle}->push_read (line => qr{\015?\012}, sub { # ... }); }, sub { $arg{timeout} }; # ...}

AnyEvent::Internals

AE::HTTP

use AnyEvent::Handle;

$h->push_write($data);

$h->push_read( chunk => $bytes, sub { my ($h,$data) = @_; # ...});$h->push_read( line => qr/EOL/, sub { ... } );

$h->unshift_read( chunk => $bytes, sub { ... } );$h->unshift_read( line => qr/EOL/, sub { ... } );

$h->on_read( sub { ... } );

$h->{rbuf}

AnyEvent::Internals

AE::HTTP

sub http_request($$@) { # ... $state{handle}->push_read ( line => qr{\015?\012}, sub { $_[1] =~ /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\015\012]*) )?/ix or return $err->("Invalid server response ($_[1])");

my %hdr = ( Status => ",$2", Reason => ",$3", URL => ",$url" );

$_[0]->unshift_read ( line => qr{(? sub { # ... } ); } ); # ...}

AnyEvent::Internals

AE::HTTP

sub http_request($$@) { # ... $_[0]->unshift_read ( line => qr{(? sub { for ("$_[1]") { y/\015//d; $hdr{lc $1} .= ",$2" while / \G ([^:\000-\037]*): [\011\040]* (( ?: [^\012]+ | \012[\011\040] )*) \012 /gxc; /\G$/ or return $err->("Garbled response headers"); } substr $_, 0, 1, '' for values %hdr; if ($hdr{location} !~ /^(?: $ | [^:\/?\#]+ : )/x) { $hdr{location} =~ s/^\.\/+//; my $url = "$rscheme://$rhost:$rport"; unless ($hdr{location} =~ s{^/}{}) { $url .= $rpath; $url =~ s{/[^/]*$}{}; } $hdr{location} = "$url/$hdr{location}"; } # ... }); # ...}

AnyEvent::Internals

AE::HTTP

sub http_request($$@) { # ... my $redirect; if ($arg{recurse}) { if ( ($hdr{Status} =~ /^30[12]$/ and $method ne "POST" ) or ($hdr{Status} == 307 and $method =~ /^(?:GET|HEAD)$/) ) { $redirect = 1; } elsif ($hdr{Status} == 303) { $method = "GET" unless $method eq "HEAD"; $redirect = 1; } } my $finish = sub { $state{handle}->destroy if $state{handle}; %state = ();

if ($redirect && exists $hdr{location}) { http_request $method => $hdr{location}, %arg, recurse => $arg{recurse} - 1, $cb; } else { $cb->($_[0], $_[1]); } }; # ...}

AnyEvent::Internals

AE::HTTP

sub http_request($$@) { # ... my $len = $hdr{"content-length"}; if ( $hdr{Status} =~ /^(?:1..|[23]04)$/ or $method eq "HEAD" or (defined $len && !$len) ) { # no body $finish->("", \%hdr); } else { # ... } # ...}

AnyEvent::Internals

AE::HTTP

sub http_request($$@) { # ... $_[0]->on_eof (undef); if ($len) { # have content length $_[0]->on_error (sub { $finish->(undef,{ Status => 599, Reason => $_[2], URL => $url }); }); $_[0]->on_read (sub { $finish->( substr(delete $_[0]{rbuf}, 0, $len, ""), \%hdr ) if $len on_error (sub { $!{ EPIPE} || !$! ? $finish->(delete $_[0]{rbuf}, \%hdr) : $finish->(undef,{ Status => 599, Reason => $_[2], URL => $url }); }); $_[0]->on_read(sub {}); } # ...}

AnyEvent::Internals

AE::HTTP

use AnyEvent::Socket;use AnyEvent::Handle;sub http_request($$@) { my $cb = pop; my ($method,$url) = (uc shift, shift); my (%arg) = ( timeout => 10, recurse => 10, @_); my %state; my %hdr = (connection => 'close'); my $err = sub { %state = (); $cb->(undef, Status => $_[1], Reason => $_[0]||599, URL => $_[2]||$url ) }; return $err->("Too many redirections") if $arg{recurse} < 0;

while (my ($k, $v) = each %{$arg{headers}}) { $hdr{lc $k} = $v };

my ($rscheme, $uauthority, $rpath, $query, $fragment) = $url =~ m|(?:([^:/?#]+):)?(?://([^/?#]*))?([^?#]*)(?:\?([^#]*))?(?:#(.*))?|; $rscheme = lc $rscheme; my $rport = $rscheme eq "http" ? 80 : $rscheme eq "https" ? 443 : return $err->("Only http and https URL schemes supported"); $uauthority =~ /^(?: .*\@ )? ([^\@:]+) (?: : (\d+) )?$/x or return $err->("Unparsable URL"); my $rhost = $1; $rport = $2 if defined $2; $hdr{host} //= defined $2 ? "$rhost:$2" : "$rhost"; $rhost =~ s/^\[(.*)\]$/$1/; $rpath .= "?$query" if length $query; $rpath =~ s%^/?%/%;

$hdr{"user-agent"} //= "AnyEvent/$AnyEvent::VERSION"; $hdr{"content-length"} = length $arg{body} if length $arg{body} or $method ne "GET";

$state{connect_guard} = tcp_connect $rhost, $rport, sub { $state{fh} = shift or return %state=(),$err->("$!"); return unless delete $state{connect_guard};

$state{handle} = new AnyEvent::Handle fh => $state{fh}, timeout => $arg{timeout}, on_error => sub { $err->($_[2]) }, on_eof => sub { $err->("Unexpected end-of-file") }, ; $state{handle}->starttls ("connect") if $rscheme eq "https";

$state{handle}->push_write ( "$method $rpath HTTP/1.0\015\012". join( "", map { defined $hdr{$_} ? "\u$_: ".delete($hdr{$_})."\015\012" : delete $hdr{$_}||() } keys %hdr )."\015\012". delete($arg{body}) );

$state{handle}->push_read (line => qr{\015?\012}, sub { $_[1] =~ /^HTTP\/([0-9\.]+) \s+ ([0-9]{3}) (?: \s+ ([^\015\012]*) )?/ix or return $err->("Invalid server response ($_[1])");

my %hdr = ( Status => ",$2", Reason => ",$3", URL => ",$url" );

$state{handle}->unshift_read (line => qr{(? for ("$_[1]") { y/\015//d; $hdr{lc $1} .= ",$2" while / \G ([^:\000-\037]*): [\011\040]* (( ?: [^\012]+ | \012[\011\040] )*) \012 /gxc; /\G$/ or return $err->("Garbled response headers"); } substr $_, 0, 1, '' for values %hdr;

if ($hdr{location} !~ /^(?: $ | [^:\/?\#]+ : )/x) { $hdr{location} =~ s/^\.\/+//; my $url = "$rscheme://$rhost:$rport"; unless ($hdr{location} =~ s{^/}{}) { $url .= $rpath; $url =~ s{/[^/]*$}{}; } $hdr{location} = "$url/$hdr{location}"; }

my $redirect; if ($arg{recurse}) { if ( ($hdr{Status} =~ /^30[12]$/ and $method ne "POST" ) or ($hdr{Status} == 307 and $method =~ /^(?:GET|HEAD)$/) ) { $redirect = 1; } elsif ($hdr{Status} == 303) { $method = "GET" unless $method eq "HEAD"; $redirect = 1; } }

my $finish = sub { $state{handle}->destroy if $state{handle}; %state = ();

if ($redirect && exists $hdr{location}) { http_request $method => $hdr{location}, %arg, recurse => $arg{recurse} - 1, $cb; } else { $cb->($_[0], $_[1]); } };

my $len = $hdr{"content-length"}; if ( $hdr{Status} =~ /^(?:1..|[23]04)$/ or $method eq "HEAD" or (defined $len && !$len) ) { # no body $finish->("", \%hdr); } else { $_[0]->on_eof (undef); if ($len) { # have content length $_[0]->on_error (sub { $finish->(undef, { Status => 599, Reason => $_[2], URL => $url }) }); $_[0]->on_read (sub {$finish->( substr(delete $_[0]{rbuf}, 0, $len, ""), \%hdr) if $len on_error (sub { $!{ EPIPE} || !$! ? $finish->(delete $_[0]{rbuf}, \%hdr) : $finish->(undef,{ Status => 599, Reason => $_[2], URL => $url }); }); $_[0]->on_read(sub {}); } } }); }); }, sub { $arg{timeout} }; defined wantarray && AnyEvent::Util::guard { %state = () }}

AnyEvent::Internals

AE::HTTPServer

my $server = AnyEvent::HTTP::Server->new( port => 80, request => sub { my $r = shift; if ($r->wants_websocket) { $r->upgrade('websocket', sub { if (my $ws = shift) { $ws->onmessage(sub { $ws->send("@_"); }); $ws->send("Hello!"); } else { warn "Upgrade failed: @_"; } }); } else { # ... } },);

AnyEvent::Internals

AE::HTTPServer

tcp_server $self->{host}, $self->{port}, sub { my $fh = shift or return warn "couldn't accept client: $!"; my ($host, $port) = @_; my $con = $self->{connection_class}->new( server => $self, fh => $fh, host => $host, port => $port, on_error => sub { my $con = shift; warn "@_"; delete $self->{con}{$con->{id}}; }, ); $self->{con}{$con->{id}} = $con;}, sub { 1024;};

AnyEvent::Internals

AE::HTTPServer

sub Connection::new { my $self = bless {}, shift; weaken (my $this = $self); my %args = @_; my $srv = $args{server}; my $fh = $args{fh}; my $h = AnyEvent::Handle::Writer->new( fh => $args{fh}, on_eof => sub { $this or return; $this->error("EOF") }, on_error => sub { $this or return; $this->error("$!") }, ); weaken($self->{srv} = $args{server}); $self->{id} = int $self; $self->{fh} = $args{fh}; $self->{h} = $h; $self->{host} = $args{host}; $self->{port} = $args{port}; $self->{r} = []; # Request queue $self->{ka_timeout} = $srv->{keep_alive_timeout} || 30; if ($srv->{keep_alive}) { $self->{touch} = AE::now; $self->ka_timer; } $self->read_header(); return $self;}

AnyEvent::Internals

AE::HTTPServer

sub Connection::ka_timer { my $self = shift; $self->{srv} or return $self->destroy; weaken (my $this = $self); $self->{ka} = AE::timer $this->{ka_timeout} + 1, 0, sub { $this or return; if (AE::now - $this->{touch} >= $this->{ka_timeout}) { $this->close; } else { $this->ka_timer; } } ;}

AnyEvent::Internals

AE::HTTPServer

sub Connection::read_header { my $self = shift; $self->{srv} or return $self->destroy; weaken (my $con = $self); $con->{h}->push_read(chunk => 3 => sub { $con or return; shift; my $pre = shift; if ($pre =~ m{^{h}->unshift_read(regex => qr{.+?>} => sub { my $xml = $pre.shift(); # Hanlde XML here $con->destroy(); }); } else { $con->{h}->unshift_read(line => sub { $con or return;shift; my $line = $pre.shift; if ($line =~ /(\S+) \040 (\S+) \040 HTTP\/(\d+\.\d+)/xso) { my ($meth, $url, $hv) = ($1, $2, $3); $con->{method} = $meth; $con->{uri} = $url; $con->{version} = $hv; $con->read_headers(); } elsif ($line eq '') { $con->read_header(); } else { $con->fatal_error(400); } }); }; });}

AnyEvent::Internals

AE::HTTPServer

sub Connection::read_headers { my ($self) = @_; $self->{srv} or return $self->destroy; weaken (my $con = $self); $self->{h}->unshift_read( line => qr{(? sub { $con or return; my ($h, $data) = @_; my $hdr = HTTP::Easy::Headers->decode($data) // return $con->fatal_error(400); my $r = $con->{srv}{request_class}->new( method => delete $con->{method}, uri => delete $con->{uri}, host => $hdr->{Host}, headers => $hdr, ); push @{ $con->{r} }, $r; weaken($r->{con} = $con); weaken($con->{r}[-1]); # ... } );}

AnyEvent::Internals

AE::HTTPServer

sub { $con or return; # ... $hdr->{connection} = lc $hdr->{connection}; if ($hdr->{connection} eq 'close' or $con->{version} < 1.1) { delete $con->{ka}; $con->{type} = 'close'; $con->{close} = 1; } elsif ($hdr->{connection} =~ /keep-alive/) { $con->{close} = 0; $con->{type} = 'keep-alive'; } elsif ($hdr->{connection} eq 'upgrade') { delete $con->{ka}; $con->{type} = 'upgrade'; $con->{close} = 0; } if (defined $hdr->{'content-length'}) { $con->{h}->unshift_read( chunk => $hdr->{'content-length'}, sub { my ($hdl, $data) = @_; $con->handle_request($r, $data); $con->read_header() if $con->{ka}; }); } else { $con->handle_request($r); $con->read_header() if $con->{ka}; } }

AnyEvent::Internals

AE::HTTPServer

sub Request::response { my $self = shift; $self->{con} or return $self->dispose; $self->{con}->response($self, @_); $self->dispose;}

sub Request::dispose { my $self = shift; return %$self = ();}

sub Request::DESTROY { my $self = shift; $self->{con} or return %$self = (); $self->{con}->response( $self, 404, '', msg => "Request not handled" );}

sub Request::wants_websocket { my $self = shift; return lc $self->{headers}{connection} eq 'upgrade' && lc $self->{headers}{upgrade} eq 'websocket' ? 1 : 0;}

AnyEvent::Internals

AE::HTTPServer

sub Request::upgrade { my $self = shift; my $cb = pop; my $type = shift; my $headers = shift || HTTP::Easy::Headers->new({}); if (lc $type eq 'websocket') { $headers->{upgrade} = 'WebSocket'; $headers->{connection} = 'Upgrade'; $headers->{'websocket-origin'} ||= $self->{headers}{origin}; $headers->{'websocket-location'} ||= do { my $loc = URI->new_abs($self->{uri}, "http://$self->{headers}{host}"); $loc = "$loc"; $loc =~ s{^http}{ws}; $loc; }; $self->{con}->response($self,101,'',headers => $headers, msg => "Web Socket Protocol Handshake"); my $ws = $self->{con}{srv}{websocket_class}->new( con => $self->{con}, ); $self->dispose; return $cb->($ws); } else { return $cb->(undef, "Unsupported upgrade type: $type"); }}

AnyEvent::Internals

AE::HTTPServer

sub Connection::response { my ($con,$r,$code,$content,%args) = @_; my $msg = $args{msg} || $HTTP::Easy::Status::MSG{$code}; my $hdr = $args{headers} || HTTP::Easy::Headers->new({});

# Resolve pipeline if (@{$con->{r}} and $con->{r}[0] == $r) { shift @{ $con->{r} }; } else { $r->{ready} = [ $code, $msg, $hdr, $content ]; return; } my $res = "HTTP/$con->{version} $code $msg\015\012"; $hdr->{'content-type'} ||= 'text/html'; if (ref $content eq 'HASH') { if ($content->{sendfile}) { $content->{size} = $hdr->{'content-length'} = -s $content->{sendfile}; } } $hdr->{connection} ||= $con->{type}; if ($code >= 400 and !length $content ) { $content = "$code $msg"; } # ...

AnyEvent::Internals

AE::HTTPServer

# ... $hdr->{'content-length'} = length $content if not (defined $hdr->{'content-length'}) and not ref $content and $code !~ /^(?:1\d\d|[23]04)$/;

$res .= $hdr->encode(); $con->{h}->push_write($res);

if (ref $content eq 'HASH') { if ($content->{sendfile}) { $con->{h}->push_sendfile( $content->{sendfile}, $content->{size}); } else { die "Bad response content"; } } else { $con->{h}->push_write($content) if length $content; }

if ($con->{close}) { $con->close(); } elsif ( @{$con->{r}} and $con->{r}[0]{ready}) { $con->response($con->{r}[0],@{$con->{r}[0]{ready}}); }}

AnyEvent::Internals

AE::Memcached

my $cv = AE::cv;$cv->begin;$cv->begin(sub { $cv->send });

my $memd = AnyEvent::Memcached->new( servers => [ '127.0.0.1:11211' ], cv => $cv, namespace => "test:",);

$memd->set("key1", "val1", cb => sub { shift or return warn "Set key1 failed: @_"; $memd->get("key1", cb => sub { my ($v,$e) = @_; $e and return warn "Get failed: $e"; warn "Got value for key1: $v"; });});

$cv->end;$cv->recv;

AnyEvent::Internals

AE::Memcached

sub set { shift->_set( set => @_) }

sub _set { my $self,$cmd,$key) = splice @_,0,3; my $cas; if ($cmd eq 'cas') { $cas = shift; } my $val = shift; # Some preparations... $self->_do( $key, "$cmd $self->{namespace}%s $flags $expire $len". ( defined $cas ? ' '.$cas : '')."\015\012$val", sub { local $_ = shift; if ($_ eq 'STORED') { return 1 } elsif ($_ eq 'NOT_STORED') { return 0 } elsif ($_ eq 'EXISTS') { return 0 } else { return undef, $_ } }, cb => $args{cb}, );}

AnyEvent::Internals

AE::Memcached

sub _do { my ($self,$key,$com,$wrk,%args) = @_; my $servers = $self->{hash}->servers($key); my %res; my %err; my $res; $_ and $_->begin for $self->{cv}, $args{cv}; my $cv = AE::cv { if ($res != -1) { $args{cb}($res); } else { $args{cb}( undef, dumper(\%err) ); } $_ and $_->end for $args{cv}, $self->{cv}; }; for my $srv ( keys %$servers ) { for my $real (@{ $servers->{$srv} }) { $cv->begin; my $cmd = $com; substr($cmd, index($cmd,'%s'),2) = $real; $self->{peers}{$srv}{con}->command( '...' ); } } return;}

AnyEvent::Internals

AE::Memcached

# ... $self->{peers}{$srv}{con}->command( $cmd, cb => sub { if (defined( local $_ = shift )) { my ($ok,$fail) = $wrk->($_); if (defined $ok) { $res{$real}{$srv} = $ok; $res = (!defined $res) || $res == $ok ? $ok : -1; } else { $err{$real}{$srv} = $fail; $res = -1; } } else { $err{$real}{$srv} = $_; $res = -1; } $cv->end; } );# ...

AnyEvent::Internals

AE::Memcached

sub Peer::command { my $self = shift; if ($self->{connected}) { return $self->{con}->command( @_ ); } else { my ($cmd,%args) = @_; $self->conntrack( command => \@_, $args{cb} ); }}

AnyEvent::Internals

AE::Memcached

sub Peer::conntrack { my $self = shift; my ($method,$args,$cb) = @_; if($self->{connecting} and $self->{failed}) { $cb and $cb->(undef, "Not connected"); return; } elsif (!$self->{connected}) { # connect } else { return $self->{con}->$method(@$args); }}

AnyEvent::Internals

AE::Memcached

sub Peer::connect { my $self = shift; $self->{connecting} and return; $self->reg_cb( connected => sub { $self->{failed} = 0; } ); $self->reg_cb( connfail => sub { $self->{failed} = 1; } ); $self->reg_cb( disconnect => sub { shift;shift; %$self or return; my $e = @_ ? "@_" : "disconnected"; for ( keys %{$self->{waitingcb}} ) { if ($self->{waitingcb}{$_}) { $self->{waitingcb}{$_}(undef,$e); } delete $self->{waitingcb}{$_}; } } ); $self->next::method(@_); return;}

AnyEvent::Internals

AE::Memcached

elsif (!$self->{connected}) { my @args = @$args; # copy to avoid rewriting my ($c,$t); weaken( $self ); weaken( $self->{waitingcb}{int $cb} = $cb ) if $cb; $c = $self->reg_cb( connected => sub { shift->unreg_me; undef $c; undef $t; $self or return; delete $self->{waitingcb}{int $cb} if $cb; return $self->{con}->$method(@args); }); $t = AE::timer $self->{timeout}, 0, sub { undef $c;undef $t; $self or return; if ($cb){ delete $self->{waitingcb}{int $cb}; $cb->(undef, "Connect timeout"); } }, ); $self->connect(); }

AnyEvent::Internals

Resources

Documentation AE

AnyEvent

AnyEvent::Intro

Authors

Marc Lehmann (MLEHMANN)

Robin Redeker (ELMEX)

Mons Anderson (MONS)

AnyEvent::Internals

JFDI

Use Perl or die;

AnyEvent::Internals

Author

Mons Anderson(aka )Rambler Internet Holding

DevConf::Perl()2010

$cv->send;