diff --git a/Web/lib/MJB/Web/Task.pm b/Web/lib/MJB/Web/Task.pm index 64551ca..d9632ad 100644 --- a/Web/lib/MJB/Web/Task.pm +++ b/Web/lib/MJB/Web/Task.pm @@ -73,6 +73,18 @@ sub system_command ( $self, $cmd, $settings = {} ){ } } + # Automatic retry on key exchange failure + if ( $settings->{retry_on_ssh_fail} ) { + if ( $err =~ /kex_exchange_identification/ ) { + $settings->{retry_on_ssh_fail_count}++; + if ( $settings->{retry_on_ssh_fail_count} < 5 ) { + $self->append_log( "--- ssh error: retrying...." ); + return $self->system_command( $cmd, $settings ); + } + } + } + + # Return to the directory we started in if we chdir'ed. if ( $settings->{return_chdir} ) { chdir $settings->{return_chdir} diff --git a/Web/lib/MJB/Web/Task/InitializeBlog.pm b/Web/lib/MJB/Web/Task/InitializeBlog.pm index ed9bc34..d1dd402 100644 --- a/Web/lib/MJB/Web/Task/InitializeBlog.pm +++ b/Web/lib/MJB/Web/Task/InitializeBlog.pm @@ -39,11 +39,11 @@ sub run ( $job, $blog_id ) { my $server = 'root@' . $host->hostname; my $domain = $blog->domain->name; - $job->system_command( [ 'scp', $config->to_string, $server . ":/etc/nginx/sites-enabled/" . $domain ] ); - $job->system_command( [ 'ssh', $server, 'mkdir -p /var/www/' . $domain . '/html' ] ); - $job->system_command( [ 'scp', $welcome->to_string, $server . ":/var/www/" . $domain . "/html/index.html" ] ); - $job->system_command( [ 'ssh', $server, 'chown -R www-data:www-data /var/www/' . $domain ] ); - $job->system_command( [ 'ssh', $server, 'systemctl reload nginx' ] ); + $job->system_command( [ 'scp', $config->to_string, $server . ":/etc/nginx/sites-enabled/" . $domain ], { retry_on_ssh_fail => 1 } ); + $job->system_command( [ 'ssh', $server, 'mkdir -p /var/www/' . $domain . '/html' ], { retry_on_ssh_fail => 1 } ); + $job->system_command( [ 'scp', $welcome->to_string, $server . ":/var/www/" . $domain . "/html/index.html" ], { retry_on_ssh_fail => 1 } ); + $job->system_command( [ 'ssh', $server, 'chown -R www-data:www-data /var/www/' . $domain ], { retry_on_ssh_fail => 1 } ); + $job->system_command( [ 'ssh', $server, 'systemctl reload nginx' ], { retry_on_ssh_fail => 1 } ); } $job->note( is_config_deployed => 1 ); diff --git a/Web/lib/MJB/Web/Task/SyncBlog.pm b/Web/lib/MJB/Web/Task/SyncBlog.pm index a7fe57a..b86e5c2 100644 --- a/Web/lib/MJB/Web/Task/SyncBlog.pm +++ b/Web/lib/MJB/Web/Task/SyncBlog.pm @@ -38,7 +38,7 @@ sub run ( $job, $blog_id ) { while ( my $server = $servers->next ) { my $dest = "root@" . $server->hostname . ":/var/www/$domain/html/"; - $job->system_command( [ qw( rsync -vrLptgoD --delete -e ), $ssh_opt, $source, $dest ]); + $job->system_command( [ qw( rsync -vrLptgoD --delete -e ), $ssh_opt, $source, $dest ], { retry_on_ssh_fail => 1 }); } $job->note( is_deploy_complete => 1 ); diff --git a/Web/lib/MJB/Web/Task/SyncBlogMedia.pm b/Web/lib/MJB/Web/Task/SyncBlogMedia.pm index 43aff19..dee92b8 100644 --- a/Web/lib/MJB/Web/Task/SyncBlogMedia.pm +++ b/Web/lib/MJB/Web/Task/SyncBlogMedia.pm @@ -24,7 +24,7 @@ sub run ( $job, $blog_id ) { while ( my $server = $servers->next ) { my $dest = "root@" . $server->hostname . ":/var/www/$domain/html/assets/media"; - $job->system_command( [ qw( rsync -vrLptgoD --delete -e ), $ssh_opt, $source, $dest ]); + $job->system_command( [ qw( rsync -vrLptgoD --delete -e ), $ssh_opt, $source, $dest ], { retry_on_ssh_fail => 1 }); } $job->note( is_deploy_complete => 1 );