Now with retry functionality.

Web/lib/MJB/Web/Task.pm

@@ -73,6 +73,18 @@ sub system_command ( $self, $cmd, $settings = {} ){
         }
     }
 
+    # Automatic retry on key exchange failure
+    if ( $settings->{retry_on_ssh_fail} ) {
+        if ( $err =~ /kex_exchange_identification/ ) {
+            $settings->{retry_on_ssh_fail_count}++;
+            if ( $settings->{retry_on_ssh_fail_count} < 5 ) {
+                $self->append_log( "--- ssh error: retrying...." );
+                return $self->system_command( $cmd, $settings );
+            }
+        }
+    }
+
+
     # Return to the directory we started in if we chdir'ed.
     if ( $settings->{return_chdir} ) {
         chdir $settings->{return_chdir}

Web/lib/MJB/Web/Task/InitializeBlog.pm

@@ -39,11 +39,11 @@ sub run ( $job, $blog_id ) {
         my $server = 'root@' . $host->hostname;
         my $domain = $blog->domain->name;
 
-        $job->system_command( [ 'scp', $config->to_string, $server . ":/etc/nginx/sites-enabled/" . $domain ] );
+        $job->system_command( [ 'scp', $config->to_string, $server . ":/etc/nginx/sites-enabled/" . $domain ], { retry_on_ssh_fail => 1 } );
-        $job->system_command( [ 'ssh', $server, 'mkdir -p /var/www/' . $domain . '/html' ] );
+        $job->system_command( [ 'ssh', $server, 'mkdir -p /var/www/' . $domain . '/html' ], { retry_on_ssh_fail => 1 } );
-        $job->system_command( [ 'scp', $welcome->to_string, $server . ":/var/www/" . $domain . "/html/index.html" ] );
+        $job->system_command( [ 'scp', $welcome->to_string, $server . ":/var/www/" . $domain . "/html/index.html" ], { retry_on_ssh_fail => 1 } );
-        $job->system_command( [ 'ssh', $server, 'chown -R www-data:www-data /var/www/' . $domain ] );
+        $job->system_command( [ 'ssh', $server, 'chown -R www-data:www-data /var/www/' . $domain ], { retry_on_ssh_fail => 1 } );
-        $job->system_command( [ 'ssh', $server, 'systemctl reload nginx' ] );
+        $job->system_command( [ 'ssh', $server, 'systemctl reload nginx' ], { retry_on_ssh_fail => 1 } );
     }
     
     $job->note( is_config_deployed => 1 );

Web/lib/MJB/Web/Task/SyncBlog.pm

@@ -38,7 +38,7 @@ sub run ( $job, $blog_id ) {
 
     while ( my $server = $servers->next ) {
         my $dest = "root@" . $server->hostname . ":/var/www/$domain/html/";
-        $job->system_command( [ qw( rsync -vrLptgoD --delete -e ), $ssh_opt, $source, $dest ]);
+        $job->system_command( [ qw( rsync -vrLptgoD --delete -e ), $ssh_opt, $source, $dest ], { retry_on_ssh_fail => 1 });
     }
 
     $job->note( is_deploy_complete => 1 );

Web/lib/MJB/Web/Task/SyncBlogMedia.pm

@@ -24,7 +24,7 @@ sub run ( $job, $blog_id ) {
 
     while ( my $server = $servers->next ) {
         my $dest = "root@" . $server->hostname . ":/var/www/$domain/html/assets/media";
-        $job->system_command( [ qw( rsync -vrLptgoD --delete -e ), $ssh_opt, $source, $dest ]);
+        $job->system_command( [ qw( rsync -vrLptgoD --delete -e ), $ssh_opt, $source, $dest ], { retry_on_ssh_fail => 1 });
     }
 
     $job->note( is_deploy_complete => 1 );