From c3d0e1384edc959959b1c00a8751c17495fd5816 Mon Sep 17 00:00:00 2001 From: Kaitlyn Parkhurst Date: Thu, 15 Sep 2022 14:28:59 -0700 Subject: [PATCH] Change up the build system. --- Web/lib/MJB/Web/Task.pm | 185 ++++++++++++++++++ Web/lib/MJB/Web/{Tasks => Task}/CreateBlog.pm | 0 Web/lib/MJB/Web/Task/DeployBlog.pm | 54 +++++ Web/lib/MJB/Web/{Tasks => Task}/PurgeBlog.pm | 0 Web/lib/MJB/Web/Tasks/DeployBlog.pm | 18 -- 5 files changed, 239 insertions(+), 18 deletions(-) create mode 100644 Web/lib/MJB/Web/Task.pm rename Web/lib/MJB/Web/{Tasks => Task}/CreateBlog.pm (100%) create mode 100644 Web/lib/MJB/Web/Task/DeployBlog.pm rename Web/lib/MJB/Web/{Tasks => Task}/PurgeBlog.pm (100%) delete mode 100644 Web/lib/MJB/Web/Tasks/DeployBlog.pm diff --git a/Web/lib/MJB/Web/Task.pm b/Web/lib/MJB/Web/Task.pm new file mode 100644 index 0000000..7530577 --- /dev/null +++ b/Web/lib/MJB/Web/Task.pm @@ -0,0 +1,185 @@ +package MJB::Web::Task; +use Mojo::Base 'Minion::Job', -signatures; +use Mojo::File qw( curfile tempfile ); +use YAML; +use IPC::Run3; +use URI; +use Storable qw( dclone ); + +sub system_command ( $self, $cmd, $settings = {} ){ + + # Change the directory, if requested. + if ( $settings->{chdir} ) { + # Throw an error if that directory doesn't exist. + die "Error: directory " . $settings->{chdir} . "doesn't exist." + unless -d $settings->{chdir}; + + # Change to that directory, or die with error. + chdir $settings->{chdir} + or die "Failed to chdir to " . $settings->{chdir} . ": $!"; + + $settings->{return_chdir} = curfile->dirname->to_string; + } + + # Run the command, capture the return code, stdout, and stderr. + #my $ret = run3( $cmd, \undef, \my $out, \my $err ); + # Mask values we don't want exposed in the logs. + my $masked_cmd = dclone($cmd); + if ( ref $settings->{mask} eq 'HASH' ) { + foreach my $key ( keys %{$settings->{mask}} ) { + my $value = $settings->{mask}{$key}; + $masked_cmd = [ map { s/\Q$key\E/$value/g; $_ } @{$masked_cmd} ]; + } + } + + # Log the lines + $self->append_log( "\n\nshell> " . join( " ", @{$masked_cmd || $cmd} ) ); + my ( $out, $err ); + my $ret = run3( $cmd, \undef, sub { + chomp $_; + # Mask values we don't want exposed in the logs. + if ( ref $settings->{mask} eq 'HASH' ) { + foreach my $key ( keys %{$settings->{mask}} ) { + my $value = $settings->{mask}{$key}; + s/\Q$key\E/$value/g; + } + } + $out .= "$_\n"; + $self->append_log( "< stdout: $_" ); + }, sub { + chomp $_; + # Mask values we don't want exposed in the logs. + if ( ref $settings->{mask} eq 'HASH' ) { + foreach my $key ( keys %{$settings->{mask}} ) { + my $value = $settings->{mask}{$key}; + s/\Q$key\E/$value/g; + } + } + $err .= "$_\n"; + $self->append_log( "<{fail_on_stderr} ) { + my @tests = @{$settings->{fail_on_stderr}}; + + while ( my $regex = shift @tests ) { + my $reason = shift @tests; + + if ( $err =~ /$regex/ ) { + $self->fail( $reason ); + $self->stop; + } + } + } + + # Return to the directory we started in if we chdir'ed. + if ( $settings->{return_chdir} ) { + chdir $settings->{return_chdir} + or die "Failed to chdir to " . $settings->{chdir} . ": $!"; + } + + return { + stdout => $out, + stderr => $err, + exitno => $ret, + }; +} + +sub process_webroot ( $job, $site, $source, $dest ) { + + if ( -d $source ) { + + chdir $source + or die "Failed to chdir to $source: $!"; + + $job->append_log( "\n\n--- Processing Static Files ---" ); + + my $files = Mojo::File->new( $source )->list_tree; + + if ( $files->size > $site->max_static_file_count ) { + $job->fail( "This site may have up to " . $site->max_static_file_count . " static files, however the webroot contains " . $files->size . " files."); + $job->stop; + } + + my $total_file_size = 0; + + foreach my $file ( $files->each ) { + # Does file exceed size allowed? + if ( $file->stat->size >= ( $site->max_static_file_size * 1024 * 1024 ) ) { + $job->fail( sprintf("This site may have static files up to %d MiB, however %s exceeds this limit.", + $site->max_static_file_size, + $file->to_string + )); + $job->stop; + } + + $total_file_size += $file->stat->size; + + # If the total file size exceeds the max_static_webroot_size, fail the job. + if ( $total_file_size >= ( $site->max_static_webroot_size * 1024 * 1024 ) ) { + $job->fail( "This site may have up to " . $site->max_static_webroot_size . + " MiB in static files, however the webroot exceeds this limit." + ); + $job->stop; + } + + Mojo::File->new( "$dest/html/" . $file->to_rel( $source )->dirname )->make_path; + $file->move_to( "$dest/html/" . $file->to_rel( $source ) ); + $job->append_log("File Processed: " . $file->to_rel( $source )); + } + $job->append_log( "--- Done Processing Static Files ---" ); + } +} + +sub append_log ( $self, @lines ){ + my @logs = @{$self->info->{notes}{logs} || []}; + + push @logs, @lines; + + $self->note( logs => \@logs ); +} + +sub checkout_repo ( $job, $site_id ) { + my $repo = $job->app->db->site($site_id)->repo; + + die "Error: No repo found for site_id: $site_id" + unless $repo; + + my $build_dir = Mojo::File->tempdir( 'build-XXXXXX', CLEANUP => 0 ); + + my $git_errors = [ + qr|^fatal: repository \'[^']+\' does not exist$| => "Does not seem to be a valid repository.", + qr|^fatal: Could not read from remote repository\.$| => "Error: Permission denied - Valid access and repo?", + qr|^fatal: | => "Error: There was an unexpected fatal error.", + ]; + + if ( $repo->ssh_key_id ) { + my $sshkey_file = tempfile; + $sshkey_file->spurt( $repo->ssh_key->private_key )->chmod( 0600 ); + $ENV{GIT_SSH_COMMAND} = 'ssh -i ' . $sshkey_file->to_string; + $job->system_command( [ 'git', 'clone', $repo->url, "$build_dir/src" ], { + fail_on_stderr => $git_errors, + }); + + } elsif ( $repo->basic_auth_id ) { + my $checkout_url = URI->new( $repo->url ); + my ( $hba_user, $hba_pass ) = ( $repo->basic_auth->username, $repo->basic_auth->password ); + $checkout_url->userinfo( "$hba_user:$hba_pass" ); + + # Supress the user's password from the job logs. + $job->system_command( [ 'git', 'clone', $checkout_url, "$build_dir/src" ], { + mask => { $hba_pass => '_password_' }, + fail_on_stderr => $git_errors, + }); + + } else { + $job->system_command( [ 'git', 'clone', $repo->url, "$build_dir/src" ], { + fail_on_stderr => $git_errors, + }); + } + + return $build_dir; +} + +1; diff --git a/Web/lib/MJB/Web/Tasks/CreateBlog.pm b/Web/lib/MJB/Web/Task/CreateBlog.pm similarity index 100% rename from Web/lib/MJB/Web/Tasks/CreateBlog.pm rename to Web/lib/MJB/Web/Task/CreateBlog.pm diff --git a/Web/lib/MJB/Web/Task/DeployBlog.pm b/Web/lib/MJB/Web/Task/DeployBlog.pm new file mode 100644 index 0000000..0c9b4e3 --- /dev/null +++ b/Web/lib/MJB/Web/Task/DeployBlog.pm @@ -0,0 +1,54 @@ +package MJB::Web::Task::DeployBlog; +use Mojo::Base 'MJB::Web::Task', -signatures; +use Mojo::File qw( curfile ); +use File::Copy::Recursive qw( dircopy ); +use IPC::Run3; + +sub run ( $job, $site_id ) { + $job->note( _mds_template => 'build_static' ); + + my $build_dir = $job->checkout_repo( $site_id ); + my $site = $job->app->db->site( $site_id ); + + $job->note( is_clone_complete => 1 ); + + # Show the user the commit we're on. + $job->system_command( [ 'git', '-C', $build_dir->child('src')->to_string, 'log', '-1' ] ); + + $build_dir->child('build')->make_path; + + + $job->system_command( [qw( podman run -ti --rm -v .:/srv/jekyll -e JEKYLL_ROOTLESS=1 docker.io/jekyll/jekyll jekyll build ) ], { + chdir => $build_dir->child('src')->to_string, + }); + + $job->process_webroot( + $site, + $build_dir->child('src')->child('_site')->to_string, + $build_dir->child('build')->to_string + ); + + #== + # Build Site Config + #== TODO: There is two different files made here, one is done by ansible -- pick one, + # probably this one. + Mojo::File->new($build_dir)->child('build')->child('site.yml')->spurt( + YAML::Dump({ + domain => $site->domain->domain, + www_dir => "$build_dir/build/", + }) + ); + + $job->note( is_build_complete => 1 ); + + # Go to the build directory and make $build_dir/. + $ENV{MARKDOWNSITE_CONFIG} = Mojo::File->new($build_dir->to_string)->child('build')->child('site.yml'); + $job->system_command( [ 'ansible-playbook', '/etc/ansible/deploy-website.yml' ] ); + + + $job->note( is_deploy_complete => 1 ); + $job->finish( ); + +} + +1; diff --git a/Web/lib/MJB/Web/Tasks/PurgeBlog.pm b/Web/lib/MJB/Web/Task/PurgeBlog.pm similarity index 100% rename from Web/lib/MJB/Web/Tasks/PurgeBlog.pm rename to Web/lib/MJB/Web/Task/PurgeBlog.pm diff --git a/Web/lib/MJB/Web/Tasks/DeployBlog.pm b/Web/lib/MJB/Web/Tasks/DeployBlog.pm deleted file mode 100644 index d0da9b2..0000000 --- a/Web/lib/MJB/Web/Tasks/DeployBlog.pm +++ /dev/null @@ -1,18 +0,0 @@ -package MJB::Web::Tasks::DeployBlog; -use Mojo::Base 'Minion::Job', -signatures; -use Mojo::File qw( curfile ); -use File::Copy::Recursive qw( dircopy ); -use IPC::Run3; - -sub run ( $job, $blog_id ) { - my $blog = $job->app->db->blog( $blog_id ); - - # Check out the blog repo to a build area. - - # Build the blog with the jekyll builder in podman - - # Use ansible to deploy the blog to the web servers. - -} - -1;