#!/usr/bin/perl -w
#
# deploy-vhost:
# Deploy a virtual host.
#
# Copyright (c) 2005 UK Citizens Online Democracy. All rights reserved.
# WWW: http://www.mysociety.org/

use strict;

use FindBin;
use lib "$FindBin::Bin/../lib";
use lib "$FindBin::Bin/../perllib";
my $mysociety_bin = $FindBin::Bin;
my $servers_dir = "$FindBin::Bin/../../servers";

use mySociety::SystemMisc qw(shell);
use mySociety::Deploy;

use Cwd;
use DBI;
use Errno;
use Getopt::Long;
use Time::Piece;
use File::Path qw(make_path);
use File::Copy;
use File::Copy::Recursive qw(dircopy);
use File::Temp qw(mktemp);
use File::Basename;
use URI::Escape;
use Data::Dumper;

our $verbose = $ENV{VERBOSE} ? 1 : 0;

#####################################################################
# Main code

# Make sure $servers_dir is up-to-date
# Do this as the maint user, not as root, to avoid the need for root SSH
shell("su maint -c 'git -C $servers_dir/vhosts pull -q origin'");

# Now, read in the configuration
mySociety::Deploy::read_conf($servers_dir);

# Check command line parameters, look up in config file
die "Specify virtual host name as first parameter, deploy/stop/update/start/remove/servers as second parameter" if scalar(@ARGV) < 2;

my $flush;
my $force;
my $thread_id = '';
GetOptions(
    flush => \$flush,
    force => \$force,
    'thread:s' => \$thread_id,
);

my $vhost = shift @ARGV;

my $action = shift @ARGV;
die "Specify 'deploy', 'stop', 'update', 'start', 'servers', 'balancers' or 'remove' as second parameter" if $action ne "deploy" && $action ne "stop" && $action ne "update" && $action ne "start" && $action ne 'remove' && $action ne 'servers' && $action ne 'balancers';

my $use_package = 0;
if ($ENV{USE_PACKAGE} eq "1" || $ENV{USE_PACKAGE} eq "true") {
    $use_package = 1;
}

my $deployment_id = $ENV{DEPLOYMENT_ID};

my $hostname = `hostname`;
chomp($hostname);

my $debian_codename = `lsb_release -cs`;
chomp($debian_codename);
my $debian_version = `lsb_release -rs`;
chomp($debian_version);
my ($debian_version_major, $debian_version_minor) = split /\./, $debian_version;

my $conf = mySociety::Deploy::setup_conf($vhost);
my $vhost_dir = $conf->{vhost_dir};
my $timestamped_deploy = $conf->{timestamped_deploy};
my $vcspath = $conf->{vcspath};

if ($action eq 'servers') {
    print join("\n", @{$conf->{servers}}) . "\n";
    exit;
}

if ($action eq 'balancers') {
    if ($conf->{balancers}) {
        print join("\n", @{$conf->{balancers}}) . "\n";
    }
    exit;
}

# Verify that this server is listed for this vhost
die "'$hostname' is not a server for '$vhost'\nValid options: @{$conf->{servers}}\n"
    if (!grep { m/^$hostname$/ } @{$conf->{servers}}) && $action ne 'remove';

# Bomb out on deploying redirect-only vhosts. These are handled by Puppet.
die "Don't use this script to deploy redirect-only vhosts: these are now handled by Puppet\n"
    if $conf->{redirects_only} && $action ne 'remove';

my $vcspath_install = $vcspath;
my $timestamp = gmtime->strftime('%Y-%m-%dT%H-%M-%S');
if ($timestamped_deploy && $action eq "deploy") {
    $vcspath_install = "$vcspath-$timestamp";
}

# Ensure that the conf_dir and private_conf_dir elements are references to a list of config dirs.
foreach my $dir (qw( conf_dir private_conf_dir )){
    my $c = $conf->{$dir};
    $c ||= [];
    $c = [$c] if (!ref($c));
    $conf->{$dir} = $c;
}

# Make database config settings
if (defined $conf->{'databases'}) {
    foreach my $database (@{$conf->{'databases'}}) {
        my $params = $mySociety::Deploy::databases->{$database};
        db_conf($params, $database);
        upgrade_db_password($params, $database) if ($params->{new_pgpw});
    }
}

my $settings_file_namepart = "settings-autogen.pl";
my $settings_file = "$vhost_dir/$settings_file_namepart";

my $cron_name = "vhost-$vhost";
$cron_name =~ s/\./-/g;
$cron_name = "/etc/cron.d/$cron_name";

# This variable is used to decide whether to restart Apache if you are using
# the touch_to_restart config option (otherwise it always restarts).
my $restart_apache = 0;

# If we're using Docker, set a suitable stack name
if ($conf->{docker}) {
    ($conf->{docker}{stack} = $vhost) =~ s/\./_/g;
}

# If daemons_internal is set and there are no servers listed
# in servers_internal, warn about deployment.
if ($conf->{daemons_internal} && !$conf->{servers_internal}) {
    print "daemons_internal is set, but there are no servers listed in servers_internal, so daemons will be deployed.\n";
}



#####################################################################
# Helper functions

sub get_db_host {
    my $host = shift;
    if ($host eq 'localhost' || $host =~ /\./) {
        # If the host is localhost or a FQDN, then it
        # shouldn't be suffixed with a domain:
        return $host;
    } else {
        # Otherwise suffix it with a ukcod domain:
        return "${host}.srv.mysociety.org";
    }
}

# Check to see whether we should be using the new secret for db passwords
sub get_pgpw_string {
    my ($params, $database) = @_;
    return $params->{new_pgpw} ? "-n $database" : $database;
}

# For each database, pull in information from the JSON and derive some
# further data as required. Store this in $conf->{database_configs}
sub db_conf {
    my ($params, $database) = @_;
    if ($params->{type} ne 'mysql' && $params->{type} ne 'psql'){
        die "unknown database type '$params->{type}'";
    }

    my $pgpw_string = get_pgpw_string($params, $database);

    my $db_config = {
        prefix           => $params->{prefix},
        host             => get_db_host($params->{host}),
        name             => $database,
        username         => $database,
        password         => pgpw($pgpw_string),
        password_escaped => uri_escape(pgpw($pgpw_string)), # RFC3986 escaped version for use in database connection URIs
    };

    # All psql databases should have a port, MySQL often don't.
    # Let's ensure there's a default ready regardless.
    # TODO: Move all JSON config validation and default setting into Deploy.pm
    if ($params->{port}) {
        $db_config->{port} = $params->{port};
    }
    elsif ($params->{type} eq 'psql') {
        $db_config->{port} = '5432';
    }
    elsif ($params->{type} eq 'mysql') {
        $db_config->{port} = '3306';
    }

    # Replica information.
    if ($params->{replica}) {
        $db_config->{replica_host} = get_db_host($params->{replica});
        $db_config->{replica_port} = $db_config->{port};
    }

    # Adapter used by Rails configuration
    if ($params->{type} eq 'mysql') {
        $db_config->{adapter} = 'mysql2';
    }
    elsif ($params->{type} eq 'psql') {
        $db_config->{adapter} = 'postgresql'
    }

    # Add all the parameters as structured data to $conf.
    push @{$conf->{database_configs}}, $db_config;

}

sub apache_graceful {
    shell("apache2ctl", "graceful") if -e "/usr/sbin/apache2";
}

sub nginx_graceful {
    shell("/bin/systemctl", "reload", "nginx");
}

sub graceful {

    # Docker doesn't use Apache
    if (!$conf->{docker}) {
        # Either gracefully restart Apache, or do a touch-to-restart if supported
        if (!$conf->{touch_to_restart} || $restart_apache) {
            apache_graceful();
        } else {
            my $touch = "$vhost_dir/$vcspath/$conf->{touch_to_restart}";
            utime(undef, undef, $touch)
                || shell("su", "-", $conf->{'user'}, "-c touch $touch")
                || apache_graceful();
        }
    }

    # ...and nginx
    nginx_graceful();
}

sub make_git_repository {
   my ($git_url_base, $git_user, $git_repository, $git_ref) = @_;

   if ($timestamped_deploy) {
       # In order to save time on subsequent deploys, keep a shared copy of the
       # repo that we can update and clone from, which means we don't need to
       # do a fresh clone on each deploy
       _make_git_repository($git_user, $git_ref, 'local-git.git', "$git_url_base$git_repository.git", 1);
       _make_git_repository($git_user, $git_ref, $vcspath_install, "$vhost_dir/local-git.git");
   } else {
       _make_git_repository($git_user, $git_ref, $git_repository, "$git_url_base$git_repository.git");
   }

}

sub _make_git_repository {
   my ($git_user, $git_ref, $vcspath_to_use, $git_url, $bare) = @_;

   my $git_dir;
   if ($bare) {
       $git_dir = "$vhost_dir/$vcspath_to_use";
   } else {
       $git_dir = "$vhost_dir/$vcspath_to_use/.git";
   }
   if (! -d $git_dir) {
       my $clone_command;
       my $submodule_command;
       if ($bare) {
           $clone_command = "cd \"$vhost_dir\" && git clone --mirror $git_url $vcspath_to_use";
       } else {
           $clone_command = "cd \"$vhost_dir\" && git clone --no-checkout $git_url $vcspath_to_use && cd \"$vcspath_to_use\" && git checkout $git_ref";
           $submodule_command = "cd \"$vhost_dir/$vcspath_to_use\" && git submodule update --init";
       }
       if ($git_user eq 'root') {
           shell($clone_command);
           shell("$submodule_command && chown -R ".$conf->{'user'}.":".$conf->{'user'}." ../") if $submodule_command;
       } else {
           shell("su", "-", $conf->{'user'}, "-c $clone_command");
           shell("su", "-", $conf->{'user'}, "-c $submodule_command") if $submodule_command;
       }
   } else {
       if (!$bare && $timestamped_deploy) {
           # Make sure we're using local copy
           git("config remote.origin.url $git_url", { dir => "$vhost_dir/$vcspath_to_use" });
       } else {
           my $current_url = git("config --get remote.origin.url", { dir => "$vhost_dir/$vcspath_to_use" });
           $current_url =~ s/\s*$//;
           die "git remote.origin.url in $git_dir is \n\t$current_url\nwhen config says\n\t$git_url\n" if ($current_url ne $git_url);
       }

       # get new stuff so various other parts of script can compare before deploying to working directory
       my $fetch_command = "git -C \"$vhost_dir/$vcspath_to_use\" fetch origin";
       if ($git_user eq 'root') {
           shell("$fetch_command && chown -R ".$conf->{'user'}.":".$conf->{'user'}." ../");
       } else {
           shell("su", "-", $conf->{'user'}, "-c $fetch_command");
       }
       if (!$bare && $git_user eq 'anon' && !$force) {
           shell("su", "-", $conf->{'user'}, "-c cd \"$vhost_dir/$vcspath_to_use\" && $mysociety_bin/git-safe-to-checkout . $git_ref")
       }
   }
}

sub update_git {
    my ($git_user, $git_repository, $git_ref) = @_;

    my $vcspath = $git_repository;
    if ($timestamped_deploy) {
        $vcspath = $vcspath_install;
    }

    die "unexpectedly missing .git directory in update stage" if (! -d "$vhost_dir/$vcspath/.git");
    my $update_command = "git fetch origin && git checkout $git_ref && git submodule update --init";
    if ($git_user eq 'root') {
        shell("cd \"$vhost_dir/$vcspath\" && $update_command && chown -R ".$conf->{'user'}.":".$conf->{'user'}." ../");
    } else {
        shell("su", "-", $conf->{'user'}, "-c cd \"$vhost_dir/$vcspath\" && $update_command");
    }
    my $hash = `cd "$vhost_dir/$vcspath" && git rev-parse --short HEAD`;
    chomp $hash;
    $git_ref =~ s{origin/}{};
    return "<https://github.com/mysociety/$git_repository/tree/$git_ref|$git_ref> branch, commit <https://github.com/mysociety/$git_repository/commit/$hash|$hash>";
}

sub update_vcspath_symlink {
    unless ($timestamped_deploy) {
        die "You can only use update_vcspath_symlink with timestamped_deploy";
    }
    # If there's a real directory there, rather than a symlink, move
    # it out of the way:
    if (-d "$vhost_dir/$vcspath" && ! -l "$vhost_dir/$vcspath") {
        rename("$vhost_dir/$vcspath", "$vhost_dir/$vcspath.moved");
    }
    shell("ln -snf $vhost_dir/$vcspath_install $vhost_dir/$vcspath");
}

sub daemons {
    my $type = shift;
    my @daemons;
    foreach my $daemon (keys %{$conf->{'daemons'}}) {
        my $daemon_mugly_file;
        my $servers;
        my $config = $conf->{'daemons'}->{$daemon};
        if (ref $config eq 'HASH') {
            $daemon_mugly_file = $config->{file};
            $servers = $config->{servers};
            $servers = [ $servers ] if $servers && ref $servers ne 'ARRAY';
        } else {
            $daemon_mugly_file = $config;
        }
        if (!$servers && $conf->{daemons_internal}) {
            $servers = $conf->{servers_internal};
        }

        next if $type eq 'banish' && (!$servers || grep { m/^$hostname$/ } @$servers);
        next if $type eq 'summon' && ($servers && !grep { m/^$hostname$/ } @$servers);

        push @daemons, {
            name => $daemon,
            file => $daemon_mugly_file,
            servers => $servers,
        };
    }
    return @daemons;
}

sub update_daemons {
    create_daemons();
    remove_daemons();
}

sub remove_daemons {
    my $all = shift;
    my $daemon_reload = 0;
    foreach my $obj (daemons($all ? 'all' : 'banish')) {
        my $daemon = $obj->{name};
        if (-e "/etc/systemd/system/${daemon}.service" || -e "/etc/init.d/$daemon") {
            shell("/bin/systemctl", "stop", "$daemon");
            shell("/bin/systemctl", "disable", "$daemon");
            unlink("/etc/init.d/$daemon") if -e "/etc/init.d/$daemon";
            unlink("/etc/systemd/system/${daemon}.service") if -e "/etc/systemd/system/${daemon}.service";
            $daemon_reload = 1;
        }
    }
    shell("/bin/systemctl", "daemon-reload") if $daemon_reload;
}

# Generate daemon init scripts from templates
sub create_daemons {
    # Check to see if we want daemons to be deployed.
    foreach my $obj (daemons('summon')) {
        my $daemon = $obj->{name};
        my $daemon_mugly_file = $obj->{file};

        # See if there's an ugly file in the servers dir
        $daemon_mugly_file =~ m#^(.*?).ugly$#;
        my $name_part = $1;
        my $systemd = $name_part =~ /\.service$/ ? 1 : '';
        my $daemon_mugly;
        my $servers_dir_ugly_file = get_conf_ugly_file_from_servers_dir($name_part);

        if (-e $servers_dir_ugly_file) {
            $daemon_mugly = $servers_dir_ugly_file;
        } else {
            my ($git_ref, $conf_dir);
            if ( $conf->{git_ref} ) {
                $git_ref = $conf->{git_ref};
                $conf_dir = $conf->{conf_dir}->[0]
            } elsif ( $conf->{private_git_ref} ) {
                $git_ref = $conf->{private_git_ref};
                $conf_dir = $conf->{private_conf_dir}->[0]
            }
            $daemon_mugly = git("show $git_ref:$conf_dir/$daemon_mugly_file");
        }

        my $fh = File::Temp->new;
        print $fh "\$daemon_name = '$daemon';\n";
        if ($systemd) {
            mugly($daemon_mugly, "/etc/systemd/system/${daemon}.service", $fh->filename);
            # In case we're transitioning from an older init script, make sure it's been removed.
            unlink("/etc/init.d/$daemon") if -e "/etc/init.d/$daemon";
        }
        else {
            mugly($daemon_mugly, "/etc/init.d/$daemon", $fh->filename);
            chmod 0755, "/etc/init.d/$daemon";
        }
        shell("/bin/systemctl", "daemon-reload");
        shell("/bin/systemctl", "enable", "$daemon");
    }
}

#####################################################################
# Make directories and conf files, check database and conf files

sub create_or_update_files {
    # Create directories and symlink to docs
    make_path($vhost_dir, { user => $conf->{user_uid}, group => $conf->{user_gid} });
    make_dir("$vhost_dir/logs",
        # Allow logs to be read on staging hosts so that tests can run
        # Protect logs on production hosts
        $conf->{staging} ? 0755 : 0750,
        $conf->{user_uid}, scalar(getgrnam('adm')));
    make_dir("$vhost_dir/applogs", 0750, $conf->{user_uid}, scalar(getgrnam('adm')));
    my $web_dir = $conf->{redirects_only} ?  "$vhost_dir/web" : "$vcspath/" . $conf->{web_dir};
    make_symlink($web_dir, "$vhost_dir/docs");
    shell("chown", "-h", "$conf->{user_uid}:$conf->{user_gid}", "$vhost_dir/docs"); # perl's chown doesn't do symlinks

    if ($use_package) {
        print "Will deploy from package.\n";
        my $build_command = "/data/mysociety/bin/build-vhost-package --vhost $vhost --debian_version $debian_codename";
        if ($deployment_id) {
            $build_command .= " --deployment_id $deployment_id";
        }
        my $vhost_package_url = `$build_command`;
        if ($? != 0) {
            die "Failed to build vhost package.";
        }
        my $vhost_package = basename($vhost_package_url);

        print "Will deploy from package $vhost_package.\n";
        print "Fetching $vhost_package..\n.";
        shell(
            "curl",
            "$vhost_package_url",
            "--output",
            "/var/tmp/$vhost_package",
        );
        print "Extracting $vhost_package...\n";
        shell(
            "tar",
            "-xzf",
            "/var/tmp/$vhost_package",
            "-C",
            "/var/tmp/",
        );
        print "Changing file ownership to $conf->{'user'}...\n";
        shell(
            "chown",
            "-R",
            $conf->{'user'} . ':' . $conf->{'user'},
            "/var/tmp/$vhost",
        );

        my $deploy_dir_package_path = $conf->{packaging}->{deploy_dir_package_path};
        die "A deploy_dir_package_path must be specified" unless $deploy_dir_package_path;

        foreach (@{$conf->{packaging}->{vhost_level_package_path_mappings} || []}) {
            my $package_path = $_->{package_path};
            if (!$package_path) {
                die "Missing 'package_path' in 'vhost_level_package_path_mappings' entry " . Dumper($_);
            }
            my $vhost_path = $_->{vhost_path};
            if (!$vhost_path) {
                die "Missing 'vhost_path' in 'vhost_level_package_path_mappings' entry " . Dumper($_);
            }
            print "Moving $package_path to vhost path $vhost_path...\n";
            my $source_path = "/var/tmp/$vhost/$package_path";
            if (-d $source_path) {
                $source_path .= "/";
            }
            shell("rsync -a --delete $source_path $vhost_dir/$vhost_path");
            shell("rm -rf $source_path");
        }

        print "Moving extracted package to $vcspath_install...\n";
        shell(
            "mv",
            "/var/tmp/$vhost/$deploy_dir_package_path",
            "$vhost_dir/$vcspath_install",
        );
    } else {
        # Initial clone, for git repository that don't exist
        my $git_url_base;
        if (exists($conf->{git_repository})) {
            if (exists($conf->{git_url_base})) {
                $git_url_base = $conf->{git_url_base};
            } elsif ($conf->{git_user} eq 'anon') {
                $git_url_base = "git://git.mysociety.org/"
            } else {
                $git_url_base = "ssh://" . $conf->{git_user} . '@git.mysociety.org/data/git/public/';
            }
            make_git_repository($git_url_base, $conf->{git_user}, $conf->{git_repository}, $conf->{git_ref});
        }
        if (exists($conf->{private_git_repository})) {
            $git_url_base = "ssh://" . $conf->{user} . '@git.mysociety.org/data/git/private/';
            make_git_repository($git_url_base, $conf->{user}, $conf->{private_git_repository}, $conf->{private_git_ref});
        }
    }

    # Make public directories
    foreach my $public_dir (@{$conf->{public_dirs}}) {
        make_dir("$vhost_dir/$public_dir");
    }
    if ($conf->{redirects_only}) {
        make_dir("$vhost_dir/web");
    }

    mySociety::Deploy::write_settings_file($settings_file, $conf);

    check_packages();
}

sub update_system_config {
    # Create the daemon
    update_daemons();

    # Common pathname for Apache and Nginx.
    my $vhost_file = "virtualhosts.d/$vhost.conf";

    # Don't do Apache things with Docker vhosts
    if (!$conf->{docker}) {
        # Create apache configuration stanza
        my $temp_file = mktemp("/tmp/deploy-vhost-apache-XXXXXXXX");
        my $apache_file = "/etc/apache2/$vhost_file";
        mugly("$servers_dir/vhosts/single-vhost.conf.ugly", $temp_file);
        if (!-e $apache_file || system("diff", "-u", $apache_file, $temp_file) != 0) {
            $restart_apache = 1;
            copy($temp_file, $apache_file);
        }
    }

    # Nginx
    my $nginx_file = "/etc/nginx/$vhost_file";
    mugly("$servers_dir/vhosts/single-vhost-nginx-https-proxy.conf.ugly", $nginx_file);
}

#####################################################################
# Shut down services, update, start services

if ($action eq "stop") {
    stop_site();
    deploy_logger("Stopped vhost $vhost");
} elsif ($action eq "update") {
    create_or_update_files();
    update_config();
    update_system_config();
    my $git_log = update_site();
    graceful();
    deploy_logger("Updated vhost $vhost ($git_log)");
} elsif ($action eq "start") {
    start_site();
    deploy_logger("Started vhost $vhost");
} elsif ($action eq "deploy") {
    create_or_update_files();
    update_config();
    update_system_config();
    my $deploy_log_message;
    if ($timestamped_deploy) {
        $deploy_log_message = update_site();
        stop_site();
        run_tasks_from_config($conf->{exec_while_down});
        update_vcspath_symlink();
    } else {
        stop_site();
        $deploy_log_message = update_site();
    }
    graceful();
    start_site();
    deploy_logger("Deployed vhost $vhost ($deploy_log_message)")
        unless $vhost =~ /testharness/;
    if (@{$conf->{servers}} > 1) {
        my @other_hosts = grep { $_ ne $hostname } @{$conf->{servers}};
        print "Note: $vhost is also on " . join (', ', @other_hosts) . "\n";
    }
    if ($flush) {
        # Memcached
        if (-e "/usr/bin/memcached") {
            # This will flush everything on the host from Memcache
            print "Flushing Memcached...\n";
            use mySociety::Memcached;
            mySociety::Memcached::set_config('localhost');
            mySociety::Memcached::flush_all();
        } else {
            print "No local Memcached to flush, skipping."
        }

        # Varnish
        if (-e "/usr/bin/varnishadm") {
            # Ban the main vhost
            print "Flushing Varnish...\n";
            print `varnishadm ban "req.http.host == $vhost"`;
            # And any aliases
            if ($conf->{aliases}) {
                foreach my $alias (@{$conf->{aliases}}) {
                    print `varnishadm ban "req.http.host == $alias"`;
                }
            }
        } else {
            print "No local Varnish to flush, skipping. You could run 'ban-vhost' from a management node instead.\n";
        }
    }
} elsif ($action eq "remove") {
    stop_site();
    remove_site();
    deploy_logger("Removed vhost $vhost");
} else {
    die "unknown action";
}

sub check_package_file {
    my ($conf_dir, $git_ref) = @_;
    if (-e "$vhost_dir/$vcspath_install/$conf_dir/packages") {
        my $tree = git("ls-tree $git_ref $conf_dir/packages");
        if ($tree =~ /^120000/) {
            $tree = git("show $git_ref:$conf_dir/packages");
        } else {
            $tree = 'packages';
        }
        my $tmp_packages = git("show $git_ref:$conf_dir/$tree");
        local $SIG{PIPE} = 'IGNORE';
        open(CHECK, "|/data/mysociety/bin/check-packages-installed");
        print CHECK $tmp_packages;
        close CHECK;
    }
}

sub check_packages {
    # Check Debian packages are all installed, if there is a conf/packages file
    foreach my $conf_dir (@{$conf->{conf_dir}}) {
        check_package_file($conf_dir, $conf->{git_ref});
    }
    foreach my $conf_dir (@{$conf->{private_conf_dir}}) {
        check_package_file($conf_dir, $conf->{private_git_ref});
    }
}

# Halt any daemons, crontabs, email systems, and the web host itself
sub stop_site {
    # Stop daemons (if present on the system)
    foreach my $daemon (keys %{$conf->{'daemons'}}) {
        if (-e "/etc/systemd/system/${daemon}.service" || -e "/etc/init.d/$daemon") {
            shell("/bin/systemctl", "stop", "$daemon");
        }
    }

    # Stop crontab
    unlink($cron_name);

    # Stop email forwarding - calls the disable-email script for each
    # user specified in the $conf->{'email'} hash.
    if ($conf->{'email'}) {
        foreach my $email_user (keys %{$conf->{'email'}}) {
            shell("su", "-", $email_user, "-c /data/mysociety/bin/disable-email activate");
        }
    }

    if (-d "$vhost_dir/docs") {
        if (-s "$servers_dir/vhosts/downs/${vhost}.html") {
            # Have to copy rather than symlink because $servers_dir probably isn't readable
            # by the vhost user.
            unlink("$vhost_dir/docs/down.html");
            copy("$servers_dir/vhosts/downs/${vhost}.html", "$vhost_dir/docs/down.html")
                or warn "can't copy $servers_dir/vhosts/downs/${vhost}.html to $vhost_dir/docs/down.html: $!";
        } elsif (-s "$servers_dir/vhosts/downs/$conf->{site}.html") {
            # Have to copy rather than symlink because $servers_dir probably isn't readable
            # by the vhost user.
            unlink("$vhost_dir/docs/down.html");
            copy("$servers_dir/vhosts/downs/$conf->{site}.html", "$vhost_dir/docs/down.html")
                or warn "can't copy $servers_dir/vhosts/downs/$conf->{site}.html to $vhost_dir/docs/down.html: $!";
        } elsif (-s "$vhost_dir/docs/down.current.html") {
            unlink("$vhost_dir/docs/down.html");
            symlink("$vhost_dir/docs/down.current.html", "$vhost_dir/docs/down.html");
        } elsif (-s "$vhost_dir/docs/down.default.html") {
            unlink("$vhost_dir/docs/down.html");
            symlink("$vhost_dir/docs/down.default.html", "$vhost_dir/docs/down.html");
        } else {
            # Down notice
            open(FHOUT, "> $vhost_dir/docs/.down.html.new")
                or die "Failed to open $vhost_dir/docs/.down.html.new for writing: $!\n";
            open(FHIN, "$servers_dir/vhosts/downs/default.html")
                or die "Failed to open $servers_dir/vhosts/downs/default.html for reading: $!\n";
            while (<FHIN>) {
                s/\{\{site}}/$conf->{server_name}/;
                print FHOUT;
            }
            close FHIN;
            close FHOUT;
            unlink("$vhost_dir/docs/down.html");
            rename("$vhost_dir/docs/.down.html.new", "$vhost_dir/docs/down.html")
                or die "failed to rename .down.html.new to down.html in $vhost_dir/docs: $!";
            chown($conf->{user_uid}, $conf->{user_gid}, "$vhost_dir/docs/down.html");
        }
    }

    if ($conf->{docker}) {
        shell("/usr/local/bin/docker-server", $vhost, "$vcspath_install/$conf->{conf_dir}->[0]", "stop");
    }
}

sub get_conf_ugly_file_from_servers_dir {
  my ($conf_ugly_part) = @_;
  my $conf_ugly_file = "$servers_dir/vhosts/$vcspath/$vhost/$conf_ugly_part.ugly";
  $conf_ugly_file = "$servers_dir/vhosts/$vcspath/$conf_ugly_part.ugly" unless -e $conf_ugly_file;
  $conf_ugly_file = "$servers_dir/vhosts/${vcspath}_$conf_ugly_part.ugly" unless -e $conf_ugly_file;
  return $conf_ugly_file;
}

# Update a config dir
sub update_conf_dir {
    my ($conf_dir, $git_ref) = @_;

    foreach my $glob (glob "$vhost_dir/$vcspath_install/$conf_dir/*-example") {
       $glob =~ m#^.*/(.*?)-example$#;

        my $name_part = $1;
        my $conf_file = "$vhost_dir/$vcspath_install/$conf_dir/$name_part"; # destination file
        my $previous_conf_file = "$vhost_dir/$vcspath/$conf_dir/$name_part"; # same on non-timestamped deploys
        my $old_umask = umask(0027);
        my $name_base = $name_part;
        $name_base =~ s/\.yml$//;

        # find source template (.ugly) file
        my $conf_ugly_part = "$conf_dir/$name_part"; # e.g. config/general
        $conf_ugly_part =~ s#/#_#g;
        my $conf_ugly_file = get_conf_ugly_file_from_servers_dir($conf_ugly_part);

        # update example we'll be comparing against, to make sure no missing entries.
        # For public conf files, use the example in the main public checkout. For
        # private, use the example in the vhost conf dir.
        my $example_file = mktemp("/tmp/deploy-vhost-$name_part-example-XXXXXXXX");
        git("show $git_ref:$conf_dir/$name_part-example", { out => $example_file });
        # check destination file hasn't been separately modified
        my $make_conf = 1;
        if (-e $conf_file) {
            # XXX detect errors
            if (system("diff", "-u", "$conf_file.deployed", "$conf_file") != 0) {
                print STDERR "Warning: $conf_file was changed directly since last update from\n";
                print STDERR "template file $conf_ugly_file, so has not been updated.\n";
                $make_conf = 0;
            }
        }
        if ($make_conf) {
            my $conf_file_tmp = $conf_file;
            $conf_file_tmp =~ s/((\.yml)?)$/.tmp$1/;

            mugly($conf_ugly_file, $conf_file_tmp);
            if ($name_base eq 'general') {
                # test contents of new configuration file before copying it over
                shell("$mysociety_bin/compareconfig.pl", $conf_file_tmp, $example_file);
            }
            if ($name_base eq 'httpd.conf') {
                if (!-e $previous_conf_file || system("diff", "-u", "$previous_conf_file.deployed", $conf_file_tmp) != 0) {
                    $restart_apache = 1;
                }
            }
            # copy into place
            copy($conf_file_tmp, "$conf_file.deployed");
            chown($conf->{user_uid}, $conf->{user_gid}, "$conf_file.deployed");
            copy($conf_file_tmp, "$conf_file");
            chown($conf->{user_uid}, $conf->{user_gid}, "$conf_file");
            unlink($conf_file_tmp) or die "couldn't unlink $conf_file_tmp";
        } else {
            if ($name_base eq 'general') {
                # test contents of configuration file before copying it over
                shell("$mysociety_bin/compareconfig.pl", $conf_file, $example_file);
            }
        }
        # clear up temporary example file in git case and reset umask
        unlink($example_file);
        umask($old_umask);
    }

    # Make sure custom logrotate files are called using
    # a cron job with a vhost-specific state file.
    my $conf_file = "$vhost_dir/$vcspath_install/$conf_dir/logrotate";
    if (-f $conf_file) {
        open CRON, ">".$cron_name."-logrotate";
        print CRON "MAILTO=root\n";
        print CRON "5 0 * * * $conf->{user} /usr/sbin/logrotate -s $vhost_dir/logrotate.state $conf_file\n";
        close CRON;

        # Make sure any symlink created by an earlier version of this script, or pre-OS-upgrade,
        # is removed.
        unlink("/etc/logrotate.d/$vhost");
    } else {
        # Make sure any cron job created by an earlier version of this script is removed.
        unlink($cron_name."-logrotate");
    }
}

# Check that config files and database files have been updated
sub update_config {
    # Generate all conf files from conf/*-example template files e.g. conf/general
    foreach my $conf_dir (@{$conf->{conf_dir}}) {
        update_conf_dir($conf_dir, $conf->{git_ref});
    }
    foreach my $conf_dir(@{$conf->{private_conf_dir}}){
        update_conf_dir($conf_dir, $conf->{private_git_ref});
    }
}

sub _run_sub_tasks_from_config {
    my $exec_extras = shift;
    my $envars = "";
    if ($use_package) {
        $envars .= "PACKAGE_DEPLOY=1; ";
    }
    foreach my $exec_extra (@{$exec_extras->{user}}) {
        print "\n\033[34m[user] $exec_extra\033[0m\n";
        shell("su", "-l", $conf->{'user'}, "-c cd \"$vhost_dir/$vcspath_install\" && $envars $exec_extra");
    }
    foreach my $exec_extra (@{$exec_extras->{root}}) {
        print "\n\033[34m[root] $exec_extra\033[0m\n";
        shell("cd '$vhost_dir/$vcspath_install' && $envars $exec_extra");
    }
}

sub run_tasks_from_config {
    my $exec_extras = shift;
    _run_sub_tasks_from_config($exec_extras->{always}) if $exec_extras->{always};
    foreach my $group (@{$exec_extras->{groups}}) {
        _run_sub_tasks_from_config($exec_extras->{$group});
    }
    _run_sub_tasks_from_config($exec_extras);
}

sub update_site {
    my $git_log = '';

    # Update from git
    my $git_repository;
    if (exists($conf->{'git_repository'})) {
        $git_repository =  $conf->{'git_repository'};
        $git_log = update_git($conf->{'git_user'}, $git_repository, $conf->{'git_ref'});
    }
    # softlink any private dirs into the public directory structure
    if (exists($conf->{'private_git_repository'})){
        my $private_git_repository = $conf->{'private_git_repository'};
        $git_log = update_git($conf->{'user'}, $private_git_repository, $conf->{'private_git_ref'});
        unless ($timestamped_deploy) {
            if (exists($conf->{'private_git_dirs'})){
                foreach my $private_dir (@{$conf->{'private_git_dirs'}}) {
                    shell("ln -snf $vhost_dir/$private_git_repository/$private_dir $vhost_dir/$git_repository/$private_dir");
                }
            }
        }
    }

    set_or_clear_ruby_version();

    # Run any extra executables
    if ($timestamped_deploy) {
        if (exists($conf->{'exec_before_down'})) {
            run_tasks_from_config($conf->{exec_before_down});
        }
    } else {
        if (exists($conf->{exec_extras})) {
            run_tasks_from_config($conf->{exec_extras});
        }
    }

    $git_log .= ", " if $git_log;
    $git_log .= "updating in $vcspath_install";

    return $git_log;
}

sub set_or_clear_ruby_version {

  # Filename to use for the local Ruby version.
  my $version_file = ".rbenv-version";

  if ($conf->{ruby_version}) {

    # Set rbenv_global to something true to activate the new, global rbenv under /opt/rbenv.
    # If it evaluates false, keep using the old per-user method.

    # Location of profile snippet depends on rbenv_global.
    my $rbenv_profile;

    if ($conf->{rbenv_global}) {

      $rbenv_profile = '/data/servers/langs/profile.rbenv.global';
      $version_file = '.ruby-version';

    } else {

      $rbenv_profile = '/data/servers/langs/.profile.rbenv';
      $version_file = '.rbenv-version';

      # copy .rbenv files from /etc/skel
      # This gives the shims and the version-specific binaries they point to.
      dircopy("/etc/skel/.rbenv", "/home/$conf->{'user'}/.rbenv");
      shell("chown", "-Rh", "$conf->{user_uid}:$conf->{user_gid}", "/home/$conf->{'user'}/.rbenv");

    }

    # Write a .(ruby|rbenv)-version file at the root of the application directory
    # This should set the version of ruby for any rbenv-enabled user when operating
    # in the directory or subdirectories
    shell("su", "-", $conf->{'user'}, "-c echo \"$conf->{'ruby_version'}\" >$vhost_dir/$vcspath_install/$version_file");

    # Create .profile.rbenv to the user's home directory to initialise rbenv.
    # This should be sourced by .profile, if it was created from /etc/skel
    copy($rbenv_profile, "/home/$conf->{'user'}/.profile.rbenv");
    shell("chown", "$conf->{user_uid}:$conf->{user_gid}", "/home/$conf->{'user'}/.profile.rbenv");

    # add a wrapper scripts to allow the user's .forward files to operate with the right ruby version
    # The logic in the mugly template checks for rbenv_global.
    foreach my $file ("run-with-rbenv-path", "run-with-rbenv-path-exec") {
        mugly("$servers_dir/vhosts/$file.ugly", "/home/$conf->{'user'}/$file");
        shell("chown", "$conf->{user_uid}:$conf->{user_gid}", "/home/$conf->{'user'}/$file");
        chmod 0755, "/home/$conf->{'user'}/$file";
    }

  } else {

    # Remove any .rbenv-version file
    unlink("$vhost_dir/$vcspath_install/$version_file");

    # Remove any .profile.rbenv file in the user's home directory
    unlink("/home/$conf->{'user'}/.profile.rbenv");

    # Remove any run-with-rbenv-path in the user's home directory
    unlink("/home/$conf->{'user'}/run-with-rbenv-path");
  }
}

# Resolve a named wrapper (e.g. "rbenv") to the actual executable path,
# or treat an unrecognised value as a literal path.
sub resolve_wrapper {
    my ($email_user, $wrapper) = @_;

    my %named_wrappers = (
        rbenv => { bin => "/home/${email_user}/run-with-rbenv-path", },
        rbenv_exec => { bin => "/home/${email_user}/run-with-rbenv-path-exec", chdir => 1 },
    );

    return $named_wrappers{$wrapper} // { bin => $wrapper };
}

sub make_dot_forward {
    my ($email_user, $script, $suffix, $wrapper) = @_;
    my $forward_file_name = defined($suffix) ? ".forward-$suffix" : ".forward";

    # remove any backup files created by the disable-email script.
    if ( -e "/home/${email_user}/${forward_file_name}.bak" ) {
        unlink "/home/${email_user}/${forward_file_name}.bak";
    }

    if (scalar @_ < 4) {
        $wrapper = 'rbenv' if $conf->{ruby_version};
    }

    if (defined $wrapper) {
        my $w = resolve_wrapper($email_user, $wrapper);

        if ($w->{chdir}) {
            $script = "/bin/sh -c 'cd $vhost_dir/$vcspath && $w->{bin} $script'";
        } else {
            $script = "$w->{bin} $vhost_dir/$vcspath/$script";
        }
    } else {
        $script = "${vhost_dir}/${vcspath}/${script}";
    }

    # Create new version.
    shell("su", "-", $email_user, "-c echo \"|$script\" >~$email_user/$forward_file_name");

    # Ensure permissions are correct: writeable only by the user, regardless of umask.
    chmod 0644, "~${email_user}/${forward_file_name}";
}

sub install_or_remove_crontab {
    my $ok = 0;
    if ($conf->{crontab}) {
        if ($conf->{crontab} eq '1' || $conf->{crontab} eq 'all') {
            $ok = 1;
        } elsif ($conf->{crontab} eq 'only-one') {
            if ($hostname eq $conf->{servers}[0]) {
                $ok = 1;
            }
        } else {
            if ($hostname eq $conf->{crontab}) {
                $ok = 1;
            }
        }
    }

    if (!$ok) {
        unlink($cron_name);
        return;
    }

    my $setup_git_cron = sub {
        my ( $conf_dir, $git_ref ) = @_;
        # if a crontab exists in the conf directory in source control, or has been created there
        # in the update_conf_dir function, just install it
        my $crontab = "$conf_dir/crontab";
        my $crontab_path = "$vhost_dir/$vcspath_install/$crontab";
        my $tmp_crontab = mktemp("/tmp/deploy-vhost-crontab-XXXXXXXX");
        if (-e $crontab_path) {
            copy($crontab_path, $tmp_crontab);
        } else {
            return unless -e "$crontab_path.ugly";
            my $tmp_crontab_ugly = git("show $git_ref:$crontab.ugly");
            mugly($tmp_crontab_ugly, $tmp_crontab);
        }

        # Check that MAILTO is valid
        open CRON, $tmp_crontab or die "can't open $tmp_crontab: $!";
        while (<CRON>) {
            if (/^MAILTO=(.*)$/) {
                system("/data/mysociety/bin/lookup_google_apps_email.sh $1");
                my $result = $? >> 8;
                if ($result == 1) {
                    unlink $tmp_crontab;
                    die "MAILTO=$1: not a valid address in crontab";
                } elsif ($result == 2) {
                    warn "Google Apps API not working; can't check MAILTO address";
                } elsif ($result != 0) {
                    warn "unexpected result code $result from lookup_google_apps_email.sh";
                }

                last;
            }
        }
        close CRON;

        # Now copy the file in place
        copy($tmp_crontab, $cron_name);
        unlink $tmp_crontab;
    };

    $setup_git_cron->( $_, $conf->{'git_ref'} )         for @{$conf->{conf_dir}};
    $setup_git_cron->( $_, $conf->{'private_git_ref'} ) for @{$conf->{private_conf_dir}};
}

sub start_site {

    if ($conf->{docker}) {
        shell("/usr/local/bin/docker-server", $vhost, "$vcspath_install/$conf->{conf_dir}->[0]", "start");
    }

    # Up notice
    unlink("$vhost_dir/docs/down.html") or do {
        # It doesn't matter if down.html has already been removed.
        die "couldn't unlink $vhost_dir/docs/down.html: $!\n" unless $!{ENOENT};
    };

    # Start daemons
    foreach my $obj (daemons('summon')) {
        my $daemon = $obj->{name};
        shell("/bin/systemctl", "start", "$daemon");
    }

    # Crontab
    install_or_remove_crontab();

    # Make .forward files
    if ($conf->{'email'}) {
        foreach my $email_user (keys %{$conf->{'email'}}) {
            my $email_conf = $conf->{'email'}->{$email_user};
            if (ref($email_conf) eq "HASH") {
                while (my ($suffix, $script) = each(%$email_conf)) {
                    if ($suffix eq '') {
                        make_dot_forward($email_user, $script);
                    } else {
                        make_dot_forward($email_user, $script, $suffix);
                    }
                }

            } elsif (ref($email_conf) eq 'ARRAY') {
                foreach my $entry (@$email_conf) {
                    die "Email entry for $email_user is missing 'script' key" unless defined $entry->{script};

                    # Pass wrapper explicitly (even if undef) to suppress ruby_version inference in make_dot_forward
                    make_dot_forward($email_user, $entry->{script}, $entry->{suffix} || undef, $entry->{wrapper} || undef);
                }

            } else {
                make_dot_forward($email_user, $email_conf);
            }
        }
    }
}

sub check_for_unpushed {
    my ($git_repository) = @_;
    my $unpushed_command = "git -C \"$vhost_dir/$git_repository\" log --branches --not --remotes --simplify-by-decoration --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit --date=relative";
    my $unpushed = `$unpushed_command`;
    print "\nThe following branches are unpushed:\n$unpushed\n" if $unpushed;
}

# Remove crontabs, vhost config etc.
sub remove_site {
    # Remove any daemons
    remove_daemons(1);

    # Remove crontab
    unlink($cron_name);

    # Remove email forwarding
    if ($conf->{'email'}) {
        foreach my $email_user (keys %{$conf->{'email'}}) {
            unlink("/home/$email_user/.forward");
        }
    }

       # Remove any .profile.rbenv file in the user's home directory
    if ($conf->{'ruby_version'}) {
      unlink("/home/$conf->{'user'}/.profile.rbenv");
    }

    # Remove Apache virtual host config
    unlink("/etc/apache2/virtualhosts.d/$vhost.conf");
    apache_graceful();

    # Remove nginx virtual host config
    unlink("/etc/nginx/virtualhosts.d/$vhost.conf");
    nginx_graceful();

    # Remove vhost user's logrotate cron job
    unlink("$cron_name-logrotate");

    print "Config files removed, please tidy $vhost_dir yourself.\n";

    if (exists($conf->{'git_repository'})) {
        check_for_unpushed($conf->{'git_repository'});
    }
    if (exists($conf->{'private_git_repository'})){
        check_for_unpushed($conf->{'private_git_repository'});
    }


}

# Upgrade database password on the fly
sub upgrade_db_password {
    my ($params, $database) = @_;

    my $db_host = get_db_host($params->{host});

    if ($params->{type} eq 'psql') {
        # Try to connect with the old password.  If we can't, assume upgrade already done.
        my $dbh = DBI->connect("dbi:Pg:dbname=$database;host=$db_host;port=$params->{port}", $database, pgpw($database), { PrintError => 0, RaiseError => 0 });
        if ($dbh) {
            # Set password to the new type
            if (!($dbh->do("ALTER ROLE \"$database\" WITH PASSWORD '".pgpw("-n $database")."';"))) {
                warn "unable to upgrade password for postgres database $database";
            }
            $dbh->disconnect();
        } else {
            if (DBI->errstr() !~ /password authentication failed/) {
                warn "unexpected error trying to connect to $database: ".DBI->errstr();
            }
        }
    } elsif ($params->{type} eq 'mysql') {
        # Try to connect with the old password.  If we can't, assume upgrade already done.
        my $dbh = DBI->connect("dbi:mysql:dbname=$database;host=$db_host", $database, pgpw($database), { PrintError => 0, RaiseError => 0 });
        if ($dbh) {
            # Set password to the new type
            if (!($dbh->do("SET PASSWORD = PASSWORD('".pgpw("-n $database")."');"))) {
                warn "unable to upgrade password for postgres database $database";
            }
            $dbh->disconnect();
        } else {
            if (DBI->err != 1045) { # 1045 = authentication failure
                warn "unexpected error trying to connect to $database: ".DBI->errstr();
            }
        }
    }
}

#####################################################################
# General functions

sub deploy_logger {
    my $message = shift;
    shell("/data/mysociety/bin/deploy-logger", $message, $thread_id);
}

sub make_dir {
    my ($dir, $chmod, $uid, $gid) = @_;
    if (! -d $dir) {
        mkdir($dir) || die "failed to mkdir '$dir'";
    }
    chown($uid || $conf->{user_uid}, $gid || $conf->{user_gid}, $dir);
    chmod $chmod, $dir if $chmod;
}

sub make_symlink {
    my ($a, $b) = @_;
    my $current = readlink($b);
    if (!$current) {
        symlink($a, $b) || die "failed to make symlink '$b' -> '$a'";
    } elsif ($current ne $a) {
        die "remove existing wrong symlink '$b' -> '$current' first, so can become '$b' -> '$a'";
    }
}

sub pgpw {
    $_ = shift;
    $_ = `/data/mysociety/bin/pgpw $_`;
    die "pgpw failed with error code $?" if $? > 0;
    s/\s+$//;
    return $_;
}

sub mugly {
    my ($in, $out, $extra_settings) = @_;
    my @args = ("$mysociety_bin/mugly", "-O", $out, "-p", $settings_file);
    push @args, "-p", $extra_settings if $extra_settings;
    if ($in =~ m{^/}) {
        shell(@args, $in);
    } else {
        local $SIG{PIPE} = 'IGNORE';
        open(MUGLY, "|-", @args, '-') or die $!;
        print MUGLY $in;
        close MUGLY;
    }
}

sub git {
    my ($command, $args) = @_;
    my $dir = $args->{dir} || "$vhost_dir/$vcspath_install";
    if ($args->{out}) {
        shell("git -C '$dir' $command > $args->{out}");
    } else {
        return `git -C '$dir' $command`;
    }
}
