feat: add s3 backup support

2016-10-30 13:16:28 -06:00
parent 7ee8d7ee73
commit cc5fac0376
16 changed files with 251 additions and 23 deletions
--- a/README.md
+++ b/README.md
@@ -17,6 +17,11 @@ sudo dokku plugin:install https://github.com/dokku/dokku-mysql.git mysql
 ## commands

 ```
+mysql:backup <name> <bucket>   Create a backup of the mysql service to an existing s3 bucket
+mysql:backup-auth <name> <aws_access_key_id> <aws_secret_access_key> Sets up authentication for backups on the mysql service
+mysql:backup-deauth <name>     Removes backup authentication for the mysql service
+mysql:backup-schedule <name> <schedule>  <aws_access_key_id> <aws_secret_access_key> <bucket> Schedules a backup of the mysql service
+mysql:backup-unschedule <name> Unschedules the backup of the mysql service
 mysql:clone <name> <new-name>  Create container <new-name> then copy data from <name> into <new-name>
 mysql:connect <name>           Connect via mysql to a mysql service
 mysql:create <name>            Create a mysql service with environment variables
@@ -172,6 +177,27 @@ OR
 - Change MYSQL_DATABASE_SCHEME to the desired setting
 - Relink the service

-## Adding a custom my.cnf
+### Backups
+
+Backups can be performed using the backup commands:
+
+```
+# setup s3 backup authentication
+dokku mysql:backup-auth lolipop AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY
+
+# remove s3 authentication
+dokku mysql:backup-deauth lolipop
+
+# backup the `lolipop` service to the `BUCKET_NAME` bucket on AWS
+dokku mysql:backup lolipop BUCKET_NAME
+
+# schedule a backup
+dokku mysql:backup-schedule lolipop CRON_SCHEDULE BUCKET_NAME
+
+# remove the scheduled backup from cron
+dokku mysql:backup-unschedule lolipop
+```
+
+### Adding a custom my.cnf

 You can create a custom my.cnf by saving it to `/var/lib/dokku/services/mysql/APP_NAME/config/my.cnf` and restarting your database
--- a/25
+++ b/25
@@ -11,6 +11,26 @@ if [[ ! -d $PLUGIN_DATA_ROOT ]]; then
 fi

 case "$1" in
+  $PLUGIN_COMMAND_PREFIX:backup)
+    "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/subcommands/backup" "$@"
+    ;;
+
+  $PLUGIN_COMMAND_PREFIX:backup-auth)
+    "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/subcommands/backup-auth" "$@"
+    ;;
+
+  $PLUGIN_COMMAND_PREFIX:backup-deauth)
+    "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/subcommands/backup-deauth" "$@"
+    ;;
+
+  $PLUGIN_COMMAND_PREFIX:backup-schedule)
+    "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/subcommands/backup-schedule" "$@"
+    ;;
+
+  $PLUGIN_COMMAND_PREFIX:backup-unschedule)
+    "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/subcommands/backup-unschedule" "$@"
+    ;;
+
  $PLUGIN_COMMAND_PREFIX:clone)
    "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/subcommands/clone" "$@"
    ;;
@@ -84,6 +104,11 @@ case "$1" in
      # shellcheck disable=SC2034
      declare desc="return $PLUGIN_COMMAND_PREFIX plugin help content"
      cat<<help_content
+    $PLUGIN_COMMAND_PREFIX:backup <name> <bucket>, Create a backup of the $PLUGIN_COMMAND_PREFIX service to an existing s3 bucket
+    $PLUGIN_COMMAND_PREFIX:backup-auth <name> <aws_access_key_id> <aws_secret_access_key>, Sets up authentication for backups on the $PLUGIN_COMMAND_PREFIX service
+    $PLUGIN_COMMAND_PREFIX:backup-deauth <name>, Removes backup authentication for the $PLUGIN_COMMAND_PREFIX service
+    $PLUGIN_COMMAND_PREFIX:backup-schedule <name> <schedule> <bucket>, Schedules a backup of the $PLUGIN_COMMAND_PREFIX service
+    $PLUGIN_COMMAND_PREFIX:backup-unschedule <name>, Unschedules the backup of the $PLUGIN_COMMAND_PREFIX service
    $PLUGIN_COMMAND_PREFIX:clone <name> <new-name>, Create container <new-name> then copy data from <name> into <new-name>
    $PLUGIN_COMMAND_PREFIX:connect <name>, Connect via mysql to a $PLUGIN_SERVICE service
    $PLUGIN_COMMAND_PREFIX:create <name>, Create a $PLUGIN_SERVICE service
--- a/71
+++ b/71
@@ -102,6 +102,68 @@ service_alternative_alias() {
  echo "$ALIAS"
 }

+service_backup() {
+  declare desc="Creates a backup of a service to an existing s3 bucket"
+  declare SERVICE="$1" BUCKET_NAME="$2"
+  local SERVICE_ROOT="$PLUGIN_DATA_ROOT/$SERVICE"
+  local AWS_ACCESS_KEY_ID_FILE="$SERVICE_ROOT/backup/AWS_ACCESS_KEY_ID"
+  local AWS_SECRET_ACCESS_KEY_FILE="$SERVICE_ROOT/backup/AWS_SECRET_ACCESS_KEY"
+
+  [[ ! -f "$AWS_ACCESS_KEY_ID_FILE" ]] && dokku_log_fail "Missing AWS_ACCESS_KEY_ID file"
+  [[ ! -f "$AWS_SECRET_ACCESS_KEY_FILE" ]] && dokku_log_fail "Missing AWS_SECRET_ACCESS_KEY file"
+
+  TMPDIR=$(mktemp -d)
+  trap 'rm -rf "$TMP_WORK_DIR" > /dev/null' RETURN INT TERM EXIT
+
+  (service_export "$SERVICE" > "${TMPDIR}/export")
+  docker run \
+      -e AWS_ACCESS_KEY_ID="$(cat "$AWS_ACCESS_KEY_ID_FILE")" \
+      -e AWS_SECRET_ACCESS_KEY="$(cat "$AWS_SECRET_ACCESS_KEY_FILE")" \
+      -e BUCKET_NAME="$BUCKET_NAME" \
+      -e BACKUP_NAME="${PLUGIN_COMMAND_PREFIX}-${SERVICE}" \
+      -v "${TMPDIR}:/backup" dokkupaas/s3backup:0.5.0-1
+}
+
+service_backup_auth() {
+  declare desc="Sets up authentication"
+  declare SERVICE="$1" AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3"
+  local SERVICE_ROOT="${PLUGIN_DATA_ROOT}/${SERVICE}"
+  local SERVICE_BACKUP_ROOT="${SERVICE_ROOT}/backup/"
+
+  mkdir -p "$SERVICE_BACKUP_ROOT"
+  echo "$AWS_ACCESS_KEY_ID" > "${SERVICE_BACKUP_ROOT}/AWS_ACCESS_KEY_ID"
+  echo "$AWS_SECRET_ACCESS_KEY" > "${SERVICE_BACKUP_ROOT}/AWS_SECRET_ACCESS_KEY"
+}
+
+service_backup_deauth() {
+  declare desc="Removes authentication"
+  declare SERVICE="$1"
+  local SERVICE_ROOT="${PLUGIN_DATA_ROOT}/${SERVICE}"
+  local SERVICE_BACKUP_ROOT="${SERVICE_ROOT}/backup/"
+
+  rm -rf "$SERVICE_BACKUP_ROOT"
+}
+
+service_backup_schedule() {
+  declare desc="schedules a backup of the service"
+  declare SERVICE="$1" SCHEDULE="$2" BUCKET_NAME="$3"
+  local DOKKU_BIN="$(which dokku)"
+  local CRON_FILE="/etc/cron.d/dokku-${PLUGIN_COMMAND_PREFIX}-${SERVICE}"
+  local TMP_CRON_FILE="${PLUGIN_DATA_ROOT}/.TMP_CRON_FILE"
+
+  echo "${SCHEDULE} dokku ${DOKKU_BIN} ${PLUGIN_COMMAND_PREFIX}:backup ${SERVICE} ${BUCKET_NAME}" > "$TMP_CRON_FILE"
+  sudo /bin/mv "$TMP_CRON_FILE" "$CRON_FILE"
+  sudo /bin/chown root:root "$CRON_FILE"
+}
+
+service_backup_unschedule() {
+  declare desc="unschedules the backup of the service"
+  declare SERVICE="$1"
+  local CRON_FILE="/etc/cron.d/dokku-${PLUGIN_COMMAND_PREFIX}-${SERVICE}"
+
+  sudo /bin/rm -f "$CRON_FILE"
+}
+
 service_enter() {
  declare desc="enters running app container of specified proc type"
  declare SERVICE="$1" && shift 1
@@ -411,6 +473,15 @@ service_version() {
  docker inspect -f '{{.Config.Image}}' "$SERVICE_NAME"
 }

+update_plugin_scheme_for_app() {
+  declare desc="Retrieves the updated plugin scheme"
+  declare APP="$1"
+  local DATABASE_SCHEME
+
+  DATABASE_SCHEME=$(config_get "$APP" "${PLUGIN_VARIABLE}_DATABASE_SCHEME" || true)
+  PLUGIN_SCHEME=${DATABASE_SCHEME:-$PLUGIN_SCHEME}
+}
+
 verify_service_name() {
  declare desc="Verifies that a service exists"
  declare SERVICE="$1"
--- a/1
+++ b/1
@@ -13,6 +13,7 @@ export PLUGIN_IMAGE=$MYSQL_IMAGE
 export PLUGIN_IMAGE_VERSION=$MYSQL_IMAGE_VERSION
 export PLUGIN_SCHEME="mysql"
 export PLUGIN_SERVICE="MySQL"
+export PLUGIN_VARIABLE="MYSQL"
 export PLUGIN_BASE_PATH="$PLUGIN_PATH"
 if [[ -n $DOKKU_API_VERSION ]]; then
  export PLUGIN_BASE_PATH="$PLUGIN_ENABLED_PATH"
--- a/6
+++ b/6
@@ -116,9 +116,3 @@ service_url() {
  local SERVICE_ALIAS="$(service_alias "$SERVICE")"
  echo "$PLUGIN_SCHEME://mysql:$PASSWORD@$SERVICE_ALIAS:${PLUGIN_DATASTORE_PORTS[0]}/$SERVICE"
 }
-
-update_plugin_scheme_for_app() {
-  local APP="$1"
-  local MYSQL_DATABASE_SCHEME=$(config_get "$APP" MYSQL_DATABASE_SCHEME)
-  PLUGIN_SCHEME=${MYSQL_DATABASE_SCHEME:-$PLUGIN_SCHEME}
-}
--- a/42
+++ b/42
@@ -2,16 +2,36 @@
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/config"
 set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x

-pull-docker-image() {
-  declare IMAGE="$1"
-  if [[ "$(docker images -q "${IMAGE}" 2> /dev/null)" == "" ]]; then
-    docker pull "${IMAGE}"
-  fi
+plugin-install() {
+  pull-docker-image() {
+    declare IMAGE="$1"
+    if [[ "$(docker images -q "${IMAGE}" 2> /dev/null)" == "" ]]; then
+      docker pull "${IMAGE}"
+    fi
+  }
+
+  pull-docker-image "${PLUGIN_IMAGE}:${PLUGIN_IMAGE_VERSION}"
+  pull-docker-image "svendowideit/ambassador:latest"
+  pull-docker-image "dokkupaas/wait:0.2"
+  pull-docker-image "dokkupaas/s3backup:0.5.0-1"
+  pull-docker-image "busybox:latest"
+
+  mkdir -p "$PLUGIN_DATA_ROOT" || echo "Failed to create $PLUGIN_SERVICE directory"
+  chown dokku:dokku "$PLUGIN_DATA_ROOT"
+
+  rm -f "/etc/sudoers.d/dokku-${PLUGIN_COMMAND_PREFIX}*"
+  _SUDOERS_FILE="/etc/sudoers.d/dokku-${PLUGIN_COMMAND_PREFIX}"
+
+  touch "$_SUDOERS_FILE"
+  cat > "$_SUDOERS_FILE" <<EOL
+%dokku ALL=(ALL) NOPASSWD:/bin/rm -f /etc/cron.d/dokku-${PLUGIN_COMMAND_PREFIX}-*
+%dokku ALL=(ALL) NOPASSWD:/bin/chown root\:root /etc/cron.d/dokku-${PLUGIN_COMMAND_PREFIX}-*
+%dokku ALL=(ALL) NOPASSWD:/bin/mv ${PLUGIN_DATA_ROOT}/.TMP_CRON_FILE /etc/cron.d/dokku-${PLUGIN_COMMAND_PREFIX}-*
+%dokku ALL=(ALL) NOPASSWD:/bin/chown 8983 $PLUGIN_DATA_ROOT/*
+%dokku ALL=(ALL) NOPASSWD:/bin/chgrp 8983 $PLUGIN_DATA_ROOT/*
+EOL
+
+  chmod 0440 "$_SUDOERS_FILE"
 }

-pull-docker-image "${PLUGIN_IMAGE}:${PLUGIN_IMAGE_VERSION}"
-pull-docker-image "svendowideit/ambassador:latest"
-pull-docker-image "dokkupaas/wait:0.2"
-
-mkdir -p "$PLUGIN_DATA_ROOT" || echo "Failed to create $PLUGIN_SERVICE directory"
-chown dokku:dokku "$PLUGIN_DATA_ROOT"
+plugin-install "$@"
--- a/subcommands/backup
+++ b/subcommands/backup
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+source "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")/config"
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_BASE_PATH/common/functions"
+source "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")/functions"
+
+mysql-backup-cmd() {
+  declare desc="creates a backup of the $PLUGIN_SERVICE service to an existing s3 bucket"
+  local cmd="$PLUGIN_COMMAND_PREFIX:backup" argv=("$@"); [[ ${argv[0]} == "$cmd" ]] && shift 1
+  declare SERVICE="$1" BUCKET_NAME="$2"
+
+  [[ -z "$SERVICE" ]] && dokku_log_fail "Please specify a name for the service"
+  [[ -z "$BUCKET_NAME" ]] && dokku_log_fail "Please specify an aws bucket for the backup"
+  verify_service_name "$SERVICE"
+  service_backup "$SERVICE" "$BUCKET_NAME"
+}
+
+mysql-backup-cmd "$@"
--- a/subcommands/backup-auth
+++ b/subcommands/backup-auth
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+source "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")/config"
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_BASE_PATH/common/functions"
+source "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")/functions"
+
+mysql-backup-auth-cmd() {
+  declare desc="sets up authentication for backups on the $PLUGIN_SERVICE service"
+  local cmd="$PLUGIN_COMMAND_PREFIX:backup-auth" argv=("$@"); [[ ${argv[0]} == "$cmd" ]] && shift 1
+  declare SERVICE="$1" AWS_ACCESS_KEY_ID="$2" AWS_SECRET_ACCESS_KEY="$3"
+
+  [[ -z "$SERVICE" ]] && dokku_log_fail "Please specify a name for the service"
+  [[ -z "$AWS_ACCESS_KEY_ID" ]] && dokku_log_fail "Please specify an aws access key id"
+  [[ -z "$AWS_SECRET_ACCESS_KEY" ]] && dokku_log_fail "Please specify an aws secret access key"
+  verify_service_name "$SERVICE"
+  service_backup_auth "$SERVICE" "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY"
+}
+
+mysql-backup-auth-cmd "$@"
--- a/subcommands/backup-deauth
+++ b/subcommands/backup-deauth
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+source "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")/config"
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_BASE_PATH/common/functions"
+source "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")/functions"
+
+mysql-backup-deauth-cmd() {
+  declare desc="removes backup authentication for the $PLUGIN_SERVICE service"
+  local cmd="$PLUGIN_COMMAND_PREFIX:backup-deauth" argv=("$@"); [[ ${argv[0]} == "$cmd" ]] && shift 1
+  declare SERVICE="$1"
+
+  [[ -z "$SERVICE" ]] && dokku_log_fail "Please specify a name for the service"
+  verify_service_name "$SERVICE"
+  service_backup_deauth "$SERVICE"
+}
+
+mysql-backup-deauth-cmd "$@"
--- a/subcommands/backup-schedule
+++ b/subcommands/backup-schedule
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+source "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")/config"
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_BASE_PATH/common/functions"
+source "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")/functions"
+
+mysql-backup-schedule-cmd() {
+  declare desc="schedules a backup of the $PLUGIN_SERVICE service"
+  local cmd="$PLUGIN_COMMAND_PREFIX:backup-schedule" argv=("$@"); [[ ${argv[0]} == "$cmd" ]] && shift 1
+  declare SERVICE="$1" SCHEDULE="$2" BUCKET_NAME="$3"
+
+  [[ -z "$SERVICE" ]] && dokku_log_fail "Please specify a name for the service"
+  [[ -z "$SCHEDULE" ]] && dokku_log_fail "Please specify a schedule for the backup"
+  [[ -z "$BUCKET_NAME" ]] && dokku_log_fail "Please specify an aws bucket for the backup"
+  verify_service_name "$SERVICE"
+  service_backup_schedule "$SERVICE" "$SCHEDULE" "$BUCKET_NAME"
+}
+
+mysql-backup-schedule-cmd "$@"
--- a/subcommands/backup-unschedule
+++ b/subcommands/backup-unschedule
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+source "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")/config"
+set -eo pipefail; [[ $DOKKU_TRACE ]] && set -x
+source "$PLUGIN_BASE_PATH/common/functions"
+source "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")/functions"
+
+mysql-backup-unschedule-cmd() {
+  declare desc="unschedules the backup of the $PLUGIN_SERVICE service"
+  local cmd="$PLUGIN_COMMAND_PREFIX:backup-unschedule" argv=("$@"); [[ ${argv[0]} == "$cmd" ]] && shift 1
+  declare SERVICE="$1"
+
+  [[ -z "$SERVICE" ]] && dokku_log_fail "Please specify a name for the service"
+  verify_service_name "$SERVICE"
+  service_backup_unschedule "$SERVICE"
+}
+
+mysql-backup-unschedule-cmd "$@"
--- a/tests/hook_pre_delete.bats
+++ b/tests/hook_pre_delete.bats
@@ -10,7 +10,7 @@ setup() {
 teardown() {
  dokku "$PLUGIN_COMMAND_PREFIX:unlink" l my_app >&2
  dokku --force "$PLUGIN_COMMAND_PREFIX:destroy" l >&2
-  rm "$DOKKU_ROOT/my_app" -rf
+  rm -rf "$DOKKU_ROOT/my_app"
 }

@test "($PLUGIN_COMMAND_PREFIX:hook:pre-delete) removes app from links file when destroying app" {
--- a/tests/service_link.bats
+++ b/tests/service_link.bats
@@ -8,7 +8,7 @@ setup() {

 teardown() {
  dokku --force "$PLUGIN_COMMAND_PREFIX:destroy" l >&2
-  rm "$DOKKU_ROOT/my_app" -rf
+  rm -rf "$DOKKU_ROOT/my_app"
 }

@test "($PLUGIN_COMMAND_PREFIX:link) error when there are no arguments" {
--- a/tests/service_promote.bats
+++ b/tests/service_promote.bats
@@ -10,7 +10,7 @@ setup() {
 teardown() {
  dokku "$PLUGIN_COMMAND_PREFIX:unlink" l my_app
  dokku --force "$PLUGIN_COMMAND_PREFIX:destroy" l >&2
-  rm "$DOKKU_ROOT/my_app" -rf
+  rm -rf "$DOKKU_ROOT/my_app"
 }

@test "($PLUGIN_COMMAND_PREFIX:promote) error when there are no arguments" {
--- a/tests/service_unlink.bats
+++ b/tests/service_unlink.bats
@@ -8,7 +8,7 @@ setup() {

 teardown() {
  dokku --force "$PLUGIN_COMMAND_PREFIX:destroy" l >&2
-  rm "$DOKKU_ROOT/my_app" -rf
+  rm -rf "$DOKKU_ROOT/my_app"
 }

@test "($PLUGIN_COMMAND_PREFIX:unlink) error when there are no arguments" {
@@ -49,6 +49,6 @@ teardown() {
@test "($PLUGIN_COMMAND_PREFIX:unlink) unsets config url from app" {
  dokku "$PLUGIN_COMMAND_PREFIX:link" l my_app >&2
  dokku "$PLUGIN_COMMAND_PREFIX:unlink" l my_app
-  config=$(dokku config:get my_app DATABASE_URL)
+  config=$(dokku config:get my_app DATABASE_URL || true)
  assert_equal "$config" ""
 }
--- a/1
+++ b/1
@@ -0,0 +1 @@
+install