first time setup and more refinements

contrib/docker-compose/.env

@@ -274,6 +274,8 @@ DB_DATABASE="pixelfed_prod"
 # See: https://docs.pixelfed.org/technical-documentation/config/#db_port
 DB_PORT="3306"
 
+ENTRYPOINT_DEBUG=0
+
 ###############################################################
 # Mail configuration
 ###############################################################

contrib/docker-compose/docker-compose.yml

@@ -4,10 +4,10 @@ version: "3"
 services:
   web:
     image: "${DOCKER_IMAGE}:${DOCKER_TAG}"
-    # build:
+    build:
-    #   context: ../..
+      context: ../..
-    #   dockerfile: contrib/docker/Dockerfile
+      dockerfile: contrib/docker/Dockerfile
-    #   target: apache-runtime
+      target: apache-runtime
     restart: unless-stopped
     env_file:
       - "./.env"
@@ -23,10 +23,10 @@ services:
 
   worker:
     image: "${DOCKER_IMAGE}:${DOCKER_TAG}"
-    # build:
+    build:
-    #   context: ../..
+      context: ../..
-    #   dockerfile: contrib/docker/Dockerfile
+      dockerfile: contrib/docker/Dockerfile
-    #   target: apache-runtime
+      target: apache-runtime
     command: gosu www-data php artisan horizon
     restart: unless-stopped
     env_file:

contrib/docker/shared/root/docker/entrypoint.d/01-permissions.sh

@@ -3,7 +3,8 @@ source /docker/helpers.sh
 
 entrypoint-set-script-name "$0"
 
-# Ensure the two Docker volumes are owned by the runtime user
+# Ensure the two Docker volumes and dot-env files are owned by the runtime user as other scripts
+# will be writing to these
 run-as-current-user chown --verbose ${RUNTIME_UID}:${RUNTIME_GID} "./.env"
 run-as-current-user chown --verbose ${RUNTIME_UID}:${RUNTIME_GID} "./bootstrap/cache"
 run-as-current-user chown --verbose ${RUNTIME_UID}:${RUNTIME_GID} "./storage"
@@ -22,5 +23,5 @@ fi
 
 for path in "${ensure_ownership_paths[@]}"; do
     log-info "Ensure ownership of [${path}] is correct"
-    run-as-current-user chown --recursive ${RUNTIME_UID}:${RUNTIME_GID} "${path}"
+    stream-prefix-command-output run-as-current-user chown --recursive ${RUNTIME_UID}:${RUNTIME_GID} "${path}"
 done

contrib/docker/shared/root/docker/entrypoint.d/05-templating.sh

@@ -49,7 +49,7 @@ find "${ENTRYPOINT_TEMPLATE_DIR}" -follow -type f -print | while read -r templat
     cat "${template_file}" | gomplate >"${output_file_path}"
 
     # Show the diff from the envsubst command
-    if [[ ${ENTRYPOINT_SHOW_TEMPLATE_DIFF} = 1 ]]; then
+    if [[ ${ENTRYPOINT_SHOW_TEMPLATE_DIFF:-1} = 1 ]]; then
-        git --no-pager diff "${template_file}" "${output_file_path}" || :
+        git --no-pager diff --color=always "${template_file}" "${output_file_path}" || :
     fi
 done

contrib/docker/shared/root/docker/entrypoint.d/11-first-time-setup.sh

@@ -3,36 +3,12 @@ source /docker/helpers.sh
 
 entrypoint-set-script-name "$0"
 
-# if the script is running in another container, wait for it to complete
+await-database-ready
-while [ -e "./storage/docker-first-time-is-running" ]; do
-    sleep 1
-done
 
-# We got the lock!
+only-once "storage:link" run-as-runtime-user php artisan storage:link
-touch "./storage/docker-first-time-is-running"
+only-once "key:generate" run-as-runtime-user php artisan key:generate
-
+only-once "initial:migrate" run-as-runtime-user php artisan migrate --force
-# Make sure to clean up on exit
+only-once "import:cities" run-as-runtime-user php artisan import:cities
-trap "rm -f ./storage/docker-first-time-is-running" EXIT
-
-if [ ! -e "./storage/docker-storage-link-has-run" ]; then
-    run-as-runtime-user php artisan storage:link
-    touch "./storage/docker-storage-link-has-run"
-fi
-
-if [ ! -e "./storage/docker-key-generate-has-run" ]; then
-    run-as-runtime-user php artisan key:generate
-    touch "./storage/docker-key-generate-has-run"
-fi
-
-if [ ! -e "./storage/docker-migrate-has-run" ]; then
-    run-as-runtime-user php artisan migrate --force
-    touch "./storage/docker-migrate-has-run"
-fi
-
-if [ ! -e "./storage/docker-import-cities-has-run" ]; then
-    run-as-runtime-user php artisan import:cities
-    touch "./storage/docker-import-cities-has-run"
-fi
 
 # if [ ! -e "./storage/docker-instance-actor-has-run" ]; then
 #     run-as-runtime-user php artisan instance:actor

contrib/docker/shared/root/docker/entrypoint.d/12-migrations.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+source /docker/helpers.sh
+
+entrypoint-set-script-name "$0"
+
+await-database-ready
+
+declare new_migrations=0
+run-as-runtime-user php artisan migrate:status | grep No && migrations=yes || migrations=no

contrib/docker/shared/root/docker/entrypoint.d/30-cache.sh

@@ -3,6 +3,6 @@ source /docker/helpers.sh
 
 entrypoint-set-script-name "$0"
 
+run-as-runtime-user php artisan config:cache
 run-as-runtime-user php artisan route:cache
 run-as-runtime-user php artisan view:cache
-run-as-runtime-user php artisan config:cache

contrib/docker/shared/root/docker/entrypoint.sh

@@ -31,6 +31,8 @@ if is-directory-empty "${ENTRYPOINT_ROOT}"; then
     exec "$@"
 fi
 
+acquire-lock
+
 # Start scanning for entrypoint.d files to source or run
 log-info "looking for shell scripts in [${ENTRYPOINT_ROOT}]"
 
@@ -50,9 +52,9 @@ find "${ENTRYPOINT_ROOT}" -follow -type f -print | sort -V | while read -r file;
             log-error-and-exit "File [${file}] is not executable (please 'chmod +x' it)"
         fi
 
-        log-info
+        log-info ""
-        log-info "Sourcing [${file}]"
+        log-info "${notice_message_color}Sourcing [${file}]${color_clear}"
-        log-info
+        log-info ""
 
         source "${file}"
 
@@ -67,9 +69,9 @@ find "${ENTRYPOINT_ROOT}" -follow -type f -print | sort -V | while read -r file;
             log-error-and-exit "File [${file}] is not executable (please 'chmod +x' it)"
         fi
 
-        log-info
+        log-info ""
-        log-info "Running [${file}]"
+        log-info "${notice_message_color}Executing [${file}]${color_clear}"
-        log-info
+        log-info ""
 
         "${file}"
         ;;
@@ -80,6 +82,8 @@ find "${ENTRYPOINT_ROOT}" -follow -type f -print | sort -V | while read -r file;
     esac
 done
 
+release-lock
+
 log-info "Configuration complete; ready for start up"
 
 exec "$@"

contrib/docker/shared/root/docker/helpers.sh

@@ -6,14 +6,16 @@ set -e -o errexit -o nounset -o pipefail
 # Some splash of color for important messages
 declare -g error_message_color="\033[1;31m"
 declare -g warn_message_color="\033[1;34m"
+declare -g notice_message_color="\033[1;34m"
 declare -g color_clear="\033[1;0m"
 
 # Current and previous log prefix
+declare -g script_name=
+declare -g script_name_previous=
 declare -g log_prefix=
-declare -g log_prefix_previous=
 
 # dot-env files to source when reading config
-declare -ra dot_env_files=(
+declare -a dot_env_files=(
     /var/www/.env.docker
     /var/www/.env
 )
@@ -21,16 +23,24 @@ declare -ra dot_env_files=(
 # environment keys seen when source dot files (so we can [export] them)
 declare -ga seen_dot_env_variables=()
 
+declare -g docker_state_path="$(readlink -f ./storage/docker)"
+declare -g docker_locks_path="${docker_state_path}/lock"
+declare -g docker_once_path="${docker_state_path}/once"
+
+declare -g runtime_username=$(id -un ${RUNTIME_UID})
+
 # @description Restore the log prefix to the previous value that was captured in [entrypoint-set-script-name ]
 # @arg $1 string The name (or path) of the entrypoint script being run
 function entrypoint-set-script-name() {
-    log_prefix_previous="${log_prefix}"
+    script_name_previous="${script_name}"
-    log_prefix="ENTRYPOINT - [$(get-entrypoint-script-name $1)] - "
+    script_name="${1}"
+
+    log_prefix="[entrypoint / $(get-entrypoint-script-name $1)] - "
 }
 
 # @description Restore the log prefix to the previous value that was captured in [entrypoint-set-script-name ]
 function entrypoint-restore-script-name() {
-    log_prefix="${log_prefix_previous}"
+    entrypoint-set-script-name "${script_name_previous}"
 }
 
 # @description Run a command as the [runtime user]
@@ -38,7 +48,7 @@ function entrypoint-restore-script-name() {
 # @exitcode 0 if the command succeeeds
 # @exitcode 1 if the command fails
 function run-as-runtime-user() {
-    run-command-as "$(id -un ${RUNTIME_UID})" "${@}"
+    run-command-as "${runtime_username}" "${@}"
 }
 
 # @description Run a command as the [runtime user]
@@ -64,9 +74,9 @@ function run-command-as() {
     log-info-stderr "👷 Running [${*}] as [${target_user}]"
 
     if [[ ${target_user} != "root" ]]; then
-        su --preserve-environment "${target_user}" --shell /bin/bash --command "${*}"
+        stream-prefix-command-output su --preserve-environment "${target_user}" --shell /bin/bash --command "${*}"
     else
-        "${@}"
+        stream-prefix-command-output "${@}"
     fi
 
     exit_code=$?
@@ -80,11 +90,62 @@ function run-command-as() {
     return $exit_code
 }
 
+# @description Streams stdout from the command and echo it
+# with log prefixing.
+# @see stream-prefix-command-output
+function stream-stdout-handler() {
+    local prefix="${1:-}"
+
+    while read line; do
+        log-info "(stdout) ${line}"
+    done
+}
+
+# @description Streams stderr from the command and echo it
+# with a bit of color and log prefixing.
+# @see stream-prefix-command-output
+function stream-stderr-handler() {
+    while read line; do
+        log-info-stderr "(${error_message_color}stderr${color_clear}) ${line}"
+    done
+}
+
+# @description Steam stdout and stderr from a command with log prefix
+# and stdout/stderr prefix. If stdout or stderr is being piped/redirected
+# it will automatically fall back to non-prefixed output.
+# @arg $@ string The command to run
+function stream-prefix-command-output() {
+    local stdout=stream-stdout-handler
+    local stderr=stream-stderr-handler
+
+    # if stdout is being piped, print it like normal with echo
+    if [ ! -t 1 ]; then
+        stdout= echo >&1 -ne
+    fi
+
+    # if stderr is being piped, print it like normal with echo
+    if [ ! -t 2 ]; then
+        stderr= echo >&2 -ne
+    fi
+
+    "$@" > >($stdout) 2> >($stderr)
+}
+
 # @description Print the given error message to stderr
 # @arg $message string A error message.
 # @stderr The error message provided with log prefix
 function log-error() {
-    echo -e "${error_message_color}${log_prefix}ERROR - ${*}${color_clear}" >/dev/stderr
+    local msg
+
+    if [[ $# -gt 0 ]]; then
+        msg="$@"
+    elif [[ ! -t 0 ]]; then
+        read msg || log-error-and-exit "[${FUNCNAME}] could not read from stdin"
+    else
+        log-error-and-exit "[${FUNCNAME}] did not receive any input arguments and STDIN is empty"
+    fi
+
+    echo -e "${error_message_color}${log_prefix}ERROR - ${msg}${color_clear}" >/dev/stderr
 }
 
 # @description Print the given error message to stderr and exit 1
@@ -94,6 +155,8 @@ function log-error() {
 function log-error-and-exit() {
     log-error "$@"
 
+    show-call-stack
+
     exit 1
 }
 
@@ -101,15 +164,35 @@ function log-error-and-exit() {
 # @arg $@ string A warning message.
 # @stderr The warning message provided with log prefix
 function log-warning() {
-    echo -e "${warn_message_color}${log_prefix}WARNING - ${*}${color_clear}" >/dev/stderr
+    local msg
+
+    if [[ $# -gt 0 ]]; then
+        msg="$@"
+    elif [[ ! -t 0 ]]; then
+        read msg || log-error-and-exit "[${FUNCNAME}] could not read from stdin"
+    else
+        log-error-and-exit "[${FUNCNAME}] did not receive any input arguments and STDIN is empty"
+    fi
+
+    echo -e "${warn_message_color}${log_prefix}WARNING - ${msg}${color_clear}" >/dev/stderr
 }
 
 # @description Print the given message to stdout unless [ENTRYPOINT_QUIET_LOGS] is set
 # @arg $@ string A info message.
 # @stdout The info message provided with log prefix unless $ENTRYPOINT_QUIET_LOGS
 function log-info() {
+    local msg
+
+    if [[ $# -gt 0 ]]; then
+        msg="$@"
+    elif [[ ! -t 0 ]]; then
+        read msg || log-error-and-exit "[${FUNCNAME}] could not read from stdin"
+    else
+        log-error-and-exit "[${FUNCNAME}] did not receive any input arguments and STDIN is empty"
+    fi
+
     if [ -z "${ENTRYPOINT_QUIET_LOGS:-}" ]; then
-        echo "${log_prefix}$*"
+        echo -e "${log_prefix}${msg}"
     fi
 }
 
@@ -117,8 +200,18 @@ function log-info() {
 # @arg $@ string A info message.
 # @stderr The info message provided with log prefix unless $ENTRYPOINT_QUIET_LOGS
 function log-info-stderr() {
+    local msg
+
+    if [[ $# -gt 0 ]]; then
+        msg="$@"
+    elif [[ ! -t 0 ]]; then
+        read msg || log-error-and-exit "[${FUNCNAME}] could not read from stdin"
+    else
+        log-error-and-exit "[${FUNCNAME}] did not receive any input arguments and STDIN is empty"
+    fi
+
     if [ -z "${ENTRYPOINT_QUIET_LOGS:-}" ]; then
-        echo "${log_prefix}$*"
+        echo -e "${log_prefix}$msg" >/dev/stderr
     fi
 }
 
@@ -196,3 +289,164 @@ function ensure-directory-exists() {
 function get-entrypoint-script-name() {
     echo "${1#"$ENTRYPOINT_ROOT"}"
 }
+
+# @description Ensure a command is only run once (via a 'lock' file) in the storage directory.
+#   The 'lock' is only written if the passed in command ($2) successfully ran.
+# @arg $1 string The name of the lock file
+# @arg $@ string The command to run
+function only-once() {
+    local name="${1:-$script_name}"
+    local file="${docker_once_path}/${name}"
+    shift
+
+    if [[ -e "${file}" ]]; then
+        log-info "Command [${*}] has already run once before (remove file [${file}] to run it again)"
+
+        return 0
+    fi
+
+    ensure-directory-exists "$(dirname "${file}")"
+
+    if ! "$@"; then
+        return 1
+    fi
+
+    touch "${file}"
+    return 0
+}
+
+# @description Best effort file lock to ensure *something* is not running in multiple containers.
+#   The script uses "trap" to clean up after itself if the script crashes
+# @arg $1 string The lock identifier
+function acquire-lock() {
+    local name="${1:-$script_name}"
+    local file="${docker_locks_path}/${name}"
+
+    ensure-directory-exists "$(dirname "${file}")"
+
+    log-info "🔑 Trying to acquire lock: ${file}: "
+    while [[ -e "${file}" ]]; do
+        log-info "🔒 Waiting on lock ${file}"
+
+        staggered-sleep
+    done
+
+    touch "${file}"
+
+    log-info "🔐 Lock acquired [${file}]"
+
+    on-trap "release-lock ${name}" EXIT INT QUIT TERM
+}
+
+# @description Release a lock aquired by [acquire-lock]
+# @arg $1 string The lock identifier
+function release-lock() {
+    local name="${1:-$script_name}"
+    local file="${docker_locks_path}/${name}"
+
+    log-info "🔓 Releasing lock [${file}]"
+
+    rm -f "${file}"
+}
+
+# @description Helper function to append multiple actions onto
+#   the bash [trap] logic
+# @arg $1 string The command to run
+# @arg $@ string The list of trap signals to register
+function on-trap() {
+    local trap_add_cmd=$1
+    shift || log-error-and-exit "${FUNCNAME} usage error"
+
+    for trap_add_name in "$@"; do
+        trap -- "$(
+            # helper fn to get existing trap command from output
+            # of trap -p
+            extract_trap_cmd() { printf '%s\n' "${3:-}"; }
+            # print existing trap command with newline
+            eval "extract_trap_cmd $(trap -p "${trap_add_name}")"
+            # print the new trap command
+            printf '%s\n' "${trap_add_cmd}"
+        )" "${trap_add_name}" ||
+            log-error-and-exit "unable to add to trap ${trap_add_name}"
+    done
+}
+
+# Set the trace attribute for the above function.
+#
+# This is required to modify DEBUG or RETURN traps because functions don't
+# inherit them unless the trace attribute is set
+declare -f -t on-trap
+
+# @description Waits for the database to be healthy and responsive
+function await-database-ready() {
+    log-info "❓ Waiting for database to be ready"
+
+    case "${DB_CONNECTION:-}" in
+    mysql)
+        while ! echo "SELECT 1" | mysql --user="$DB_USERNAME" --password="$DB_PASSWORD" --host="$DB_HOST" "$DB_DATABASE" --silent >/dev/null; do
+            staggered-sleep
+        done
+        ;;
+
+    pgsql)
+        while ! echo "SELECT 1" | psql --user="$DB_USERNAME" --password="$DB_PASSWORD" --host="$DB_HOST" "$DB_DATABASE" >/dev/null; do
+            staggered-sleep
+        done
+        ;;
+
+    sqlsrv)
+        log-warning "Don't know how to check if SQLServer is *truely* ready or not - so will just check if we're able to connect to it"
+
+        while ! timeout 1 bash -c "cat < /dev/null > /dev/tcp/${DB_HOST}/${DB_PORT}"; do
+            staggered-sleep
+        done
+        ;;
+
+    sqlite)
+        log-info "sqlite are always ready"
+        ;;
+
+    *)
+        log-error-and-exit "Unknown database type: [${DB_CONNECT}]"
+        ;;
+    esac
+
+    log-info "✅ Successfully connected to database"
+}
+
+# @description sleeps between 1 and 3 seconds to ensure a bit of randomness
+#   in multiple scripts/containers doing work almost at the same time.
+function staggered-sleep() {
+    sleep $(get-random-number-between 1 3)
+}
+
+# @description Helper function to get a random number between $1 and $2
+# @arg $1 int Minimum number in the range (inclusive)
+# @arg $2 int Maximum number in the range (inclusive)
+function get-random-number-between() {
+    local -i from=${1:-1}
+    local -i to="${2:-10}"
+
+    shuf -i "${from}-${to}" -n 1
+}
+
+# @description Helper function to show the bask call stack when something
+#   goes wrong. Is super useful when needing to debug an issue
+function show-call-stack() {
+    local stack_size=${#FUNCNAME[@]}
+    local func
+    local lineno
+    local src
+
+    # to avoid noise we start with 1 to skip the get_stack function
+    for ((i = 1; i < $stack_size; i++)); do
+        func="${FUNCNAME[$i]}"
+        [ x$func = x ] && func=MAIN
+
+        lineno="${BASH_LINENO[$((i - 1))]}"
+        src="${BASH_SOURCE[$i]}"
+        [ x"$src" = x ] && src=non_file_source
+
+        log-error "  at: ${func} ${src}:${lineno}"
+    done
+}

contrib/docker/shared/root/docker/install/base.sh

@@ -23,6 +23,7 @@ declare -ra standardPackages=(
     libzip-dev
     locales
     locales-all
+    moreutils
     nano
     procps
     software-properties-common
@@ -63,6 +64,8 @@ declare -ra videoProcessing=(
 declare -ra databaseDependencies=(
     libpq-dev
     libsqlite3-dev
+    mariadb-client
+    postgresql-client
 )
 
 apt-get update