v3.5.0

===================================================================== --- Changes ------------------------------ - AWS (the cli wrapper) now checks for it's required variables *on run*. This accomodates scwrypts which may need to run in multiple regions, but make the downstream scwrypt responsible for adding AWS_REGION to the REQUIRED_ENV list! - Got rid of all kinds of hackiness surrounding postgres password evals between both the postgres library and the rds library --- Bug Fixes ---------------------------- - scwrypts --update now pulls tags for proper versioning
2023-08-24 17:09:30 -06:00 · 2023-08-24 17:09:30 -06:00 · e199e9bf91
commit e199e9bf91
parent 4c161aba49
11 changed files with 99 additions and 84 deletions
--- a/1
+++ b/1
@ -85,6 +85,7 @@ __RUN() {
 			--update )
 				cd "$SCWRYPTS__ROOT__scwrypts"
 				git fetch --quiet origin main
+				git fetch --quiet origin main --tags
 				local SYNC_STATUS=$?

 				git diff --exit-code origin/main -- . >&2
--- a/zsh/cloud/aws/rds/create-backup
+++ b/zsh/cloud/aws/rds/create-backup
@ -12,13 +12,7 @@ CREATE_BACKUP() {
 	local DB_HOST DB_PORT DB_NAME DB_USER DB_PASS
 	RDS__GET_DATABASE_CREDENTIALS $@ || return 1

-	PG_DUMP \
-		--host $DB_HOST \
-		--port $DB_PORT \
-		--name $DB_NAME \
-		--user $DB_USER \
-		--pass $DB_PASS \
-		;
+	PG_DUMP
 }

 #####################################################################
--- a/zsh/cloud/aws/rds/interactive-login
+++ b/zsh/cloud/aws/rds/interactive-login
@ -12,13 +12,7 @@ RDS_INTERACTIVE_LOGIN() {
 	local DB_HOST DB_PORT DB_NAME DB_USER DB_PASS
 	RDS__GET_DATABASE_CREDENTIALS $@ || return 1

-	POSTGRES__LOGIN_INTERACTIVE \
-		--host $DB_HOST \
-		--port $DB_PORT \
-		--name $DB_NAME \
-		--user $DB_USER \
-		--pass $DB_PASS \
-		;
+	POSTGRES__LOGIN_INTERACTIVE
 }


--- a/zsh/cloud/aws/rds/load-backup
+++ b/zsh/cloud/aws/rds/load-backup
@ -12,13 +12,7 @@ LOAD_BACKUP() {
 	local DB_HOST DB_PORT DB_NAME DB_USER DB_PASS
 	RDS__GET_DATABASE_CREDENTIALS $@ || return 1

-	PG_RESTORE \
-		--host $DB_HOST \
-		--port $DB_PORT \
-		--name $DB_NAME \
-		--user $DB_USER \
-		--pass $DB_PASS \
-		;
+	PG_RESTORE
 }


--- a/zsh/lib/cloud/aws/cli.module.zsh
+++ b/zsh/lib/cloud/aws/cli.module.zsh
@ -4,15 +4,13 @@ DEPENDENCIES+=(
 	aws
 )

-REQUIRED_ENV+=(
-	AWS_ACCOUNT
-	AWS_PROFILE
-	AWS_REGION
-)
+REQUIRED_ENV+=()

 #####################################################################

 AWS() {
+	REQUIRED_ENV=(AWS_REGION AWS_ACCOUNT AWS_PROFILE) CHECK_ENVIRONMENT || return 1
+
 	aws \
 		--profile $AWS_PROFILE \
 		--region  $AWS_REGION \
--- a/zsh/lib/cloud/aws/rds.module.zsh
+++ b/zsh/lib/cloud/aws/rds.module.zsh
@ -86,14 +86,9 @@ RDS__GET_DATABASE_CREDENTIALS() {
 		user-input     ) _RDS_AUTH__userinput ;;
 	esac

-	STATUS
-	STATUS "host     : $DB_HOST"
-	STATUS "type     : $DB_TYPE"
-	STATUS "port     : $DB_PORT"
-	STATUS "database : $DB_NAME"
-	STATUS "username : $DB_USER"
-	[[ $PRINT_PASSWORD -eq 1 ]] && STATUS "password : $DB_PASS"
-	STATUS
+	[[ $PRINT_PASSWORD -eq 1 ]] && INFO "password : $DB_PASS"
+
+	return 0
 }

 _RDS_AUTH__iam() {
@ -108,10 +103,10 @@ _RDS_AUTH__iam() {
 _RDS_AUTH__secretsmanager() {
 	local CREDENTIALS=$(_RDS__GET_SECRETSMANAGER_CREDENTIALS)
 	echo $CREDENTIALS | jq -e '.pass' >/dev/null 2>&1 \
-		&& DB_PASS="'$(echo $CREDENTIALS | jq -r '.pass' | sed "s/'/'\"'\"'/g")'"
+		&& DB_PASS="$(echo $CREDENTIALS | jq -r '.pass')"
 	
 	echo $CREDENTIALS | jq -e '.password' >/dev/null 2>&1 \
-		&& DB_PASS="'$(echo $CREDENTIALS | jq -r '.password' | sed "s/'/'\"'\"'/g")'"
+		&& DB_PASS="$(echo $CREDENTIALS | jq -r '.password')"
 	
 	echo $CREDENTIALS | jq -e '.user' >/dev/null 2>&1 \
 		&& DB_USER=$(echo $CREDENTIALS | jq -r '.user')
--- a/zsh/lib/db/postgres.module.zsh
+++ b/zsh/lib/db/postgres.module.zsh
@ -4,7 +4,6 @@ DEPENDENCIES+=(
 	pg_dump
 	pg_restore
 	psql
-	pgcli
 )

 REQUIRED_ENV+=()
@ -12,22 +11,22 @@ REQUIRED_ENV+=()
 #####################################################################

 PSQL() {
-	[[ ${#ARGS[@]} -eq 0 ]] && POSTGRES__SET_LOGIN_ARGS $@
-
-	eval PGPASSWORD=$_PASS psql ${_ARGS[@]}
+	POSTGRES__SET_LOGIN_ARGS $@
+	eval PGPASSWORD=$(printf '%q ' "$DB_PASS") psql ${PSQL_ARGS[@]}
 }

 #####################################################################

 PG_DUMP() {
-	local _HOST _NAME _PORT _USER _FILE
-	local DATA_DIR _PASS _ARGS=()
+	local DATA_DIR
 	POSTGRES__SET_LOGIN_ARGS --verbose $@

 	local OUTPUT_FILE="$DATA_DIR/backup.$(date '+%Y-%m-%d.%H-%M')"

+
+
 	STATUS "
-	making backup of : $_USER@$_HOST:$_PORT/$_NAME
+	making backup of : $DB_USER@$DB_HOST:$DB_PORT/$DB_NAME

 	    (compressed) : '$OUTPUT_FILE.dump'
 	      (safe-raw) : '$OUTPUT_FILE.sql'
@ -36,23 +35,33 @@ PG_DUMP() {

 	: \
 		&& STATUS "creating compressed backup..." \
-		&& eval PGPASSWORD=$_PASS pg_dump ${_ARGS[@]} --format custom --file "$OUTPUT_FILE.dump" \
+		&& eval PGPASSWORD=$(printf '%q ' "$DB_PASS") psql ${PSQL_ARGS[@]} \
+			--format custom \
+			--file "$OUTPUT_FILE.dump" \
+			--verbose \
 		&& SUCCESS "completed compressed backup" \
 		&& STATUS "creating raw backup..." \
-		&& eval PGPASSWORD=$_PASS pg_dump ${_ARGS[@]} > "$OUTPUT_FILE.raw.sql" \
+		&& pg_restore -f "$OUTPUT_FILE.raw.sql" "$OUTPUT_FILE.dump" \
 		&& SUCCESS "completed raw backup" \
 		&& STATUS "creating single-transaction raw backup..." \
-		&& { echo "BEGIN;"; cat "$OUTPUT_FILE.raw.sql"; echo "END;" } > "$OUTPUT_FILE.sql" \
+		&& { echo "BEGIN;\n"; cat "$OUTPUT_FILE.raw.sql"; echo "\nEND;" } > "$OUTPUT_FILE.sql" \
 		&& SUCCESS "completed single-transaction raw backup" \
-		|| { ERROR "error creating backup for '$_HOST/$_NAME' (see above)"; return 1; }
+		|| { ERROR "error creating backup for '$DB_HOST/$DB_NAME' (see above)"; return 1; }
+
+	SUCCESS "
+	completed backup : $DB_USER@$DB_HOST:$DB_PORT/$DB_NAME
+
+	    (compressed) : '$OUTPUT_FILE.dump'
+	      (safe-raw) : '$OUTPUT_FILE.sql'
+	           (raw) : '$OUTPUT_FILE.raw.sql'
+	 "
 }

 #####################################################################

 PG_RESTORE() {
-	local _HOST _NAME _PORT _USER
-	local _PASS _ARGS=()
-	local _FILE
+	local _ARGS=()
+	local FILE
 	POSTGRES__SET_LOGIN_ARGS $@

 	local INPUT_FILE=$(find "$DATA_DIR"/backup.* -type f | FZF 'select database file to restore')
@ -72,7 +81,7 @@ PG_RESTORE() {
 	[[ $INPUT_FILE =~ \\.dump$ ]] && RAW=0

 	STATUS "
-	loading backup for : $_USER@$_HOST:$_PORT/$_NAME
+	loading backup for : $DB_USER@$DB_HOST:$DB_PORT/$DB_NAME

 	              file : '$INPUT_FILE'
 	"
@ -92,7 +101,7 @@ PG_RESTORE() {
 	}

 	[[ $RAW -eq 0 ]] && {
-		PGPASSWORD="$_PASS" pg_restore ${_ARGS[@]} \
+		eval PGPASSWORD=$(printf '%q ' "$DB_PASS") pg_restore ${PSQL_ARGS[@]} \
 			--verbose \
 			--format custom \
 			--single-transaction \
@ -101,8 +110,8 @@ PG_RESTORE() {
 	}

 	[[ $EXIT_CODE -eq 0 ]] \
-		&& SUCCESS "finished restoring backup for '$_HOST/$_NAME'" \
-		|| ERROR "error restoring backup for '$_HOST/$_NAME' (see above)" \
+		&& SUCCESS "finished restoring backup for '$DB_HOST/$DB_NAME'" \
+		|| ERROR "error restoring backup for '$DB_HOST/$DB_NAME' (see above)" \
 		;

 	return $EXIT_CODE
@ -111,48 +120,70 @@ PG_RESTORE() {
 #####################################################################

 POSTGRES__LOGIN_INTERACTIVE() {
-	local _PASS _ARGS=()
+	DEPENDENCIES=(pgcli) CHECK_ENVIRONMENT --optional \
+		&& COMMAND=pgcli || COMMAND=psql
+
+	[[ $COMMAND =~ psql ]] && WARNING "using 'psql' instead"
+
 	POSTGRES__SET_LOGIN_ARGS $@

-	STATUS "performing login  : $_USER@$_HOST:$_PORT/$_NAME"
-	STATUS "working directory : $DATA_DIR"
+	STATUS "
+	performing login  : $DB_USER@$DB_HOST:$DB_PORT/$DB_NAME
+	working directory : $DATA_DIR
+	 "

-	eval PGPASSWORD=$_PASS pgcli ${_ARGS[@]}
+	eval PGPASSWORD=$(printf '%q ' "$DB_PASS") $COMMAND ${PSQL_ARGS[@]}
 }

 #####################################################################

 POSTGRES__SET_LOGIN_ARGS() {
+	# allow for manual override with PSQL_ARGS
+	[[ ${#PSQL_ARGS[@]} -gt 0 ]] && return 0
+
+	local DATA_DIR_PREFIX
+
 	while [[ $# -gt 0 ]]
 	do
 		case $1 in
-			--host ) _ARGS+=(-h $2); _HOST="$2"; shift 1 ;;
-			--name ) _ARGS+=(-d $2); _NAME="$2"; shift 1 ;;
-			--port ) _ARGS+=(-p $2); _PORT="$2"; shift 1 ;;
-			--user ) _ARGS+=(-U $2); _USER="$2"; shift 1 ;;
+			-h | --host ) DB_HOST="$2"; shift 1 ;;
+			-p | --port ) DB_PORT="$2"; shift 1 ;;
+			-d | --name ) DB_NAME="$2"; shift 1 ;;
+			-U | --user ) DB_USER="$2"; shift 1 ;;
+			-P | --pass ) DB_PASS="$2"; shift 1 ;;

-			--pass ) _PASS="$2"; shift 1 ;;
+			--file  ) PSQL_FILE="$2";  shift 1 ;;

-			--file ) _FILE="$2"; shift 1 ;;
+			--data-dir-prefix ) DATA_DIR_PREFIX="$2"; shift 1 ;;

-			* ) _ARGS+=($1) ;;
+			* ) PSQL_ARGS+=($1) ;;
 		esac
 		shift 1
 	done

-	[ $_FILE ] && [ ! -f "$_FILE" ] && {
-		ERROR "no such file '$_FILE'"
-		exit 1
-	}
+	[ $PSQL_FILE ] && [ ! -f "$PSQL_FILE" ] \
+		&& ERROR "no such file available:\n'$PSQL_FILE'"

-	[ $_HOST ] && [ $_NAME ] \
-		&& DATA_DIR="$SCWRYPTS_DATA_PATH/db/$_HOST/$_NAME" \
+	CHECK_ERRORS
+
+	##########################################
+
+	[ $DATA_DIR_PREFIX ] && {
+		DATA_DIR="$SCWRYPTS_DATA_PATH/$DATA_DIR_PREFIX"
+	} || {
+		[ $DB_HOST ] && [ $DB_NAME ] \
+			&& DATA_DIR="$SCWRYPTS_DATA_PATH/db/$DB_HOST/$DB_NAME" \
 			|| DATA_DIR="$EXECUTION_DIR/temp-db" \
 			;
+	}

-	[ ! -d "$DATA_DIR" ] && mkdir -p "$DATA_DIR"
+	mkdir -p "$DATA_DIR"
 	cd "$DATA_DIR"

-	return 0
-}
+	[ $DB_HOST ] || DB_HOST=127.0.0.1
+	[ $DB_PORT ] || DB_PORT=5432
+	[ $DB_NAME ] || DB_NAME=postgres
+	[ $DB_USER ] || DB_USER=postgres

+	PSQL_ARGS+=(-h $DB_HOST -p $DB_PORT -d $DB_NAME -U $DB_USER)
+}
--- a/zsh/lib/utils/dependencies.zsh
+++ b/zsh/lib/utils/dependencies.zsh
@ -1,5 +1,6 @@
 __CHECK_DEPENDENCIES() {
 	local DEP ERROR=0
+	[ ! $E ] && E=ERROR

 	DEPENDENCIES=($(echo $DEPENDENCIES | sed 's/ \+/\n/g' | sort -u))

@ -13,7 +14,7 @@ __CHECK_DEPENDENCY() {
 	local DEPENDENCY="$1"
 	[ ! $DEPENDENCY ] && return 1
 	command -v $DEPENDENCY >/dev/null 2>&1 || {
-		ERROR "'$1' required but not available on PATH $(__CREDITS $1)"
+		$E "application '$1' "$([[ $OPTIONAL -eq 1 ]] && echo preferred || echo required)" but not available on PATH $(__CREDITS $1)"
 		return 1
 	}
 }
--- a/zsh/lib/utils/environment.zsh
+++ b/zsh/lib/utils/environment.zsh
@ -32,7 +32,7 @@ __CHECK_ENV_VAR() {
 		[ $DEFAULT_VALUE ] && $NAME="$DEFAULT_VALUE"
 		return 0
 	} || {
-		 ERROR "'$NAME' required"
+		ERROR "variable '$NAME' required"
 		return 1
 	}
 }
--- a/zsh/lib/utils/io.zsh
+++ b/zsh/lib/utils/io.zsh
@ -99,7 +99,7 @@ Yn() {
 	PROMPT "$@ [Yn]"
 	[ $CI ] && { echo y; return 0; }

-	local Yn; READ -k Yn; echo
+	local Yn; READ -k Yn; echo >&2
 	[[ $Yn =~ [nN] ]] && return 1 || return 0
 }

@ -107,7 +107,7 @@ yN() {
 	PROMPT "$@ [yN]"
 	[ $CI ] && { echo y; return 0; }

-	local yN; READ -k yN; echo
+	local yN; READ -k yN; echo >&2
 	[[ $yN =~ [yY] ]] && return 0 || return 1
 }

--- a/zsh/lib/utils/utils.module.zsh
+++ b/zsh/lib/utils/utils.module.zsh
@ -18,6 +18,12 @@ source ${0:a:h}/environment.zsh
 #####################################################################

 CHECK_ENVIRONMENT() {
+	local OPTIONAL=0
+	[[ $1 =~ --optional ]] && OPTIONAL=1
+
+	[[ $OPTIONAL -eq 1 ]] \
+		&& E=WARNING || E=ERROR
+
 	local ENVIRONMENT_STATUS=0

 	__CHECK_DEPENDENCIES $DEPENDENCIES
@ -55,9 +61,9 @@ CHECK_ENVIRONMENT() {

 	##########################################

-	[[ ENVIRONMENT_STATUS -eq 0 ]] || {
+	[[ ENVIRONMENT_STATUS -ne 0 ]] && [[ $OPTIONAL -eq 0 ]] && {
 		ERROR_MESSAGE=$(echo $ERROR_MESSAGE | sed '1d; s/^/   /')
-		ERROR "environment errors found (see above)\n$ERROR_MESSAGE"
+		$E "environment errors found (see above)\n$ERROR_MESSAGE"
 	}

 	[[ $MISSING_ENVIRONMENT_VARIABLES -ne 0 ]] && {
@ -67,10 +73,11 @@ CHECK_ENVIRONMENT() {
 		"
 	}

-	[[ $ENVIRONMENT_STATUS -eq 0 ]] || {
-		[[ $NO_EXIT -eq 1 ]] && return $ENVIRONMENT_STATUS
+	[[ $ENVIRONMENT_STATUS -ne 0 ]] && [[ $NO_EXIT -ne 1 ]] && [[ $OPTIONAL -eq 0 ]] && {
 		exit $ENVIRONMENT_STATUS
 	}
+
+	return $ENVIRONMENT_STATUS
 }

 CHECK_ENVIRONMENT