v3.0.0 "The Great Overhaul"

=====================================================================

Notice the major version change which comes with breaking changes to
2.x! Reconstructs "library" functions for both python and zsh scwrypts,
with changes to virtualenv naming conventions (you'll need to refresh
all virtualenv with the appropriate scwrypt).

--- Changes ------------------------------

- changed a naming convention across zsh scripts, particularly
  removing underscores where there is no need to avoid naming clash
  (e.g. 'zsh/lib/utils/io.zsh' renames '__STATUS' to 'STATUS')

- moved clients reliant on py.lib.http to the py.lib.http module

- python scripts now rely on py.lib.scwrypts.execute

- updated package.json in zx scripts to include `type = module`

- 'scwrypts --list' commandline argument now includes additional
  relevant data for each scwrypt

- environment variables no longer add themselves to be staged in the
  '.env.template'

--- New Features -------------------------

- new 'use' syntax for disjoint import within zsh scripts; took me
  a very long time to convince myself this would be necessary

- introduced scwrypt "groups" to allow portable module creation;
  (i.e. ability add your own scripts from another repo!)

- py.lib.scwrypts.io provides a combined IO stream for quick, hybrid
  use of input/output files and stdin/stdout

- py.lib.fzf provides a wrapper to provide similar functionality to
  zsh/utils/io.zsh including fzf_(head|tail)

- improved efficiency of various scwrypts; notably reducing runtime
  of scwrypts/environment sync

- improved scwrypts CLI by adding new options for exact scwrypt
  matching, better filtering, and prettier/more-detailed interfaces

--- New Scripts --------------------------

- py/twilio )
    basic SMS integration with twilio
     - send-sms

- py/directus )
    interactive directus GET query
     - get-items

- py/discord )
    post message to discord channel or webhook
     - post-message
This commit is contained in:
2023-02-21 18:44:27 -07:00
parent 7617c938b1
commit 76a746a53e
196 changed files with 3472 additions and 2053 deletions

View File

@ -1,14 +0,0 @@
_DEPENDENCIES+=()
_REQUIRED_ENV+=()
source ${0:a:h}/../common.zsh
#####################################################################
_SORT_ENV() {
local ENV_FILE="$1"
_SED -i "/^# /d; /^$/d" "$ENV_FILE"
_SED -i "s/^[A-Z]/export &/; s/^[^#=]\\+$/&=/" "$ENV_FILE"
LC_COLLATE=C sort -uo "$ENV_FILE" "$ENV_FILE"
}
_SED() { sed --follow-symlinks $@; }

View File

@ -1,37 +1,33 @@
#!/bin/zsh
_DEPENDENCIES+=()
_REQUIRED_ENV+=()
source ${0:a:h}/common.zsh
DEPENDENCIES+=()
REQUIRED_ENV+=()
use scwrypts/environment-files
CHECK_ENVIRONMENT
#####################################################################
__PROMPT 'choose an environment to copy'
TEMPLATE_ENV_NAME=$(__SELECT_ENV)
[ ! $TEMPLATE_ENV_NAME ] && __ABORT
PROMPT 'choose an environment to copy'
TEMPLATE_ENV_NAME=$(SCWRYPTS__SELECT_ENV)
[ ! $TEMPLATE_ENV_NAME ] && ABORT
__STATUS "selected '$TEMPLATE_ENV_NAME'"
STATUS "selected '$TEMPLATE_ENV_NAME'"
__PROMPT 'enter new environment name'
ENV_NAME=$(echo '' | __FZF_HEAD 'new environment')
[ ! $ENV_NAME ] && __ABORT
PROMPT 'enter new environment name'
ENV_NAME=$(echo '' | FZF_HEAD 'new environment')
[ ! $ENV_NAME ] && ABORT
TEMPLATE_ENV_FILE=$(__GET_ENV_FILE $TEMPLATE_ENV_NAME)
ENV_FILE=$(__GET_ENV_FILE $ENV_NAME)
TEMPLATE_ENV_FILE=$(SCWRYPTS__GET_ENV_FILE $TEMPLATE_ENV_NAME 2>/dev/null)
ENV_FILE=$(SCWRYPTS__GET_ENV_FILE $ENV_NAME)
[ -f "$ENV_FILE" ] && __FAIL 2 "'$ENV_NAME' already exists"
[ -f "$ENV_FILE" ] && FAIL 2 "'$ENV_NAME' already exists"
__STATUS "creating environment '$ENV_NAME'"
cp "$TEMPLATE_ENV_FILE" "$ENV_FILE" \
&& __SUCCESS "created '$ENV_NAME'" \
|| __FAIL 3 "unable to create '$ENV_NAME'"
STATUS "creating environment '$ENV_NAME'"
cat "$TEMPLATE_ENV_FILE" \
| sed 's/ from.*//' \
> "$ENV_FILE" \
&& SCWRYPTS__RUN --name scwrypts/environment/synchronize --group scwrypts --type zsh -- --no-prompt \
&& SUCCESS "created '$ENV_NAME'" \
|| FAIL 3 "something went wrong creating '$ENV_NAME'"
__STATUS 'stripping inherited values'
_SED -i 's/ # inherited from.*$//' "$ENV_FILE" 2>/dev/null
__RUN_SCWRYPT zsh/scwrypts/environment/synchronize -- --no-prompt \
|| __FAIL 4 'failed to run environment sync'
__RUN_SCWRYPT zsh/scwrypts/environment/edit -- $ENV_NAME \
|| __FAIL 4 'failed to edit new environment'
;
__SUCCESS "finished copy environment '$TEMPLATE_ENV_NAME > $ENV_NAME'"
SUCCESS "finished copy environment '$TEMPLATE_ENV_NAME > $ENV_NAME'"

View File

@ -1,25 +1,28 @@
#!/bin/zsh
_DEPENDENCIES+=()
_REQUIRED_ENV+=()
source ${0:a:h}/common.zsh
DEPENDENCIES+=()
REQUIRED_ENV+=()
use scwrypts/environment-files
CHECK_ENVIRONMENT
#####################################################################
__PROMPT 'choose an environment to delete'
ENV_NAME=$(__SELECT_ENV)
[ ! $ENV_NAME ] && __ABORT
PROMPT 'choose an environment to delete'
ENV_NAME=$(SCWRYPTS__SELECT_ENV)
[ ! $ENV_NAME ] && ABORT
ENV_FILE=$(__GET_ENV_FILE $ENV_NAME)
ENV_FILE=$(SCWRYPTS__GET_ENV_FILE $ENV_NAME)
__STATUS "preparing to remove '$ENV_NAME'"
STATUS "preparing to remove '$ENV_NAME'"
__WARNING
__WARNING "the '$ENV_NAME' environment will be removed"
__WARNING 'configured options and stored credentials will be lost forever'
__WARNING
WARNING "
the '$ENV_NAME' environment will be removed
configured options and stored credentials will be lost forever
"
__yN 'continue?' || __ABORT
yN 'continue?' || ABORT
__STATUS "removing environment"
STATUS "removing environment"
rm "$ENV_FILE" \
&& __SUCCESS "removed '$ENV_NAME'" \
|| __FAIL 3 "unable to remove '$ENV_FILE'; is it protected?"
&& SUCCESS "removed '$ENV_NAME'" \
|| FAIL 3 "unable to remove '$ENV_FILE'; is it protected?"

View File

@ -1,7 +1,10 @@
#!/bin/zsh
_DEPENDENCIES+=()
_REQUIRED_ENV+=()
source ${0:a:h}/common.zsh
DEPENDENCIES+=()
REQUIRED_ENV+=()
use scwrypts/environment-files
CHECK_ENVIRONMENT
#####################################################################
[ $1 ] && ENV_NAME="$1"
@ -9,34 +12,23 @@ source ${0:a:h}/common.zsh
[ ! $1 ] && {
[ $SCWRYPTS_ENV ] \
&& ENV_NAME=$SCWRYPTS_ENV \
|| ENV_NAME=$(__SELECT_OR_CREATE_ENV)
|| ENV_NAME=$(SCWRYPTS__SELECT_OR_CREATE_ENV)
}
[ ! $ENV_NAME ] && __ABORT
[ ! $ENV_NAME ] && ABORT
ENV_FILE=$(__GET_ENV_FILE $ENV_NAME)
[ ! -f $ENV_FILE ] && {
__STATUS "Creating '$ENV_NAME'..." \
&& cp $__ENV_TEMPLATE $ENV_FILE \
&& __RUN_SCWRYPT zsh/scwrypts/environment/synchronize -- --no-prompt \
&& __SUCCESS "created '$ENV_NAME'" \
|| { __ERROR "failed to create '$ENV_FILE'"; exit 1; }
ENV_FILE=$(SCWRYPTS__GET_ENV_FILE $ENV_NAME 2>/dev/null)
[ ! -f "$ENV_FILE" ] && {
STATUS "Creating '$ENV_NAME'..." \
&& touch "$ENV_FILE" \
&& SCWRYPTS__RUN --name scwrypts/environment/synchronize --group scwrypts --type zsh -- --no-prompt \
&& SUCCESS "created '$ENV_NAME'" \
|| { ERROR "failed to create '$ENV_FILE'"; exit 1; }
}
__EDIT $ENV_FILE
_SORT_ENV $ENV_FILE
EDIT $ENV_FILE
while read line
do
ENV_VAR=$(echo "$line" | _SED 's/=.*$//; s/^export //')
grep -q "$ENV_VAR" $__ENV_TEMPLATE || {
((NEW_VAR+=1))
echo "export $ENV_VAR=" >> $__ENV_TEMPLATE
__STATUS "detected new variable '$ENV_VAR'"
}
done < $ENV_FILE
__RUN_SCWRYPT zsh/scwrypts/environment/synchronize -- --no-prompt \
|| __FAIL 4 'failed to run environment sync' \
SCWRYPTS__RUN --name scwrypts/environment/synchronize --group scwrypts --type zsh -- --no-prompt \
|| FAIL 4 'failed to run environment sync' \
;
__SUCCESS "environment '$ENV_NAME' successfully modified"
SUCCESS "environment '$ENV_NAME' successfully modified"

View File

@ -1,7 +1,8 @@
#!/bin/zsh
_DEPENDENCIES+=()
_REQUIRED_ENV+=()
source ${0:a:h}/common.zsh
#!/bin/zsh
DEPENDENCIES+=()
REQUIRED_ENV+=()
CHECK_ENVIRONMENT
#####################################################################
__CHECK_REQUIRED_ENV $@

View File

@ -1,154 +1,170 @@
#!/bin/zsh
_DEPENDENCIES+=()
_REQUIRED_ENV+=()
source ${0:a:h}/common.zsh
#!/bin/zsh
DEPENDENCIES+=()
REQUIRED_ENV+=()
use scwrypts/environment-files
CHECK_ENVIRONMENT
#####################################################################
_SYNCHRONIZE() {
SYNCHRONIZE() {
while [[ $# -gt 0 ]]
do
case $1 in
--no-prompt ) SLIENT=1; shift 1 ;;
* ) __WARNING "argument '$1' not recognized"
* ) WARNING "argument '$1' not recognized"
shift 1 ;;
esac
done
local TEMPLATE_FILE
[ ! $SLIENT ] && {
__yN 'change the template before sync?' && __EDIT $__ENV_TEMPLATE
yN 'change the template(s) before sync?' && {
EDIT $(
for TEMPLATE_FILE in $(SCWRYPTS__GET_ENV_TEMPLATE_FILES)
do
[ -f "$TEMPLATE_FILE" ] && echo "$TEMPLATE_FILE"
[ -f "$TEMPLATE_FILE.descriptions" ] && echo "$TEMPLATE_FILE.descriptions"
done
)
}
for TEMPLATE_FILE in $(SCWRYPTS__GET_ENV_TEMPLATE_FILES)
do
{
echo '#!/bin/zsh'
cat "$TEMPLATE_FILE" \
| sed '
/__[a-z_]\+=$/d;
/^#/d; /^$/d;
s/^[A-Z]/export &/; s/^[^#=]\\+$/&=/;
' \
| LC_COLLATE=C sort -u
} > "$TEMPLATE_FILE.temp"
mv "$TEMPLATE_FILE.temp" "$TEMPLATE_FILE"
done
}
_SORT_ENV "$__ENV_TEMPLATE"
_SED -i '/__[a-z_]\+=$/d' "$__ENV_TEMPLATE"
git add $__ENV_TEMPLATE >/dev/null 2>&1
local ENVIRONMENTS ENVIRONMENT_FILES
local FILE NAME ENVIRONMENT_FILE
ENVIRONMENTS=$(__GET_ENV_NAMES | sort -r)
ENVIRONMENTS=($(SCWRYPTS__GET_ENV_NAMES | sort -r))
ENVIRONMENT_FILES=($(
for NAME in ${ENVIRONMENTS[@]}
do
SCWRYPTS__GET_ENV_FILE $NAME
done
))
_CLEAR_INHERITED_VARIABLES
_INSERT_NEW_VARIABLES
_REMOVE_OLD_VARIABLES
_SORT_AND_CASCADE
_ADD_DESCRIPTIONS
STATUS 'generating working environment files...'
for FILE in ${ENVIRONMENT_FILES[@]}
do
GENERATE_TEMP_ENVIRONMENT_FILE "$FILE"
done
__SUCCESS 'finished sync!'
STATUS 'cascading environment values to children...'
for NAME in ${ENVIRONMENTS[@]}
do
CASCADE_ENVIRONMENT $NAME
done
STATUS 'cleaning up working space...'
for FILE in ${ENVIRONMENT_FILES[@]}
do
CLEANUP_ENVIRONMENT_FILE "$FILE"
done
SUCCESS 'finished sync!'
}
#####################################################################
_CLEAR_INHERITED_VARIABLES() {
for ENV_NAME in $(echo $ENVIRONMENTS)
do
ENV_FILE=$(__GET_ENV_FILE $ENV_NAME)
_SED -i 's/ # inherited from.*//' "$ENV_FILE"
done
}
_INSERT_NEW_VARIABLES() {
__STATUS 'inserting new environment variables...'
local ENV_NAME ENV_FILE line
while read line
do
for ENV_NAME in $(echo $ENVIRONMENTS)
do
ENV_FILE=$(__GET_ENV_FILE $ENV_NAME)
grep -q "$line" $ENV_FILE || {
echo $line >> $ENV_FILE && __STATUS "added '$line' to '$ENV_NAME'"
}
done
done < <(_SED -n '/^./p' "$__ENV_TEMPLATE")
}
_REMOVE_OLD_VARIABLES() {
__STATUS 'removing old environment variables...'
local ENV_NAME ENV_FILE line
for ENV_NAME in $(echo $ENVIRONMENTS)
do
ENV_FILE=$(__GET_ENV_FILE $ENV_NAME)
while read line
do
ENV_VAR=$(echo "$line" | _SED 's/=.*/=/')
echo $ENV_VAR | grep -q '__[a-z_]\+=' && continue
grep -q "$ENV_VAR" "$__ENV_TEMPLATE" || {
_SED -i "\\%$ENV_VAR%d" "$ENV_FILE"
echo "$ENV_VAR" | grep -qv '^#' \
&& __WARNING "removed unwanted '$ENV_VAR' from '$ENV_NAME'"
}
done < $ENV_FILE
done
}
_SORT_AND_CASCADE() {
local ENV_NAM ENV_FILE
for ENV_NAME in $(echo $ENVIRONMENTS)
do
ENV_FILE=$(__GET_ENV_FILE $ENV_NAME)
_CASCADE_ENVIRONMENT $ENV_NAME
done
for ENV_NAME in $(echo $ENVIRONMENTS)
do
ENV_FILE=$(__GET_ENV_FILE $ENV_NAME)
_SORT_ENV "$ENV_FILE"
done
}
_CASCADE_ENVIRONMENT() {
CASCADE_ENVIRONMENT() {
local PARENT_NAME="$1"
local PARENT_FILE=$(__GET_ENV_FILE $PARENT_NAME)
local PARENT_FILE="$(SCWRYPTS__GET_ENV_FILE $PARENT_NAME).temp"
local CHILD_NAMES=$(echo $ENVIRONMENTS | grep "^$PARENT_NAME\\.")
[ ! $CHILD_NAMES ] && return 0
local CHILD_NAMES=($(echo $ENVIRONMENTS | sed 's/ \+/\n/g' |grep "^$PARENT_NAME\\."))
[[ ${#CHILD_NAMES[@]} -eq 0 ]] && return 0
__STATUS "cascading '$PARENT_NAME' to children"
for CHILD_NAME in $(echo $CHILD_NAMES)
local PARENT_VARIABLES="$(READ_PARENT_VARIABLES "$PARENT_NAME" "$PARENT_FILE")"
[ ! $PARENT_VARIABLES ] && return 0
local CHILD_NAME CHILD_FILE CONTENT
for CHILD_NAME in ${CHILD_NAMES[@]}
do
__SUCCESS "detected child '$CHILD_NAME'"
STATUS "propagating environment '$PARENT_NAME' to child '$CHILD_NAME'"
CHILD_FILE="$(SCWRYPTS__GET_ENV_FILE $CHILD_NAME).temp"
CONTENT=$(cat "$CHILD_FILE")
echo "$PARENT_VARIABLES" | ADD_LINES
echo "$CONTENT" > "$CHILD_FILE"
done
local PARENT_VAR VAR_PATTERN CHILD_NAME CHILD_FILE
while read PARENT_VAR
do
VAR_PATTERN=$(echo "$PARENT_VAR" | _SED 's/=.*/=/; s/\//\/\//g')
__STATUS "propagating '$(echo $VAR_PATTERN | _SED 's/^export \([^=]*\)=/\1/')' to children"
PARENT_VAR+=" # inherited from $PARENT_NAME"
for CHILD_NAME in $(echo $CHILD_NAMES)
do
CHILD_FILE=$(__GET_ENV_FILE $CHILD_NAME)
_SED -i "/^$VAR_PATTERN/d" "$CHILD_FILE"
echo $PARENT_VAR >> "$CHILD_FILE"
done
done < <(_SED -n '/^[^#][^=]*=[^#]\+$/p' "$PARENT_FILE")
__SUCCESS "finished '$PARENT_NAME' propagation"
}
_ADD_DESCRIPTIONS() {
__STATUS 'updating descriptions'
while read DESCRIPTION_LINE
do
ENV_VAR=$(echo $DESCRIPTION_LINE | _SED 's/ \+| .*$//')
DESCRIPTION=$(echo $DESCRIPTION_LINE | _SED 's/^.* | //')
for ENV_NAME in $(echo $ENVIRONMENTS)
do
_SED -i "/^export $ENV_VAR=/i # $DESCRIPTION" "$(__GET_ENV_FILE $ENV_NAME)"
done
done < <(_SED -n '/^[^ ]\+ \+| /p' "$__ENV_TEMPLATE.descriptions")
for ENV_NAME in $(echo $ENVIRONMENTS)
do
_SED -i "/^# /i \ " "$(__GET_ENV_FILE $ENV_NAME)"
_SED -i "s/^ $//" "$(__GET_ENV_FILE $ENV_NAME)"
done
SUCCESS "finished '$PARENT_NAME' propagation"
}
#####################################################################
_SYNCHRONIZE $@
GENERATE_TEMP_ENVIRONMENT_FILE() {
local FILE="$1"
local CONTENT=$(GENERATE_TEMPLATE)
READ_POPULATED_VARIABLES "$FILE" | ADD_LINES
echo "$CONTENT" > "$FILE.temp"
}
ADD_LINES() {
local LINE VARIABLE SHORT VALUE
while read LINE
do
VARIABLE=$(echo $LINE | sed 's/=.*$//')
echo $CONTENT | grep -qi "^$VARIABLE" || {
echo $LINE | grep -qi '__[a-z_]\+=' || {
WARNING "skipping variable $(echo $LINE | sed 's/^export //; s/=.*//')
(must be included in a template before it can be added)"
continue
}
SHORT=$(echo "$VARIABLE" | sed 's/__[a-z].*//')
CONTENT=$(echo "$CONTENT" | sed "/^$SHORT/a $LINE")
}
CONTENT=$(echo "$CONTENT" | sed "s%^$VARIABLE.*$%$LINE%")
done
}
READ_POPULATED_VARIABLES() {
local FILE="$1"
cat "$FILE" \
| grep -v '^#' \
| grep -v '=$' \
| grep -v '^$' \
| grep -v ' # from ' \
| awk '/^[^=]+$/{printf "%s_____",$0;next}7' \
| sed 's/\(_____\)\(export\)/\1\n\2/; s/\(_____\)$/\1\n/' \
| sed 's/^.*_____.*$/_____&/' \
| sed -z 's/[\n
] *_____/_____/g' \
| grep -v '^$' \
;
}
READ_PARENT_VARIABLES() {
local PARENT_NAME="$1"
local PARENT_FILE="$2"
READ_POPULATED_VARIABLES "$PARENT_FILE" \
| sed 's/_____/ /g; s/\s\+/ /g' \
| sed 's/( /(/; s/ )/)/' \
| sed "s/$/ # from $PARENT_NAME/" \
| grep -v '__[a-z_]\+=' \
;
}
CLEANUP_ENVIRONMENT_FILE() {
cat "$1.temp" \
| sed 's/_____$//g; s/_____/\n/g' \
> "$1"
rm "$1.temp"
}
#####################################################################