v3.0.0 "The Great Overhaul"

=====================================================================

Notice the major version change which comes with breaking changes to
2.x! Reconstructs "library" functions for both python and zsh scwrypts,
with changes to virtualenv naming conventions (you'll need to refresh
all virtualenv with the appropriate scwrypt).

--- Changes ------------------------------

- changed a naming convention across zsh scripts, particularly
  removing underscores where there is no need to avoid naming clash
  (e.g. 'zsh/lib/utils/io.zsh' renames '__STATUS' to 'STATUS')

- moved clients reliant on py.lib.http to the py.lib.http module

- python scripts now rely on py.lib.scwrypts.execute

- updated package.json in zx scripts to include `type = module`

- 'scwrypts --list' commandline argument now includes additional
  relevant data for each scwrypt

- environment variables no longer add themselves to be staged in the
  '.env.template'

--- New Features -------------------------

- new 'use' syntax for disjoint import within zsh scripts; took me
  a very long time to convince myself this would be necessary

- introduced scwrypt "groups" to allow portable module creation;
  (i.e. ability add your own scripts from another repo!)

- py.lib.scwrypts.io provides a combined IO stream for quick, hybrid
  use of input/output files and stdin/stdout

- py.lib.fzf provides a wrapper to provide similar functionality to
  zsh/utils/io.zsh including fzf_(head|tail)

- improved efficiency of various scwrypts; notably reducing runtime
  of scwrypts/environment sync

- improved scwrypts CLI by adding new options for exact scwrypt
  matching, better filtering, and prettier/more-detailed interfaces

--- New Scripts --------------------------

- py/twilio )
    basic SMS integration with twilio
     - send-sms

- py/directus )
    interactive directus GET query
     - get-items

- py/discord )
    post message to discord channel or webhook
     - post-message

py/lib/__init__.py

@@ -0,0 +1,6 @@
+import py.lib.data
+import py.lib.fzf
+import py.lib.http
+import py.lib.redis
+import py.lib.scwrypts
+import py.lib.twilio

py/lib/data/__init__.py

@@ -0,0 +1 @@
+import py.lib.data.converter

py/lib/data/converter.py

@@ -2,18 +2,13 @@ import csv
 import json
 import yaml
 
-from py.lib.data.io import get_stream
 
-
-def convert(input_file, input_type, output_file, output_type):
+def convert(input_stream, input_type, output_stream, output_type):
     if input_type == output_type:
         raise ValueError('input type and output type are the same')
 
-    with get_stream(input_file) as input_stream:
-        data = convert_input(input_stream, input_type)
-
-    with get_stream(output_file, 'w+') as output_stream:
-        _write_output(output_stream, output_type, data)
+    data = convert_input(input_stream, input_type)
+    write_output(output_stream, output_type, data)
 
 
 def convert_input(stream, input_type):
@@ -28,7 +23,8 @@ def convert_input(stream, input_type):
             'yaml': _read_yaml,
             }[input_type](stream)
 
-def _write_output(stream, output_type, data):
+
+def write_output(stream, output_type, data):
     supported_output_types = {'csv', 'json', 'yaml'}
 
     if output_type not in supported_output_types:
@@ -40,6 +36,7 @@ def _write_output(stream, output_type, data):
             'yaml': _write_yaml,
             }[output_type](stream, data)
 
+
 #####################################################################
 
 def _read_csv(stream):

py/lib/fzf/__init__.py

@@ -0,0 +1 @@
+from py.lib.fzf.client import fzf, fzf_tail, fzf_head

py/lib/fzf/client.py

@@ -0,0 +1,61 @@
+from pyfzf.pyfzf import FzfPrompt
+
+FZF_PROMPT = None
+
+
+def fzf( # pylint: disable=too-many-arguments
+        choices=None,
+        prompt=None,
+        fzf_options='',
+        delimiter='\n',
+        return_type=str,
+        force_list=False,
+        ):
+    global FZF_PROMPT # pylint: disable=global-statement
+
+    if choices is None:
+        choices = []
+
+    if not isinstance(return_type, type):
+        raise ValueError(f'return_type must be a valid python type; "{return_type}" is not a type')
+
+    if FZF_PROMPT is None:
+        FZF_PROMPT = FzfPrompt()
+
+    options = ' '.join({
+        '-i',
+        '--layout=reverse',
+        '--ansi',
+        '--height=30%',
+        f'--prompt "{prompt} : "' if prompt is not None else '',
+        fzf_options,
+        })
+
+    selections = [
+            return_type(selection)
+            for selection in FZF_PROMPT.prompt(choices, options, delimiter)
+            ]
+
+    if not force_list:
+        if len(selections) == 0:
+            return None
+
+        if len(selections) == 1:
+            return selections[0]
+
+    return selections
+
+
+def fzf_tail(*args, **kwargs):
+    return _fzf_print(*args, **kwargs)[-1]
+
+def fzf_head(*args, **kwargs):
+    return _fzf_print(*args, **kwargs)[0]
+
+def _fzf_print(*args, fzf_options='', **kwargs):
+    return fzf(
+            *args,
+            **kwargs,
+            fzf_options = f'--print-query {fzf_options}',
+            force_list = True,
+            )

py/lib/http/__init__.py

@@ -1 +1,5 @@
 from py.lib.http.client import get_request_client
+
+import py.lib.http.directus
+import py.lib.http.discord
+import py.lib.http.linear

py/lib/http/directus/__init__.py

@@ -0,0 +1,2 @@
+from py.lib.http.directus.client import *
+from py.lib.http.directus.constant import *

py/lib/http/directus/client.py

@@ -0,0 +1,56 @@
+from py.lib.http import get_request_client
+from py.lib.scwrypts import getenv
+
+
+REQUEST     = None
+COLLECTIONS = None
+FIELDS      = {}
+
+
+def request(method, endpoint, **kwargs):
+    global REQUEST # pylint: disable=global-statement
+
+    if REQUEST is None:
+        REQUEST = get_request_client(
+                base_url = getenv("DIRECTUS__BASE_URL"),
+                headers = {
+                    'Authorization': f'bearer {getenv("DIRECTUS__API_TOKEN")}',
+                    }
+                )
+
+    return REQUEST(method, endpoint, **kwargs)
+
+def graphql(query, system=False):
+    return request(
+            'POST',
+            'graphql' if system is True else 'graphql/system',
+            json={'query': query},
+            )
+
+
+def get_collections():
+    global COLLECTIONS # pylint: disable=global-statement
+
+    if COLLECTIONS is None:
+        COLLECTIONS = [
+            item['collection']
+            for item in request(
+                'GET',
+                'collections?limit=-1&fields[]=collection',
+                ).json()['data']
+            ]
+
+    return COLLECTIONS
+
+
+def get_fields(collection):
+    if FIELDS.get(collection) is None:
+        FIELDS[collection] = [
+                item['field']
+                for item in request(
+                    'GET',
+                    f'fields/{collection}?limit=-1&fields[]=field',
+                    ).json()['data']
+                ]
+
+    return FIELDS[collection]

py/lib/http/directus/constant.py

@@ -0,0 +1,25 @@
+FILTER_OPERATORS = {
+        '_eq',
+        '_neq',
+        '_lt',
+        '_lte',
+        '_gt',
+        '_gte',
+        '_in',
+        '_nin',
+        '_null',
+        '_nnull',
+        '_contains',
+        '_ncontains',
+        '_starts_with',
+        '_ends_with',
+        '_nends_with',
+        '_between',
+        '_nbetween',
+        '_empty',
+        '_nempty',
+        '_intersects',
+        '_nintersects',
+        '_intersects_bbox',
+        '_nintersects_bbox',
+        }

py/lib/http/discord/__init__.py

@@ -0,0 +1,2 @@
+from py.lib.http.discord.client import *
+from py.lib.http.discord.send_message import *

py/lib/http/discord/client.py

@@ -0,0 +1,20 @@
+from py.lib.http import get_request_client
+from py.lib.scwrypts import getenv
+
+REQUEST = None
+
+def request(method, endpoint, **kwargs):
+    global REQUEST # pylint: disable=global-statement
+
+    if REQUEST is None:
+        headers = {}
+
+        if (token := getenv("DISCORD__BOT_TOKEN", required = False)) is not None:
+            headers['Authorization'] = f'Bot {token}'
+
+        REQUEST = get_request_client(
+                base_url = 'https://discord.com/api',
+                headers = headers,
+                )
+
+    return REQUEST(method, endpoint, **kwargs)

py/lib/http/discord/send_message.py

@@ -0,0 +1,34 @@
+from py.lib.scwrypts import getenv
+from py.lib.http.discord import request
+
+def send_message(content, channel_id=None, webhook=None, avatar_url=None, **kwargs):
+    if channel_id is None:
+        channel_id = getenv('DISCORD__DEFAULT_CHANNEL_ID', required=False)
+
+    if avatar_url is None:
+        avatar_url = getenv('DISCORD__DEFAULT_AVATAR_URL', required=False)
+
+    endpoint = None
+
+    if webhook is not None:
+        endpoint = f'webhooks/{webhook}'
+    elif channel_id is not None:
+        endpoint = f'channels/{channel_id}/messages'
+    else:
+        raise ValueError('must provide target channel_id or webhook')
+
+
+    return request(
+            method = 'POST',
+            endpoint = endpoint,
+            json = {
+                key: value
+                for key, value in {
+                    'content': content,
+                    'username': 'wrobot',
+                    'avatar_url': avatar_url,
+                    **kwargs,
+                    }.items()
+                if value is not None
+                },
+            )

py/lib/http/linear/__init__.py

@@ -0,0 +1 @@
+from py.lib.http.linear.client import *

py/lib/http/linear/client.py

@@ -0,0 +1,20 @@
+from py.lib.http import get_request_client
+from py.lib.scwrypts import getenv
+
+REQUEST = None
+
+def request(method, endpoint, **kwargs):
+    global REQUEST # pylint: disable=global-statement
+
+    if REQUEST is None:
+        REQUEST = get_request_client(
+                base_url = 'https://api.linear.app',
+                headers = {
+                    'Authorization': f'bearer {getenv("LINEAR__API_TOKEN")}',
+                    }
+                )
+
+    return REQUEST(method, endpoint, **kwargs)
+
+def graphql(query):
+    return request('POST', 'graphql', json={'query': query})

py/lib/linear/__init__.py

@@ -1 +0,0 @@
-from py.lib.linear.client import request, graphql

py/lib/linear/client.py

@@ -1,13 +0,0 @@
-from py.lib.http import get_request_client
-from py.lib.scwrypts import getenv
-
-
-request = get_request_client(
-        base_url = 'https://api.linear.app',
-        headers = {
-            'Authorization': f'bearer {getenv("LINEAR__API_TOKEN")}',
-            }
-        )
-
-def graphql(query):
-    return request('POST', 'graphql', json={'query': query})

py/lib/redis/__init__.py

@@ -1 +1 @@
-
+from py.lib.redis.client import get_client

py/lib/redis/client.py

@@ -2,14 +2,18 @@ from redis import StrictRedis
 
 from py.lib.scwrypts import getenv
 
+CLIENT = None
 
-class RedisClient(StrictRedis):
-    def __init__(self):
-        super().__init__(
+def get_client():
+    global CLIENT # pylint: disable=global-statement
+
+    if CLIENT is None:
+        print('getting redis client')
+        CLIENT = StrictRedis(
                 host = getenv('REDIS_HOST'),
                 port = getenv('REDIS_PORT'),
                 password = getenv('REDIS_AUTH', required=False),
                 decode_responses = True,
                 )
 
-Client = RedisClient()
+    return CLIENT

py/lib/scwrypts/__init__.py

@@ -1,3 +1,6 @@
+from py.lib.scwrypts.execute import execute
 from py.lib.scwrypts.getenv import getenv
 from py.lib.scwrypts.interactive import interactive
 from py.lib.scwrypts.run import run
+
+import py.lib.scwrypts.io

py/lib/scwrypts/exceptions.py

@@ -1,3 +1,16 @@
-class MissingVariableError(Exception):
+from argparse import ArgumentError
+
+
+class MissingVariableError(EnvironmentError):
     def init(self, name):
         super().__init__(f'Missing required environment variable "{name}"')
+
+
+class ImportedExecutableError(ImportError):
+    def __init__(self):
+        super().__init__('executable only; must run through scwrypts')
+
+
+class MissingFlagAndEnvironmentVariableError(EnvironmentError, ArgumentError):
+    def __init__(self, flags, env_var):
+        super().__init__(f'must provide at least one of : {{ flags: {flags} OR {env_var} }}')

py/lib/scwrypts/execute.py

@@ -0,0 +1,23 @@
+from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
+
+from py.lib.scwrypts.io import get_combined_stream, add_io_arguments
+
+
+def execute(main, description=None, parse_args=None, toggle_input=True, toggle_output=True):
+    if parse_args is None:
+        parse_args = []
+
+    parser = ArgumentParser(
+            description = description,
+            formatter_class = ArgumentDefaultsHelpFormatter,
+            )
+
+    add_io_arguments(parser, toggle_input, toggle_output)
+
+    for a in parse_args:
+        parser.add_argument(*a[0], **a[1])
+
+    args = parser.parse_args()
+
+    with get_combined_stream(args.input_file, args.output_file) as stream:
+        return main(args, stream)

py/lib/scwrypts/getenv.py

@@ -1,16 +1,15 @@
 from os import getenv as os_getenv
 
 from py.lib.scwrypts.exceptions import MissingVariableError
-from py.lib.scwrypts.run import run
 
 
 def getenv(name, required=True):
     value = os_getenv(name, None)
 
-    if value == None:
-        run('zsh/scwrypts/environment/stage-variables', name)
-
     if required and not value:
         raise MissingVariableError(name)
 
+    if value == '':
+        value = None
+
     return value

py/lib/scwrypts/interactive.py

@@ -1,11 +1,22 @@
 from bpython import embed
 
 
-def interactive(function):
-    def main(*args, **kwargs):
-        print('preparing interactive environment...')
-        local_vars = function(*args, **kwargs)
-        print('environment ready; user, GO! :)')
-        embed(local_vars)
+def interactive(variable_descriptions):
+    def outer(function):
 
-    return main
+        def inner(*args, **kwargs):
+
+            print('\npreparing interactive environment...\n')
+
+            local_vars = function(*args, **kwargs)
+
+            print('\n\n'.join([
+                f'>>> {x}' for x in variable_descriptions
+                ]))
+            print('\nenvironment ready; user, GO! :)\n')
+
+            embed(local_vars)
+
+        return inner
+
+    return outer

py/lib/scwrypts/io.py

@@ -30,6 +30,9 @@ def get_stream(filename=None, mode='r', encoding='utf-8', verbose=False, **kwarg
 
         yield stdin if is_read else stdout
 
+        if not is_read:
+            stdout.flush()
+
 
 def add_io_arguments(parser, toggle_input=True, toggle_output=True):
     if toggle_input:
@@ -49,3 +52,35 @@ def add_io_arguments(parser, toggle_input=True, toggle_output=True):
                 help     = 'path to output file; omit for stdout',
                 required = False,
                 )
+
+
+@contextmanager
+def get_combined_stream(input_file=None, output_file=None):
+    with get_stream(input_file, 'r') as input_stream, get_stream(output_file, 'w+') as output_stream:
+        yield CombinedStream(input_stream, output_stream)
+
+
+class CombinedStream:
+    def __init__(self, input_stream, output_stream):
+        self.input = input_stream
+        self.output = output_stream
+
+    def read(self, *args, **kwargs):
+        return self.input.read(*args, **kwargs)
+
+    def readline(self, *args, **kwargs):
+        return self.input.readline(*args, **kwargs)
+
+    def readlines(self, *args, **kwargs):
+        return self.input.readlines(*args, **kwargs)
+
+    def write(self, *args, **kwargs):
+        return self.output.write(*args, **kwargs)
+
+    def writeline(self, line):
+        x = self.output.write(f'{line}\n')
+        self.output.flush()
+        return x
+
+    def writelines(self, *args, **kwargs):
+        return self.output.writelines(*args, **kwargs)

py/lib/scwrypts/run.py

@@ -7,8 +7,9 @@ def run(scwrypt_name, *args):
     DEPTH = int(getenv('SUBSCWRYPT', '0'))
     DEPTH += 1
 
-    SCWRYPTS_EXE = Path(__file__).parents[2] / 'scwrypts'
+    SCWRYPTS_EXE = Path(__file__).parents[3] / 'scwrypts'
     ARGS = ' '.join([str(x) for x in args])
+    print(f'SUBSCWRYPT={DEPTH} {SCWRYPTS_EXE} {scwrypt_name} -- {ARGS}')
 
     print(f'\n {"--"*DEPTH} ({DEPTH}) BEGIN SUBSCWRYPT : {Path(scwrypt_name).name}')
     subprocess_run(

py/lib/twilio/__init__.py

@@ -0,0 +1,2 @@
+from py.lib.twilio.client import get_client
+from py.lib.twilio.send_sms import send_sms

py/lib/twilio/client.py

@@ -0,0 +1,18 @@
+from twilio.rest import Client
+
+from py.lib.scwrypts import getenv
+
+CLIENT = None
+
+def get_client():
+    global CLIENT # pylint: disable=global-statement
+
+    if CLIENT is None:
+        print('loading client')
+        CLIENT = Client(
+                username = getenv('TWILIO__API_KEY'),
+                password = getenv('TWILIO__API_SECRET'),
+                account_sid = getenv('TWILIO__ACCOUNT_SID'),
+                )
+
+    return CLIENT

py/lib/twilio/send_sms.py

@@ -0,0 +1,57 @@
+from json import dumps
+from time import sleep
+
+from py.lib.twilio.client import get_client
+
+
+def send_sms(to, from_, body, max_char_count=300, stream=None):
+    '''
+    abstraction for twilio.client.messages.create which will break
+    messages into multi-part SMS rather than throwing an error or
+    requiring the use of MMS data
+
+    @param to               messages.create parameter
+    @param from_            messages.create parameter
+    @param body             messages.create parameter
+    @param max_char_count   1 ≤ N ≤ 1500 (default 300)
+    @param stream           used to report success/failure (optional)
+
+    @return   a list of twilio MessageInstance objects
+    '''
+    client = get_client()
+    messages = []
+
+    max_char_count = max(1, min(max_char_count, 1500))
+
+    total_sms_parts = 1 + len(body) // max_char_count
+    contains_multiple_parts = total_sms_parts > 1
+
+    for i in range(0, len(body), max_char_count):
+        msg_body = body[i:i+max_char_count]
+        current_part = 1 + i // max_char_count
+
+        if contains_multiple_parts:
+            msg_body = f'{current_part}/{total_sms_parts}\n{msg_body}'
+
+        message = client.messages.create(
+                to = to,
+                from_ = from_,
+                body = msg_body,
+                )
+
+        messages.append(message)
+
+        if stream is not None:
+            stream.writeline(
+                    dumps({
+                        'sid': message.sid,
+                        'to': to,
+                        'from': from_,
+                        'body': msg_body,
+                        })
+                    )
+
+        if contains_multiple_parts:
+            sleep(2 if max_char_count <= 500 else 5)
+
+    return messages