refactor py/lib into python-scwrypts subproject

py/lib/.gitignore

@@ -0,0 +1 @@
+dist/

py/lib/README.md

@@ -0,0 +1,3 @@
+# Python Scwrypts
+[![Generic Badge](https://img.shields.io/badge/python->=3.9-informational.svg)](https://python.org)
+<br>

py/lib/__init__.py

@@ -1,6 +0,0 @@
-import py.lib.data
-import py.lib.fzf
-import py.lib.http
-import py.lib.redis
-import py.lib.scwrypts
-import py.lib.twilio

py/lib/data/__init__.py

@@ -1 +0,0 @@
-import py.lib.data.converter

py/lib/fzf/__init__.py

@@ -1 +0,0 @@
-from py.lib.fzf.client import fzf, fzf_tail, fzf_head

py/lib/http/__init__.py

@@ -1,5 +0,0 @@
-from py.lib.http.client import get_request_client
-
-import py.lib.http.directus
-import py.lib.http.discord
-import py.lib.http.linear

py/lib/http/directus/__init__.py

@@ -1,2 +0,0 @@
-from py.lib.http.directus.client import *
-from py.lib.http.directus.constant import *

py/lib/http/discord/__init__.py

@@ -1,2 +0,0 @@
-from py.lib.http.discord.client import *
-from py.lib.http.discord.send_message import *

py/lib/http/linear/__init__.py

@@ -1 +0,0 @@
-from py.lib.http.linear.client import *

py/lib/pyproject.toml

@@ -0,0 +1,61 @@
+[project]
+name = 'scwrypts'
+description = 'scwrypts library and invoker'
+license = 'GPL-3.0-or-later'
+
+readme = 'README.md'
+requires-python = '>=3.10'
+
+authors = [
+	{ name='yage', email='yage@yage.io' },
+]
+
+
+classifiers = [
+	'Programming Language :: Python :: 3',
+	'Programming Language :: Python :: 3.10',
+	'Programming Language :: Python :: 3.11',
+	'Programming Language :: Python :: 3.12',
+	'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)',
+	]
+
+dynamic = ['version']
+
+dependencies = [
+	'bpython',
+	'pyfzf',
+	'pyyaml',
+	'redis',
+	'twilio',
+]
+
+[project.optional-dependencies]
+dev = [
+	'pylint',
+]
+
+test = [
+	'pytest',
+	'mergedeep',
+]
+
+[project.urls]
+homepage = 'https://github.com/wrynegade/scwrypts'
+issues = 'https://github.com/wrynegade/scwrypts/issues'
+
+[build-system]
+requires = [
+	'hatchling',
+	'versioningit',
+]
+build-backend = 'hatchling.build'
+
+[tool.hatch.version]
+source = 'versioningit'
+
+[tool.hatch.build.targets.wheel]
+packages = ['./']
+
+[tool.versioningit]
+match = ['v*']
+

py/lib/redis/__init__.py

@@ -1 +0,0 @@
-from py.lib.redis.client import get_client

py/lib/scwrypts/__init__.py

@@ -1,6 +1,9 @@
-from py.lib.scwrypts.execute import execute
-from py.lib.scwrypts.getenv import getenv
-from py.lib.scwrypts.interactive import interactive
-from py.lib.scwrypts.run import run
+'''
+scwrypts
 
-import py.lib.scwrypts.io
+python library functions and invoker for scwrypts
+'''
+
+from .scwrypts.execute import execute
+from .scwrypts.interactive import interactive
+from .scwrypts.scwrypts import scwrypts

py/lib/scwrypts/data/__init__.py

@@ -0,0 +1 @@
+from .converter import convert

py/lib/scwrypts/data/converter.py

@@ -4,9 +4,6 @@ import yaml
 
 
 def convert(input_stream, input_type, output_stream, output_type):
-    if input_type == output_type:
-        raise ValueError('input type and output type are the same')
-
     data = convert_input(input_stream, input_type)
     write_output(output_stream, output_type, data)
 

py/lib/scwrypts/data/test_converter.py

@@ -0,0 +1,75 @@
+from io import StringIO
+#from string import ascii_letters, digits
+from unittest.mock import patch
+
+from pytest import raises
+
+from scwrypts.test import generate
+
+from .converter import convert
+
+GENERATE_OPTIONS = {
+        'depth': 1,
+        'minimum': -999999,
+        'maximum':  999999,
+        'dict_key_types': {str, int},
+        'csv_columns_minimum': 10,
+        'csv_columns_maximum': 64,
+        'csv_rows_minimum': 10,
+        'csv_rows_maximum': 64,
+        }
+
+INPUT_TYPES  = {'csv', 'json', 'yaml'}
+OUTPUT_TYPES = {'csv', 'json', 'yaml'}
+
+
+def test_convert_to_csv():
+    for input_type in INPUT_TYPES:
+        input_stream = generate(input_type, {
+            **GENERATE_OPTIONS,
+            'data_types': {bool,int,float,str},
+            })
+
+        if isinstance(input_stream, str):
+            input_stream = StringIO(input_stream)
+
+
+        convert(input_stream, input_type, StringIO(), 'csv')
+
+def test_convert_to_json():
+    for input_type in INPUT_TYPES:
+        input_stream = generate(input_type, GENERATE_OPTIONS)
+
+        if isinstance(input_stream, str):
+            input_stream = StringIO(input_stream)
+
+        convert(input_stream, input_type, StringIO(), 'json')
+
+def test_convert_to_yaml():
+    for input_type in INPUT_TYPES:
+        input_stream = generate(input_type, GENERATE_OPTIONS)
+
+        if isinstance(input_stream, str):
+            input_stream = StringIO(input_stream)
+
+        convert(input_stream, input_type, StringIO(), 'yaml')
+
+
+def test_convert_deep_json_to_yaml():
+    input_stream = StringIO(generate('json', {**GENERATE_OPTIONS, 'depth': 4}))
+    convert(input_stream, 'json', StringIO(), 'yaml')
+
+def test_convert_deep_yaml_to_json():
+    input_stream = generate('yaml', {**GENERATE_OPTIONS, 'depth': 4})
+    convert(input_stream, 'yaml', StringIO(), 'json')
+
+
+def test_convert_output_unsupported():
+    for input_type in list(INPUT_TYPES):
+        with raises(ValueError):
+            convert(StringIO(), input_type, StringIO(), generate(str))
+
+def test_convert_input_unsupported():
+    for output_type in list(OUTPUT_TYPES):
+        with raises(ValueError):
+            convert(StringIO(), generate(str), StringIO(), output_type)

py/lib/scwrypts/env.py

@@ -1,6 +1,6 @@
 from os import getenv as os_getenv
 
-from py.lib.scwrypts.exceptions import MissingVariableError
+from .scwrypts.exceptions import MissingVariableError
 
 
 def getenv(name, required=True):

py/lib/scwrypts/fzf/__init__.py

@@ -0,0 +1 @@
+from .client import fzf, fzf_tail, fzf_head

py/lib/scwrypts/http/__init__.py

@@ -0,0 +1 @@
+from .client import get_request_client

py/lib/scwrypts/http/directus/__init__.py

@@ -1,3 +1,5 @@
+from .client import *
+
 FILTER_OPERATORS = {
         '_eq',
         '_neq',

py/lib/scwrypts/http/directus/client.py

@@ -1,5 +1,6 @@
-from py.lib.http import get_request_client
-from py.lib.scwrypts import getenv
+from scwrypts.env import getenv
+
+from .. import get_request_client
 
 
 REQUEST     = None

py/lib/scwrypts/http/discord/__init__.py

@@ -0,0 +1,2 @@
+from .client import *
+from .send_message import *

py/lib/scwrypts/http/discord/client.py

@@ -1,5 +1,5 @@
-from py.lib.http import get_request_client
-from py.lib.scwrypts import getenv
+from scwrypts.env import getenv
+from scwrypts.http import get_request_client
 
 REQUEST = None
 

py/lib/scwrypts/http/discord/send_message.py

@@ -1,5 +1,5 @@
-from py.lib.scwrypts import getenv
-from py.lib.http.discord import request
+from scwrypts.env import getenv
+from .client import request
 
 def send_message(content, channel_id=None, webhook=None, username=None, avatar_url=None, **kwargs):
     if username is None:

py/lib/scwrypts/http/linear/__init__.py

@@ -0,0 +1 @@
+from .client import *

py/lib/scwrypts/http/linear/client.py

@@ -1,5 +1,6 @@
-from py.lib.http import get_request_client
-from py.lib.scwrypts import getenv
+from scwrypts.env import getenv
+
+from .. import get_request_client
 
 REQUEST = None
 

py/lib/scwrypts/io/__init__.py

@@ -0,0 +1 @@
+from .combined_io_stream import get_combined_stream, add_io_arguments

py/lib/scwrypts/io/combined_io_stream.py

@@ -2,7 +2,7 @@ from contextlib import contextmanager
 from pathlib import Path
 from sys import stdin, stdout, stderr
 
-from py.lib.scwrypts.getenv import getenv
+from scwrypts.env import getenv
 
 
 @contextmanager
@@ -34,8 +34,8 @@ def get_stream(filename=None, mode='r', encoding='utf-8', verbose=False, **kwarg
             stdout.flush()
 
 
-def add_io_arguments(parser, toggle_input=True, toggle_output=True):
-    if toggle_input:
+def add_io_arguments(parser, allow_input=True, allow_output=True):
+    if allow_input:
         parser.add_argument(
                 '-i', '--input-file',
                 dest     = 'input_file',
@@ -44,7 +44,7 @@ def add_io_arguments(parser, toggle_input=True, toggle_output=True):
                 required = False,
                 )
 
-    if toggle_output:
+    if allow_output:
         parser.add_argument(
                 '-o', '--output-file',
                 dest     = 'output_file',

py/lib/scwrypts/redis/__init__.py

@@ -0,0 +1 @@
+from .client import get_client

py/lib/scwrypts/redis/client.py

@@ -1,6 +1,6 @@
 from redis import StrictRedis
 
-from py.lib.scwrypts import getenv
+from scwrypts.env import getenv
 
 CLIENT = None
 

py/lib/scwrypts/run.py

@@ -1,22 +0,0 @@
-from os import getenv
-from pathlib import Path
-from subprocess import run as subprocess_run
-
-
-def run(scwrypt_name, *args):
-    DEPTH = int(getenv('SUBSCWRYPT', '0'))
-    DEPTH += 1
-
-    SCWRYPTS_EXE = Path(__file__).parents[3] / 'scwrypts'
-    ARGS = ' '.join([str(x) for x in args])
-    print(f'SUBSCWRYPT={DEPTH} {SCWRYPTS_EXE} {scwrypt_name} -- {ARGS}')
-
-    print(f'\n {"--"*DEPTH} ({DEPTH}) BEGIN SUBSCWRYPT : {Path(scwrypt_name).name}')
-    subprocess_run(
-        f'SUBSCWRYPT={DEPTH} {SCWRYPTS_EXE} {scwrypt_name} -- {ARGS}',
-        shell=True,
-        executable='/bin/zsh',
-        check=False,
-        )
-
-    print(f' {"--"*DEPTH} ({DEPTH}) END SUBSCWRYPT   : {Path(scwrypt_name).name}\n')

py/lib/scwrypts/scwrypts/exceptions.py

@@ -6,11 +6,11 @@ class MissingVariableError(EnvironmentError):
         super().__init__(f'Missing required environment variable "{name}"')
 
 
-class ImportedExecutableError(ImportError):
-    def __init__(self):
-        super().__init__('executable only; must run through scwrypts')
-
-
 class MissingFlagAndEnvironmentVariableError(EnvironmentError, ArgumentError):
     def __init__(self, flags, env_var):
         super().__init__(f'must provide at least one of : {{ flags: {flags} OR {env_var} }}')
+
+
+class MissingScwryptsExecutableError(EnvironmentError):
+    def __init__(self):
+        super().__init__(f'scwrypts must be installed and available on your PATH')

py/lib/scwrypts/scwrypts/execute.py

@@ -1,9 +1,12 @@
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 
-from py.lib.scwrypts.io import get_combined_stream, add_io_arguments
+from scwrypts.io import get_combined_stream, add_io_arguments
 
 
-def execute(main, description=None, parse_args=None, toggle_input=True, toggle_output=True):
+def execute(main, description=None, parse_args=None, allow_input=True, allow_output=True):
+    '''
+    API to initiate a python-based scwrypt
+    '''
     if parse_args is None:
         parse_args = []
 
@@ -12,7 +15,7 @@ def execute(main, description=None, parse_args=None, toggle_input=True, toggle_o
             formatter_class = ArgumentDefaultsHelpFormatter,
             )
 
-    add_io_arguments(parser, toggle_input, toggle_output)
+    add_io_arguments(parser, allow_input, allow_output)
 
     for a in parse_args:
         parser.add_argument(*a[0], **a[1])

py/lib/scwrypts/scwrypts/interactive.py

@@ -2,6 +2,9 @@ from bpython import embed
 
 
 def interactive(variable_descriptions):
+    '''
+    main() decorator to drop to interactive python environment upon completion
+    '''
     def outer(function):
 
         def inner(*args, **kwargs):

py/lib/scwrypts/scwrypts/scwrypts.py

@@ -0,0 +1,30 @@
+from os import getenv
+from shutil import which
+from subprocess import run
+
+from .exceptions import MissingScwryptsExecutableError
+
+
+def scwrypts(name, group, _type, *args, log_level=None):
+    '''
+    invoke non-python scwrypts from python
+    '''
+    executable = which('scwrypts')
+    if executable is None:
+        raise MissingScwryptsExecutableError()
+
+    pre_args = ''
+
+    if log_level is not None:
+        pre_args += '--log-level {log_level}'
+
+    depth = getenv('SUBSCWRYPT', '')
+    if depth != '':
+        depth = int(depth) + 1
+
+    return run(
+        f'SUBSCWRYPT={depth} {executable} --name {name} --group {group} --type {_type} {pre_args} -- {" ".join(args)}',
+        shell=True,
+        executable='/bin/zsh',
+        check=False,
+        )

py/lib/scwrypts/test/__init__.py

@@ -0,0 +1 @@
+from .random_generator import generate

py/lib/scwrypts/test/exceptions.py

@@ -0,0 +1,10 @@
+class GeneratorError(Exception):
+    pass
+
+class NoDataTypeError(GeneratorError, ValueError):
+    def __init__(self):
+        super().__init__('must provide at least one data type (either "data_type" or "data_types")')
+
+class BadGeneratorTypeError(GeneratorError, ValueError):
+    def __init__(self, data_type):
+        super().__init__(f'no generator exists for data type "{data_type}"')

py/lib/scwrypts/test/random_generator.py

@@ -0,0 +1,240 @@
+from csv import writer, QUOTE_NONNUMERIC
+from io import StringIO
+from json import dumps
+from random import randint, uniform, choice
+from re import sub
+from string import printable
+from yaml import safe_dump
+
+from .exceptions import NoDataTypeError, BadGeneratorTypeError
+
+
+SUPPORTED_DATA_TYPES = None
+
+DEFAULT_OPTIONS = {
+        'data_types': None,
+        'minimum':  0,
+        'maximum': 64,
+        'depth': 1,
+        'character_set': None,
+        'bool_nullable': False,
+        'str_length': None,
+        'str_minimum_length': 0,
+        'str_maximum_length': 32,
+        'list_length': 8,
+        'set_length': 8,
+        'dict_length': 8,
+        'dict_key_types': {int, float, chr, str},
+        'csv_columns': None,
+        'csv_columns_minimum': 1,
+        'csv_columns_maximum': 16,
+        'csv_rows': None,
+        'csv_rows_minimum': 2,
+        'csv_rows_maximum': 16,
+        'json_initial_type': dict,
+        'yaml_initial_type': dict,
+        }
+
+def generate(data_type=None, options=None):
+    '''
+    generate random data with the call of a function
+        use data_type to generate a single value
+
+        use options to set generation options (key = type, value = kwargs)
+
+        use options.data_types and omit data_type to generate a random type
+    '''
+    if options is None:
+        options = {**DEFAULT_OPTIONS}
+    else:
+        options = DEFAULT_OPTIONS | options
+
+    if data_type is None and options['data_types'] is None:
+        raise NoDataTypeError()
+
+    if data_type is None and options['data_types'] is not None:
+        return generate(data_type=choice(list(options['data_types'])), options=options)
+
+    if not isinstance(data_type, str):
+        data_type = data_type.__name__
+
+    if data_type not in Generator.get_supported_data_types():
+        raise BadGeneratorTypeError(data_type)
+
+    return getattr(Generator, f'_{data_type}')(options)
+
+#####################################################################
+
+class Generator:
+
+    @classmethod
+    def get_supported_data_types(cls):
+        global SUPPORTED_DATA_TYPES  # pylint: disable=global-statement
+        if SUPPORTED_DATA_TYPES is None:
+            SUPPORTED_DATA_TYPES = {
+                    sub('^_', '', data_type)
+                    for data_type, method in Generator.__dict__.items()
+                    if isinstance(method, staticmethod)
+                    }
+
+        return SUPPORTED_DATA_TYPES
+
+
+
+    @staticmethod
+    def _bool(options):
+        return choice([True, False, None]) if options['bool_nullable'] else choice([True, False])
+
+
+    @staticmethod
+    def _int(options):
+        return randint(options['minimum'], options['maximum'])
+
+
+    @staticmethod
+    def _float(options):
+        return uniform(options['minimum'], options['maximum'])
+
+
+    @staticmethod
+    def _chr(options):
+        character_set = options['character_set']
+        return choice(character_set) if character_set is not None else chr(randint(0,65536))
+
+
+    @staticmethod
+    def _str(options):
+        length = options['str_length']
+        if length is None:
+            length = generate(int, {
+                'minimum': options['str_minimum_length'],
+                'maximum': options['str_maximum_length'],
+                })
+        return ''.join((generate(chr, options) for _ in range(length)))
+
+
+    @staticmethod
+    def _list(options):
+        if options['depth'] <= 0:
+            return []
+
+        options['depth'] -= 1
+
+        if options['data_types'] is None:
+            options['data_types'] = {bool, int, float, chr, str}
+
+        return [ generate(None, options) for _ in range(options['list_length']) ]
+
+
+    @staticmethod
+    def _set(options):
+        if options['depth'] <= 0:
+            return set()
+
+        options['depth'] -= 1
+
+        if options['data_types'] is None:
+            options['data_types'] = {bool, int, float, chr, str}
+
+        set_options = options | {'data_types': options['data_types'] - {list, dict, set}}
+
+        return { generate(None, set_options) for _ in range(options['set_length']) }
+
+
+    @staticmethod
+    def _dict(options):
+        if options['depth'] <= 0:
+            return {}
+
+        options['depth'] -= 1
+
+        if options['data_types'] is None:
+            options['data_types'] = {bool, int, float, chr, str, list, set, dict}
+
+        if len(options['data_types']) == 0:
+            return {}
+
+        key_options = options | {'data_types': options['dict_key_types']}
+        return {
+                generate(None, key_options): generate(None, options)
+                for _ in range(options['dict_length'])
+                }
+
+
+    @staticmethod
+    def _csv(options):
+        '''
+        creates a StringIO object containing csv data
+        '''
+        if options['data_types'] is None:
+            options['data_types'] = {int, float, str}
+
+        columns = options['csv_columns']
+        if columns is None:
+            columns = max(1, generate(int, {
+                'minimum': options['csv_columns_minimum'],
+                'maximum': options['csv_columns_maximum'],
+                }))
+
+        rows = options['csv_rows']
+        if rows is None:
+            rows = max(1, generate(int, {
+                'minimum': options['csv_rows_minimum'],
+                'maximum': options['csv_rows_maximum'],
+                }))
+
+        if options['character_set'] is None:
+            options['character_set'] = printable
+
+        csv = StringIO()
+        csv_writer = writer(csv, quoting=QUOTE_NONNUMERIC)
+
+        options['list_length'] = columns
+
+        for line in [ generate(list, options) for _ in range(rows) ]:
+            csv_writer.writerow(line)
+
+        csv.seek(0)
+        return csv
+
+
+    @staticmethod
+    def _json(options):
+        '''
+        creates a str containing json data
+        '''
+        if options['data_types'] is None:
+            options['data_types'] = {bool, int, float, str, list, dict}
+
+        if options['character_set'] is None:
+            options['character_set'] = printable
+
+        options['dict_key_types'] = { int, float, str }
+
+        data = generate(options['json_initial_type'], options)
+        return dumps(data)
+
+
+    @staticmethod
+    def _yaml(options):
+        '''
+        creates a StringIO object containing yaml data
+        '''
+        if options['data_types'] is None:
+            options['data_types'] = {bool, int, float, str, list, dict}
+
+        if options['character_set'] is None:
+            options['character_set'] = printable
+
+        options['dict_key_types'] = { int, float, str }
+
+        yaml = StringIO()
+        safe_dump(generate(options['yaml_initial_type'], options), yaml, default_flow_style=False)
+
+        yaml.seek(0)
+        return yaml
+
+
+#####################################################################
+if __name__ == '__main__':
+    print(generate('json', {'depth': 3}))

py/lib/scwrypts/test/test_random_generator.py

@@ -0,0 +1,44 @@
+from os import getenv
+from random import randint
+
+from .random_generator import generate, Generator
+
+
+ITERATIONS = int(getenv('PYTEST_ITERATIONS__scwrypts__test__random_generator', getenv('PYTEST_ITERATIONS', '999')))
+
+
+def test_generate():  # generators should be quick and "just work" (no Exceptions)
+    for data_type in Generator.get_supported_data_types():
+        for _ in range(ITERATIONS):
+            generate(data_type)
+
+
+def test_generate_depth_deep():
+    for data_type in Generator.get_supported_data_types():
+        generate(data_type, {'depth': 4})
+
+def test_generate_depth_shallow():
+    for data_type in Generator.get_supported_data_types():
+        generate(data_type, {'depth': randint(-999, 0)})
+
+
+def test_generate_range_all():
+    for data_type in Generator.get_supported_data_types():
+        generate(data_type, {'minimum': -99, 'maximum': 99})
+
+def test_generate_range_positive():
+    for data_type in Generator.get_supported_data_types():
+        generate(data_type, {'minimum':   1, 'maximum': 99})
+
+def test_generate_range_zero():
+    for data_type in Generator.get_supported_data_types():
+        generate(data_type, {'minimum':   3, 'maximum':  3})
+
+def test_generate_range_negative():
+    for data_type in Generator.get_supported_data_types():
+        generate(data_type, {'minimum': -99, 'maximum': -1})
+
+
+def test_generate_bool_nullable():
+    for data_type in Generator.get_supported_data_types():
+        generate(data_type, {'bool': {'nullable': True}})

py/lib/scwrypts/twilio/__init__.py

@@ -0,0 +1,2 @@
+from .client import get_client
+from .send_sms import send_sms

py/lib/scwrypts/twilio/client.py

@@ -1,6 +1,6 @@
 from twilio.rest import Client
 
-from py.lib.scwrypts import getenv
+from scwrypts.env import getenv
 
 CLIENT = None
 

py/lib/scwrypts/twilio/send_sms.py

@@ -1,7 +1,7 @@
 from json import dumps
 from time import sleep
 
-from py.lib.twilio.client import get_client
+from .client import get_client
 
 
 def send_sms(to, from_, body, max_char_count=300, stream=None):

py/lib/twilio/__init__.py

@@ -1,2 +0,0 @@
-from py.lib.twilio.client import get_client
-from py.lib.twilio.send_sms import send_sms