From 34f3d1e2a723405bf4172617e76a7288dc17cf60 Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Sat, 29 Jan 2022 20:45:48 -0600 Subject: [PATCH 01/12] Added basic support for PHP. --- scripttease/lib/snippets/mappings.py | 5 +++-- scripttease/lib/snippets/php.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 scripttease/lib/snippets/php.py diff --git a/scripttease/lib/snippets/mappings.py b/scripttease/lib/snippets/mappings.py index 9d0f922..338ee58 100644 --- a/scripttease/lib/snippets/mappings.py +++ b/scripttease/lib/snippets/mappings.py @@ -5,6 +5,7 @@ from .django import django from .messages import messages from .mysql import mysql from .pgsql import pgsql +from .php import php from .posix import posix from .python import python from .ubuntu import ubuntu @@ -56,6 +57,6 @@ def merge_dictionaries(first: dict, second: dict) -> dict: MAPPINGS = { - 'centos': merge(centos, django, messages, mysql, pgsql, posix, python), - 'ubuntu': merge(ubuntu, django, messages, mysql, pgsql, posix, python), + 'centos': merge(centos, django, messages, mysql, pgsql, php, posix, python), + 'ubuntu': merge(ubuntu, django, messages, mysql, pgsql, php, posix, python), } diff --git a/scripttease/lib/snippets/php.py b/scripttease/lib/snippets/php.py new file mode 100644 index 0000000..867b32a --- /dev/null +++ b/scripttease/lib/snippets/php.py @@ -0,0 +1,3 @@ +php = { + 'module': "phpenmod {{ args[0] }}", +} From 4ca551d2c9c569afdd49cfafdabc3a3045b1f567 Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Sat, 29 Jan 2022 20:46:05 -0600 Subject: [PATCH 02/12] Updated dependencies. --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 03baa6f..297385e 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,8 @@ setup( "jinja2", "pygments", "python-commonkit", + "pyyaml", + "tabulate", ], # dependency_links=[ # "https://github.com/develmaycare/superpython", From 6abcede7e88c7c6ccbb2c2581530d6fc8d699cb4 Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Sat, 29 Jan 2022 20:46:43 -0600 Subject: [PATCH 03/12] Added `load_variables()` to all. --- scripttease/lib/contexts.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripttease/lib/contexts.py b/scripttease/lib/contexts.py index 116ec7c..ebc8e5c 100644 --- a/scripttease/lib/contexts.py +++ b/scripttease/lib/contexts.py @@ -10,6 +10,7 @@ log = logging.getLogger(__name__) # Exports __all__ = ( + "load_variables", "Context", "Variable", ) @@ -85,6 +86,15 @@ class Context(object): return v + def append(self, variable): + """Append a variable to the context. + + :param variable: The variable to be added to the context. + :type variable: scripttease.lib.contexts.Variable + + """ + self.variables[variable.name] = variable + def mapping(self): """Get the context as a dictionary. From 2b990ff4b77749ebcff056bffa49210809edd0a3 Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Sat, 29 Jan 2022 20:47:10 -0600 Subject: [PATCH 04/12] Added sudo support to Template output. --- scripttease/lib/loaders/base.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/scripttease/lib/loaders/base.py b/scripttease/lib/loaders/base.py index 0d067ca..7335b5f 100644 --- a/scripttease/lib/loaders/base.py +++ b/scripttease/lib/loaders/base.py @@ -305,7 +305,7 @@ class Snippet(object): self.kwargs = kwargs or dict() self.name = name - sudo = self.kwargs.get("sudo", None) + sudo = self.kwargs.pop("sudo", None) if isinstance(sudo, Sudo): self.sudo = sudo elif type(sudo) is str: @@ -503,17 +503,28 @@ class Sudo(object): class Template(object): PARSER_JINJA = "jinja2" + PARSER_PYTHON = "python" PARSER_SIMPLE = "simple" def __init__(self, source, target, backup=True, parser=PARSER_JINJA, **kwargs): self.backup_enabled = backup self.context = kwargs.pop("context", dict()) + self.kwargs = kwargs + self.parser = parser self.locations = kwargs.pop("locations", list()) self.source = os.path.expanduser(source) self.target = target - self.kwargs = kwargs + sudo = self.kwargs.pop("sudo", None) + if isinstance(sudo, Sudo): + self.sudo = sudo + elif type(sudo) is str: + self.sudo = Sudo(enabled=True, user=sudo) + elif sudo is True: + self.sudo = Sudo(enabled=True) + else: + self.sudo = Sudo() def __getattr__(self, item): return self.kwargs.get(item) @@ -537,6 +548,10 @@ class Template(object): return content + if self.parser == self.PARSER_PYTHON: + content = read_file(template) + return content % self.context + try: return parse_jinja_template(template, self.context) except TemplateNotFound: @@ -554,7 +569,8 @@ class Template(object): # TODO: Backing up a template's target is currently specific to bash. if self.backup_enabled: - lines.append('if [[ -f "%s" ]]; then mv %s %s.b; fi;' % (self.target, self.target, self.target)) + command = "%s mv %s %s.b" % (self.sudo, self.target, self.target) + lines.append('if [[ -f "%s" ]]; then %s fi;' % (self.target, command.lstrip())) # Get the content; e.g. parse the template. content = self.get_content() @@ -563,12 +579,15 @@ class Template(object): if content.startswith("#!"): _content = content.split("\n") first_line = _content.pop(0) - lines.append('echo "%s" > %s' % (first_line, self.target)) - lines.append("cat >> %s << EOF" % self.target) + command = '%s echo "%s" > %s' % (self.sudo, first_line, self.target) + lines.append(command.lstrip()) + command = "%s cat >> %s << EOF" % (self.sudo, self.target) + lines.append(command.lstrip()) lines.append("\n".join(_content)) lines.append("EOF") else: - lines.append("cat > %s << EOF" % self.target) + command = "%s cat >> %s << EOF" % (self.sudo, self.target) + lines.append(command.lstrip()) lines.append(content) lines.append("EOF") From 8e42390807eedd85219ff69ab9fef251d7f894f2 Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Sat, 29 Jan 2022 20:47:39 -0600 Subject: [PATCH 05/12] Updated interim CLI to ignore explanations and screenshots in command output. --- sandbox/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sandbox/cli.py b/sandbox/cli.py index c87096b..81054fb 100755 --- a/sandbox/cli.py +++ b/sandbox/cli.py @@ -121,8 +121,6 @@ This command is used to parse configuration files and output the commands. help="Load variables from a file." ) - - # Access to the version number requires special consideration, especially # when using sub parsers. The Python 3.3 behavior is different. See this # answer: http://stackoverflow.com/questions/8521612/argparse-optional-subparser-for-version @@ -215,6 +213,10 @@ This command is used to parse configuration files and output the commands. else: commands = list() for snippet in loader.get_snippets(): + # Skip explanations and screenshots. They don't produce usable statements. + if snippet.name in ("explain", "screenshot"): + continue + statement = snippet.get_statement() if statement is not None: commands.append(statement) From 291dcdfd5fd04f67a0e3f0df88922ede1d27b46b Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Sat, 29 Jan 2022 20:47:56 -0600 Subject: [PATCH 06/12] Added support for explanations and screenshots. --- scripttease/lib/snippets/messages.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scripttease/lib/snippets/messages.py b/scripttease/lib/snippets/messages.py index 65ad7c4..e75554e 100644 --- a/scripttease/lib/snippets/messages.py +++ b/scripttease/lib/snippets/messages.py @@ -7,6 +7,22 @@ messages = { 'clear;' ], 'echo': 'echo "{{ args[0] }}"', + 'explain': "{{ args[0] }}", + 'screenshot': [ + '{% if output == "md" %}', + "![{% if caption %}{{ caption }}]({{ args[0] }})", + '{% elif output == "rst" %}', + '.. figure:: {{ args[0] }}', + '{% if caption %}\n :alt: {{ caption }}{% endif %}', + '{% if height %}\n :height: {{ height }}{% endif %}', + '{% if width %}\n :width: {{ width }}{% endif %}' + '\n', + '{% else %}', + '{{ caption }}{% endif %}' + '{% if classes %} class={{ classes }}{% endif %}' + '{% if height %} height=', + '{% endif %}', + ], 'slack': [ "curl -X POST -H 'Content-type: application/json' --data", '{"text": "{{ args[0] }}"}', From 57236f5f3e4645f50c00574722949a2598fdb13b Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Sat, 29 Jan 2022 20:48:15 -0600 Subject: [PATCH 07/12] Updated nextcloud inventory to use new PHP support. --- .../data/inventory/nextcloud/steps.ini | 35 ++----------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/scripttease/data/inventory/nextcloud/steps.ini b/scripttease/data/inventory/nextcloud/steps.ini index a569658..ebe04a4 100644 --- a/scripttease/data/inventory/nextcloud/steps.ini +++ b/scripttease/data/inventory/nextcloud/steps.ini @@ -50,38 +50,9 @@ apache.enable_module: rewrite [enable SSL engine] apache.enable_module: ssl -[enable php ctype] -run: "phpenmod ctype" - -[enable php curl] -run: "phpenmod curl" - -[enable php dom] -run: "phpenmod dom" - -[enable php GD] -run: "phpenmod gd" - -[enable php JSON] -run: "phpenmod json" - -[enable php PGSQL] -run: "phpenmod pdo_pgsql" - -[enable php SimpleXML] -run: "phpenmod simplexml" - -[enable php posix] -run: "phpenmod posix" - -[enable php XMLReader] -run: "phpenmod xmlreader" - -[enable php XMLWriter] -run: "phpenmod xmlwriter" - -[enable php zip] -run: "phpenmod zip" +[enable php modules] +php.module: $item +items: ctype, curl, dom, gd, json, pdo_pgsql, posix, simplexml, xmlreader, xmlwriter, zip ;PHP module libxml (Linux package libxml2 must be >=2.7.0) ;php -i | grep -i libxml From 10c1c88b0bf897d4fe5a1e2da233a27fa7bfd83c Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Sat, 29 Jan 2022 20:48:31 -0600 Subject: [PATCH 08/12] Updated nextcloud inventory to use new PHP support. --- scripttease/lib/snippets/messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripttease/lib/snippets/messages.py b/scripttease/lib/snippets/messages.py index e75554e..861979b 100644 --- a/scripttease/lib/snippets/messages.py +++ b/scripttease/lib/snippets/messages.py @@ -7,7 +7,7 @@ messages = { 'clear;' ], 'echo': 'echo "{{ args[0] }}"', - 'explain': "{{ args[0] }}", + 'explain': "{{ args[0] }} ", 'screenshot': [ '{% if output == "md" %}', "![{% if caption %}{{ caption }}]({{ args[0] }})", From 7ecdb3c1f3a4ae5556d8fc4401867c0c1e9856b5 Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Tue, 1 Feb 2022 21:46:38 -0600 Subject: [PATCH 09/12] Worked on new docs/tutorial output. --- sandbox/cli.py | 111 ++++++++++++++-- scripttease/data/inventory/radicale/steps.ini | 1 + scripttease/lib/contexts.py | 52 +------- scripttease/lib/loaders/__init__.py | 5 +- scripttease/lib/loaders/base.py | 122 +++++++++++++++++- scripttease/lib/snippets/messages.py | 18 +-- 6 files changed, 232 insertions(+), 77 deletions(-) diff --git a/sandbox/cli.py b/sandbox/cli.py index 81054fb..79ca964 100755 --- a/sandbox/cli.py +++ b/sandbox/cli.py @@ -1,17 +1,17 @@ #! /usr/bin/env python from argparse import ArgumentParser, RawDescriptionHelpFormatter -from commonkit import highlight_code, smart_cast +from commonkit import highlight_code, indent, smart_cast from commonkit.logging import LoggingHelper from commonkit.shell import EXIT +from markdown import markdown import sys sys.path.insert(0, "../") from scripttease.constants import LOGGER_NAME -from scripttease.lib.contexts import load_variables, Context -from scripttease.lib.loaders.ini import INILoader -from scripttease.lib.loaders.yaml import YMLLoader +from scripttease.lib.contexts import Context +from scripttease.lib.loaders import load_variables, INILoader, YMLLoader from scripttease.version import DATE as VERSION_DATE, VERSION DEBUG = 10 @@ -59,10 +59,11 @@ This command is used to parse configuration files and output the commands. ) parser.add_argument( - "-d", - "--docs", - action="store_true", - dest="docs_enabled", + "-d=", + "--docs=", + choices=["html", "markdown", "plain", "rst"], + # default="markdown", + dest="docs", help="Output documentation instead of code." ) @@ -172,7 +173,7 @@ This command is used to parse configuration files and output the commands. filters[key].append(value) - # Handle options. + # Handle global command options. options = dict() if args.options: for token in args.options: @@ -208,8 +209,96 @@ This command is used to parse configuration files and output the commands. exit(EXIT.ERROR) # Generate output. - if args.docs_enabled: - pass + if args.docs: + output = list() + for snippet in loader.get_snippets(): + if snippet is None: + continue + + if snippet.name == "explain": + output.append(snippet.args[0]) + output.append("") + elif snippet.name == "screenshot": + if args.docs == "html": + b = list() + b.append('") + elif args.docs == "plain": + output.append(snippet.args[0]) + output.append("") + elif args.docs == "rst": + output.append(".. figure:: %s" % snippet.args[0]) + + if snippet.caption: + output.append(indent(":alt: %s" % snippet.caption)) + + if snippet.height: + output.append(indent(":height: %s" % snippet.height)) + + if snippet.width: + output.append(indent(":width: %s" % snippet.width)) + + output.append("") + else: + if snippet.caption: + output.append("![%s](%s)" % (snippet.caption, snippet.args[0])) + else: + output.append("![](%s)" % (snippet.args[0])) + + output.append("") + elif snippet.name == "template": + if args.docs == "plain": + output.append("+++") + output.append(snippet.get_content()) + output.append("+++") + elif args.docs == "rst": + output.append(".. code-block:: %s" % snippet.get_target_language()) + output.append("") + output.append(indent(snippet.get_content())) + output.append("") + else: + output.append("```%s" % snippet.get_target_language()) + output.append(snippet.get_content()) + output.append("```") + else: + statement = snippet.get_statement(include_comment=False, include_register=False, include_stop=False) + if statement is not None: + line = snippet.comment.replace("#", "") + output.append("%s:" % line.capitalize()) + output.append("") + if args.docs == "plain": + output.append("---") + output.append(statement) + output.append("---") + output.append("") + elif args.docs == "rst": + output.append(".. code-block:: bash") + output.append("") + output.append(indent(statement)) + output.append("") + else: + output.append("```bash") + output.append(statement) + output.append("```") + output.append("") + + if args.docs == "html": + print(markdown("\n".join(output), extensions=['fenced_code'])) + else: + print("\n".join(output)) else: commands = list() for snippet in loader.get_snippets(): diff --git a/scripttease/data/inventory/radicale/steps.ini b/scripttease/data/inventory/radicale/steps.ini index d4f5e2e..bd23eb8 100644 --- a/scripttease/data/inventory/radicale/steps.ini +++ b/scripttease/data/inventory/radicale/steps.ini @@ -24,6 +24,7 @@ system: yes [create the systemd service file for radicale] template: radicale.service /etc/systemd/system/radicale.service +lang: ini [start the radicale service] start: radicale diff --git a/scripttease/lib/contexts.py b/scripttease/lib/contexts.py index ebc8e5c..632d2c7 100644 --- a/scripttease/lib/contexts.py +++ b/scripttease/lib/contexts.py @@ -1,58 +1,16 @@ # Imports -from commonkit import smart_cast -from configparser import ParsingError, RawConfigParser import logging -import os log = logging.getLogger(__name__) # Exports __all__ = ( - "load_variables", "Context", "Variable", ) -# Functions - - -def load_variables(path): - """Load variables from an INI file. - - :param path: The path to the INI file. - :type path: str - - :rtype: list[scripttease.lib.contexts.Variable] - - """ - if not os.path.exists(path): - log.warning("Variables file does not exist: %s" % path) - return list() - - ini = RawConfigParser() - try: - ini.read(path) - except ParsingError as e: - log.warning("Failed to parse %s variables file: %s" % (path, str(e))) - return list() - - variables = list() - for variable_name in ini.sections(): - _value = None - kwargs = dict() - for key, value in ini.items(variable_name): - if key == "value": - _value = smart_cast(value) - continue - - kwargs[key] = smart_cast(value) - - variables.append(Variable(variable_name, _value, **kwargs)) - - return variables - # Classes @@ -108,11 +66,10 @@ class Context(object): return d - class Variable(object): """An individual variable.""" - def __init__(self, name, value, **kwargs): + def __init__(self, name, value, environment=None, **kwargs): """Initialize a variable. :param name: The name of the variable. @@ -120,13 +77,18 @@ class Variable(object): :param value: The value of the variable. + :param environment: The environment in which the variable is used. + :type environment: str + kwargs are available as dynamic attributes. """ - self.attributes = kwargs + self.environment = environment self.name = name self.value = value + self.attributes = kwargs + def __getattr__(self, item): return self.attributes.get(item) diff --git a/scripttease/lib/loaders/__init__.py b/scripttease/lib/loaders/__init__.py index 4a0d831..527b4ef 100644 --- a/scripttease/lib/loaders/__init__.py +++ b/scripttease/lib/loaders/__init__.py @@ -1,3 +1,6 @@ """ The job of a loader is to collect commands and their arguments from a text file. -""" \ No newline at end of file +""" +from .base import filter_snippets, load_variables +from .ini import INILoader +from .yaml import YMLLoader diff --git a/scripttease/lib/loaders/base.py b/scripttease/lib/loaders/base.py index 7335b5f..96661b2 100644 --- a/scripttease/lib/loaders/base.py +++ b/scripttease/lib/loaders/base.py @@ -1,21 +1,111 @@ # Imports -from commonkit import parse_jinja_string, parse_jinja_template, pick, read_file, smart_cast, split_csv, File +from commonkit import any_list_item, parse_jinja_string, parse_jinja_template, pick, read_file, smart_cast, split_csv, \ + File +from configparser import ParsingError, RawConfigParser from jinja2.exceptions import TemplateError, TemplateNotFound import logging import os +from ..contexts import Variable from ..snippets.mappings import MAPPINGS log = logging.getLogger(__name__) # Exports + __all__ = ( + "filter_snippets", + "load_variables", "BaseLoader", "Snippet", "Sudo", "Template", ) +# Functions + + +def filter_snippets(snippets, environments=None, tags=None): + """Filter snippets based on the given criteria. + + :param snippets: The snippets to be filtered. + :type snippets: list[scripttease.lib.loaders.base.Snippet] + + :param environments: Environment names to be matched. + :type environments: list[str] + + :param tags: Tag names to be matched. + :type tags: list[str] + + """ + filtered = list() + for snippet in snippets: + if environments is not None and len(snippet.environments) > 0: + if not any_list_item(environments, snippet.environments): + continue + + if tags is not None: + if not any_list_item(tags, snippet.tags): + continue + + filtered.append(snippet) + + return filtered + + +def load_variables(path, env=None): + """Load variables from an INI file. + + :param path: The path to the INI file. + :type path: str + + :param env: The environment name of variables to return. + :type env: str + + :rtype: list[scripttease.lib.contexts.Variable] + + """ + if not os.path.exists(path): + log.warning("Variables file does not exist: %s" % path) + return list() + + ini = RawConfigParser() + try: + ini.read(path) + except ParsingError as e: + log.warning("Failed to parse %s variables file: %s" % (path, str(e))) + return list() + + variables = list() + for variable_name in ini.sections(): + if ":" in variable_name: + variable_name, _environment = variable_name.split(":") + else: + _environment = None + variable_name = variable_name + + kwargs = { + 'environment': _environment, + } + _value = None + for key, value in ini.items(variable_name): + if key == "value": + _value = smart_cast(value) + continue + + kwargs[key] = smart_cast(value) + + variables.append(Variable(variable_name, _value, **kwargs)) + + if env is not None: + filtered_variables = list() + for var in variables: + if var.environment and var.environment == env or var.environment is None: + filtered_variables.append(var) + + return filtered_variables + + return variables # Classes @@ -305,6 +395,9 @@ class Snippet(object): self.kwargs = kwargs or dict() self.name = name + self.environments = kwargs.pop("environments", list()) + self.tags = kwargs.pop("tags", list()) + sudo = self.kwargs.pop("sudo", None) if isinstance(sudo, Sudo): self.sudo = sudo @@ -509,14 +602,14 @@ class Template(object): def __init__(self, source, target, backup=True, parser=PARSER_JINJA, **kwargs): self.backup_enabled = backup self.context = kwargs.pop("context", dict()) - self.kwargs = kwargs - + self.name = "template" self.parser = parser + self.language = kwargs.pop("lang", None) self.locations = kwargs.pop("locations", list()) self.source = os.path.expanduser(source) self.target = target - sudo = self.kwargs.pop("sudo", None) + sudo = kwargs.pop("sudo", None) if isinstance(sudo, Sudo): self.sudo = sudo elif type(sudo) is str: @@ -526,6 +619,8 @@ class Template(object): else: self.sudo = Sudo() + self.kwargs = kwargs + def __getattr__(self, item): return self.kwargs.get(item) @@ -603,6 +698,25 @@ class Template(object): return "\n".join(lines) + def get_target_language(self): + if self.language is not None: + return self.language + + if self.target.endswith(".conf"): + return "conf" + elif self.target.endswith(".ini"): + return "ini" + elif self.target.endswith(".php"): + return "php" + elif self.target.endswith(".py"): + return "python" + elif self.target.endswith(".sh"): + return "bash" + elif self.target.endswith(".yml"): + return "yaml" + else: + return "text" + def get_template(self): """Get the template path. diff --git a/scripttease/lib/snippets/messages.py b/scripttease/lib/snippets/messages.py index 861979b..f01447a 100644 --- a/scripttease/lib/snippets/messages.py +++ b/scripttease/lib/snippets/messages.py @@ -7,22 +7,8 @@ messages = { 'clear;' ], 'echo': 'echo "{{ args[0] }}"', - 'explain': "{{ args[0] }} ", - 'screenshot': [ - '{% if output == "md" %}', - "![{% if caption %}{{ caption }}]({{ args[0] }})", - '{% elif output == "rst" %}', - '.. figure:: {{ args[0] }}', - '{% if caption %}\n :alt: {{ caption }}{% endif %}', - '{% if height %}\n :height: {{ height }}{% endif %}', - '{% if width %}\n :width: {{ width }}{% endif %}' - '\n', - '{% else %}', - '{{ caption }}{% endif %}' - '{% if classes %} class={{ classes }}{% endif %}' - '{% if height %} height=', - '{% endif %}', - ], + 'explain': None, + 'screenshot': None, 'slack': [ "curl -X POST -H 'Content-type: application/json' --data", '{"text": "{{ args[0] }}"}', From 7a0c450662d8faef314c895d7ee9006879a256fc Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Tue, 1 Feb 2022 21:55:02 -0600 Subject: [PATCH 10/12] Better handling for explain snippet in docs output. --- sandbox/cli.py | 4 ++++ scripttease/data/inventory/radicale/steps.ini | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/sandbox/cli.py b/sandbox/cli.py index 79ca964..23ad89a 100755 --- a/sandbox/cli.py +++ b/sandbox/cli.py @@ -216,6 +216,10 @@ This command is used to parse configuration files and output the commands. continue if snippet.name == "explain": + if snippet.header: + output.append("## %s" % snippet.name.title()) + output.append("") + output.append(snippet.args[0]) output.append("") elif snippet.name == "screenshot": diff --git a/scripttease/data/inventory/radicale/steps.ini b/scripttease/data/inventory/radicale/steps.ini index bd23eb8..041c1de 100644 --- a/scripttease/data/inventory/radicale/steps.ini +++ b/scripttease/data/inventory/radicale/steps.ini @@ -1,12 +1,23 @@ +[introduction] +explain: "In this tutorial, we are going to install Radicale." +header: yes + [make sure a maintenance root exists] mkdir: /var/www/maint/www group: www-data owner: www-data recursive: yes +[about maintenance root] +explain: "The maintenance root is used to register an SSL certificate (below) before the site is completed and (later) after the site is live." + [install radicale] pip3: radicale +;[install radicale screenshot] +;screenshot: images/install.png +;caption: Radical Installed + [create radicale configuration directory] mkdir: /etc/radicale/config owner: radicale From 55dbe0e5a8e233882d1ddb16c6d86cd7c80e0138 Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Wed, 2 Feb 2022 20:10:11 -0600 Subject: [PATCH 11/12] Added support for passing through excluded kwargs so that custom parsers (like django) can be informed of implementation-specific attributes. --- VERSION.txt | 2 +- sandbox/cli.py | 17 +++++++---------- scripttease/constants.py | 11 +++++++++++ scripttease/lib/loaders/base.py | 22 ++++++++++++++++++---- scripttease/lib/snippets/django.py | 14 ++------------ scripttease/version.py | 2 +- 6 files changed, 40 insertions(+), 28 deletions(-) diff --git a/VERSION.txt b/VERSION.txt index 295ec49..8fc3e14 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -6.8.2 \ No newline at end of file +6.8.3 \ No newline at end of file diff --git a/sandbox/cli.py b/sandbox/cli.py index 23ad89a..960643b 100755 --- a/sandbox/cli.py +++ b/sandbox/cli.py @@ -226,9 +226,7 @@ This command is used to parse configuration files and output the commands. if args.docs == "html": b = list() b.append('") + output.append("") elif args.docs == "plain": output.append(snippet.args[0]) output.append("") @@ -247,7 +246,7 @@ This command is used to parse configuration files and output the commands. output.append(".. figure:: %s" % snippet.args[0]) if snippet.caption: - output.append(indent(":alt: %s" % snippet.caption)) + output.append(indent(":alt: %s" % snippet.caption or snippet.comment)) if snippet.height: output.append(indent(":height: %s" % snippet.height)) @@ -257,11 +256,7 @@ This command is used to parse configuration files and output the commands. output.append("") else: - if snippet.caption: - output.append("![%s](%s)" % (snippet.caption, snippet.args[0])) - else: - output.append("![](%s)" % (snippet.args[0])) - + output.append("![%s](%s)" % (snippet.caption or snippet.comment, snippet.args[0])) output.append("") elif snippet.name == "template": if args.docs == "plain": @@ -306,8 +301,10 @@ This command is used to parse configuration files and output the commands. else: commands = list() for snippet in loader.get_snippets(): - # Skip explanations and screenshots. They don't produce usable statements. + # Explanations and screenshots don't produce usable statements but may be added as comments. if snippet.name in ("explain", "screenshot"): + commands.append("# %s" % snippet.args[0]) + commands.append("") continue statement = snippet.get_statement() diff --git a/scripttease/constants.py b/scripttease/constants.py index 26d0a12..ae45aae 100644 --- a/scripttease/constants.py +++ b/scripttease/constants.py @@ -1 +1,12 @@ +EXCLUDED_KWARGS = [ + "cd", + "comment", + "environments", + "prefix", + "register", + "shell", + "stop", + "tags", +] + LOGGER_NAME = "script-tease" diff --git a/scripttease/lib/loaders/base.py b/scripttease/lib/loaders/base.py index 96661b2..665a665 100644 --- a/scripttease/lib/loaders/base.py +++ b/scripttease/lib/loaders/base.py @@ -6,6 +6,7 @@ from configparser import ParsingError, RawConfigParser from jinja2.exceptions import TemplateError, TemplateNotFound import logging import os +from ...constants import EXCLUDED_KWARGS from ..contexts import Variable from ..snippets.mappings import MAPPINGS @@ -114,7 +115,8 @@ def load_variables(path, env=None): class BaseLoader(File): """Base class for loading a command file.""" - def __init__(self, path, context=None, locations=None, mappings=None, profile="ubuntu", **kwargs): + def __init__(self, path, context=None, excluded_kwargs=None, locations=None, mappings=None, profile="ubuntu", + **kwargs): """Initialize the loader. :param path: The path to the command file. @@ -124,6 +126,13 @@ class BaseLoader(File): converted to a ``dict`` when passed to a Snippet or Template. :type context: scripttease.lib.contexts.Context + :param excluded_kwargs: For commands that support ad hoc sub-commands (like Django), this is a list of keyword + argument names that must be removed. Defaults to the names of common command attributes. + If your implementation requires custom but otherwise standard command attributes, you'll + need to import the ``EXCLUDED_KWARGS`` constant and add your attribute names before + passing it to the loader. + :type excluded_kwargs: list[str] + :param locations: A list of paths where templates and other external files may be found. The ``templates/`` directory in which the command file exists is added automatically. :type locations: list[str] @@ -140,6 +149,7 @@ class BaseLoader(File): """ self.context = context + self.excluded_kwargs = excluded_kwargs or EXCLUDED_KWARGS self.is_loaded = False self.locations = locations or list() self.mappings = mappings or MAPPINGS @@ -365,7 +375,7 @@ class Snippet(object): """ - def __init__(self, name, args=None, content=None, context=None, kwargs=None, parser=None): + def __init__(self, name, args=None, content=None, context=None, excluded_kwargs=None, kwargs=None, parser=None): """Initialize a snippet. :param name: The canonical name of the snippet. @@ -380,6 +390,9 @@ class Snippet(object): :param context: Additional context variables used to render the command. :type context: dict + :param excluded_kwargs: See parameter description for BaseLoader. + :type excluded_kwargs: list[str] + :param kwargs: The keyword arguments found in the config file. These may be specific to the command or one of the common options. They are accessible as dynamic attributes of the Snippet instance. :type kwargs: dict @@ -389,6 +402,7 @@ class Snippet(object): """ self.args = args or list() + self.excluded_kwargs = excluded_kwargs or EXCLUDED_KWARGS self.parser = parser self.content = content self.context = context or dict() @@ -453,7 +467,7 @@ class Snippet(object): args.append(arg.replace("$item", item)) if self.parser: - statement = self.parser(self, args=args) + statement = self.parser(self, args=args, excluded_kwargs=self.excluded_kwargs) else: statement = self._parse(args=args) @@ -485,7 +499,7 @@ class Snippet(object): a.append("%s &&" % self.prefix) if self.parser: - statement = self.parser(self) + statement = self.parser(self, excluded_kwargs=self.excluded_kwargs) else: statement = self._parse() diff --git a/scripttease/lib/snippets/django.py b/scripttease/lib/snippets/django.py index 21bf612..3d2ae8f 100644 --- a/scripttease/lib/snippets/django.py +++ b/scripttease/lib/snippets/django.py @@ -1,20 +1,10 @@ from commonkit import parse_jinja_string +from ...constants import EXCLUDED_KWARGS -DJANGO_EXCLUDED_KWARGS = [ - "cd", - "comment", - "environments", - "prefix", - "register", - "shell", - "stop", - "tags", - # "venv", # ? -] def django_command_parser(snippet, args=None, excluded_kwargs=None): - _excluded_kwargs = excluded_kwargs or DJANGO_EXCLUDED_KWARGS + _excluded_kwargs = excluded_kwargs or EXCLUDED_KWARGS # We need to remove the common options so any remaining keyword arguments are converted to switches for the # management command. diff --git a/scripttease/version.py b/scripttease/version.py index e8a2fdc..fef3ff8 100644 --- a/scripttease/version.py +++ b/scripttease/version.py @@ -2,4 +2,4 @@ DATE = "2021-01-26" VERSION = "6.8.2" MAJOR = 6 MINOR = 8 -PATCH = 2 +PATCH = 3 From d72584af67bf0ffe569fc288bf616a10d7d4d945 Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Tue, 8 Feb 2022 18:56:23 -0600 Subject: [PATCH 12/12] Updated docs. --- .gitignore | 1 + help/docs/commands/django.md | 118 ++++++++++++++++ help/docs/commands/messages.md | 111 +++++++++++++++ help/docs/commands/mysql.md | 64 +++++++++ help/docs/commands/pgsql.md | 57 ++++++++ help/docs/commands/php.md | 9 ++ help/docs/commands/posix.md | 148 ++++++++++++++++++++ help/docs/commands/python.md | 20 +++ help/docs/config/command-file.md | 208 +++++++++++++++++++++++++++++ help/docs/config/variables.md | 79 +++++++++++ help/docs/index.md | 58 ++++++++ help/docs/profiles/centos.md | 124 +++++++++++++++++ help/docs/profiles/ubuntu.md | 124 +++++++++++++++++ help/mkdocs.yml | 26 ++++ scripttease/lib/snippets/mysql.py | 2 +- scripttease/lib/snippets/pgsql.py | 2 +- scripttease/lib/snippets/ubuntu.py | 8 +- 17 files changed, 1153 insertions(+), 6 deletions(-) create mode 100644 help/docs/commands/django.md create mode 100644 help/docs/commands/messages.md create mode 100644 help/docs/commands/mysql.md create mode 100644 help/docs/commands/pgsql.md create mode 100644 help/docs/commands/php.md create mode 100644 help/docs/commands/posix.md create mode 100644 help/docs/commands/python.md create mode 100644 help/docs/config/command-file.md create mode 100644 help/docs/config/variables.md create mode 100644 help/docs/index.md create mode 100644 help/docs/profiles/centos.md create mode 100644 help/docs/profiles/ubuntu.md create mode 100644 help/mkdocs.yml diff --git a/.gitignore b/.gitignore index 1552802..7d7c3c2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ _scraps build dist docs/build +help/site htmlcov tmp.* tmp diff --git a/help/docs/commands/django.md b/help/docs/commands/django.md new file mode 100644 index 0000000..1c0a72a --- /dev/null +++ b/help/docs/commands/django.md @@ -0,0 +1,118 @@ +# Django + +Summary: Work with Django management commands. + +## Common Options for Django Commands + +You will generally want to include `cd` to change to the project directory and `prefix` to load the virtual environment. + +```yaml +- collect static files: + django: static + cd: /path/to/project/source + prefix: source ../python/bin/activate +``` + +## Automatic Conversion of Django Command Switches + +Options provided in the command configuration file are automatically converted to command line switches. + +```yaml +- run database migrations: + django: migrate + settings: tenants.example_com.settings + +- dump some data: + django: dumpdata + indent: 4 + natural_foreign: yes + natural_primary: yes +``` + +## Available Commands + +### check + +```ini +[run django checks] +django: check +``` + +```yaml +- run django checks: + django: check +``` + +### dumpdata + +Dump fixture data. + +- app: Required. The name of the app. +- model: Optional. A model name within the app. +- path: The path to the JSON file. When a model is provided, this defaults to `fixtures/app/model.json`. Otherwise, it is `fixtures/app/initial.json`. + +```ini +[dump project data] +django: dumpdata +app: projects + +[dump project categories] +django: dumpdata +app: projects +model: Category +path: local/projects/fixtures/default-categories.json +``` + +### loaddata + +Load fixture data. + +- path: The path to the JSON file. When a model is provided, this defaults to `fixtures/app/model.json`. Otherwise, it is `fixtures/app/initial.json`. + +### migrate + +Run database migrations. + +```ini +[run database migrations] +django: migrate +``` + +```yaml +- run database migrations: + django: migrate +``` + +### static + +Collect static files. + +```ini +[collect static files] +django: static +``` + +```yaml +- collect static files: + django: static +``` + +## Custom or Ad Hoc Commands + +It is possible to work with any Django management command provided the parameters may be specified as a switch. + +```ini +[run any django command] +django: command_name +first_option_name: asdf +second_option_name: 1234 +third_option_name: yes +``` + +```yaml +- run any django command: + django: command_name + first_option_name: asdf + second_option_name: 1234 + third_option_name: yes +``` diff --git a/help/docs/commands/messages.md b/help/docs/commands/messages.md new file mode 100644 index 0000000..3fff90f --- /dev/null +++ b/help/docs/commands/messages.md @@ -0,0 +1,111 @@ +# Messages + +Summary: Send feedback to users. + +## Available Commands + +### dialog + +Use the dialog CLI to display a message. + +- `height`: The height of the dialog box. Default: `15` +- `title`: An optional title to display as part of the dialog box. Default: `Message`. +- `width`: The width of the dialog box. Default: `100` + +```ini +[send some feedback] +dialog: "This is a message." +``` + +```yaml +- send some feedback: + dialog: "This is a message." +``` + +!!! warning + + The dialog command line utility must be installed. + +### explain + +Provide an explanation. When generating code this is added as a comment. When documentation is generated, it is output as text. + +```ini +[introduction] +explain: "These steps will set up a Radicale CalDav/CardDav server." +header: Introduction +``` + +The `header` option is not used in comments, but makes documentation more readable and facilitates the creation of tutorials or install guides that re-use the defined steps. + +### echo + +Display a simple message. + +```ini +[send some feedback] +echo: "This is a message." +``` + +```yaml +- send some feedback: + echo: "This is a message." +``` + +### slack + +Send a message via Slack. + +- `url`: Required. The URL to which the message should be sent. + +```ini +[send some feedback] +slack: "This is a message." +url: https://subdomain.slack.com/path/to/your/integration +``` + +```yaml +- send some feedback: + slack: "This is a message." + url: https://subdomain.slack.com/path/to/your/integration +``` + +!!! note + You could easily define a variable for the Slack URL and set ``url: {{ slack_url }}`` to save some typing. See [variables](../config/variables.md). + +### screenshot + +Like `explain` above, a screenshot adds detail to comments or documentation, but does not produce a command statement. + +```ini +[login screenshot after successful install] +screenshot: images/login.png +caption: Login Page +height: 50% +width: 50% +``` + +The value of `screenshot` may be relative to the command file or a full URL to the image. If `caption` is omitted the section (comment) is used. + +### twist + +Send a message via [Twist](https://twist.com). + +- `title`: The title of the message. Default: `Notice` +- `url`: Required. The URL to which the message should be sent. + +```ini +[send some feedback] +twist: "This is a message." +url: https://subdomain.twist.com/path/to/your/integration +``` + +```yaml +- send some feedback: + twist: "This is a message." + url: https://subdomain.twist.com/path/to/your/integration +``` + +!!! note + + As with Slack, you could easily define a variable for the Twist URL and set ``url: {{ twist_url }}``. See [variables](../config/variables.md). diff --git a/help/docs/commands/mysql.md b/help/docs/commands/mysql.md new file mode 100644 index 0000000..bd539d7 --- /dev/null +++ b/help/docs/commands/mysql.md @@ -0,0 +1,64 @@ +# MySQL + +Summary: Work with MySQL (and Maria) databases. + +## Common Options + +- `admin_pass`: The password off the admin-authorized user. +- `admin_user`: The user name of the admin-authorized user. Default: `root` +- `host`: The host name. Default: `localhost` +- `port`: The TCP port. Default: `3306` + +## Available Commands + +### mysql.create + +Create a database. Argument is the database name. + +- `owner`: The user name that owns the database. + +```ini +[create the database] +mysql.create: database_name +``` + +### mysql.drop + +Drop a database. Argument is the database name. + +### mysql.dump + +Dump the database schema. Argument is the database name. + +- `path`: The path to the dump file. Default: `dump.sql` + +### mysql.exec + +Execute an SQL statement. Argument is the SQL statement. + +- `database`: The name of the database where the statement will be executed. Default: `default` + +### mysql.exists + +Determine if a database exists. Argument is the database name. + +### mysql.grant + +Grant privileges to a user. Argument is the privileges to be granted. + +- `database`: The database name where privileges are granted. +- `user`: The user name for which the privileges are provided. + +### mysql.user.create + +Create a user. Argument is the user name. + +- `password`: The user's password. + +### mysql.user.drop + +Remove a user. Argument is the user name. + +### mysql.user.exists + +Determine if a user exists. Argument is the user name. diff --git a/help/docs/commands/pgsql.md b/help/docs/commands/pgsql.md new file mode 100644 index 0000000..4f9e423 --- /dev/null +++ b/help/docs/commands/pgsql.md @@ -0,0 +1,57 @@ +# PostgreSQL + +Summary: Work with Postgres databases. + +## Common Options + +- `admin_pass`: The password off the admin-authorized user. +- `admin_user`: The user name of the admin-authorized user. Default: `postgres` +- `host`: The host name. Default: `localhost` +- `port`: The TCP port. Default: `5432` + +## Available Commands + +### pgsql.create + +Create a database. Argument is the database name. + +- `owner`: The user name that owns the database. + +```ini +[create the database] +pgsql.create: database_name +``` + +### pgsql.drop + +Drop a database. Argument is the database name. + +### pgsql.dump + +Dump the database schema. Argument is the database name. + +- `path`: The path to the dump file. Default: `dump.sql` + +### pgsql.exec + +Execute an SQL statement. Argument is the SQL statement. + +- `database`: The name of the database where the statement will be executed. Default: `default` + +### pgsql.exists + +Determine if a database exists. Argument is the database name. + +### pgsql.user.create + +Create a user. Argument is the user name. + +- `password`: The user's password. + +### pgsql.user.drop + +Remove a user. Argument is the user name. + +### pgsql.user.exists + +Determine if a user exists. Argument is the user name. diff --git a/help/docs/commands/php.md b/help/docs/commands/php.md new file mode 100644 index 0000000..12c5a6b --- /dev/null +++ b/help/docs/commands/php.md @@ -0,0 +1,9 @@ +# PHP + +Summary: Work with PHP. + +## Available Commands + +### module + +Enable a PHP module. Argument is the module name. diff --git a/help/docs/commands/posix.md b/help/docs/commands/posix.md new file mode 100644 index 0000000..b08419d --- /dev/null +++ b/help/docs/commands/posix.md @@ -0,0 +1,148 @@ +# POSIX + +Summary: Work with common POSIX-compliant commands.. + +## Available Commands + +### append + +Append content to a file. Argument is the file name. + +- `content`: The content to be appended. + +### archive + +Create an archive (tarball). Argument is the target file or directory. + +- `absolute`: Don't strip leading slashes from file names. +- `view`: View the progress. +- `exclude`: Exclude file name patterns. +- `strip`: Strip component paths to the given depth (integer). +- `to`: The path to where the archive will be created. + +### copy + +Copy a file or directory. First argument is the target file/directory. Second argument is the destination. + +- `overwrite`: Overwrite an existing target. +- `recursive`: Copy directories recursively. + +### dir + +Create a directory. Argument is the path. + +- `group`: Set the group to the given group name. +- `mode`: Set the mode on the path. +- `owner`: Set the owner to the given owner name. +- `recursive`: Create the full path even if intermediate directories do not exist. + +### extract + +Extract an archive (tarball). Argument is the path to the archive file. + +- `absolute`: Strip leading slashes from file names. +- `view`: View the progress. +- `exclude`: Exclude file name patterns. +- `strip`: Strip component paths to the given depth (integer). +- `to`: The path to where the archive will be extracted. Defaults to the current working directory. + +### file + +Create a file. Argument is the path. + +- `content`: The content of the file. Otherwise, an empty file is created. +- `group`: Set the group to the given group name. +- `mode`: Set the mode on the path. +- `owner`: Set the owner to the given owner name. + +### link + +Create a symlink. First argument is the target. Second argument is the destination. + +- `force`: Force creation of the link. + +### move + +Move a file or directory. First argument is the target. Second argument is the desitnation. + +### perms + +Set permissions on a file or directory. Argument is the path. + +- `group`: Set the group to the given group name. +- `mode`: Set the mode on the path. +- `owner`: Set the owner to the given owner name. +- `recursive`: Apply permission recursively (directories only). + +### push + +Push (rsync) a path to a remote server. First argument is the local path. Second argument is the remote path. + +- `delete`: Delete existing files/directories. +- `host`: The host name. Required. +- `key_file`: Use the given SSL (private) key. Required. +- `links`: Copy symlinks. +- `exclude`: Exclude patterns from the given (local) file. +- `port`: The TCP port on the host. Default: `22` +- `recursive`: Operate recursively on directories. +- `user`: The user name. Required. + +### remove + +Remove a file or directory. Argument is the path. + +- `force`: Force the removal. +- `recursive`: Remove (directories) rescurisvely. + +### rename + +Rename a file or directory. First argument is the target. Second argument is the destination. + +### replace + +Replace something in a file. First argument is the path. + +- `backup`: Create a backup. +- `delimiiter`: The sed delimiter. Default: `/` +- `find`: The text to be found. Required. +- `sub`: The text to be replaced. Required. + +### scopy + +Copy a file to a remote server. First argument is the local file name. Second argument is the remote destination. + +- `key_file`: The private key file to use for the connection. +- `host`: The host name. Required. +- `port`: The TCP port. Default: `22` +- `user`: The user name. Required. + +### ssl + +Use Let's Encrypt (certbot) to acquire an SSL certificate. Argument is the domain name. + +- `email`: The email address for "agree tos". Default: `webmaster@domain_name` +- `webroot`: The webroot to use. Default: `/var/www/maint/www` + +### sync + +Sync (rsync) local files and directories. First argument is the target. Second argument is the destination. + +- `delete`: Delete existing files/directories. +- `links`: Copy symlinks. +- `exclude`: Exclude patterns from the given (local) file. +- `recursive`: Operate recursively on directories. + +### touch + +Touch a file, whether it exists or not. Argument is the path. + +### wait + +Wait for n number of seconds before continuing. Argument is the number of seconds. + +### write + +Write to a file. Argument is the path. + +- `content`: The content to write to the file. Replaces existing content. + diff --git a/help/docs/commands/python.md b/help/docs/commands/python.md new file mode 100644 index 0000000..2f94e5f --- /dev/null +++ b/help/docs/commands/python.md @@ -0,0 +1,20 @@ +# Python + +Summary: Work with Python. + +## Available Commands + +### pip + +Use the pip command. Argument is the package name. + +- `op`: The operation; `install` (the default), `remove`, or `updgrade`. +- `venv`: The name of the virtual environment to use. + +### pip3 + +Use Python3 pip. See pip above. + +### virtualenv + +Create a python virtual environment. Argument is the environment name. diff --git a/help/docs/config/command-file.md b/help/docs/config/command-file.md new file mode 100644 index 0000000..299ddc0 --- /dev/null +++ b/help/docs/config/command-file.md @@ -0,0 +1,208 @@ +# Command File + +A command file contains the metadata about the commands to be generated. INI and YAML formats are supported. + +In an INI file, each section is a command. With YAML, each top-level list item is a command. + +## The Comment/Description + +With INI files, the section name is the command comment. + +```ini +[this becomes the command comment] +; ... +``` + +With YAML, each command is a list item and the item name becomes the command comment: + +```yaml +- this becomes the command comment: + # ... +``` + +## First Option and Arguments + +The first variable in the section is the command name. It's value contains the required arguments. + +```ini +[restart the postfix service] +restart: postfix +``` + +```yaml +- restart the postfix service: + restart: postfix +``` + +## Formatting Notes + +With both INI and YAML files, the formatting rules are: + +- The first part of each command is the INI section or YAML item and is used as the comment. +- The command name *must* be the *first* option in the section. +- The arguments for the command appear as the value of the first option in the section. Arguments are separated by a + space. +- Arguments that should be treated as a single value should be enclosed in double quotes. +- `yes` and `no` are interpreted as boolean values. `maybe` is interpreted as `None`. +- List values, where required, are separated by commas when appearing in INI files, but are a `[standard, list, of, values]` in a YAML file. + +## Additional Attributes + +Additional variables in the section are generally optional parameters that inform or control how the command should be executed and are sometimes used to add switches to the statement. + +!!! warning + + This is not always the case, so consult the documentation for the command in question, because some parameters that appear after the first line are actually required. + +## Common Attributes + +A number of common options are recognized. Some of these have no bearing on statement generation but may be used for filtering. Others may be optionally included, and a few may only be used programmatically. + +### cd + +The `cd` option sets the directory (path) from which the statement should be executed. It is included by default when the statement is generated, but may be suppressed using `cd=False`. + +```ini +[create a python virtual environment] +virtualenv: python +cd: /path/to/project +``` + +### comment + +The comment comes from the section name (INI) or list name (YAML). It is included by default when the statement is generated, by may be suppressed using `include_comment=False`. + +```ini +[this becomes the comment] +; ... +``` + +```yaml +- this becomes the comment: + # ... +``` + +### env + +The `env` option indicates the target environment (or environments) in which the statement should run. This is not used in command generation, but may be used for filtering. + +```yaml +- set up the database: + pgsql.create: example_com + env: [staging, live] +``` + +This option may be given as `environments`, `environs`, `envs`, or simply `env`. It may be a list or CSV string. + +### prefix + +The `prefix` option is used to define a statement to be executed before the main statement is executed. + +```ini +[migrate the database] +django: migrate +cd: /path/to/project/source +prefix: source ../python/bin/activate +``` + +### register + +`register` defines the name of a variable to which the result of the statement should be saved. It is included by default when the statement is generated, but may be suppressed using `include_register=False`. + +```yaml +- check apache configuration: + apache: test + register: apache_ok +``` + +### shell + +The `shell` defines the shell to be used for command execution. It is not used for statement generation, but may be used programmatically -- for example, with Python's subprocess module. Some commands (such as Django management commands) need a shell to be explicitly defined. + +```ini +[run django checks] +django: check +cd: /path/to/project/source +prefix: source ../python/bin/activate +shell: /bin/bash +``` + +!!! note + + As this option is intended for programmatic use, it would be better to define a default shell for all command execution and use this option only when the default should be overridden. + +### stop + +A `yes` indicates processing should stop if the statement fails to execute with success. It is included by default when the statement is generated, but may be suppressed. Additionally, when [register](#register) is defined, this option will use the result of the command to determine success. This option is also useful for programmatic execution. + +```yaml +- check apache configuration: + apache: test + register: apache_ok + stop: yes +``` + +!!! warning + + Some commands do not provide an zero or non-zero exit code on success or failure. You should verify that the `stop` will actually be used. + +### sudo + +The `sudo` option may be defined as `yes` or a username. This will cause the statement to be generated with sudo. + +```ini +[install apache] +install: apache2 +sudo: yes +``` + +!!! note + + When present, sudo is always generated as part of the statement. For programmatic use, it may be better to control how and when sudo is applied using some other mechanism. If sudo should be used for all statements, it can be passed as a global option. + +### tags + +`tags` is a comma separated list (INI) or list (YAML) of tag names. These may be used for filtering. + +```yaml +- install apache: + install: apache2 + tags: [apache, web] + +- enable wsgi: + apache.enable: mod_wsgi + tags: [apache, web] + +- restart apache: + apache.restart: + tags: [apache, web] + +- run django checks: + django: check + tags: [django, python] + +- apply database migrations: + django: migrate + tags: [django, python] +``` + +## Ad Hoc Options + +Options that are not recognized as common or as part of those specific to a command are still processed by the loader. This makes it possible to define your own options based on the needs of a given implementation. + +For example, suppose you are implementing a deployment system where some commands should run locally, but most should run on the remote server. + +```ini +[run tests to validate the system] +run: make tests +local: yes +stop: yes + +[install apache] +install: apache2 +remote: yes + +; and so on ... +``` + +This will be of no use as a generated script since the generator does not know about `local` and `remote`, but these could be used programmatically to control whether Python subprocess or an SSH client is invoked. diff --git a/help/docs/config/variables.md b/help/docs/config/variables.md new file mode 100644 index 0000000..03480b0 --- /dev/null +++ b/help/docs/config/variables.md @@ -0,0 +1,79 @@ +# Variables File + +A variables file contains variable definitions that may be used as the context for parsing a [command file](command-file.md) *before* the actual commands are generated. + +Unlike a command file, the INI format is the only supported format for a variables file. + +## The Variable Name + +The variable name is defined in the section: + +```ini +[domain_name] +value: example.com +``` + +## The Variable Value + +As seen in the example above, the value is defined by simply adding a variable parameter: + +```ini +[domain_name] +value: example.com + +[database_host] +value: db1.example.com +``` + +!!! note + + This is the minimum definition for all variables. + +## Defining An Environment + +You may define an environment for any given variable that may be used for filtering variables. This is done by adding the environment name to the variable name: + +```ini +[database_host:development] +comment: Local host used in development. +value: localhost + +[database_host:live] +value: db1.example.com +``` + +In this way, variables of the same name may be supported across different deployment environments. + +## Adding Comments + +As demonstrated in the example above, you may comment on a variable by adding a `comment:` attribute to the section. + +## Defining Tags + +Tags may be defined for any variable as a comma separated list. This is useful for filtering. + +```ini +[database_host:development] +value: localhost +tags: database + +[database_host:live] +value: db1.example.com +tags: database + +[domain_name] +value: example.app +tags: application +``` + +## Other Attributes + +Any other variable defined in the section is dynamically available. + +```ini +[domain_name] +value: example.app +other: test +``` + +The value of `other` is `test`. diff --git a/help/docs/index.md b/help/docs/index.md new file mode 100644 index 0000000..0e9f4ad --- /dev/null +++ b/help/docs/index.md @@ -0,0 +1,58 @@ +# Python Script Tease + +## Overview + +Script Tease is a library and command line tool for generating Bash commands programmatically and (especially) using configuration files. + +The primary focus (and limit) is to convert plain text instructions (in INI or YAML format) into valid command line statements for a given platform. It does *not* provide support for executing those statements. + +## Concepts + +### Command Generation + +Script Tease may be used in two (2) ways: + +1. Using the library to programmatically define commands and export them as command line statements. +2. Using the `tease` command to generate commands from a configuration file. See [command file configuration](config/command-file.md). + +This documentation focuses on the second method, but the developer docs may be used in your own implementation. + +### Self-Documenting + +The format of INI and YAML files is self-documenting. The command comment is this section (INI) or start of a list item (YAML). This ensures that all commands have a basic description of their purpose or intent. + +### Snippets + +An *snippet* is simply a tokenized command that may be customized based on the instructions found in a command file. Related snippets are collected into groups and then merged into a larger set that define the capabilities of a specific operating system. + +!!! note + At present, the only fully defined operating systems are for Cent OS and Ubuntu. + +Snippets are defined in Python dictionaries. These include a "canonical" command name as the key and either a string or list which define the command. In both cases, the contents are parsed as Jinja templates. There are various approaches to evaluating a snippet. + +First: The snippet is a simple mapping of command name and command snippet. This is easy. Find the command name in the dictionary, and we have the snippet to be used. For example the `append` command in the `posix` dictionary. + +Second: The snippet is a mapping of command name and a list of snippets to be combined. Find the command name in the dictionary, and iterate through the snippets. For example, many of the commands in the `posix` dictionary takes this form. Command identification is the same as the first condition. + +Third: The command is a mapping to informal sub-commands. Examples include `apache` and `system` in the `ubuntu` dictionary. There are a couple of ways to handle this in the config file: + +- Use the outer command as the command with the inner command as the first (and perhaps only) argument. For example `apache: reload` or `system: upgrade`. +- Use a "dotted path" to find the command. For example: `apache.reload: (implicity True)` or `system.upgrade: (implicitly True)`. Or `apache.enable_site: example.com`. + +The first approach complicates things when detecting actual sub-commands (below). Script Tease supports both of these approaches. + +Fourth: The command also expects a sub-command. In some cases, the sub-command may be implicit, like `pip install`. In other cases, a number of sub-commands may be pre-defined, but ad hoc sub-commands should also be supported as with Django commands. + +Fifth: Builds upon the third and fourth conditions where the commands have lots of options, some of which may be defined at runtime. Postgres and MySQL may use be presented as informal sub-commands, but there are lots of options and challenges in building the final command. Django management commands have a number of standard options, specific options, and must also support ad hoc commands. + +## Terms and Definitions + +command +: When used in Script Tease documentation, this is a command instance which contains the properties and parameters for a command line statement. + +statement +: A specific statement (string) to be executed. A *statement* is contained within a *command*. + +## License + +Python Script Tease is released under the BSD 3 clause license. diff --git a/help/docs/profiles/centos.md b/help/docs/profiles/centos.md new file mode 100644 index 0000000..639355a --- /dev/null +++ b/help/docs/profiles/centos.md @@ -0,0 +1,124 @@ +# CentOS + +## Available Commands + +The `centos` profile incorporates commands from [Django](../commands/django.md), [messages](../commands/messages.md), [MySQL](../commands/mysql.md), [PHP](../commands/php.md), [POSIX](../commands/posix.md), [Postgres](../commands/pgsql.md), and [Python](../commands/python.md). + +### apache + +Work with Apache. + +- `apache.disable_module: module_name` (not supported) +- `apache.disable_site: site_name` (not supported) +- `apache.enable_module: module_name` (not supported) +- `apache.enable_site: site_name` (not supported) +- `apache.reload` +- `apache.restart` +- `apache.start` +- `apache.stop` +- `apache.test` + +### install + +Install a system package. + +```ini +[install apache] +install: apache2 +``` + +### reload + +Reload a service. + +```ini +[reload postgres] +reload: postgresql +``` + +### restart + +Restart a service: + +```ini +[restart postgres] +restart: postgresql +``` + +### run + +Run any shell command. + +```ini +[run a useless listing command] +run: "ls -ls" +``` + +Note that commands with arguments will need to be in quotes. + +### start + +Start a service: + +```ini +[start postgres] +start: postgresql +``` + +### stop + +Stop a service: + +```ini +[stop postgres] +stop: postgresql +``` + +### system + +With with the system. + +- `system.reboot` +- `system.update` +- `system.upgrade` + +### uninstall + +Uninstall a package. + +```ini +[remove libxyz development package] +uninstall: libxyz-dev +``` + +### upgrade + +Upgrade a package. + +```ini +[upgrade libxyz development package] +upgrade: libxyz-dev +``` + +### user + +Create a user: + +- `groups`: A comma separated list of groups to which the user should be added. +- `home`: The user's home directory. +- `login`: The shell to use. +- `system`: Create as a system user. + +```ini +[create the deploy user] +user.add: deploy +groups: www-data +home: /var/www +``` + +Remove a user: + +```ini +[remove bob] +user.remove: bob +``` diff --git a/help/docs/profiles/ubuntu.md b/help/docs/profiles/ubuntu.md new file mode 100644 index 0000000..299313f --- /dev/null +++ b/help/docs/profiles/ubuntu.md @@ -0,0 +1,124 @@ +# Ubuntu + +## Available Commands + +The `ubuntu` profile incorporates commands from [Django](../commands/django.md), [messages](../commands/messages.md), [MySQL](../commands/mysql.md), [PHP](../commands/php.md), [POSIX](../commands/posix.md), [Postgres](../commands/pgsql.md), and [Python](../commands/python.md). + +### apache + +Work with Apache. + +- `apache.disable_module: module_name` +- `apache.disable_site: site_name` +- `apache.enable_module: module_name` +- `apache.enable_site: site_name` +- `apache.reload` +- `apache.restart` +- `apache.start` +- `apache.stop` +- `apache.test` + +### install + +Install a system package. + +```ini +[install apache] +install: apache2 +``` + +### reload + +Reload a service. + +```ini +[reload postgres] +reload: postgresql +``` + +### restart + +Restart a service: + +```ini +[restart postgres] +restart: postgresql +``` + +### run + +Run any shell command. + +```ini +[run a useless listing command] +run: "ls -ls" +``` + +Note that commands with arguments will need to be in quotes. + +### start + +Start a service: + +```ini +[start postgres] +start: postgresql +``` + +### stop + +Stop a service: + +```ini +[stop postgres] +stop: postgresql +``` + +### system + +With with the system. + +- `system.reboot` +- `system.update` +- `system.upgrade` + +### uninstall + +Uninstall a package. + +```ini +[remove libxyz development package] +uninstall: libxyz-dev +``` + +### upgrade + +Upgrade a package. + +```ini +[upgrade libxyz development package] +upgrade: libxyz-dev +``` + +### user + +Create a user: + +- `groups`: A comma separated list of groups to which the user should be added. +- `home`: The user's home directory. +- `login`: The shell to use. +- `system`: Create as a system user. + +```ini +[create the deploy user] +user.add: deploy +groups: www-data +home: /var/www +``` + +Remove a user: + +```ini +[remove bob] +user.remove: bob +``` diff --git a/help/mkdocs.yml b/help/mkdocs.yml new file mode 100644 index 0000000..3a906e2 --- /dev/null +++ b/help/mkdocs.yml @@ -0,0 +1,26 @@ +site_name: Script Tease +copyright: Copyright © Pleasant Tents, LLC. All rights reserved. +markdown_extensions: + - toc: + permalink: True + - admonition + - attr_list + - def_list +nav: + - Home: index.md + - Configuration: + - Command File: config/command-file.md + - Variables: config/variables.md + - Commands: + - Django: commands/django.md + - Messages: commands/messages.md + - MySQL: commands/mysql.md + - PHP: commands/php.md + - Postgres: commands/pgsql.md + - POSIX: commands/posix.md + - Python: commands/python.md + - Profiles: + - Ubuntu: profiles/ubuntu.md +# - Developer Reference: /developers/ +repo_name: GitLab +repo_url: https://git.sixgunsoftware.com/python-scripttease diff --git a/scripttease/lib/snippets/mysql.py b/scripttease/lib/snippets/mysql.py index c841dfe..eda4fd1 100644 --- a/scripttease/lib/snippets/mysql.py +++ b/scripttease/lib/snippets/mysql.py @@ -54,7 +54,7 @@ mysql = { '{% if admin_pass %}--password="{{ admin_pass }}"{% endif %}', '--host={{ host|default("localhost") }}', '--port={{ port|default("3306") }}', - '--execute="GRANT {{ args[0] }} ON {{ database }}.* TO \'{{ user }}\'@\'{{ host|default("localhost") }}\'"' + '--execute="GRANT {{ args[0] }} ON {{ database|default("default") }}.* TO \'{{ user }}\'@\'{{ host|default("localhost") }}\'"' ], 'user': { 'create': [ diff --git a/scripttease/lib/snippets/pgsql.py b/scripttease/lib/snippets/pgsql.py index d5ce2a7..7da25e2 100644 --- a/scripttease/lib/snippets/pgsql.py +++ b/scripttease/lib/snippets/pgsql.py @@ -57,7 +57,7 @@ pgsql = { '--host={{ host|default("localhost") }}', '--port={{ port|default("5432") }}', "--column-inserts", - '--file={{ file_name|default("dump.sql") }}', + '--file={{ path|default("dump.sql") }}', "{{ args[0] }}" ], 'exec': [ diff --git a/scripttease/lib/snippets/ubuntu.py b/scripttease/lib/snippets/ubuntu.py index e2a65cc..9679320 100644 --- a/scripttease/lib/snippets/ubuntu.py +++ b/scripttease/lib/snippets/ubuntu.py @@ -1,9 +1,9 @@ ubuntu = { 'apache': { - 'disable': '{% if args[0].startswith("mod_") %}a2dismod{% else %}a2dissite{% endif %} {{ args[0] }}', - 'disable_module': "a2dissite {{ args[0] }}", - 'disable_site': "a2dismod {{ args[0] }}", - 'enable': '{% if args[0].startswith("mod_") %}a2enmod{% else %}a2ensite{% endif %} {{ args[0] }}', + # 'disable': '{% if args[0].startswith("mod_") %}a2dismod{% else %}a2dissite{% endif %} {{ args[0] }}', + 'disable_module': "a2dismod {{ args[0] }}", + 'disable_site': "a2dissite {{ args[0] }}", + # 'enable': '{% if args[0].startswith("mod_") %}a2enmod{% else %}a2ensite{% endif %} {{ args[0] }}', 'enable_module': "a2enmod {{ args[0] }}", 'enable_site': "a2ensite {{ args[0] }}", 'reload': "service apache2 reload",