From e5b7b3dbe049ef1ad25656ebc43942d62e8da8d6 Mon Sep 17 00:00:00 2001 From: Shawn Davis Date: Thu, 23 Jul 2020 16:23:10 -0400 Subject: [PATCH] Added support for template command. --- VERSION.txt | 2 +- scripttease/cli/__init__.py | 4 +- scripttease/library/commands/__init__.py | 1 + scripttease/library/commands/templates.py | 196 ++++++++++++++++++++++ scripttease/library/overlays/posix.py | 16 ++ scripttease/library/overlays/ubuntu.py | 8 +- scripttease/parsers/ini.py | 49 ++++-- tests/examples/kitchen_sink.ini | 11 ++ tests/examples/templates/bad.j2.txt | 1 + tests/examples/templates/good.j2.txt | 3 + tests/examples/templates/simple.sh.txt | 5 + tests/examples/templates/simple.txt | 3 + tests/test_library_commands_templates.py | 113 +++++++++++++ tests/test_library_overlays_ubuntu.py | 6 + tests/test_parsers_ini.py | 3 + 15 files changed, 400 insertions(+), 21 deletions(-) create mode 100644 scripttease/library/commands/templates.py create mode 100644 tests/examples/templates/bad.j2.txt create mode 100644 tests/examples/templates/good.j2.txt create mode 100644 tests/examples/templates/simple.sh.txt create mode 100644 tests/examples/templates/simple.txt create mode 100644 tests/test_library_commands_templates.py diff --git a/VERSION.txt b/VERSION.txt index 564bbb7..b1a9275 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -6.0.1-d \ No newline at end of file +6.3.0-d \ No newline at end of file diff --git a/scripttease/cli/__init__.py b/scripttease/cli/__init__.py index fc6ff13..ffc0a62 100644 --- a/scripttease/cli/__init__.py +++ b/scripttease/cli/__init__.py @@ -18,13 +18,13 @@ def main_command(): """Process script configurations.""" __author__ = "Shawn Davis " - __date__ = "2020-07-22" + __date__ = "2020-07-23" __help__ = """NOTES This command is used to parse configuration files and output the commands. """ - __version__ = "6.0.1-d" + __version__ = "6.2.0-d" # Main argument parser from which sub-commands are created. parser = ArgumentParser(description=__doc__, epilog=__help__, formatter_class=RawDescriptionHelpFormatter) diff --git a/scripttease/library/commands/__init__.py b/scripttease/library/commands/__init__.py index 1084628..d4107d6 100644 --- a/scripttease/library/commands/__init__.py +++ b/scripttease/library/commands/__init__.py @@ -1,2 +1,3 @@ from .base import Command, ItemizedCommand +from .templates import Template # from .factory import command_factory diff --git a/scripttease/library/commands/templates.py b/scripttease/library/commands/templates.py new file mode 100644 index 0000000..1f5597e --- /dev/null +++ b/scripttease/library/commands/templates.py @@ -0,0 +1,196 @@ +# Imports + +from jinja2.exceptions import TemplateError, TemplateNotFound +import logging +import os +from superpython.utils import parse_jinja_template, read_file +from ...constants import LOGGER_NAME +from .base import Command + +log = logging.getLogger(LOGGER_NAME) + +# Exports + +__all__ = ( + "Template", +) + +# Classes + + +class Template(Command): + """Parse a template.""" + + PARSER_JINJA = "jinja2" + PARSER_SIMPLE = "simple" + + def __init__(self, source, target, backup=True, lines=False, parser=PARSER_JINJA, pythonic=False, **kwargs): + """Initialize the command. + + :param source: The template source file. + :type source: str + + :param target: The path to the output file. + :type target: str + + :param backup: Indicates a copy of an existing file should be madee. + :type backup: bool + + :param parser: The parser to use. + :type parser: str + + :param pythonic: Use a Python one-liner to write the file. Requires Python installation, obviously. This is + useful when the content of the file cannot be handled with a cat command; for example, shell + script templates. + :type pythonic: bool + + """ + # Base parameters need to be captured, because all others are assumed to be switches for the management command. + self._kwargs = { + 'comment': kwargs.pop("comment", None), + 'cd': kwargs.pop("cd", None), + 'environments': kwargs.pop("environments", None), + 'function': kwargs.pop("function", None), + # 'local': kwargs.pop("local", False), + 'prefix': kwargs.pop("prefix", None), + 'shell': kwargs.pop("shell", "/bin/bash"), + 'stop': kwargs.pop("stop", False), + 'sudo': kwargs.pop('sudo', False), + 'tags': kwargs.pop("tags", None), + } + + self.backup_enabled = backup + self.context = kwargs.pop("context", dict()) + self.parser = parser or self.PARSER_JINJA + self.pythonic = pythonic + self.line_by_line = lines + self.locations = kwargs.pop("locations", list()) + self.source = source + self.target = target + + # Remaining kwargs are added to the context. + # print(_kwargs['comment'], kwargs) + self.context.update(kwargs) + + super().__init__("# template: %s" % source, **self._kwargs) + + def get_content(self): + """Parse the template. + + :rtype: str | None + + """ + template = self.get_template() + + if self.parser == self.PARSER_SIMPLE: + content = read_file(template) + for key, value in self.context.items(): + replace = "$%s$" % key + content = content.replace(replace, str(value)) + + return content + + try: + return parse_jinja_template(template, self.context) + except TemplateNotFound: + log.error("Template not found: %s" % template) + return None + except TemplateError as e: + log.error("Could not parse %s template: %s" % (template, e)) + return None + + def get_statement(self, cd=False, suppress_comment=False): + """Override to get the statement based on the parser.""" + if self.parser == self.PARSER_JINJA: + return self._get_jinja2_statement(cd=cd).statement + elif self.parser == self.PARSER_SIMPLE: + return self._get_simple_statement(cd=cd).statement + else: + log.error("Unknown or unsupported template parser: %s" % self.parser) + return None + + def get_template(self): + """Get the template path. + + :rtype: str + + """ + source = self.source + for location in self.locations: + _source = os.path.join(location, self.source) + if os.path.exists(_source): + return _source + + return source + + def _get_command(self, content): + """Get the cat command.""" + output = list() + + # TODO: Template backup is not system safe, but is specific to bash. + if self.backup_enabled: + output.append('if [[ -f "%s" ]]; then mv %s %s.b; fi;' % (self.target, self.target, self.target)) + + if content.startswith("#!"): + _content = content.split("\n") + first_line = _content.pop(0) + output.append('echo "%s" > %s' % (first_line, self.target)) + output.append("cat >> %s << EOF" % self.target) + output.append("\n".join(_content)) + output.append("EOF") + else: + output.append("cat > %s << EOF" % self.target) + output.append(content) + output.append("EOF") + + statement = "\n".join(output) + + return Command(statement, **self._kwargs) + + # # BUG: This still does not seem to work, possibly because a shell script includes EOF? The work around is to use + # # get_content(), self.target, and write the file manually. + # if self.line_by_line: + # a = list() + # a.append('touch %s' % self.target) + # for i in content.split("\n"): + # i = i.replace('"', r'\"') + # a.append('echo "%s" >> %s' % (i, self.target)) + # + # output.append("\n".join(a)) + # elif self.pythonic: + # target_file = File(self.target) + # script_file = "write_%s_template.py" % target_file.name.replace("-", "_") + # + # a = list() + # a.append('content = """%s' % content) + # a.append('"""') + # a.append("") + # a.append('with open("%s", "w") as f:' % self.target) + # a.append(' f.write(content)') + # a.append(' f.close()') + # a.append('') + # output.append('cat > %s < %s << EOF" % self.target) + # output.append(content) + # output.append("EOF") + # + # statement = "\n".join(output) + # return Command(statement, **self._kwargs) + + # noinspection PyUnusedLocal + def _get_jinja2_statement(self, cd=False): + """Parse a Jinja2 template.""" + content = self.get_content() + return self._get_command(content) + + # noinspection PyUnusedLocal + def _get_simple_statement(self, cd=False): + """Parse a "simple" template.""" + content = self.get_content() + + return self._get_command(content) diff --git a/scripttease/library/overlays/posix.py b/scripttease/library/overlays/posix.py index f785b45..95f8339 100644 --- a/scripttease/library/overlays/posix.py +++ b/scripttease/library/overlays/posix.py @@ -18,6 +18,7 @@ __all__ = ( "move", "perms", "remove", + "rename", "rsync", "run", "scopy", @@ -371,6 +372,20 @@ def remove(path, force=False, recursive=False, **kwargs): return Command(" ".join(statement), **kwargs) +def rename(from_name, to_name, **kwargs): + """Rename of a file or directory. + + :param from_name: The name (or path) of the existing file. + :type from_name: str + + :param to_name: The name (or path) of the new file. + :type to_name: str + + """ + kwargs.setdefault("comment", "rename %s" % from_name) + return move(from_name, to_name, **kwargs) + + def rsync(source, target, delete=False, exclude=None, host=None, key_file=None, links=True, port=22, recursive=True, user=None, **kwargs): """Initialize the command. @@ -633,6 +648,7 @@ POSIX_MAPPINGS = { 'move': move, 'perms': perms, 'remove': remove, + 'rename': rename, 'rsync': rsync, 'run': run, 'scopy': scopy, diff --git a/scripttease/library/overlays/ubuntu.py b/scripttease/library/overlays/ubuntu.py index aa9a3d1..9285492 100644 --- a/scripttease/library/overlays/ubuntu.py +++ b/scripttease/library/overlays/ubuntu.py @@ -1,6 +1,6 @@ # Imports -from ..commands import Command +from ..commands import Command, Template from .common import COMMON_MAPPINGS from .django import DJANGO_MAPPINGS from .pgsql import PGSQL_MAPPINGS @@ -31,6 +31,7 @@ __all__ = ( "system_update", "system_upgrade", "system_uninstall", + "template", "Function", ) @@ -181,6 +182,10 @@ def system_upgrade(**kwargs): return Command("apt-get upgrade -y", **kwargs) +def template(source, target, backup=True, parser=None, **kwargs): + return Template(source, target, backup=backup, parser=parser, **kwargs) + + MAPPINGS = { 'apache': apache, 'apache.disable_module': apache_disable_module, @@ -202,6 +207,7 @@ MAPPINGS = { 'update': system_update, 'uninstall': system_uninstall, 'upgrade': system_upgrade, + 'template': template, } MAPPINGS.update(COMMON_MAPPINGS) diff --git a/scripttease/parsers/ini.py b/scripttease/parsers/ini.py index f69ea78..a636025 100644 --- a/scripttease/parsers/ini.py +++ b/scripttease/parsers/ini.py @@ -5,6 +5,8 @@ import logging from superpython.utils import parse_jinja_template, read_file, smart_cast, split_csv import os from ..constants import LOGGER_NAME +from ..library.commands import ItemizedCommand +from ..library.commands.templates import Template from .base import Parser log = logging.getLogger(LOGGER_NAME) @@ -63,6 +65,19 @@ class Config(Parser): if command is not None: if isinstance(command, self.factory.overlay.Function): self._functions.append(command) + elif isinstance(command, Template): + self._load_template(command) + self._commands.append(command) + elif isinstance(command, ItemizedCommand): + itemized_template = False + for c in command.get_commands(): + if isinstance(c, Template): + itemized_template = True + self._load_template(c) + self._commands.append(c) + + if not itemized_template: + self._commands.append(command) else: self._commands.append(command) @@ -141,20 +156,20 @@ class Config(Parser): log.error("Failed to parse %s: %s" % (self.path, e)) return None - # def _load_template(self, command): - # """Load additional resources for a template command. - # - # :param command: The template command. - # :type command: Template - # - # """ - # # This may produce problems if template kwargs are the same as the given context. - # if self.context is not None: - # command.context.update(self.context) - # - # # Custom locations come before default locations. - # command.locations += self.locations - # - # # This allows template files to be specified relative to the configuration file. - # command.locations.append(os.path.join(self.directory, "templates")) - # command.locations.append(self.directory) + def _load_template(self, command): + """Load additional resources for a template command. + + :param command: The template command. + :type command: Template + + """ + # This may produce problems if template kwargs are the same as the given context. + if self.context is not None: + command.context.update(self.context) + + # Custom locations come before default locations. + command.locations += self.locations + + # This allows template files to be specified relative to the configuration file. + command.locations.append(os.path.join(self.directory, "templates")) + command.locations.append(self.directory) diff --git a/tests/examples/kitchen_sink.ini b/tests/examples/kitchen_sink.ini index 6f28335..3576f8a 100644 --- a/tests/examples/kitchen_sink.ini +++ b/tests/examples/kitchen_sink.ini @@ -190,3 +190,14 @@ archive: /var/www/domains/example_com [extract a file archive] extract: /var/www/domains/example_com.tgz + +[create a file from a template] +template: good.j2.txt tests/tmp/good.txt +testing: {{ testing }} +times: 123 + +[create a bunch of files using templates] +template: $item.txt tests/tmp/$item +items: simple.sh.txt, simple.txt +testing: {{ testing }} +times: 123 diff --git a/tests/examples/templates/bad.j2.txt b/tests/examples/templates/bad.j2.txt new file mode 100644 index 0000000..3a5ef69 --- /dev/null +++ b/tests/examples/templates/bad.j2.txt @@ -0,0 +1 @@ +{% if testing %}{{ testing }} \ No newline at end of file diff --git a/tests/examples/templates/good.j2.txt b/tests/examples/templates/good.j2.txt new file mode 100644 index 0000000..f5c5fd9 --- /dev/null +++ b/tests/examples/templates/good.j2.txt @@ -0,0 +1,3 @@ +I am testing? {{ testing }} + +How many times? {{ times }} \ No newline at end of file diff --git a/tests/examples/templates/simple.sh.txt b/tests/examples/templates/simple.sh.txt new file mode 100644 index 0000000..31376e2 --- /dev/null +++ b/tests/examples/templates/simple.sh.txt @@ -0,0 +1,5 @@ +#! /usr/bin/bash + +echo "I am testing? $testing$"; + +echo "How many times? $times$"; diff --git a/tests/examples/templates/simple.txt b/tests/examples/templates/simple.txt new file mode 100644 index 0000000..413cf8c --- /dev/null +++ b/tests/examples/templates/simple.txt @@ -0,0 +1,3 @@ +I am testing? $testing$ + +How many times? $times$ diff --git a/tests/test_library_commands_templates.py b/tests/test_library_commands_templates.py new file mode 100644 index 0000000..1b8f4b4 --- /dev/null +++ b/tests/test_library_commands_templates.py @@ -0,0 +1,113 @@ +from scripttease.library.commands.base import Command, ItemizedCommand, Sudo +from scripttease.library.commands.templates import Template + + +class TestTemplate(object): + + def test_get_content(self): + context = { + 'testing': "yes", + 'times': 123, + } + t = Template( + "tests/examples/templates/simple.txt", + "tests/tmp/simple.txt", + backup=False, + context=context, + parser=Template.PARSER_SIMPLE + ) + content = t.get_content() + assert "I am testing? yes" in content + assert "How many times? 123" in content + + context = { + 'testing': "yes", + 'times': 123, + } + t = Template( + "tests/examples/templates/simple.sh.txt", + "tests/tmp/simple.sh", + backup=False, + context=context, + parser=Template.PARSER_SIMPLE + ) + content = t.get_content() + assert "I am testing? yes" in content + assert "How many times? 123" in content + + context = { + 'testing': "yes", + 'times': 123, + } + t = Template( + "tests/examples/templates/good.j2.txt", + "tests/tmp/good.txt", + backup=False, + context=context + ) + content = t.get_content() + assert "I am testing? yes" in content + assert "How many times? 123" in content + + t = Template("tests/examples/templates/nonexistent.j2.txt", "test/tmp/nonexistent.txt") + assert t.get_content() is None + + t = Template("tests/examples/templates/bad.j2.txt", "test/tmp/nonexistent.txt") + assert t.get_content() is None + + def test_get_statement(self): + context = { + 'testing': "yes", + 'times': 123, + } + t = Template( + "tests/examples/templates/simple.txt", + "tests/tmp/simple.txt", + context=context, + parser=Template.PARSER_SIMPLE + ) + s = t.get_statement() + assert "I am testing? yes" in s + assert "How many times? 123" in s + + context = { + 'testing': "yes", + 'times': 123, + } + t = Template( + "tests/examples/templates/simple.sh.txt", + "tests/tmp/simple.txt", + context=context, + parser=Template.PARSER_SIMPLE + ) + s = t.get_statement() + assert "I am testing? yes" in s + assert "How many times? 123" in s + + context = { + 'testing': "yes", + 'times': 123, + } + t = Template( + "tests/examples/templates/good.j2.txt", + "tests/tmp/good.txt", + context=context + ) + s = t.get_statement() + assert "I am testing? yes" in s + assert "How many times? 123" in s + + t = Template( + "tests/examples/templates/simple.txt", + "tests/tmp/simple.txt", + parser="nonexistent" + ) + assert t.get_statement() is None + + def test_get_template(self): + t = Template( + "simple.txt", + "tests/tmp/simple.txt", + locations=["tests/examples/templates"] + ) + assert t.get_template() == "tests/examples/templates/simple.txt" diff --git a/tests/test_library_overlays_ubuntu.py b/tests/test_library_overlays_ubuntu.py index 486645c..5692c77 100644 --- a/tests/test_library_overlays_ubuntu.py +++ b/tests/test_library_overlays_ubuntu.py @@ -1,4 +1,5 @@ import pytest +from scripttease.library.commands.templates import Template from scripttease.library.overlays.ubuntu import * @@ -84,3 +85,8 @@ def test_system_install(): def test_system_uninstall(): c = system_uninstall("lftp") assert "apt-get uninstall -y lftp" in c.get_statement() + + +def test_template(): + t = template("/path/to/source.txt", "/path/to/target.txt") + assert isinstance(t, Template) diff --git a/tests/test_parsers_ini.py b/tests/test_parsers_ini.py index a2c9f37..d3b0b06 100644 --- a/tests/test_parsers_ini.py +++ b/tests/test_parsers_ini.py @@ -29,6 +29,9 @@ class TestConfig(object): c = Config("tests/examples/kitchen_sink.ini") assert c.load() is True + c = Config("tests/examples/kitchen_sink.ini", context={'testing': "yes"}) + assert c.load() is True + c = Config("tests/examples/bad_command.ini") assert c.load() is False