Refactored snippets and loaders to lib.

development
Shawn Davis 3 years ago
parent 6562f2f3df
commit fba0c89d36
  1. 0
      scripttease/lib/__init__.py
  2. 0
      scripttease/lib/loaders/__init__.py
  3. 117
      scripttease/lib/loaders/base.py
  4. 9
      scripttease/lib/loaders/ini.py
  5. 6
      scripttease/lib/loaders/yaml.py
  6. 0
      scripttease/lib/snippets/__init__.py
  7. 4
      scripttease/lib/snippets/centos.py
  8. 0
      scripttease/lib/snippets/django.py
  9. 0
      scripttease/lib/snippets/mappings.py
  10. 0
      scripttease/lib/snippets/messages.py
  11. 0
      scripttease/lib/snippets/mysql.py
  12. 2
      scripttease/lib/snippets/pgsql.py
  13. 14
      scripttease/lib/snippets/posix.py
  14. 6
      scripttease/lib/snippets/python.py
  15. 4
      scripttease/lib/snippets/ubuntu.py

@ -4,7 +4,7 @@ from commonkit import parse_jinja_string, parse_jinja_template, pick, read_file,
from jinja2.exceptions import TemplateError, TemplateNotFound from jinja2.exceptions import TemplateError, TemplateNotFound
import logging import logging
import os import os
from ..library.snippets.mappings import MAPPINGS from ..snippets.mappings import MAPPINGS
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -12,12 +12,17 @@ log = logging.getLogger(__name__)
__all__ = ( __all__ = (
"BaseLoader", "BaseLoader",
"Snippet",
"Sudo",
"Template",
) )
# Classes # Classes
class BaseLoader(File): class BaseLoader(File):
"""Base class for loading a command file."""
def __init__(self, path, context=None, locations=None, mappings=None, profile="ubuntu", **kwargs): def __init__(self, path, context=None, locations=None, mappings=None, profile="ubuntu", **kwargs):
"""Initialize the loader. """Initialize the loader.
@ -25,14 +30,16 @@ class BaseLoader(File):
:param path: The path to the command file. :param path: The path to the command file.
:type path: str :type path: str
:param context: Global context that may be used when to parse the command file, snippets, and templates. :param context: Global context that may be used when to parse the command file, snippets, and templates. This is
:type context: dict converted to a ``dict`` when passed to a Snippet or Template.
:type context: scripttease.lib.contexts.Context
:param locations: A list of paths where templates and other external files may be found. The directory in which :param locations: A list of paths where templates and other external files may be found. The ``templates/``
the command file exists is added automatically. directory in which the command file exists is added automatically.
:type locations: list[str] :type locations: list[str]
:param mappings: A mapping of canonical command names and their snippets, organized by ``profile``. :param mappings: A mapping of canonical command names and their snippets, organized by ``profile``. The profile
is typically an operating system such as ``centos`` or ``ubuntu``.
:type mappings: dict :type mappings: dict
:param profile: The profile (operating system or platform) to be used. :param profile: The profile (operating system or platform) to be used.
@ -42,7 +49,7 @@ class BaseLoader(File):
be supplied as defaults for snippet processing. be supplied as defaults for snippet processing.
""" """
self.context = context or dict() self.context = context
self.is_loaded = False self.is_loaded = False
self.locations = locations or list() self.locations = locations or list()
self.mappings = mappings or MAPPINGS self.mappings = mappings or MAPPINGS
@ -53,10 +60,26 @@ class BaseLoader(File):
super().__init__(path) super().__init__(path)
# Always include the path to the current file in locations. # Always include the path to the current file in locations.
self.locations.insert(0, self.directory) self.locations.insert(0, os.path.join(self.directory, "templates"))
# command.locations.append(os.path.join(self.directory, "templates"))
def get_context(self):
"""Get the context for parsing command snippets.
:rtype: dict
"""
d = self.options.copy()
if self.context is not None:
d.update(self.context.mapping().copy())
return d
def get_snippets(self): def get_snippets(self):
"""Get the snippets found in a config file.
:rtype: list[scripttease.lib.loaders.base.Snippet]
"""
a = list() a = list()
for canonical_name, args, kwargs in self.snippets: for canonical_name, args, kwargs in self.snippets:
@ -66,12 +89,27 @@ class BaseLoader(File):
return a return a
def find_snippet(self, name, *args, **kwargs): def find_snippet(self, name, *args, **kwargs):
"""Find a named snippet that was defined in a config file.
:param name: The canonical name (or dotted name) of the snippet.
:type name: str
:rtype: scripttease.lib.loaders.base.Snippet | scripttease.lib.loaders.base.Template
``args`` and ``kwargs`` are passed to instantiate the snippet.
.. important::
The snippet may be invalid; always check ``snippet.is_valid``.
"""
# Templates require special handling. # Templates require special handling.
if name == "template": if name == "template":
source = args[0] source = args[0]
target = args[1] target = args[1]
kwargs['locations'] = self.locations kwargs['locations'] = self.locations
return Template(source, target, **kwargs) context = kwargs.copy()
context.update(self.get_context())
return Template(source, target, context=context, **kwargs)
# Convert args to a list so we can update it below. # Convert args to a list so we can update it below.
_args = list(args) _args = list(args)
@ -85,6 +123,9 @@ class BaseLoader(File):
log.error("Command not found in mappings: %s" % name) log.error("Command not found in mappings: %s" % name)
return Snippet(name, args=_args, kwargs=kwargs) return Snippet(name, args=_args, kwargs=kwargs)
# Get the global context for use in snippet instantiation below.
context = self.get_context()
# Formal or informal sub-commands exist in a dictionary. # Formal or informal sub-commands exist in a dictionary.
if type(self.mappings[self.profile][name]) is dict: if type(self.mappings[self.profile][name]) is dict:
try: try:
@ -100,27 +141,39 @@ class BaseLoader(File):
parser = self.mappings[self.profile][name].get('_parser', None) parser = self.mappings[self.profile][name].get('_parser', None)
_name = "%s.%s" % (name, sub) _name = "%s.%s" % (name, sub)
return Snippet(_name, args=_args, content=snippet, context=self.context, kwargs=kwargs, parser=parser) return Snippet(_name, args=_args, content=snippet, context=context, kwargs=kwargs, parser=parser)
# Django allows pre-defined as well as adhoc commands. The name of the command is provided as the first # Django allows pre-defined as well as adhoc commands. The name of the command is provided as the first
# argument in the config file. The following statements are only invoked if the possible_sub_command is not # argument in the config file. The following statements are only invoked if the possible_sub_command is not
# in the django dictionary. # in the django dictionary. The "command" entry in the django dictionary is for handling ad hoc commands.
if name == "django": if name == "django":
sub = _args.pop(0) sub = _args.pop(0)
kwargs['_name'] = sub kwargs['_name'] = sub
print(kwargs)
snippet = self.mappings[self.profile]['django']['command'] snippet = self.mappings[self.profile]['django']['command']
parser = self.mappings[self.profile]['django']['_parser'] parser = self.mappings[self.profile]['django']['_parser']
return Snippet("django.%s" % sub, args=_args, content=snippet, context=self.context, kwargs=kwargs, return Snippet("django.%s" % sub, args=_args, content=snippet, context=context, kwargs=kwargs,
parser=parser) parser=parser)
log.warning("Sub-command could not be determined for: %s" % name) log.warning("Sub-command could not be determined for: %s" % name)
return Snippet(name, args=list(args), context=self.context, kwargs=kwargs) return Snippet(name, args=list(args), context=context, kwargs=kwargs)
# The found snippet should just be a string. # The found snippet should just be a string.
return Snippet(name, args=list(args), content=self.mappings[self.profile][name], kwargs=kwargs) return Snippet(name, args=list(args), content=self.mappings[self.profile][name], context=context, kwargs=kwargs)
def find_snippet_by_dotted_name(self, name, *args, **kwargs): def find_snippet_by_dotted_name(self, name, *args, **kwargs):
"""Find a snippet using it's dotted name.
:param name: The dotted name of the snippet.
:param args:
:param kwargs:
``args`` and ``kwargs`` are passed to instantiate the snippet.
.. important::
The snippet may be invalid; always check ``snippet.is_valid``.
"""
# This may not exist. If so, None is the value of the Snippet.content attribute. # This may not exist. If so, None is the value of the Snippet.content attribute.
snippet = pick(name, self.mappings[self.profile]) snippet = pick(name, self.mappings[self.profile])
@ -129,7 +182,7 @@ class BaseLoader(File):
parser = pick(builder_name, self.mappings[self.profile]) parser = pick(builder_name, self.mappings[self.profile])
# Return the snippet instance. # Return the snippet instance.
return Snippet(name, args=list(args), parser=parser, content=snippet, kwargs=kwargs) return Snippet(name, args=list(args), parser=parser, content=snippet, context=self.get_context(), kwargs=kwargs)
def load(self): def load(self):
"""Load the command file. """Load the command file.
@ -147,7 +200,7 @@ class BaseLoader(File):
""" """
if self.context is not None: if self.context is not None:
try: try:
return parse_jinja_template(self.path, self.context) return parse_jinja_template(self.path, self.context.mapping())
except Exception as e: except Exception as e:
log.error("Failed to process %s file as template: %s" % (self.path, e)) log.error("Failed to process %s file as template: %s" % (self.path, e))
return None return None
@ -156,7 +209,7 @@ class BaseLoader(File):
# noinspection PyMethodMayBeStatic # noinspection PyMethodMayBeStatic
def _get_key_value(self, key, value): def _get_key_value(self, key, value):
"""Process a key/value pair from an INI section. """Process a key/value pair.
:param key: The key to be processed. :param key: The key to be processed.
:type key: str :type key: str
@ -166,9 +219,24 @@ class BaseLoader(File):
:rtype: tuple :rtype: tuple
:returns: The key and value, both of which may be modified from the originals. :returns: The key and value, both of which may be modified from the originals.
This handles special names in the following manner:
- ``environments``, ``environs``, ``envs``, and ``env`` are treated as a CSV list of environment names
if provided as a string. These are normalized to the keyword ``environments``.
- ``func`` and ``function`` are normalized to the keyword ``function``. The value is the name of the function to
be defined.
- ``groups`` is assumed to be a CSV list of groups if provided as a string.
- ``items`` is assumed to be a CSV list if provided as a string. These are used to create an "itemized" command.
- ``tags`` is assumed to be a CSV list oif provided as a string.
All other keys are used as is. Values provided as a CSV list are smart cast to a Python value.
""" """
if key in ("environments", "environs", "envs", "env"): if key in ("environments", "environs", "envs", "env"):
_key = "environments" _key = "environments"
if type(value) in (list, tuple):
_value = value
else:
_value = split_csv(value) _value = split_csv(value)
elif key in ("func", "function"): elif key in ("func", "function"):
_key = "function" _key = "function"
@ -187,6 +255,9 @@ class BaseLoader(File):
_value = split_csv(value) _value = split_csv(value)
elif key == "tags": elif key == "tags":
_key = "tags" _key = "tags"
if type(value) in (list, tuple):
_value = value
else:
_value = split_csv(value) _value = split_csv(value)
else: else:
_key = key _key = key
@ -199,8 +270,8 @@ class Snippet(object):
"""A snippet is a pseudo-command which collects the content of the snippet as well as the parameters that may be """A snippet is a pseudo-command which collects the content of the snippet as well as the parameters that may be
used to create an executable statement. used to create an executable statement.
The purpose of a snippet is *not* to provide command execution, but to capture the results of a command requested in The purpose of a snippet is *not* to provide command execution, but to capture the parameters of a command defined
a command configuration file. in a configuration file.
""" """
@ -434,10 +505,10 @@ class Template(object):
PARSER_JINJA = "jinja2" PARSER_JINJA = "jinja2"
PARSER_SIMPLE = "simple" PARSER_SIMPLE = "simple"
def __init__(self, source, target, backup=True, parser=None, **kwargs): def __init__(self, source, target, backup=True, parser=PARSER_JINJA, **kwargs):
self.backup_enabled = backup self.backup_enabled = backup
self.context = kwargs.pop("context", dict()) self.context = kwargs.pop("context", dict())
self.parser = parser or self.PARSER_JINJA self.parser = parser
self.locations = kwargs.pop("locations", list()) self.locations = kwargs.pop("locations", list())
self.source = os.path.expanduser(source) self.source = os.path.expanduser(source)
self.target = target self.target = target

@ -17,14 +17,20 @@ __all__ = (
class INILoader(BaseLoader): class INILoader(BaseLoader):
"""Load commands from an INI file."""
def load(self): def load(self):
"""Load the INI file.
:rtype: bool
"""
if not self.exists: if not self.exists:
return False return False
if self.context is not None: if self.context is not None:
try: try:
content = parse_jinja_template(self.path, self.context) content = parse_jinja_template(self.path, self.get_context())
except Exception as e: except Exception as e:
log.error("Failed to process %s INI file as template: %s" % (self.path, e)) log.error("Failed to process %s INI file as template: %s" % (self.path, e))
return False return False
@ -65,6 +71,7 @@ class INILoader(BaseLoader):
except IndexError: except IndexError:
pass pass
except TypeError: except TypeError:
# noinspection PyTypeChecker
args.append(True) args.append(True)
else: else:
_key, _value = self._get_key_value(key, value) _key, _value = self._get_key_value(key, value)

@ -17,8 +17,14 @@ __all__ = (
class YMLLoader(BaseLoader): class YMLLoader(BaseLoader):
"""Load commands from an YAML file."""
def load(self): def load(self):
"""Load the YAML file.
:rtype: bool
"""
if not self.exists: if not self.exists:
return False return False

@ -20,9 +20,11 @@ centos = {
'uninstall': "yum remove -y {{ args[0] }}", 'uninstall': "yum remove -y {{ args[0] }}",
'upgrade': "yum upgrade -y {{ args[0] }}", 'upgrade': "yum upgrade -y {{ args[0] }}",
'user': { 'user': {
'create': [ 'add': [
"adduser {{ args[0] }}", "adduser {{ args[0] }}",
"{% if home %}--home {{ home }}{% endif %}", "{% if home %}--home {{ home }}{% endif %}",
"{% if login %}--shell {{ login }}{% endif %}",
"{% if system %}--system{% endif %}",
"{% if groups %}&& {% for group in groups %}gpasswd -a {{ args[0] }} {{ group }};{% endfor %}{% endif %}" "{% if groups %}&& {% for group in groups %}gpasswd -a {{ args[0] }} {{ group }};{% endfor %}{% endif %}"
], ],
'remove': "userdel -r {{ args[0] }}", 'remove': "userdel -r {{ args[0] }}",

@ -84,7 +84,7 @@ pgsql = {
'-U {{ admin_user|default("postgres") }}', '-U {{ admin_user|default("postgres") }}',
'--host={{ host|default("localhost") }}', '--host={{ host|default("localhost") }}',
'--port={{ port|default("5432") }}', '--port={{ port|default("5432") }}',
"-DRS {{ args[0] }}", "-DRS {{ args[0] }}", # no create db or roles, and not a superuser
'{% if password %}&& psql -U {{ admin_user|default("postgres") }} ' '{% if password %}&& psql -U {{ admin_user|default("postgres") }} '
'--host={{ host|default("localhost") }} ' '--host={{ host|default("localhost") }} '
'--port={{ port|default("5432") }} ' '--port={{ port|default("5432") }} '

@ -14,6 +14,14 @@ posix = {
"{% if recursive %}-R{% endif %}", "{% if recursive %}-R{% endif %}",
"{{ args[0] }} {{ args[1] }}" "{{ args[0] }} {{ args[1] }}"
], ],
'dir': [
"mkdir",
"{% if recursive %}-p{% endif %}",
"{% if mode %}-m {{ mode }}{% endif %}",
"{{ args[0] }}",
"{% if group %}&& chgrp -R {{ group }} {{ args[0] }}{% endif %}",
"{% if owner %}&& chown -R {{ owner }} {{ args[0] }}{% endif %}"
],
'extract': [ 'extract': [
"tar", "tar",
"-xz", "-xz",
@ -23,6 +31,12 @@ posix = {
"{% if strip %}--script-components {{ strip }}{% endif %}", "{% if strip %}--script-components {{ strip }}{% endif %}",
'-f {{ args[0] }} {{ to|default("./") }}', '-f {{ args[0] }} {{ to|default("./") }}',
], ],
'file': [
"{% if content %}cat > {{ args[0] }} << EOF\n{{ content }}\nEOF{% else %}touch {{ args[0] }}{% endif %}",
"{% if mode %}&& chmod {{ mode }} {{ args[0 }}{% endif %}",
"{% if group %}&& chgrp {{ group }} {{ args[0] }}{% endif %}",
"{% if owner %}&& chown{{ owner }} {{ args[0] }}{% endif %}"
],
'link': [ 'link': [
"ln -s", "ln -s",
"{% if force %}-f{% endif %}", "{% if force %}-f{% endif %}",

@ -6,6 +6,12 @@ python = {
'{% if op == "upgrade" %}--upgrade{% endif %}', '{% if op == "upgrade" %}--upgrade{% endif %}',
"{{ args[0] }}", "{{ args[0] }}",
], ],
'pip3': [
"{% if venv %}source {{ venv }}/bin/activate &&{%- endif %}",
'pip3 {{ op|default("install") }}',
'{% if op == "upgrade"%}--upgrade{% endif %}',
"{{ args[0] }}",
],
# 'pip': { # 'pip': {
# 'install': [ # 'install': [
# "{% if venv %}source {{ venv }} &&{% endif %}", # "{% if venv %}source {{ venv }} &&{% endif %}",

@ -28,9 +28,11 @@ ubuntu = {
'user': { 'user': {
# The gecos switch eliminates the prompts. # The gecos switch eliminates the prompts.
# TODO: Deal with user password when creating a user in ubuntu. # TODO: Deal with user password when creating a user in ubuntu.
'create': [ 'add': [
"adduser {{ args[0] }} --gecos --disabled-password", "adduser {{ args[0] }} --gecos --disabled-password",
"{% if home %}--home {{ home }}{% endif %}", "{% if home %}--home {{ home }}{% endif %}",
"{% if login %}--shell {{ login }}{% endif %}",
"{% if system %}--system{% endif %}",
"{% if groups %}&& {% for group in groups %}adduser {{ args[0] }} {{ group }};{% endfor %}{% endif %}" "{% if groups %}&& {% for group in groups %}adduser {{ args[0] }} {{ group }};{% endfor %}{% endif %}"
], ],
Loading…
Cancel
Save