Improved support for explain commands.

sandbox/cli.py

@@ -1,10 +1,11 @@
 #! /usr/bin/env python
 
 from argparse import ArgumentParser, RawDescriptionHelpFormatter
-from commonkit import highlight_code, indent, smart_cast
+from commonkit import highlight_code, indent, smart_cast, write_file
 from commonkit.logging import LoggingHelper
 from commonkit.shell import EXIT
 from markdown import markdown
+import os
 import sys
 
 sys.path.insert(0, "../")
@@ -12,6 +13,7 @@ sys.path.insert(0, "../")
 from scripttease.constants import LOGGER_NAME
 from scripttease.lib.contexts import Context
 from scripttease.lib.loaders import load_variables, INILoader, YMLLoader
+from scripttease.variables import PATH_TO_SCRIPT_TEASE
 from scripttease.version import DATE as VERSION_DATE, VERSION
 
 DEBUG = 10
@@ -183,44 +185,82 @@ This command is used to parse configuration files and output the commands.
             except ValueError:
                 options[token] = True
 
+    # The path may have been given as a file name (steps.ini), path, or an inventory name.
+    input_locations = [
+        args.path,
+        os.path.join(PATH_TO_SCRIPT_TEASE, "data", "inventory", args.path, "steps.ini"),
+        os.path.join(PATH_TO_SCRIPT_TEASE, "data", "inventory", args.path, "steps.yml"),
+    ]
+    path = None
+    for location in input_locations:
+        if os.path.exists(location):
+            path = location
+            break
+
+    if path is None:
+        log.warning("Path does not exist: %s" % args.path)
+        exit(EXIT.INPUT)
+
     # Load the commands.
-    if args.path.endswith(".ini"):
+    if path.endswith(".ini"):
         loader = INILoader(
-            args.path,
+            path,
             context=context,
             locations=args.template_locations,
             profile=args.profile,
             **options
         )
-    elif args.path.endswith(".yml"):
+    elif path.endswith(".yml"):
         loader = YMLLoader(
-            args.path,
+            path,
             context=context,
             locations=args.template_locations,
             profile=args.profile,
             **options
         )
     else:
-        log.error("Unsupported file format: %s" % args.path)
+        log.error("Unsupported file format: %s" % path)
         exit(EXIT.ERROR)
 
     # noinspection PyUnboundLocalVariable
     if not loader.load():
+        log.error("Failed to load the input file: %s" % path)
+        exit(EXIT.ERROR)
+
+    # Validate snippets before continuing.
+    valid = list()
+    for snippet in loader.get_snippets():
+        if snippet.is_valid:
+            valid.append(True)
+        else:
+            log.error("Invalid snippet: %s" % snippet.name)
+            valid.append(False)
+
+    if not all(valid):
         exit(EXIT.ERROR)
 
     # Generate output.
     if args.docs:
         output = list()
         for snippet in loader.get_snippets():
-            if snippet is None:
+
-                continue
+            # Will this every happen?
+            # if snippet is None:
+            #     continue
 
             if snippet.name == "explain":
                 if snippet.header:
-                    output.append("## %s" % snippet.name.title())
+                    if args.docs == "plain":
+                        output.append("***** %s *****" % snippet.name.title())
+                    elif args.docs == "rst":
+                        output.append(snippet.name.title())
+                        output.append("=" * len(snippet.name))
+                    else:
+                        output.append("## %s" % snippet.name.title())
+
                     output.append("")
 
-                output.append(snippet.args[0])
+                output.append(snippet.content)
                 output.append("")
             elif snippet.name == "screenshot":
                 if args.docs == "html":
@@ -272,6 +312,7 @@ This command is used to parse configuration files and output the commands.
                     output.append("```%s" % snippet.get_target_language())
                     output.append(snippet.get_content())
                     output.append("```")
+                    output.append("")
             else:
                 statement = snippet.get_statement(include_comment=False, include_register=False, include_stop=False)
                 if statement is not None:
@@ -295,16 +336,21 @@ This command is used to parse configuration files and output the commands.
                         output.append("")
 
         if args.docs == "html":
-            print(markdown("\n".join(output), extensions=['fenced_code']))
+            _output = markdown("\n".join(output), extensions=['fenced_code'])
         else:
-            print("\n".join(output))
+            _output = "\n".join(output)
+
+        print(_output)
+
+        if args.output_file:
+            write_file(args.output_file, _output)
     else:
         commands = list()
         for snippet in loader.get_snippets():
             # Explanations and screenshots don't produce usable statements but may be added as comments.
             if snippet.name in ("explain", "screenshot"):
-                commands.append("# %s" % snippet.args[0])
+                # commands.append("# %s" % snippet.content)
-                commands.append("")
+                # commands.append("")
                 continue
 
             statement = snippet.get_statement()
@@ -317,6 +363,9 @@ This command is used to parse configuration files and output the commands.
         else:
             print("\n".join(commands))
 
+        if args.output_file:
+            write_file(args.output_file, "\n".join(commands))
+
     exit(EXIT.OK)
 
 

scripttease/data/inventory/nextcloud/steps.ini

@@ -4,9 +4,15 @@ system.update:
 [install apache]
 install: apache2
 
+[explain certbot]
+explain: We'll be using Let's Encrypt to get a free SSL certificate. The certbot command is used for this.
+
 [install certbot]
 install: certbot
 
+[explain maint]
+explain: There are various ways to allow certbot to verify you are authorized to create an SSL cert for a domain. The easiest (in our opinion) is to use the webroot option. To support this, we set up a generic document root where certbot can create the auto-discovery file. See the configuration for the virtual host for how this is used. For now, we just need to create the directory.
+
 [make sure a maintenance root exists]
 mkdir: /var/www/maint/www
 group: {{ apache_group }}
@@ -14,7 +20,7 @@ owner: {{ apache_user }}
 recursive: yes
 
 [disable the default site]
-apache.disable: 000-default
+apache.disable_site: 000-default
 
 [install postgres]
 install: postgresql
@@ -25,7 +31,7 @@ items: php, libapache2-mod-php, php-pgsql
 
 [install php modules]
 install: $item
-items: php-curl, php-dom, php-gd, php-json, php-mbstring, php-pdo-pgsql, php-zip
+items: php-curl, php-dom, php-gd, php-imagick, php-json, php-mbstring, php-pdo-pgsql, php-zip
 
 [create the document root for the domain]
 dir: /var/www/{{ domain_tld }}/www
@@ -33,16 +39,16 @@ group: {{ apache_group }}
 owner: {{ apache_user }}
 recursive: yes
 
-[prevent browsing of document root]
+;[prevent browsing of document root]
-file: /var/www/{{ domain_tld }}/www/index.html
+;file: /var/www/{{ domain_tld }}/www/index.html
-group: {{ apache_group }}
+;group: {{ apache_group }}
-owner: {{ apache_user }}
+;owner: {{ apache_user }}
 
 [create the initial apache config file]
 template: httpd.conf /etc/apache2/sites-available/{{ domain_name }}.conf
 
 [enable the site]
-apache.enable: {{ domain_name }}
+apache.enable_site: {{ domain_name }}
 
 [enable mod rewrite]
 apache.enable_module: rewrite
@@ -52,7 +58,7 @@ apache.enable_module: ssl
 
 [enable php modules]
 php.module: $item
-items: ctype, curl, dom, gd, json, pdo_pgsql, posix, simplexml, xmlreader, xmlwriter, zip
+items: ctype, curl, dom, json, gd, imagick, pdo_pgsql, posix, simplexml, xmlreader, xmlwriter, zip
 
 ;PHP module libxml (Linux package libxml2 must be >=2.7.0)
 ;php -i | grep -i libxml
@@ -93,7 +99,8 @@ cd: /var/www/{{ domain_tld }}/www{{ install_path }}
 
 ; createuser -U postgres -DRS {{ install_path }}_nextcloud
 ; createdb -U postgres -O diff6_nextcloud diff6_nextcloud
-; psql -U postgres -c "ALTER USER diff6_nextcloud WITH ENCRYPTED PASSWORD '******'"
+; psql -U postgres -c "ALTER USER diff6_nextcloud WITH ENCRYPTED PASSWORD '*****'"
+; psql -U postgres -c "ALTER USER cloud_diff6_com WITH ENCRYPTED PASSWORD 'SMZdUXVOMr'"
 
 ; https://docs.nextcloud.com/server/latest/admin_manual/installation/source_installation.html
 ; Recommended packages:

scripttease/data/inventory/nextcloud/templates/httpd.conf

@@ -24,5 +24,25 @@
     SSLCertificateKeyFile /etc/letsencrypt/live/{{ domain_name }}/privkey.pem
     SSLCertificateFile /etc/letsencrypt/live/{{ domain_name }}/fullchain.pem
 
+    <Directory /var/www/{{ domain_tld }}/www{{ install_path }}>
+        Options +FollowSymlinks
+        AllowOverride All
+
+        <IfModule mod_dav.c>
+            Dav off
+        </IfModule>
+
+        <IfModule mod_rewrite.c>
+            RewriteEngine on
+            RewriteRule ^\.well-known/carddav {{ install_path }}remote.php/dav [R=301,L]
+            RewriteRule ^\.well-known/caldav {{ install_path }}remote.php/dav [R=301,L]
+            RewriteRule ^\.well-known/webfinger {{ install_path }}index.php/.well-known/webfinger [R=301,L]
+            RewriteRule ^\.well-known/nodeinfo {{ install_path }}index.php/.well-known/nodeinfo [R=301,L]
+        </IfModule>
+
+        SetEnv HOME /var/www/{{ domain_tld }}/www{{ install_path }}
+        SetEnv HTTP_HOME /var/www/{{ domain_tld}}/www{{ install_path}}
+    </Directory>
+
 </VirtualHost>
 {% endif %}

scripttease/data/inventory/radicale/steps.ini

@@ -1,5 +1,5 @@
 [introduction]
-explain: "In this tutorial, we are going to install Radicale."
+explain: In this tutorial, we are going to install Radicale.
 header: yes
 
 [make sure a maintenance root exists]
@@ -9,14 +9,14 @@ owner: www-data
 recursive: yes
 
 [about maintenance root]
-explain: "The maintenance root is used to register an SSL certificate (below) before the site is completed and (later) after the site is live."
+explain: The maintenance root is used to register an SSL certificate (below) before the site is completed and (later) after the site is live.
 
 [install radicale]
 pip3: radicale
 
-;[install radicale screenshot]
+[install radicale screenshot]
-;screenshot: images/install.png
+screenshot: images/install.png
-;caption: Radical Installed
+caption: Radical Installed
 
 [create radicale configuration directory]
 mkdir: /etc/radicale/config
@@ -41,14 +41,14 @@ lang: ini
 start: radicale
 
 [disable the default site]
-apache.disable: default
+apache.disable_site: default
 
 [create the initial apache config file]
 template: httpd.conf /etc/apache2/sites-available/{{ domain_name }}.conf
 
 [enable the site]
-apache.enable: {{ domain_name }}
+apache.enable_site: {{ domain_name }}
-
+;
 [reload apache]
 apache.reload:
 

scripttease/lib/loaders/base.py

@@ -211,6 +211,12 @@ class BaseLoader(File):
             context.update(self.get_context())
             return Template(source, target, context=context, **kwargs)
 
+        # Explanations have had the content split into lots of arg strings. We just need to reconstitute this as the
+        # snippet's content.
+        if name == "explain":
+            content = " ".join(list(args))
+            return Snippet("explain", content=content, kwargs=kwargs)
+
         # Convert args to a list so we can update it below.
         _args = list(args)
 
@@ -578,9 +584,16 @@ class Snippet(object):
 
             return " ".join(a)
 
+        # try:
+        #     return parse_jinja_string(self.content, context)
+        # except TypeError as e:
+        #     log.error("Failed to build command statement for %s: %s" % (self.name, e))
+        #     return None
+
         return parse_jinja_string(self.content, context)
 
 
+
 class Sudo(object):
     """Helper class for defining sudo options."""
 
@@ -679,7 +692,7 @@ class Template(object):
         # TODO: Backing up a template's target is currently specific to bash.
         if self.backup_enabled:
             command = "%s mv %s %s.b" % (self.sudo, self.target, self.target)
-            lines.append('if [[ -f "%s" ]]; then %s fi;' % (self.target, command.lstrip()))
+            lines.append('if [[ -f "%s" ]]; then %s; fi;' % (self.target, command.lstrip()))
 
         # Get the content; e.g. parse the template.
         content = self.get_content()
@@ -690,12 +703,12 @@ class Template(object):
             first_line = _content.pop(0)
             command = '%s echo "%s" > %s' % (self.sudo, first_line, self.target)
             lines.append(command.lstrip())
-            command = "%s cat >> %s << EOF" % (self.sudo, self.target)
+            command = "%s cat > %s << EOF" % (self.sudo, self.target)
             lines.append(command.lstrip())
             lines.append("\n".join(_content))
             lines.append("EOF")
         else:
-            command = "%s cat >> %s << EOF" % (self.sudo, self.target)
+            command = "%s cat > %s << EOF" % (self.sudo, self.target)
             lines.append(command.lstrip())
             lines.append(content)
             lines.append("EOF")

scripttease/lib/loaders/ini.py

@@ -26,6 +26,7 @@ class INILoader(BaseLoader):
 
         """
         if not self.exists:
+            log.warning("Input file does not exist: %s" % self.path)
             return False
 
         if self.context is not None:
@@ -59,6 +60,11 @@ class INILoader(BaseLoader):
                 if count == 0:
                     command_name = key
 
+                    # Explanations aren't processed like commands, so the text need not be surrounded by double quotes.
+                    # if command_name == "explain":
+                    #     args.append(value)
+                    #     continue
+
                     # Arguments surrounded by quotes are considered to be one argument. All others are split into a
                     # list to be passed to the callback. It is also possible that this is a call where no arguments are
                     # present, so the whole thing is wrapped to protect against an index error. A TypeError is raised in

scripttease/lib/snippets/messages.py

@@ -7,8 +7,8 @@ messages = {
         'clear;'
     ],
     'echo': 'echo "{{ args[0] }}"',
-    'explain': None,
+    'explain': "{{ args[0] }}",  # not used, but supports is_valid
-    'screenshot': None,
+    'screenshot': "{{ args[0] }}",  # not used, but supports is_valid
     'slack': [
         "curl -X POST -H 'Content-type: application/json' --data",
         '{"text": "{{ args[0] }}"}',

scripttease/lib/snippets/php.py

@@ -1,3 +1,5 @@
 php = {
-    'module': "phpenmod {{ args[0] }}",
+    'php': {
+        'module': "phpenmod {{ args[0] }}",
+    },
 }

setup.py

@@ -29,7 +29,9 @@ setup(
     packages=find_packages(exclude=["tests", "tests.*"]),
     include_package_data=True,
     install_requires=[
+        "colorama",
         "jinja2",
+        "Markdown",
         "pygments",
         "python-commonkit",
         "pyyaml",