From 26a67e9419d96b7f92871e8b93dba00306c5df0b Mon Sep 17 00:00:00 2001 From: Mark Vartanyan Date: Sat, 20 Jul 2019 02:23:08 +0300 Subject: [PATCH] Feeding dotenv via stdin: use "-" as the data file --- CHANGELOG.md | 2 +- README.md | 13 +++++-- j2cli/cli.py | 62 +++++++++++++++++++++++++++------- j2cli/context.py | 6 ++-- misc/_doc/README.md.j2 | 7 +++- setup.py | 2 +- tests/render-test.py | 13 +++++-- tests/resources/data-empty.env | 0 8 files changed, 82 insertions(+), 23 deletions(-) create mode 100644 tests/resources/data-empty.env diff --git a/CHANGELOG.md b/CHANGELOG.md index d770aa4..ab80cc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -## 0.3.11 (2019-08-18) +## 0.3.12 (2019-08-18) * Fix: use `env` format from stdin ## 0.3.10 (2019-06-07) diff --git a/README.md b/README.md index 5461a8a..b34a436 100644 --- a/README.md +++ b/README.md @@ -127,12 +127,17 @@ Or even read environment variables from a file: $ j2 --format=env config.j2 data.env +Or pipe it: (note that you'll have to use the "-" in this particular case): + + $ j2 --format=env config.j2 - < data.env + # Reference `j2` accepts the following arguments: * `template`: Jinja2 template file to render -* `data`: (optional) path to the data used for rendering. The default is `-`: use stdin +* `data`: (optional) path to the data used for rendering. + The default is `-`: use stdin. Specify it explicitly when using env! Options: @@ -166,7 +171,7 @@ Render directly from the current environment variable values: $ j2 config.j2 -Or alternatively, read the values from a file: +Or alternatively, read the values from a dotenv file: ``` NGINX_HOSTNAME=localhost @@ -179,7 +184,9 @@ And render with: $ j2 config.j2 data.env $ env | j2 --format=env config.j2 -This is especially useful with Docker to link containers together. +If you're going to pipe a dotenv file into `j2`, you'll need to use "-" as the second argument to explicitly: + + $ j2 config.j2 - < data.env ### ini INI data input format. diff --git a/j2cli/cli.py b/j2cli/cli.py index 883ec73..fae229a 100644 --- a/j2cli/cli.py +++ b/j2cli/cli.py @@ -121,12 +121,12 @@ def render_command(cwd, environ, stdin, argv): parser.add_argument('--undefined', action='store_true', dest='undefined', help='Allow undefined variables to be used in templates (no error will be raised)') parser.add_argument('-o', metavar='outfile', dest='output_file', help="Output to a file instead of stdout") parser.add_argument('template', help='Template file to process') - parser.add_argument('data', nargs='?', default='-', help='Input data path') + parser.add_argument('data', nargs='?', default=None, help='Input data file path; "-" to use stdin') args = parser.parse_args(argv) # Input: guess format if args.format == '?': - if args.data == '-': + if args.data is None or args.data == '-': args.format = 'env' else: args.format = { @@ -138,10 +138,27 @@ def render_command(cwd, environ, stdin, argv): }[os.path.splitext(args.data)[1]] # Input: data - if args.data == '-' and args.format == 'env' and (stdin is None or stdin.isatty()): - input_data_f = None + # We always expect a file; + # unless the user wants 'env', and there's no input file provided. + if args.format == 'env': + # With the "env" format, if no dotenv filename is provided, we have two options: + # either the user wants to use the current environment, or he's feeding a dotenv file at stdin. + # Depending on whether we have data at stdin, we'll need to choose between the two. + # + # The problem is that in Linux, you can't reliably determine whether there is any data at stdin: + # some environments would open the descriptor even though they're not going to feed any data in. + # That's why many applications would ask you to explicitly specify a '-' when stdin should be used. + # + # And this is what we're going to do here as well. + # The script, however, would give the user a hint that they should use '-' + if args.data == '-': + input_data_f = stdin + elif args.data == None: + input_data_f = None + else: + input_data_f = open(args.data) else: - input_data_f = stdin if args.data == '-' else open(args.data) + input_data_f = stdin if args.data is None or args.data == '-' else open(args.data) # Python 2: Encode environment variables as unicode if sys.version_info[0] == 2 and args.format == 'env': @@ -183,7 +200,25 @@ def render_command(cwd, environ, stdin, argv): renderer.register_tests(customize.extra_tests()) # Render - result = renderer.render(args.template, context) + try: + result = renderer.render(args.template, context) + except jinja2.exceptions.UndefinedError as e: + # When there's data at stdin, tell the user they should use '-' + try: + stdin_has_data = stdin is not None and not stdin.isatty() + if args.format == 'env' and args.data == None and stdin_has_data: + extra_info = ( + "\n\n" + "If you're trying to pipe a .env file, please run me with a '-' as the data file name:\n" + "$ {cmd} {argv} -".format(cmd=os.path.basename(sys.argv[0]), argv=' '.join(sys.argv[1:])) + ) + e.args = (e.args[0] + extra_info,) + e.args[1:] + except: + # The above code is so optional that any, ANY, error, is ignored + pass + + # Proceed + raise # -o if args.output_file: @@ -199,11 +234,14 @@ def render_command(cwd, environ, stdin, argv): def main(): """ CLI Entry point """ - output = render_command( - os.getcwd(), - os.environ, - sys.stdin, - sys.argv[1:] - ) + try: + output = render_command( + os.getcwd(), + os.environ, + sys.stdin, + sys.argv[1:] + ) + except SystemExit: + return 1 outstream = getattr(sys.stdout, 'buffer', sys.stdout) outstream.write(output) diff --git a/j2cli/context.py b/j2cli/context.py index 2b14bc8..f8035d4 100644 --- a/j2cli/context.py +++ b/j2cli/context.py @@ -102,7 +102,7 @@ def _parse_env(data_string): $ j2 config.j2 - Or alternatively, read the values from a file: + Or alternatively, read the values from a dotenv file: ``` NGINX_HOSTNAME=localhost @@ -115,7 +115,9 @@ def _parse_env(data_string): $ j2 config.j2 data.env $ env | j2 --format=env config.j2 - This is especially useful with Docker to link containers together. + If you're going to pipe a dotenv file into `j2`, you'll need to use "-" as the second argument to explicitly: + + $ j2 config.j2 - < data.env """ # Parse if isinstance(data_string, basestring): diff --git a/misc/_doc/README.md.j2 b/misc/_doc/README.md.j2 index 8f7fc06..d068d7f 100644 --- a/misc/_doc/README.md.j2 +++ b/misc/_doc/README.md.j2 @@ -127,12 +127,17 @@ Or even read environment variables from a file: $ j2 --format=env config.j2 data.env +Or pipe it: (note that you'll have to use the "-" in this particular case): + + $ j2 --format=env config.j2 - < data.env + # Reference `j2` accepts the following arguments: * `template`: Jinja2 template file to render -* `data`: (optional) path to the data used for rendering. The default is `-`: use stdin +* `data`: (optional) path to the data used for rendering. + The default is `-`: use stdin. Specify it explicitly when using env! Options: diff --git a/setup.py b/setup.py index 68d46fb..71410d6 100755 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ setup( name='j2cli', - version='0.3.11', + version='0.3.12b', author='Mark Vartanyan', author_email='kolypto@gmail.com', diff --git a/tests/render-test.py b/tests/render-test.py index d0f44f4..791764a 100644 --- a/tests/render-test.py +++ b/tests/render-test.py @@ -70,6 +70,7 @@ def test_ini(self): self._testme_std(['--format=ini', 'resources/nginx.j2', 'resources/data.ini']) # Stdin self._testme_std(['--format=ini', 'resources/nginx.j2'], stdin=open('resources/data.ini')) + self._testme_std(['--format=ini', 'resources/nginx.j2', '-'], stdin=open('resources/data.ini')) def test_json(self): # Filename @@ -78,6 +79,7 @@ def test_json(self): self._testme_std(['--format=json', 'resources/nginx.j2', 'resources/data.json']) # Stdin self._testme_std(['--format=json', 'resources/nginx.j2'], stdin=open('resources/data.json')) + self._testme_std(['--format=json', 'resources/nginx.j2', '-'], stdin=open('resources/data.json')) def test_yaml(self): try: @@ -92,19 +94,24 @@ def test_yaml(self): self._testme_std(['--format=yaml', 'resources/nginx.j2', 'resources/data.yml']) # Stdin self._testme_std(['--format=yaml', 'resources/nginx.j2'], stdin=open('resources/data.yml')) + self._testme_std(['--format=yaml', 'resources/nginx.j2', '-'], stdin=open('resources/data.yml')) def test_env(self): # Filename - self._testme_std(['resources/nginx-env.j2', 'resources/data.env']) + self._testme_std(['--format=env', 'resources/nginx-env.j2', 'resources/data.env']) + self._testme_std([ 'resources/nginx-env.j2', 'resources/data.env']) # Format self._testme_std(['--format=env', 'resources/nginx-env.j2', 'resources/data.env']) + self._testme_std([ 'resources/nginx-env.j2', 'resources/data.env']) # Stdin - self._testme_std(['--format=env', 'resources/nginx-env.j2'], stdin=open('resources/data.env')) + self._testme_std(['--format=env', 'resources/nginx-env.j2', '-'], stdin=open('resources/data.env')) + self._testme_std([ 'resources/nginx-env.j2', '-'], stdin=open('resources/data.env')) # Environment! + # In this case, it's not explicitly provided, but implicitly gotten from the environment env = dict(NGINX_HOSTNAME='localhost', NGINX_WEBROOT='/var/www/project', NGINX_LOGS='/var/log/nginx/') self._testme_std(['--format=env', 'resources/nginx-env.j2'], env=env) - self._testme_std(['--format=env', 'resources/nginx-env.j2'], env=env) + self._testme_std([ 'resources/nginx-env.j2'], env=env) def test_import_env(self): # Import environment into a variable diff --git a/tests/resources/data-empty.env b/tests/resources/data-empty.env new file mode 100644 index 0000000..e69de29