From 5d637a9fd7c663d5113e06602af3d6dd7a240eb7 Mon Sep 17 00:00:00 2001 From: Matt W <768067+diranged@users.noreply.github.com> Date: Thu, 17 Feb 2022 11:44:18 -0800 Subject: [PATCH] Fix YAML Anchor handling (#516) After we switched to using a more modern version of the YAML library, we ran into a problem where YAML Merge Maps started to fail. Others have seen the same problem: * https://github.com/zerwes/hiyapyco/pull/14/files * https://github.com/ssato/python-anyconfig/issues/87 * https://github.com/yaml/pyyaml/issues/64 * https://github.com/saltstack/salt/issues/2092 * https://github.com/zerwes/hiyapyco/issues/7 In our case, we need to patch the Amazon-provided `construct_mapping` method to run through the `flatten_mapping()` function. --- .circleci/config.yml | 1 + .travis.yml | 39 -------------------------------------- examples/anchors.yaml | 23 ++++++++++++++++++++++ kingpin/test/test_utils.py | 8 ++++++++ kingpin/utils.py | 21 +++++++++++++++++++- kingpin/version.py | 2 +- 6 files changed, 53 insertions(+), 41 deletions(-) delete mode 100644 .travis.yml create mode 100644 examples/anchors.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index db9529a9..1ef09259 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,7 @@ jobs: - image: circleci/python:3.7 environment: DRY: true + USER: bogus-string steps: - checkout - save_cache: diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cd642e47..00000000 --- a/.travis.yml +++ /dev/null @@ -1,39 +0,0 @@ -language: python -python: - - "3.7" -env: - global: - # pypi api token split in 2 because of travisci issue https://travis-ci.community/t/travis-encrypt-data-too-large-for-pypi-tokens-with-older-repos/5792/5 - - secure: "gZ9TDdXbvpw+84CC1Hx4nS44XpDqlao6E1fw44g4DIAKtFwdBd/eDntexWjPCP7X1qVuY3ssswKTVV2gAKUzDPROhZUVTv6L6nrNruKn75MXKNyb4XTf6fatfUptWRA9kVu/mFi8M6Ptp1ySULvEmI0bUSuGZ9AuQISwjfhTqPA=" - - secure: "dpoI6IGPp78nrvuZpgvqkt+PbVIPKb5QrnR+XGOXKRHYpSwNb6DOeZ55tzNZ16e3T/3LVTFZ+VZsn8iv26uUYGBQctP5HBh4zzO8qOuZKgqkN2tmxCifcUxgcWh+Prjj71Toj/+gUdy8ph7zEcLfj3Q2bRTdCfuM56/MoI6Y/5E=" -# https://github.com/travis-ci/travis-ci/issues/7940#issuecomment-310759657 -before_install: - - sudo rm -f /etc/boto.cfg -install: - - pip install pip --upgrade - - make build -script: make test -before_deploy: - - make kingpin.zip -deploy: - - provider: releases - api_key: - secure: "TMRRd3PeZIRf4wvD2Bh+ykvvBVIztE6M6JE89WBb/CbaIbdUqoDoldYjybYbbXDPPPs+ybVOYZTwMylx6TDy40WsYlzlaabnbTZedvUWDC3GcqD3E4I5XMBVBColP1cKZqHYaa/p23V9QfJFoXCJzGJ5VWOXclj0A0NEvb+oUpU=" - file: kingpin.zip - overwrite: true - skip_cleanup: true - on: - tags: true - python: '3.7' - all_branches: true - condition: $TRAVIS_TAG =~ ^(v[0-9]+.[0-9]+.[0-9]+[a-z]?)|pre_release$ - repo: Nextdoor/kingpin - - provider: pypi - user: __token__ - password: $TOKEN_1$TOKEN_2 - on: - tags: true - python: '3.7' - all_branches: true - condition: $TRAVIS_TAG =~ ^v[0-9]+.[0-9]+.[0-9]+[a-z]?$ - repo: Nextdoor/kingpin diff --git a/examples/anchors.yaml b/examples/anchors.yaml new file mode 100644 index 00000000..c4e476b7 --- /dev/null +++ b/examples/anchors.yaml @@ -0,0 +1,23 @@ +actor: group.Sync +desc: main stage +options: + acts: + - &Stage1 + actor: group.Async + desc: stage 1 + options: + acts: + - actor: rightscale.server_array.Clone + desc: copy serverA + options: {dest: serverA, source: kingpin-integration-testing} + - actor: rightscale.server_array.Clone + desc: copy serverB + options: {dest: serverB, source: kingpin-integration-testing} + - actor: rightscale.server_array.Clone + desc: copy serverC + options: {dest: serverC, source: kingpin-integration-testing} + - &Stage2 + <<: *Stage1 + desc: stage 2 + - <<: *Stage2 + desc: stage 3 diff --git a/kingpin/test/test_utils.py b/kingpin/test/test_utils.py index 058066fa..82816556 100644 --- a/kingpin/test/test_utils.py +++ b/kingpin/test/test_utils.py @@ -142,6 +142,14 @@ def test_convert_script_to_dict(self): ret = utils.convert_script_to_dict(instance, {}) self.assertIsInstance(ret, dict) + # Should definitly support YAML with anchors + dirname, filename = os.path.split(os.path.abspath(__file__)) + examples = "%s/../../examples" % dirname + simple = "%s/anchors.yaml" % examples + instance = open(simple) + ret = utils.convert_script_to_dict(instance, {}) + self.assertIsInstance(ret, dict) + def test_convert_script_to_dict_bad_name(self): instance = io.StringIO() # Empty buffer will fail json instance.name = "Somefile.HAHA" diff --git a/kingpin/utils.py b/kingpin/utils.py index 6eb2c1a1..a2e91da9 100644 --- a/kingpin/utils.py +++ b/kingpin/utils.py @@ -31,9 +31,12 @@ import re import sys import io +import cfn_tools from io import IOBase from json.decoder import JSONDecodeError -import cfn_tools +from cfn_tools.yaml_loader import CfnYamlLoader +from cfn_tools.yaml_loader import construct_mapping as aws_construct_mapping + from tornado import gen from tornado import ioloop @@ -59,6 +62,22 @@ # THREADPOOL_SIZE = 10 # THREADPOOL = futures.ThreadPoolExecutor(THREADPOOL_SIZE) +# https://github.com/yaml/pyyaml/issues/64 +# https://github.com/zerwes/hiyapyco/issues/7 +# +# The AWS-provided cfn_tools code does not call the `flatten_mapping()` +# function in their constructor_mapping func. Because of this, YAML Merge +# anchors fail to parse. This little hack below overrides the function to make +# sure that the YAML parsing of merged maps works properly. +def construct_mapping(self, node, deep=False): + self.flatten_mapping(node) + mapping = aws_construct_mapping(self, node, deep) + return mapping + + +# Override the constructor reference for "tag:yaml.org,2002:map" to ours above. +CfnYamlLoader.add_constructor("tag:yaml.org,2002:map", construct_mapping) + def str_to_class(string): """Method that converts a string name into a usable Class name diff --git a/kingpin/version.py b/kingpin/version.py index 58296b91..99160295 100644 --- a/kingpin/version.py +++ b/kingpin/version.py @@ -13,4 +13,4 @@ # Copyright 2018 Nextdoor.com, Inc -__version__ = "2.0.0" +__version__ = "2.0.1"