From 8bbf33397b2ab29f413f5e63413170a8170642d5 Mon Sep 17 00:00:00 2001
From: Rafael Lima <52034564+rafael-lima-tw@users.noreply.github.com>
Date: Fri, 23 Jul 2021 10:22:27 -0300
Subject: [PATCH 01/78] Update documentation with the new github tap
 functionality (#755)

---
 docs/connectors/taps/github.rst | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/docs/connectors/taps/github.rst b/docs/connectors/taps/github.rst
index c4fa7b05e..d2b9162c7 100644
--- a/docs/connectors/taps/github.rst
+++ b/docs/connectors/taps/github.rst
@@ -33,6 +33,7 @@ Example YAML for ``tap-github``:
     name: "Github"                         # Name of the tap
     type: "tap-github"                     # !! THIS SHOULD NOT CHANGE !!
     owner: "somebody@foo.com"              # Data owner to contact
+    sync_period: "*/90 * * * *"            # Period in which the tap will run
     #send_alert: False                     # Optional: Disable all configured alerts on this tap
 
 
@@ -41,8 +42,29 @@ Example YAML for ``tap-github``:
     # ------------------------------------------------------------------------------
     db_conn:
       access_token: "<ACCESS_TOKEN>"            # Github access token with at least the repo scope
-      repository: "transferwise/pipelinewise"   # Path to one or multiple repositories that you want to extract data from
+      organization: "gnome"                     # The organization you want to extract the data from
+                                                # Required when repos_include/repository isn't present
+                                                # OR
+                                                # Required when repos_exclude contains wildcard matchers
+                                                # OR
+                                                # Required when repos_include/repository contains wildcard matchers
+      repos_include: "gnome* polari"            # Allow list strategy to extract selected repos data from organization.
                                                 # Each repo path should be space delimited.
+                                                # Supports wildcard matching
+                                                # Values also valid: singer-io/tap-github another-org/tap-octopus
+                                                # Org prefix not allowed when organization is present
+      repos_exclude: "*tests* api-docs"         # Deny list to extract all repos from organization except the ones listed.
+                                                # Each repo path should be space delimited.
+                                                # Supports wildcard matching
+                                                # Requires organization
+                                                # Org prefix not allowed in repos_exclude
+      repository: "gnome/gnome-software"        # (DEPRECATED) Path to one or multiple repositories that you want to extract data from organization (has priority over repos_exclude))
+                                                # Each repo path should be space delimited.
+                                                # Org prefix not allowed when organization is present
+      include_archived: false                   # Optional: true/false to include archived repos. Default false
+      include_disabled: false                   # Optional: true/false to include disabled repos. Default false
+      max_rate_limit_wait_seconds: 600          # Optional: Max time to wait if you hit the github api limit. Default to 600s
+
 
     # ------------------------------------------------------------------------------
     # Destination (Target) - Target properties

From 9766a4d9fffdcf8db8675ad2b03d6f85440f03bc Mon Sep 17 00:00:00 2001
From: Rafael Lima <52034564+rafael-lima-tw@users.noreply.github.com>
Date: Mon, 2 Aug 2021 10:08:38 -0300
Subject: [PATCH 02/78] Ee 000 update tap GitHub schema (#756)

---
 pipelinewise/cli/schemas/tap.json            |  64 ++++++
 pylintrc                                     |   4 +-
 tests/units/cli/resources/tap-github.yml     |  31 +++
 tests/units/cli/test_cli_utils_tap_github.py | 212 +++++++++++++++++++
 4 files changed, 309 insertions(+), 2 deletions(-)
 create mode 100644 tests/units/cli/resources/tap-github.yml
 create mode 100644 tests/units/cli/test_cli_utils_tap_github.py

diff --git a/pipelinewise/cli/schemas/tap.json b/pipelinewise/cli/schemas/tap.json
index 6c665e350..575f52b28 100644
--- a/pipelinewise/cli/schemas/tap.json
+++ b/pipelinewise/cli/schemas/tap.json
@@ -232,6 +232,67 @@
         }
       },
       "required": ["type"]
+    },
+    "is_tap_github": {
+      "required": ["type"],
+      "properties": {
+        "type": {
+          "enum": ["tap-github"]
+        }
+      }
+    },
+    "tap_github": {
+      "anyOf": [
+        {
+          "not": {
+            "$ref": "#/definitions/is_tap_github"
+          }
+        },
+        {
+          "required": ["db_conn"],
+          "properties": {
+            "db_conn": {
+              "type": "object",
+              "required": ["access_token", "start_date"],
+              "properties": {
+                "access_token": {
+                  "type": "string"
+                },
+                "start_date": {
+                  "type": "string",
+                  "format": "date-time"
+                },
+                "organization": {
+                  "type": "string"
+                },
+                "repos_include": {
+                  "type": "string"
+                },
+                "repos_exclude": {
+                  "type": "string"
+                },
+                "repository": {
+                  "type": "string"
+                },
+                "include_archived": {
+                  "type": "boolean",
+                  "default": false
+                },
+                "include_disabled": {
+                  "type": "boolean",
+                  "default": false
+                },
+                "max_rate_limit_wait_seconds": {
+                  "type": "integer",
+                  "default": 600,
+                  "minimum": 600,
+                  "maximum": 3600
+                }
+              }
+            }
+          }
+        }
+      ]
     }
   },
   "type": "object",
@@ -304,6 +365,9 @@
   "allOf": [
     {
       "$ref": "#/definitions/tap_mongo_implies_ft_and_lb"
+    },
+    {
+      "$ref": "#/definitions/tap_github"
     }
   ],
   "required": [
diff --git a/pylintrc b/pylintrc
index 15e7bc7c8..67e5096d5 100644
--- a/pylintrc
+++ b/pylintrc
@@ -455,10 +455,10 @@ module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
 module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
 
 # Regular expression matching correct method names
-method-rgx=[a-z_][a-z0-9_]{2,30}$
+method-rgx=[a-z_][a-z0-9_]{2,80}$
 
 # Naming hint for method names
-method-name-hint=[a-z_][a-z0-9_]{2,30}$
+method-name-hint=[a-z_][a-z0-9_]{2,80}$
 
 # Regular expression which should only match function or class names that do
 # not require a docstring.
diff --git a/tests/units/cli/resources/tap-github.yml b/tests/units/cli/resources/tap-github.yml
new file mode 100644
index 000000000..1a4f41c30
--- /dev/null
+++ b/tests/units/cli/resources/tap-github.yml
@@ -0,0 +1,31 @@
+id: "github"
+name: "Github"
+type: "tap-github"
+owner: "somebody@foo.com"
+sync_period: "*/90 * * * *"
+
+db_conn:
+  access_token: "<ACCESS_TOKEN>"
+  start_date: "2021-07-14T00:00:00Z"
+  organization: "gnome"
+  repos_include: "gnome* polari"
+  repos_exclude: "*tests* api-docs"
+  repository: "gnome/gnome-software"
+  include_archived: false
+  include_disabled: false
+  max_rate_limit_wait_seconds: 600
+
+target: "snowflake"
+batch_size_rows: 20000
+
+schemas:
+  - source_schema: "my_db"
+    target_schema: "repl_my_db"
+    target_schema_select_permissions:
+      - grp_stats
+    tables:
+      - table_name: "table_one"
+        replication_method: "INCREMENTAL"
+        replication_key: "last_update"
+      - table_name: "table_two"
+        replication_method: "LOG_BASED"
diff --git a/tests/units/cli/test_cli_utils_tap_github.py b/tests/units/cli/test_cli_utils_tap_github.py
new file mode 100644
index 000000000..6a85893bd
--- /dev/null
+++ b/tests/units/cli/test_cli_utils_tap_github.py
@@ -0,0 +1,212 @@
+import os
+import pytest
+
+from unittest import TestCase
+import pipelinewise.cli as cli
+
+TAP_GITHUB_YAML = '{}/resources/tap-github.yml'.format(os.path.dirname(__file__))
+
+
+# pylint: disable=no-self-use,too-many-public-methods,fixme
+class TestUtils(TestCase):
+    """
+    Unit Tests for Tap Github PipelineWise CLI utility functions
+    """
+
+    def assert_json_is_invalid(self, schema, invalid_yaml):
+        """Simple assertion to check if validate function exits with error"""
+        with pytest.raises(SystemExit) as pytest_wrapped_e:
+            cli.utils.validate(invalid_yaml, schema)
+
+        self.assertEqual(pytest_wrapped_e.type, SystemExit)
+        self.assertEqual(pytest_wrapped_e.value.code, 1)
+
+    def test_should_pass_with_valid_json_schema(self):
+        """
+        Test Should pass with valid json schema
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        self.assertIsNone(cli.utils.validate(actual_yaml, schema))
+
+    def test_should_pass_if_organization_and_repos_include_missing_but_repository_exists(self):
+        """
+        Test should pass if organization and repos include missing but repository exists
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        del actual_yaml['db_conn']['organization']
+        del actual_yaml['db_conn']['repos_include']
+
+        self.assertIsNone(cli.utils.validate(actual_yaml, schema))
+
+    def test_should_pass_if_organization_and_repository_missing_but_repos_include_exists(self):
+        """
+        Test should pass if organization and repository missing but repos_include exists
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        del actual_yaml['db_conn']['organization']
+        del actual_yaml['db_conn']['repository']
+
+        self.assertIsNone(cli.utils.validate(actual_yaml, schema))
+
+    # Todo: make schema pass this test scenario
+    # def test_should_fail_if_organization_and_repository_and_repos_include_missing(self):
+    #     """
+    #     validation fails if organization, repository and repos include are all missing
+    #     """
+    #     schema = cli.utils.load_schema('tap')
+    #
+    #     actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+    #     del actual_yaml['db_conn']['organization']
+    #     del actual_yaml['db_conn']['repository']
+    #     del actual_yaml['db_conn']['repos_include']
+    #
+    #     self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_access_token_is_missing(self):
+        """
+        Test Should fail when access token is missing
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        del actual_yaml['db_conn']['access_token']
+
+        self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_start_date_is_missing(self):
+        """
+        Test should fail when start date is missing
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        del actual_yaml['db_conn']['start_date']
+
+        self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_access_token_is_not_string(self):
+        """
+        Test should fail when access token is not string
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        actual_yaml['db_conn']['access_token'] = 123456
+
+        self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_start_date_is_not_string(self):
+        """
+        Test should fail when start date is not string
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        actual_yaml['db_conn']['start_date'] = 123456
+
+        self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_organization_is_not_string(self):
+        """
+        Test should fail when organization is not string
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        actual_yaml['db_conn']['organization'] = []
+
+        self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_repos_include_is_not_string(self):
+        """
+        Test should fail when repos include is not string
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        actual_yaml['db_conn']['repos_include'] = []
+
+        self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_repos_exclude_is_not_string(self):
+        """
+        Test should fail when repos exclude is not string
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        actual_yaml['db_conn']['repos_include'] = {}
+
+        self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_repository_is_not_string(self):
+        """
+        Test should fail when repository is not string
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        actual_yaml['db_conn']['repository'] = {}
+
+        self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_include_archived_is_not_boolean(self):
+        """
+        Test should fail when include archived is not boolean
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        actual_yaml['db_conn']['include_archived'] = 'false'
+
+        self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_include_disabled_is_not_boolean(self):
+        """
+        Test should fail when include disabled is not boolean
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        actual_yaml['db_conn']['include_archived'] = 'false'
+
+        self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_max_rate_limit_wait_seconds_is_not_integer(self):
+        """
+        Test should fail when max rate limit wait seconds is not integer
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        actual_yaml['db_conn']['max_rate_limit_wait_seconds'] = '111'
+
+        self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_max_rate_limit_wait_seconds_is_above_max(self):
+        """
+        Test should fail when max rate limit wait seconds is above the max
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        actual_yaml['db_conn']['max_rate_limit_wait_seconds'] = 4000
+
+        self.assert_json_is_invalid(schema, actual_yaml)
+
+    def test_should_fail_when_max_rate_limit_wait_seconds_is_below_minx(self):
+        """
+        Test should fail when max rate limit wait seconds is below the min
+        """
+        schema = cli.utils.load_schema('tap')
+
+        actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
+        actual_yaml['db_conn']['max_rate_limit_wait_seconds'] = 30
+
+        self.assert_json_is_invalid(schema, actual_yaml)

From e895d27b771f0b233164588511afe9d15dceeb29 Mon Sep 17 00:00:00 2001
From: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
Date: Tue, 3 Aug 2021 11:13:30 +0300
Subject: [PATCH 03/78] do not log json object (#764)

---
 pipelinewise/cli/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pipelinewise/cli/utils.py b/pipelinewise/cli/utils.py
index 46c54b96a..d67484362 100644
--- a/pipelinewise/cli/utils.py
+++ b/pipelinewise/cli/utils.py
@@ -275,8 +275,8 @@ def validate(instance, schema):
         # Serialise vault encrypted objects to string
         schema_safe_inst = json.loads(json.dumps(instance, cls=AnsibleJSONEncoder))
         jsonschema.validate(instance=schema_safe_inst, schema=schema)
-    except jsonschema.exceptions.ValidationError as exc:
-        LOGGER.critical('Invalid object %s', exc)
+    except jsonschema.exceptions.ValidationError:
+        LOGGER.critical('json object doesn\'t match schema %s', schema)
         sys.exit(1)
 
 

From 164d13def246a631601cac5b5c5b4b03b1629bde Mon Sep 17 00:00:00 2001
From: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
Date: Tue, 3 Aug 2021 14:48:22 +0300
Subject: [PATCH 04/78] AP-1037 Optimise docker image and build a ci image
 (#765)

---
 .circleci/config.yml | 24 ++++++++++++++++++++++++
 .dockerignore        | 23 +++++++++++++++++++++--
 Dockerfile           | 20 ++++++++++----------
 Dockerfile.ci        | 13 +++++++++++++
 install.sh           | 10 ++++++++--
 5 files changed, 76 insertions(+), 14 deletions(-)
 create mode 100644 Dockerfile.ci

diff --git a/.circleci/config.yml b/.circleci/config.yml
index b536a9737..88a3d1a8f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -153,6 +153,21 @@ jobs:
             DEPLOY_DOCKERFILE: "./Dockerfile"
             DEPLOY_IMAGE_ADDITIONAL_TAGS: "latest"
 
+  upload_ci_docker_image:
+    <<: *docker_k8s_deployer
+    steps:
+      - checkout
+      - setup_remote_docker
+      - *attach_workspace
+      - run:
+          name: Push and release the new ci docker image to artifactory
+          command: k8s-deployment docker-build jfrog-cli-docker-push
+          environment:
+            DEPLOY_SLACK_CHANNEL: "#analytics-platform-builds"
+            DEPLOY_IMAGE_NAME: "pipelinewise-ci"
+            DEPLOY_DOCKERFILE: "./Dockerfile.ci"
+            DEPLOY_IMAGE_ADDITIONAL_TAGS: "latest"
+
   promote_docker_image:
     <<: *docker_k8s_deployer
     steps:
@@ -183,6 +198,15 @@ workflows:
               only:
                 - master
 
+      - upload_ci_docker_image:
+          context: kubernetes-staging
+          requires:
+            - e2e_tests
+          filters:
+            branches:
+              only:
+                - master
+
       - promote_docker_image:
           context: promote-build
           requires:
diff --git a/.dockerignore b/.dockerignore
index 06acfc141..3276c3701 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,10 +1,29 @@
 **/.git
 **/.virtualenvs
+**/.github
 **/.circleci
 **/.pytest_cache
 **/__pycache__
 *.egg-info
 *.egg/
-docs
-dev-project
 *.rpm
+**/venv
+**/.venv
+**/.coverage
+
+bin
+dev-project
+docs
+scripts
+tests
+test-reports
+
+.coveragerc
+.pre-commit-config.yaml
+.gitignore
+.style.yapf
+.yapfignore
+CONTRIBUTING.md
+CHANGELOG.md
+pylintrc
+pytest.ini
diff --git a/Dockerfile b/Dockerfile
index 49c126cd5..9ad1de1f1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,21 +1,21 @@
 FROM python:3.7-slim-buster
 
-RUN apt-get -qq update && apt-get -qqy install \
+ARG connectors=all
+COPY . /app
+
+RUN apt-get -qq update \
+    && apt-get -qqy --no-install-recommends install \
         apt-utils \
         alien \
         libaio1 \
         mongo-tools \
         mbuffer \
         wget \
-    && pip install --upgrade pip
-
-ARG connectors=all
-COPY . /app
-
-# Install Oracle Instant Client for tap-oracle if its in the connectors list
-RUN bash -c "if grep -q \"tap-oracle\" <<< \"$connectors\"; then wget https://download.oracle.com/otn_software/linux/instantclient/193000/oracle-instantclient19.3-basiclite-19.3.0.0.0-1.x86_64.rpm -O /app/oracle-instantclient.rpm && alien -i /app/oracle-instantclient.rpm --scripts && rm -rf /app/oracle-instantclient.rpm ; fi"
-
-RUN cd /app \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip install -U --no-cache-dir pip \
+    # Install Oracle Instant Client for tap-oracle if its in the connectors list
+    && bash -c "if grep -q \"tap-oracle\" <<< \"$connectors\"; then wget https://download.oracle.com/otn_software/linux/instantclient/193000/oracle-instantclient19.3-basiclite-19.3.0.0.0-1.x86_64.rpm -O /app/oracle-instantclient.rpm && alien -i /app/oracle-instantclient.rpm --scripts && rm -rf /app/oracle-instantclient.rpm ; fi" \
+    && cd /app \
     && ./install.sh --connectors=$connectors --acceptlicenses --nousage --notestextras \
     && ln -s /root/.pipelinewise /app/.pipelinewise
 
diff --git a/Dockerfile.ci b/Dockerfile.ci
new file mode 100644
index 000000000..d6f620011
--- /dev/null
+++ b/Dockerfile.ci
@@ -0,0 +1,13 @@
+FROM python:3.7-slim-buster
+
+COPY . /app
+
+RUN apt-get -qq update \
+    && apt-get -qqy --no-install-recommends install apt-utils alien libaio1 wget \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip install -U --no-cache-dir pip \
+    && cd /app \
+    && ./install.sh --connectors=none --acceptlicenses --nousage --notestextras \
+    && ln -s /root/.pipelinewise /app/.pipelinewise
+
+ENTRYPOINT ["/app/entrypoint.sh"]
diff --git a/install.sh b/install.sh
index 1bb5ca109..36b25d2a4 100755
--- a/install.sh
+++ b/install.sh
@@ -123,7 +123,7 @@ print_installed_connectors() {
 # Parse command line arguments
 for arg in "$@"; do
     case $arg in
-        # Auto accept license agreemnets. Useful if PipelineWise installed by an automated script
+        # Auto accept license agreements. Useful if PipelineWise installed by an automated script
         --acceptlicenses)
             ACCEPT_LICENSES="YES"
             ;;
@@ -200,6 +200,9 @@ if [[ -z $CONNECTORS ]]; then
         install_connector $i
     done
 
+# don't install any connectors if --connectors=none passed
+elif [[ $CONNECTORS == "none" ]]; then
+  echo "No connectors will be installed"
 
 # Install every available connectors if --connectors=all passed
 elif [[ $CONNECTORS == "all" ]]; then
@@ -227,7 +230,10 @@ echo "--------------------------------------------------------------------------
 echo "PipelineWise installed successfully in $((end_time-start_time)) seconds"
 echo "--------------------------------------------------------------------------"
 
-print_installed_connectors
+if [[ $CONNECTORS != "none" ]]; then
+  print_installed_connectors
+fi
+
 if [[ $NO_USAGE != "YES" ]]; then
     echo
     echo "To start CLI:"

From c4f858768c2a6b013f7d6ff07c74e8d573933e2f Mon Sep 17 00:00:00 2001
From: Diego Furtado <diego.furtado@transferwise.com>
Date: Wed, 4 Aug 2021 09:08:01 +0100
Subject: [PATCH 05/78] EE-1104 - 0.35.0v (#760)

---
 CHANGELOG.md | 13 +++++++++++++
 README.md    |  2 +-
 setup.py     |  2 +-
 3 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 55efc2e5b..1dd168ecd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,16 @@
+0.35.0 (2021-08-04)
+-------------------
+- Support `"none"` as a value for `--connectors` in `install.sh` script to install a stripped down Pipelinewise without any connectors.
+- Optimize Dockerfile
+- Do not log invalid json objects if they fail validation against json schema. 
+- Replace `github-tap` with fork `pipelinewise-tap-github` version `1.0.0` 
+- Add schema validation for github tap
+- Increase batch_size_rows from 1M to 5M
+- Increase split_file_chunk_size_mb from 2500 to 5000
+- Add latest tag to docker image
+- Bump `pipelinewise-tap-s3-csv` from `1.2.1` to `1.2.2`
+- Update pymongo requirement from `<3.12,>=3.10` to `>=3.10,<3.13`
+
 0.34.1 (2021-07-15)
 -------------------
 - Bump `pipelinewise-target-snowflake` from `1.13.0` to `1.13.1`
diff --git a/README.md b/README.md
index 6394a421e..682f40b18 100644
--- a/README.md
+++ b/README.md
@@ -55,7 +55,7 @@ consumes data from taps and do something with it, like load it into a file, API
 | Tap       | **[Google Analytics](https://github.com/transferwise/pipelinewise-tap-google-analytics)** | Extra | [![PyPI version](https://badge.fury.io/py/pipelinewise-tap-google-analytics.svg)](https://badge.fury.io/py/tap-adwords) | Extracts data from Google Analytics |
 | Tap       | **[Oracle](https://github.com/transferwise/pipelinewise-tap-oracle)** | Extra | [![PyPI version](https://badge.fury.io/py/pipelinewise-tap-oracle.svg)](https://badge.fury.io/py/pipelinewise-tap-oracle) | Extracts data from Oracle databases. Supporting Log-Based, Key-Based Incremental and Full Table replications |
 | Tap       | **[Zuora](https://github.com/transferwise/pipelinewise-tap-zuora)** | Extra | [![PyPI version](https://badge.fury.io/py/pipelinewise-tap-zuora.svg)](https://badge.fury.io/py/pipelinewise-tap-zuora) | Extracts data from Zuora database using AQAA and REST extraction API with Key-Based incremental replications |
-| Tap       | **[GitHub](https://github.com/singer-io/tap-github)** |       | [![PyPI version](https://badge.fury.io/py/tap-github.svg)](https://badge.fury.io/py/tap-github) | Extracts data from GitHub API using Personal Access Token and Key-Based incremental replications |
+| Tap       | **[GitHub](https://github.com/transferwise/pipelinewise-tap-github)** |       | [![PyPI version](https://badge.fury.io/py/pipelinewise-tap-github.svg)](https://badge.fury.io/py/pipelinewise-tap-github) | Extracts data from GitHub API using Personal Access Token and Key-Based incremental replications |
 | Tap       | **[Shopify](https://github.com/singer-io/tap-shopify)** | Extra | [![PyPI version](https://badge.fury.io/py/tap-shopify.svg)](https://badge.fury.io/py/tap-shopify) | Extracts data from Shopify API using Personal App API Password and date based incremental replications |
 | Tap       | **[Slack](https://github.com/transferwise/pipelinewise-tap-slack)** |       | [![PyPI version](https://badge.fury.io/py/pipelinewise-tap-slack.svg)](https://badge.fury.io/py/pipelinewise-tap-slack) | Extracts data from a Slack API using Bot User Token and Key-Based incremental replications |
 | Tap       | **[Mixpanel](https://github.com/transferwise/pipelinewise-tap-mixpanel)** |       | [![PyPI version](https://badge.fury.io/py/pipelinewise-tap-mixpanel.svg)](https://badge.fury.io/py/pipelinewise-tap-mixpanel) | Extracts data from the Mixpanel API. |
diff --git a/setup.py b/setup.py
index 4803cd0ab..8babe7c90 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      version='0.34.1',
+      version='0.35.0',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',

From 604f6d466c3354d5754bbf9efbc10cb3bc947596 Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Mon, 9 Aug 2021 15:52:13 +0100
Subject: [PATCH 06/78] Bump tap-github from 1.0.0 to 1.0.1 (#767)

---
 singer-connectors/tap-github/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singer-connectors/tap-github/requirements.txt b/singer-connectors/tap-github/requirements.txt
index ed1090806..f919e5a32 100644
--- a/singer-connectors/tap-github/requirements.txt
+++ b/singer-connectors/tap-github/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-github==1.0.0
+pipelinewise-tap-github==1.0.1

From 938a609d3e1adf86d43eedffb8cad9acac5e7149 Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Fri, 13 Aug 2021 12:12:05 +0200
Subject: [PATCH 07/78] v0.35.1 (#769)

---
 CHANGELOG.md                                     | 7 +++++++
 docs/conf.py                                     | 2 +-
 setup.py                                         | 2 +-
 singer-connectors/tap-jira/requirements.txt      | 2 +-
 singer-connectors/tap-kafka/requirements.txt     | 2 +-
 singer-connectors/target-s3-csv/requirements.txt | 2 +-
 6 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1dd168ecd..180888133 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+0.35.1 (2021-08-13)
+-------------------
+- Bump `pipelinewise-tap-github` from `1.0.0` to `1.0.1`
+- Bump `pipelinewise-tap-kafka` from `4.0.0` to `4.0.1`
+- Bump `tap-jira` from `2.0.0` to `2.0.1`
+- Bump `pipelinewise-target-s3-csv` from `1.4.0` to `1.5.0`
+
 0.35.0 (2021-08-04)
 -------------------
 - Support `"none"` as a value for `--connectors` in `install.sh` script to install a stripped down Pipelinewise without any connectors.
diff --git a/docs/conf.py b/docs/conf.py
index 3cb714e3e..f3ffc0f7c 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -25,7 +25,7 @@ def setup(app):
 project = 'PipelineWise'
 copyright = f'{datetime.datetime.now().year}, Wise Ltd.'
 author = 'Wise'
-version = '0.34.0'
+version = '0.35.1'
 
 
 # -- General configuration ---------------------------------------------------
diff --git a/setup.py b/setup.py
index 8babe7c90..4c168471b 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      version='0.35.0',
+      version='0.35.1',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',
diff --git a/singer-connectors/tap-jira/requirements.txt b/singer-connectors/tap-jira/requirements.txt
index ec8ace747..38e8dda4a 100644
--- a/singer-connectors/tap-jira/requirements.txt
+++ b/singer-connectors/tap-jira/requirements.txt
@@ -1 +1 @@
-tap-jira==2.0.0
+tap-jira==2.0.1
diff --git a/singer-connectors/tap-kafka/requirements.txt b/singer-connectors/tap-kafka/requirements.txt
index 493da5f1d..2d30a0cf5 100644
--- a/singer-connectors/tap-kafka/requirements.txt
+++ b/singer-connectors/tap-kafka/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-kafka==4.0.0
+pipelinewise-tap-kafka==4.0.1
diff --git a/singer-connectors/target-s3-csv/requirements.txt b/singer-connectors/target-s3-csv/requirements.txt
index 8812bfac6..72ee8686f 100644
--- a/singer-connectors/target-s3-csv/requirements.txt
+++ b/singer-connectors/target-s3-csv/requirements.txt
@@ -1 +1 @@
-pipelinewise-target-s3-csv==1.4.0
+pipelinewise-target-s3-csv==1.5.0

From e5ce034408fce7a98163f9fccc5118ca26d20dab Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Aug 2021 11:43:43 +0200
Subject: [PATCH 08/78] Bump jinja2 from 2.11.3 to 3.0.1 (#719)

Bumps [jinja2](https://github.com/pallets/jinja) from 2.11.3 to 3.0.1.
- [Release notes](https://github.com/pallets/jinja/releases)
- [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst)
- [Commits](https://github.com/pallets/jinja/compare/2.11.3...3.0.1)

---
updated-dependencies:
- dependency-name: jinja2
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 4c168471b..ce86f6552 100644
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,7 @@
           'tabulate==0.8.9',
           'PyYAML==5.4.1',
           'ansible==4.2.0',
-          'Jinja2==2.11.3',
+          'Jinja2==3.0.1',
           'joblib==1.0.0',
           'PyMySQL==0.7.11',
           'psycopg2-binary==2.8.6',

From 19ad08c9f1a004ef784c9aee82dae4b6b2f601f6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Aug 2021 12:44:48 +0200
Subject: [PATCH 09/78] Bump pre-commit from 2.13.0 to 2.14.0 (#776)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index ce86f6552..9d80c3a52 100644
--- a/setup.py
+++ b/setup.py
@@ -31,7 +31,7 @@
           'singer-encodings==0.0.*',
           'messytables==0.15.*',
           'python-pidfile==3.0.0',
-          'pre-commit==2.13.0',
+          'pre-commit==2.14.0',
           'pymongo>=3.10,<3.13',
           'tzlocal>=2.0,<2.2',
           'slackclient>=2.7,<2.10',

From b5829d8f2a7ef076b4aaa8df149d69b5fdb3f209 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 16 Aug 2021 15:06:59 +0200
Subject: [PATCH 10/78] Bump python-dotenv from 0.18.0 to 0.19.0 (#774)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 9d80c3a52..5120880aa 100644
--- a/setup.py
+++ b/setup.py
@@ -42,7 +42,7 @@
               'pytest==6.2.4',
               'pytest-dependency==0.4.0',
               'pytest-cov==2.12.1',
-              'python-dotenv==0.18.0',
+              'python-dotenv==0.19.0',
               'mock==4.0.3',
               'pylint==2.8.3',
               'unify==0.5'

From 8a582f12f06cd6d2afa501c7eff5ed9e39f47c8b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 17 Aug 2021 10:20:39 +0200
Subject: [PATCH 11/78] Bump ansible from 4.2.0 to 4.4.0 (#773)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 5120880aa..623143b2e 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@
           'argparse==1.4.0',
           'tabulate==0.8.9',
           'PyYAML==5.4.1',
-          'ansible==4.2.0',
+          'ansible==4.4.0',
           'Jinja2==3.0.1',
           'joblib==1.0.0',
           'PyMySQL==0.7.11',

From 6aff16312cd669bb5e98234e7110da919507c999 Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Tue, 17 Aug 2021 15:02:17 +0200
Subject: [PATCH 12/78] Bump pipelinewise-tap-github from 1.0.1 to 1.0.2 (#777)

---
 singer-connectors/tap-github/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singer-connectors/tap-github/requirements.txt b/singer-connectors/tap-github/requirements.txt
index f919e5a32..1830c2385 100644
--- a/singer-connectors/tap-github/requirements.txt
+++ b/singer-connectors/tap-github/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-github==1.0.1
+pipelinewise-tap-github==1.0.2

From 1e90571c6d2221675d0acaf246b8294bafa0baf1 Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Tue, 17 Aug 2021 15:57:21 +0200
Subject: [PATCH 13/78] v0.35.2 (#778)

---
 CHANGELOG.md | 5 +++++
 docs/conf.py | 2 +-
 setup.py     | 2 +-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 180888133..63f60dc7f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+0.35.2 (2021-08-17)
+-------------------
+- Bump `pipelinewise-tap-github` from `1.0.1` to `1.0.2`
+- Update a few vulnerable or outdated dependencies to latest
+
 0.35.1 (2021-08-13)
 -------------------
 - Bump `pipelinewise-tap-github` from `1.0.0` to `1.0.1`
diff --git a/docs/conf.py b/docs/conf.py
index f3ffc0f7c..a97e64c18 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -25,7 +25,7 @@ def setup(app):
 project = 'PipelineWise'
 copyright = f'{datetime.datetime.now().year}, Wise Ltd.'
 author = 'Wise'
-version = '0.35.1'
+version = '0.35.2'
 
 
 # -- General configuration ---------------------------------------------------
diff --git a/setup.py b/setup.py
index 623143b2e..9928e7ee9 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      version='0.35.1',
+      version='0.35.2',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',

From 37e9c1796681210126e674c4e465ce2599a5cefb Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Wed, 25 Aug 2021 14:12:14 +0200
Subject: [PATCH 14/78] Bumpy pylint from 2.8.3 to 2.10.2 (#784)

---
 pipelinewise/cli/alert_sender.py              |  2 +-
 pipelinewise/cli/commands.py                  |  2 +-
 pipelinewise/cli/config.py                    | 20 ++++++++++---------
 pipelinewise/cli/pipelinewise.py              | 10 ++++++----
 pipelinewise/cli/utils.py                     | 18 ++++++++++-------
 pipelinewise/fastsync/commons/split_gzip.py   |  2 +-
 .../fastsync/commons/target_snowflake.py      |  2 +-
 .../fastsync/commons/transform_utils.py       |  5 +++++
 pipelinewise/fastsync/commons/utils.py        |  4 ++--
 setup.py                                      |  2 +-
 tests/end_to_end/helpers/assertions.py        |  6 +++---
 tests/end_to_end/helpers/env.py               |  4 ++--
 tests/end_to_end/test_target_snowflake.py     |  3 +--
 tests/units/cli/test_alert_sender.py          |  3 +--
 tests/units/cli/test_cli.py                   |  2 +-
 tests/units/cli/test_cli_utils.py             |  2 +-
 tests/units/cli/test_cli_utils_tap_github.py  |  3 ++-
 tests/units/cli/test_commands.py              |  2 +-
 tests/units/cli/test_config.py                |  2 +-
 .../commons/test_fastsync_tap_mysql.py        |  2 +-
 20 files changed, 54 insertions(+), 42 deletions(-)

diff --git a/pipelinewise/cli/alert_sender.py b/pipelinewise/cli/alert_sender.py
index b51c4208d..88a4ce7e9 100644
--- a/pipelinewise/cli/alert_sender.py
+++ b/pipelinewise/cli/alert_sender.py
@@ -39,7 +39,7 @@ class AlertSender:
     def __init__(self, alert_handlers: Dict = None) -> None:
         # Initialise alert_handlers as empty dictionary if None provided
         if not alert_handlers:
-            self.alert_handlers = dict()
+            self.alert_handlers = {}
         else:
             self.alert_handlers = alert_handlers
 
diff --git a/pipelinewise/cli/commands.py b/pipelinewise/cli/commands.py
index 8fa0bf162..e824987cb 100644
--- a/pipelinewise/cli/commands.py
+++ b/pipelinewise/cli/commands.py
@@ -338,7 +338,7 @@ def run_command(command: str, log_file: str = None, line_callback: callable = No
 
         # Start command
         with Popen(shlex.split(piped_command), stdout=PIPE, stderr=STDOUT) as proc:
-            with open(log_file_running, 'a+') as logfile:
+            with open(log_file_running, 'a+', encoding='utf-8') as logfile:
                 stdout = ''
                 while True:
                     line = proc.stdout.readline()
diff --git a/pipelinewise/cli/config.py b/pipelinewise/cli/config.py
index 7ba093fb2..bccdad371 100644
--- a/pipelinewise/cli/config.py
+++ b/pipelinewise/cli/config.py
@@ -22,8 +22,8 @@ def __init__(self, config_dir):
         self.logger = logging.getLogger(__name__)
         self.config_dir = config_dir
         self.config_path = os.path.join(self.config_dir, 'config.json')
-        self.global_config = dict()
-        self.targets = dict()
+        self.global_config = {}
+        self.targets = {}
 
     @classmethod
     # pylint: disable=too-many-locals
@@ -54,6 +54,7 @@ def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None):
             utils.validate(instance=global_config, schema=global_config_schema)
             config.global_config = global_config or {}
 
+        # pylint: disable=E1136,E1137  # False positive when loading vault encrypted YAML
         # Load every target yaml into targets dictionary
         for yaml_file in target_yamls:
             config.logger.info('LOADING TARGET: %s', yaml_file)
@@ -102,8 +103,8 @@ def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None):
             taps[tap_id] = tap_data
 
         # Link taps to targets
-        for target_key in targets:
-            targets[target_key]['taps'] = [tap for tap in taps.values() if tap['target'] == target_key]
+        for target_key, target in targets.items():
+            target['taps'] = [tap for tap in taps.values() if tap['target'] == target_key]
 
         # Final structure is ready
         config.targets = targets
@@ -173,9 +174,10 @@ def save_main_config_json(self):
         targets = []
 
         # Generate dictionary for config.json
-        for key in self.targets:
+        for target_tuple in self.targets.items():
+            target = target_tuple[1]
             taps = []
-            for tap in self.targets[key].get('taps'):
+            for tap in target.get('taps'):
                 taps.append({
                     'id': tap.get('id'),
                     'name': tap.get('name'),
@@ -187,10 +189,10 @@ def save_main_config_json(self):
                 })
 
             targets.append({
-                'id': self.targets[key].get('id'),
-                'name': self.targets[key].get('name'),
+                'id': target.get('id'),
+                'name': target.get('name'),
                 'status': 'ready',
-                'type': self.targets[key].get('type'),
+                'type': target.get('type'),
                 'taps': taps
             })
         main_config = {**self.global_config, **{'targets': targets}}
diff --git a/pipelinewise/cli/pipelinewise.py b/pipelinewise/cli/pipelinewise.py
index 5db67826f..db1abd7e4 100644
--- a/pipelinewise/cli/pipelinewise.py
+++ b/pipelinewise/cli/pipelinewise.py
@@ -842,7 +842,7 @@ def update_state_file(line: str) -> str:
                 nonlocal start, state
 
                 if start is None or time() - start >= 2:
-                    with open(tap.state, 'w') as state_file:
+                    with open(tap.state, 'w', encoding='utf-8') as state_file:
                         state_file.write(line)
 
                     # Update start time to be the current time.
@@ -870,7 +870,7 @@ def update_state_file_with_extra_log(line: str) -> str:
 
         # update the state file one last time to make sure it always has the last state message.
         if state is not None:
-            with open(tap.state, 'w') as statefile:
+            with open(tap.state, 'w', encoding='utf-8') as statefile:
                 statefile.write(state)
 
     def run_tap_fastsync(self, tap: TapParams, target: TargetParams, transform: TransformParams):
@@ -1074,7 +1074,7 @@ def stop_tap(self):
         """
         pidfile_path = self.tap['files']['pidfile']
         try:
-            with open(pidfile_path) as pidf:
+            with open(pidfile_path, encoding='utf-8') as pidf:
                 pid = int(pidf.read())
                 parent = psutil.Process(pid)
 
@@ -1218,6 +1218,7 @@ def validate(self):
         vault_secret = self.args.secret
 
         target_ids = set()
+        # pylint: disable=E1136,E1137  # False positive when loading vault encrypted YAML
         # Validate target json schemas and that no duplicate IDs exist
         for yaml_file in target_yamls:
             self.logger.info('Started validating %s', yaml_file)
@@ -1232,6 +1233,7 @@ def validate(self):
             self.logger.info('Finished validating %s', yaml_file)
 
         tap_ids = set()
+        # pylint: disable=E1136,E1137  # False positive when loading vault encrypted YAML
         # Validate tap json schemas, check that every tap has valid 'target' and that no duplicate IDs exist
         for yaml_file in tap_yamls:
             self.logger.info('Started validating %s', yaml_file)
@@ -1393,7 +1395,7 @@ def _print_tap_run_summary(self, status, start_time, end_time):
 
             # Append the summary to the right log file
             if log_file_to_write_summary:
-                with open(log_file_to_write_summary, 'a') as logfile:
+                with open(log_file_to_write_summary, 'a', encoding='utf-8') as logfile:
                     logfile.write(summary)
 
     # pylint: disable=unused-variable
diff --git a/pipelinewise/cli/utils.py b/pipelinewise/cli/utils.py
index d67484362..1d3dc4ff0 100644
--- a/pipelinewise/cli/utils.py
+++ b/pipelinewise/cli/utils.py
@@ -25,7 +25,7 @@
 from ansible.parsing.dataloader import DataLoader
 from ansible.parsing.vault import (VaultLib, get_file_vault_secret, is_encrypted_file)
 from ansible.parsing.yaml.loader import AnsibleLoader
-from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
+from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleVaultEncryptedUnicode
 
 from . import tap_properties
 
@@ -74,7 +74,7 @@ def is_json_file(path):
     """
     try:
         if os.path.isfile(path):
-            with open(path) as jsonfile:
+            with open(path, encoding='utf-8') as jsonfile:
                 if json.load(jsonfile):
                     return True
         return False
@@ -89,7 +89,7 @@ def load_json(path):
     try:
         LOGGER.debug('Parsing file at %s', path)
         if os.path.isfile(path):
-            with open(path) as jsonfile:
+            with open(path, encoding='utf-8') as jsonfile:
                 return json.load(jsonfile)
         else:
             LOGGER.debug('No file at %s', path)
@@ -115,7 +115,7 @@ def save_json(data, path):
     """
     try:
         LOGGER.debug('Saving JSON %s', path)
-        with open(path, 'w') as jsonfile:
+        with open(path, 'w', encoding='utf-8') as jsonfile:
             return json.dump(data, jsonfile, cls=AnsibleJSONEncoder, indent=4, sort_keys=True)
     except Exception as exc:
         raise Exception(f'Cannot save JSON {path} {exc}') from exc
@@ -138,7 +138,7 @@ def is_yaml_file(path):
     """
     try:
         if os.path.isfile(path):
-            with open(path) as yamlfile:
+            with open(path, encoding='utf-8') as yamlfile:
                 if yaml.safe_load(yamlfile):
                     return True
         return False
@@ -182,7 +182,7 @@ def load_yaml(yaml_file, vault_secret=None):
 
     data = None
     if os.path.isfile(yaml_file):
-        with open(yaml_file, 'r') as stream:
+        with open(yaml_file, 'r', encoding='utf-8') as stream:
             # Render environment variables using jinja templates
             contents = stream.read()
             template = Template(contents)
@@ -204,6 +204,9 @@ def load_yaml(yaml_file, vault_secret=None):
     else:
         LOGGER.debug('No file at %s', yaml_file)
 
+    if isinstance(data, AnsibleMapping):
+        data = dict(data)
+
     return data
 
 
@@ -295,6 +298,7 @@ def delete_keys_from_dict(dic, keys):
         return dic
     if isinstance(dic, list):
         return [v for v in (delete_keys_from_dict(v, keys) for v in dic) if v]
+    # pylint: disable=C0325  # False positive on tuples
     return {k: v for k, v in ((k, delete_keys_from_dict(v, keys)) for k, v in dic.items()) if k not in keys}
 
 
@@ -502,7 +506,7 @@ def find_errors_in_log_file(file, max_errors=10, error_pattern=None):
 
     errors = []
     if file and os.path.isfile(file):
-        with open(file) as file_object:
+        with open(file, encoding='utf-8') as file_object:
             for line in file_object:
                 if len(re.findall(error_pattern, line)) > 0:
                     errors.append(line)
diff --git a/pipelinewise/fastsync/commons/split_gzip.py b/pipelinewise/fastsync/commons/split_gzip.py
index 9905d5eb9..8d7838915 100644
--- a/pipelinewise/fastsync/commons/split_gzip.py
+++ b/pipelinewise/fastsync/commons/split_gzip.py
@@ -109,7 +109,7 @@ def _activate_chunk_file(self):
             if self.compress:
                 self.chunk_file = gzip.open(self.chunk_filename, self.mode)
             else:
-                self.chunk_file = builtins.open(self.chunk_filename, self.mode)
+                self.chunk_file = builtins.open(self.chunk_filename, self.mode, encoding='utf-8')
 
     @staticmethod
     def _bytes_to_megabytes(size: int) -> float:
diff --git a/pipelinewise/fastsync/commons/target_snowflake.py b/pipelinewise/fastsync/commons/target_snowflake.py
index f5ee65d3c..685490b46 100644
--- a/pipelinewise/fastsync/commons/target_snowflake.py
+++ b/pipelinewise/fastsync/commons/target_snowflake.py
@@ -115,7 +115,7 @@ def upload_to_s3(self, file, tmp_dir=None):
             )
 
             # Upload to s3
-            extra_args = {'ACL': s3_acl} if s3_acl else dict()
+            extra_args = {'ACL': s3_acl} if s3_acl else {}
 
             # Send key and iv in the metadata, that will be required to decrypt and upload the encrypted file
             extra_args['Metadata'] = {
diff --git a/pipelinewise/fastsync/commons/transform_utils.py b/pipelinewise/fastsync/commons/transform_utils.py
index 164af4b35..6dc9a300c 100644
--- a/pipelinewise/fastsync/commons/transform_utils.py
+++ b/pipelinewise/fastsync/commons/transform_utils.py
@@ -119,6 +119,7 @@ def get_trans_in_sql_flavor(
         return trans_map
 
     @classmethod
+    # pylint: disable=W0238  # False positive when it is used by another classmethod
     def __conditions_to_sql(
             cls,
             transform_conditions: List[Dict],
@@ -187,6 +188,7 @@ def __conditions_to_sql(
         return ' AND '.join(conditions)
 
     @classmethod
+    # pylint: disable=W0238  # False positive when it is used by another classmethod
     def __safe_column(cls, col: str, sql_flavor: SQLFlavor):
         # Make the field id safe in case it's a reserved word
         if sql_flavor == SQLFlavor.SNOWFLAKE:
@@ -204,6 +206,7 @@ def __safe_column(cls, col: str, sql_flavor: SQLFlavor):
         return column
 
     @classmethod
+    # pylint: disable=W0238  # False positive when it is used by another classmethod
     def __hash_to_sql(cls, column: str, sql_flavor: SQLFlavor) -> str:
         """
         convert HASH transformation into the right sql string
@@ -231,6 +234,7 @@ def __hash_to_sql(cls, column: str, sql_flavor: SQLFlavor) -> str:
         return trans
 
     @classmethod
+    # pylint: disable=W0238  # False positive when it is used by another classmethod
     def __hash_skip_first_to_sql(cls, transform_type: TransformationType, column: str, sql_flavor: SQLFlavor) -> str:
         """
         convert HASH-SKIP-FIRST-n transformation into the right sql string
@@ -261,6 +265,7 @@ def __hash_skip_first_to_sql(cls, transform_type: TransformationType, column: st
         return trans
 
     @classmethod
+    # pylint: disable=W0238  # False positive when it is used by another classmethod
     def __mask_date_to_sql(cls, column: str, sql_flavor: SQLFlavor) -> str:
         """
         convert MASK-DATE transformation into the right sql string
diff --git a/pipelinewise/fastsync/commons/utils.py b/pipelinewise/fastsync/commons/utils.py
index 20729c66a..3beb3efdb 100644
--- a/pipelinewise/fastsync/commons/utils.py
+++ b/pipelinewise/fastsync/commons/utils.py
@@ -35,13 +35,13 @@ def get_cpu_cores():
 
 
 def load_json(path):
-    with open(path) as fil:
+    with open(path, encoding='utf-8') as fil:
         return json.load(fil)
 
 
 def save_dict_to_json(path, data):
     LOGGER.info('Saving new state file to %s', path)
-    with open(path, 'w') as fil:
+    with open(path, 'w', encoding='utf-8') as fil:
         fil.write(json.dumps(data))
 
 
diff --git a/setup.py b/setup.py
index 9928e7ee9..bb0090e14 100644
--- a/setup.py
+++ b/setup.py
@@ -44,7 +44,7 @@
               'pytest-cov==2.12.1',
               'python-dotenv==0.19.0',
               'mock==4.0.3',
-              'pylint==2.8.3',
+              'pylint==2.10.2',
               'unify==0.5'
           ]
       },
diff --git a/tests/end_to_end/helpers/assertions.py b/tests/end_to_end/helpers/assertions.py
index 8cf316b8b..810129cde 100644
--- a/tests/end_to_end/helpers/assertions.py
+++ b/tests/end_to_end/helpers/assertions.py
@@ -57,7 +57,7 @@ def assert_command_success(return_code, stdout, stderr, log_path=None):
         failed_log_path = f'{log_path}.failed'
         # Load failed log file if exists
         if os.path.isfile(failed_log_path):
-            with open(failed_log_path, 'r') as file:
+            with open(failed_log_path, 'r', encoding='utf-8') as file:
                 failed_log = file.read()
 
         print(f'STDOUT: {stdout}\nSTDERR: {stderr}\nFAILED LOG: {failed_log}')
@@ -81,14 +81,14 @@ def assert_state_file_valid(target_name, tap_name, log_path=None):
     if log_path:
         success_log_path = f'{log_path}.success'
         state_in_log = None
-        with open(success_log_path, 'r') as log_f:
+        with open(success_log_path, 'r', encoding='utf-8') as log_f:
             state_log_pattern = re.search(r'\nINFO STATE emitted from target: (.+\n)', '\n'.join(log_f.readlines()))
             if state_log_pattern:
                 state_in_log = state_log_pattern.groups()[-1]
 
         # If the emitted state message exists in the log then compare it to the actual state file
         if state_in_log:
-            with open(state_file, 'r') as state_f:
+            with open(state_file, 'r', encoding='utf-8') as state_f:
                 assert state_in_log == ''.join(state_f.readlines())
 
 
diff --git a/tests/end_to_end/helpers/env.py b/tests/end_to_end/helpers/env.py
index fddd9c26a..a4a88e818 100644
--- a/tests/end_to_end/helpers/env.py
+++ b/tests/end_to_end/helpers/env.py
@@ -277,7 +277,7 @@ def _init_test_project_dir(self, project_dir):
         templates = glob.glob(f'{project_dir}/*.yml.template')
         for template_path in templates:
             # Replace env vars in template
-            with open(template_path, 'r') as f_template:
+            with open(template_path, 'r', encoding='utf-8') as f_template:
                 yaml = f_template.read()
 
                 # Detect if every env var configured for the template
@@ -293,7 +293,7 @@ def _init_test_project_dir(self, project_dir):
                         yaml = yaml.replace(f'${{{var}}}', self._all_env_vars_to_dict().get(var))
 
                     # Write the template replaced YAML file
-                    with open(yaml_path, 'w+') as f_render:
+                    with open(yaml_path, 'w+', encoding='utf-8') as f_render:
                         f_render.write(yaml)
 
                 # Delete if exists but not configured
diff --git a/tests/end_to_end/test_target_snowflake.py b/tests/end_to_end/test_target_snowflake.py
index bdf8143cb..0d0015832 100644
--- a/tests/end_to_end/test_target_snowflake.py
+++ b/tests/end_to_end/test_target_snowflake.py
@@ -262,13 +262,12 @@ def test_replicate_pg_to_sf_with_archive_load_files(self):
             'public.country': 1,    # FULL_TABLE : fastsync only
             'public2.wearehere': 1  # FULL_TABLE : fastsync only
         }
-        for schema_table in expected_archive_files_count:
+        for schema_table, expected_archive_files in expected_archive_files_count.items():
             schema, table = schema_table.split('.')
             files_in_s3_archive = s3_client.list_objects(
                 Bucket=s3_bucket,
                 Prefix=('{}/postgres_to_sf_archive_load_files/{}'.format(archive_s3_prefix, table))).get('Contents')
 
-            expected_archive_files = expected_archive_files_count[schema_table]
             if files_in_s3_archive is None or len(files_in_s3_archive) != expected_archive_files:
                 raise Exception('files_in_archive for {} is {}. Expected archive files count: {}'.format(
                     table,
diff --git a/tests/units/cli/test_alert_sender.py b/tests/units/cli/test_alert_sender.py
index 9ca7ad852..344cdfcd0 100644
--- a/tests/units/cli/test_alert_sender.py
+++ b/tests/units/cli/test_alert_sender.py
@@ -3,8 +3,7 @@
 from unittest.mock import patch
 from slack.errors import SlackApiError
 
-import pipelinewise.cli.alert_handlers.errors as errors
-
+from pipelinewise.cli.alert_handlers import errors
 from pipelinewise.cli.alert_sender import AlertHandler, AlertSender
 from pipelinewise.cli.alert_handlers.slack_alert_handler import SlackAlertHandler
 from pipelinewise.cli.alert_handlers.victorops_alert_handler import VictoropsAlertHandler
diff --git a/tests/units/cli/test_cli.py b/tests/units/cli/test_cli.py
index 31dc805a6..e426edf9c 100644
--- a/tests/units/cli/test_cli.py
+++ b/tests/units/cli/test_cli.py
@@ -7,10 +7,10 @@
 import pidfile
 from pathlib import Path
 
-import pipelinewise.cli as cli
 import pytest
 from unittest.mock import patch
 from tests.units.cli.cli_args import CliArgs
+from pipelinewise import cli
 from pipelinewise.cli.pipelinewise import PipelineWise
 
 RESOURCES_DIR = '{}/resources'.format(os.path.dirname(__file__))
diff --git a/tests/units/cli/test_cli_utils.py b/tests/units/cli/test_cli_utils.py
index 6d885699a..20039c619 100644
--- a/tests/units/cli/test_cli_utils.py
+++ b/tests/units/cli/test_cli_utils.py
@@ -1,7 +1,7 @@
 import os
 import re
 
-import pipelinewise.cli as cli
+from pipelinewise import cli
 import pytest
 
 VIRTUALENVS_DIR = './virtualenvs-dummy'
diff --git a/tests/units/cli/test_cli_utils_tap_github.py b/tests/units/cli/test_cli_utils_tap_github.py
index 6a85893bd..5f6db4d5a 100644
--- a/tests/units/cli/test_cli_utils_tap_github.py
+++ b/tests/units/cli/test_cli_utils_tap_github.py
@@ -2,12 +2,13 @@
 import pytest
 
 from unittest import TestCase
-import pipelinewise.cli as cli
+from pipelinewise import cli
 
 TAP_GITHUB_YAML = '{}/resources/tap-github.yml'.format(os.path.dirname(__file__))
 
 
 # pylint: disable=no-self-use,too-many-public-methods,fixme
+# pylint: disable=E1136,E1137  # False positive when loading vault encrypted YAML
 class TestUtils(TestCase):
     """
     Unit Tests for Tap Github PipelineWise CLI utility functions
diff --git a/tests/units/cli/test_commands.py b/tests/units/cli/test_commands.py
index 0746c1118..7d70669ca 100644
--- a/tests/units/cli/test_commands.py
+++ b/tests/units/cli/test_commands.py
@@ -1,8 +1,8 @@
 import os
 import sys
 import pytest
-import pipelinewise.cli.commands as commands
 
+from pipelinewise.cli import commands
 from pipelinewise.cli.errors import StreamBufferTooLargeException
 
 
diff --git a/tests/units/cli/test_config.py b/tests/units/cli/test_config.py
index b3517b0b9..70b786c66 100644
--- a/tests/units/cli/test_config.py
+++ b/tests/units/cli/test_config.py
@@ -1,7 +1,7 @@
 import os
 import shutil
 
-import pipelinewise.cli as cli
+from pipelinewise import cli
 import pytest
 
 PIPELINEWISE_TEST_HOME = '/tmp/.pipelinewise'
diff --git a/tests/units/fastsync/commons/test_fastsync_tap_mysql.py b/tests/units/fastsync/commons/test_fastsync_tap_mysql.py
index 75377d4b0..eafe4f616 100644
--- a/tests/units/fastsync/commons/test_fastsync_tap_mysql.py
+++ b/tests/units/fastsync/commons/test_fastsync_tap_mysql.py
@@ -2,7 +2,7 @@
 from unittest.mock import patch
 import pymysql
 
-import pipelinewise.fastsync.commons.tap_mysql as tap_mysql
+from pipelinewise.fastsync.commons import tap_mysql
 from pipelinewise.fastsync.commons.tap_mysql import FastSyncTapMySql
 
 

From ca92f8d40b716a6b8bc06c581bb3e9a65e19a443 Mon Sep 17 00:00:00 2001
From: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
Date: Mon, 30 Aug 2021 10:28:40 +0300
Subject: [PATCH 15/78] AP-1045 Issue-578 Fastync Mongodb-Reshift pair doesn't
 exit - Fix Fastync pairing logic (#787)

---
 docs/concept/fastsync.rst        |   9 ++-
 pipelinewise/cli/config.py       |   5 +-
 pipelinewise/cli/constants.py    |  17 +++++
 pipelinewise/cli/pipelinewise.py | 109 ++++++++++++++++++-------------
 pipelinewise/fastsync/README.md  |  19 ++----
 tests/units/cli/test_cli.py      |  80 +++++++++++++++--------
 tests/units/cli/test_config.py   |  48 +++++++++-----
 7 files changed, 183 insertions(+), 104 deletions(-)
 create mode 100644 pipelinewise/cli/constants.py

diff --git a/docs/concept/fastsync.rst b/docs/concept/fastsync.rst
index 179e2bf24..d5acd5be5 100644
--- a/docs/concept/fastsync.rst
+++ b/docs/concept/fastsync.rst
@@ -45,4 +45,11 @@ Fast Sync exists only between the following tap and target components:
 +----------------------------+----------------------------------+
 | :ref:`tap-mongodb`         | **->** :ref:`target-postgres`    |
 +----------------------------+----------------------------------+
-
+| :ref:`tap-mysql`           | **->** :ref:`target-bigquery`    |
++----------------------------+----------------------------------+
+| :ref:`tap-postgres`        | **->** :ref:`target-bigquery`    |
++----------------------------+----------------------------------+
+| :ref:`tap-s3-csv`          | **->** :ref:`target-bigquery`    |
++----------------------------+----------------------------------+
+| :ref:`tap-mongodb`         | **->** :ref:`target-bigquery`    |
++----------------------------+----------------------------------+
diff --git a/pipelinewise/cli/config.py b/pipelinewise/cli/config.py
index bccdad371..bd1f3ea0e 100644
--- a/pipelinewise/cli/config.py
+++ b/pipelinewise/cli/config.py
@@ -6,6 +6,8 @@
 import sys
 import json
 
+from typing import Dict
+
 from pipelinewise.utils import safe_column_name
 from . import utils
 
@@ -130,7 +132,7 @@ def get_tap_dir(self, target_id, tap_id):
         return os.path.join(self.config_dir, target_id, tap_id)
 
     @staticmethod
-    def get_connector_files(connector_dir):
+    def get_connector_files(connector_dir: str) -> Dict:
         """
         Returns the absolute paths of a tap/target configuration files
         """
@@ -141,6 +143,7 @@ def get_connector_files(connector_dir):
             'state': os.path.join(connector_dir, 'state.json'),
             'transformation': os.path.join(connector_dir, 'transformation.json'),
             'selection': os.path.join(connector_dir, 'selection.json'),
+            'pidfile': os.path.join(connector_dir, 'pipelinewise.pid')
         }
 
     def save(self):
diff --git a/pipelinewise/cli/constants.py b/pipelinewise/cli/constants.py
new file mode 100644
index 000000000..b676c81ec
--- /dev/null
+++ b/pipelinewise/cli/constants.py
@@ -0,0 +1,17 @@
+import enum
+
+
+class ConnectorType(enum.Enum):
+    """
+    Enums for various Singer connector type names
+    Todo: add more
+    """
+    TAP_MYSQL = 'tap-mysql'
+    TAP_POSTGRES = 'tap-postgres'
+    TAP_MONGODB = 'tap-mongodb'
+    TAP_S3_CSV = 'tap-s3-csv'
+
+    TARGET_BIGQUERY = 'target-bigquery'
+    TARGET_POSTGRES = 'target-postgres'
+    TARGET_SNOWFLAKE = 'target-snowflake'
+    TARGET_REDSHIFT = 'target-redshift'
diff --git a/pipelinewise/cli/pipelinewise.py b/pipelinewise/cli/pipelinewise.py
index db1abd7e4..6ffdcbbfb 100644
--- a/pipelinewise/cli/pipelinewise.py
+++ b/pipelinewise/cli/pipelinewise.py
@@ -13,17 +13,44 @@
 
 from datetime import datetime
 from time import time
-from typing import Dict, Optional, List
+from typing import Dict, Optional, List, Any
 from joblib import Parallel, delayed, parallel_backend
 from tabulate import tabulate
 
 from . import utils
+from .constants import ConnectorType
 from . import commands
 from .commands import TapParams, TargetParams, TransformParams
 from .config import Config
 from .alert_sender import AlertSender
 from .alert_handlers.base_alert_handler import BaseAlertHandler
 
+FASTSYNC_PAIRS = {
+    ConnectorType.TAP_MYSQL: {
+        ConnectorType.TARGET_SNOWFLAKE,
+        ConnectorType.TARGET_REDSHIFT,
+        ConnectorType.TARGET_POSTGRES,
+        ConnectorType.TARGET_BIGQUERY
+    },
+    ConnectorType.TAP_POSTGRES: {
+        ConnectorType.TARGET_SNOWFLAKE,
+        ConnectorType.TARGET_REDSHIFT,
+        ConnectorType.TARGET_POSTGRES,
+        ConnectorType.TARGET_BIGQUERY
+    },
+    ConnectorType.TAP_S3_CSV: {
+        ConnectorType.TARGET_SNOWFLAKE,
+        ConnectorType.TARGET_REDSHIFT,
+        ConnectorType.TARGET_POSTGRES,
+        ConnectorType.TARGET_BIGQUERY
+    },
+    ConnectorType.TAP_MONGODB: {
+        ConnectorType.TARGET_SNOWFLAKE,
+        ConnectorType.TARGET_POSTGRES,
+        ConnectorType.TARGET_BIGQUERY
+    },
+}
+
 
 # pylint: disable=too-many-lines,too-many-instance-attributes,too-many-public-methods
 class PipelineWise:
@@ -115,7 +142,12 @@ def create_consumable_target_config(self, target_config, tap_inheritable_config)
             raise Exception(f'Cannot merge JSON files {dict_a} {dict_b} - {exc}') from exc
 
     # pylint: disable=too-many-statements,too-many-branches,too-many-nested-blocks,too-many-locals,too-many-arguments
-    def create_filtered_tap_properties(self, target_type, tap_type, tap_properties, tap_state, filters,
+    def create_filtered_tap_properties(self,
+                                       target_type: ConnectorType,
+                                       tap_type: ConnectorType,
+                                       tap_properties: str,
+                                       tap_state: str,
+                                       filters: Dict[str, Any],
                                        create_fallback=False):
         """
         Create a filtered version of tap properties file based on specific filter conditions.
@@ -132,11 +164,10 @@ def create_filtered_tap_properties(self, target_type, tap_type, tap_properties,
         """
         # Get filter conditions with default values from input dictionary
         # Nothing selected by default
-        f_selected = filters.get('selected', None)
-        f_target_type = filters.get('target_type', None)
-        f_tap_type = filters.get('tap_type', None)
+        f_selected: bool = filters.get('selected', False)
+        f_tap_target_pairs: Dict = filters.get('tap_target_pairs', {})
         f_replication_method = filters.get('replication_method', None)
-        f_initial_sync_required = filters.get('initial_sync_required', None)
+        f_initial_sync_required: bool = filters.get('initial_sync_required', False)
 
         # Lists of tables that meet and don't meet the filter criteria
         filtered_tap_stream_ids = []
@@ -200,8 +231,7 @@ def create_filtered_tap_properties(self, target_type, tap_type, tap_properties,
                 # pylint: disable=too-many-boolean-expressions
                 if (
                         (f_selected is None or selected == f_selected) and
-                        (f_target_type is None or target_type in f_target_type) and
-                        (f_tap_type is None or tap_type in f_tap_type) and
+                        (f_tap_target_pairs is None or target_type in f_tap_target_pairs.get(tap_type, set())) and
                         (f_replication_method is None or replication_method in f_replication_method) and
                         (f_initial_sync_required is None or initial_sync_required == f_initial_sync_required)
                 ):
@@ -254,10 +284,10 @@ def create_filtered_tap_properties(self, target_type, tap_type, tap_properties,
                                                                        suffix='.json')[1]
                 utils.save_json(fallback_properties, temp_fallback_properties_path)
 
-                return temp_properties_path, \
-                       filtered_tap_stream_ids, \
-                       temp_fallback_properties_path, \
-                       fallback_filtered_stream_ids
+                return (temp_properties_path,
+                        filtered_tap_stream_ids,
+                        temp_fallback_properties_path,
+                        fallback_filtered_stream_ids)
 
             # Fallback not required: Save only the filtered properties JSON
             temp_properties_path = utils.create_temp_file(dir=self.get_temp_dir(),
@@ -318,21 +348,6 @@ def get_connector_python_bin(self, connector_type):
         """
         return os.path.join(self.venv_dir, connector_type, 'bin', 'python')
 
-    @classmethod
-    def get_connector_files(cls, connector_dir):
-        """
-        Get connector file paths
-        """
-        return {
-            'config': os.path.join(connector_dir, 'config.json'),
-            'inheritable_config': os.path.join(connector_dir, 'inheritable_config.json'),
-            'properties': os.path.join(connector_dir, 'properties.json'),
-            'state': os.path.join(connector_dir, 'state.json'),
-            'transformation': os.path.join(connector_dir, 'transformation.json'),
-            'selection': os.path.join(connector_dir, 'selection.json'),
-            'pidfile': os.path.join(connector_dir, 'pipelinewise.pid')
-        }
-
     def get_targets(self):
         """
         Get every target
@@ -353,14 +368,14 @@ def get_target(self, target_id: str) -> Dict:
         self.logger.debug('Getting %s target', target_id)
         targets = self.get_targets()
 
-        target = next((item for item in targets if item['id'] == target_id), False)
+        target = next((item for item in targets if item['id'] == target_id), None)
 
         if not target:
             raise Exception(f'Cannot find {target_id} target')
 
         target_dir = self.get_target_dir(target_id)
         if os.path.isdir(target_dir):
-            target['files'] = self.get_connector_files(target_dir)
+            target['files'] = Config.get_connector_files(target_dir)
         else:
             raise Exception(f'Cannot find target at {target_dir}')
 
@@ -385,21 +400,21 @@ def get_taps(self, target_id):
 
         return taps
 
-    def get_tap(self, target_id, tap_id):
+    def get_tap(self, target_id: str, tap_id: str) -> Dict:
         """
         Get tap by id from a specific target
         """
         self.logger.debug('Getting %s tap from target %s', tap_id, target_id)
         taps = self.get_taps(target_id)
 
-        tap = next((item for item in taps if item['id'] == tap_id), False)
+        tap = next((item for item in taps if item['id'] == tap_id), None)
 
         if not tap:
             raise Exception(f'Cannot find {tap_id} tap in {target_id} target')
 
         tap_dir = self.get_tap_dir(target_id, tap_id)
         if os.path.isdir(tap_dir):
-            tap['files'] = self.get_connector_files(tap_dir)
+            tap['files'] = Config.get_connector_files(tap_dir)
         else:
             raise Exception(f'Cannot find tap at {tap_dir}')
 
@@ -424,7 +439,7 @@ def merge_schemas(self, old_schema, new_schema):
             for new_stream_idx, new_stream in enumerate(new_streams):
                 new_tap_stream_id = new_stream['tap_stream_id']
 
-                old_stream = next((item for item in old_streams if item['tap_stream_id'] == new_tap_stream_id), False)
+                old_stream = next((item for item in old_streams if item['tap_stream_id'] == new_tap_stream_id), None)
 
                 # Is this a new stream?
                 if not old_stream:
@@ -562,7 +577,7 @@ def make_default_selection(self, schema, selection_file):
             streams = schema['streams']
             for stream_idx, stream in enumerate(streams):
                 tap_stream_id = stream.get('tap_stream_id')
-                tap_stream_sel = False
+                tap_stream_sel = None
                 for sel in selection:
                     if 'tap_stream_id' in sel and tap_stream_id.lower() == sel['tap_stream_id'].lower():
                         tap_stream_sel = sel
@@ -739,7 +754,7 @@ def detect_tap_status(self, target_id, tap_id):
         self.logger.debug('Detecting %s tap status in %s target', tap_id, target_id)
         tap_dir = self.get_tap_dir(target_id, tap_id)
         log_dir = self.get_tap_log_dir(target_id, tap_id)
-        connector_files = self.get_connector_files(tap_dir)
+        connector_files = Config.get_connector_files(tap_dir)
         status = {
             'currentStatus': 'unknown',
             'lastStatus': 'unknown',
@@ -973,13 +988,13 @@ def run_tap(self):
             tap_properties_singer,
             singer_stream_ids
         ) = self.create_filtered_tap_properties(
-            target_type,
-            tap_type,
+            ConnectorType(target_type),
+            ConnectorType(tap_type),
             tap_properties,
-            tap_state, {
+            tap_state,
+            {
                 'selected': True,
-                'target_type': ['target-snowflake', 'target-redshift', 'target-postgres', 'target-bigquery'],
-                'tap_type': ['tap-mysql', 'tap-postgres', 'tap-s3-csv', 'tap-mongodb'],
+                'tap_target_pairs': FASTSYNC_PAIRS,
                 'initial_sync_required': True
             },
             create_fallback=True)
@@ -1346,13 +1361,13 @@ def _is_initial_sync_required(self, replication_method: str, stream_bookmark: Di
         :return: Boolean, True if needs initial sync, False otherwise
         """
         return replication_method == self.FULL_TABLE \
-                or (replication_method == self.INCREMENTAL and
-                    'replication_key_value' not in stream_bookmark and
-                    'modified_since' not in stream_bookmark) \
-                or (replication_method == self.LOG_BASED and
-                    'lsn' not in stream_bookmark and
-                    'log_pos' not in stream_bookmark and
-                    'token' not in stream_bookmark)
+               or (replication_method == self.INCREMENTAL and
+                   'replication_key_value' not in stream_bookmark and
+                   'modified_since' not in stream_bookmark) \
+               or (replication_method == self.LOG_BASED and
+                   'lsn' not in stream_bookmark and
+                   'log_pos' not in stream_bookmark and
+                   'token' not in stream_bookmark)
 
     # pylint: disable=unused-argument
     def _exit_gracefully(self, sig, frame, exit_code=1):
diff --git a/pipelinewise/fastsync/README.md b/pipelinewise/fastsync/README.md
index 7c7d650d9..28c44eeeb 100644
--- a/pipelinewise/fastsync/README.md
+++ b/pipelinewise/fastsync/README.md
@@ -13,17 +13,10 @@ components and uses it automatically whenever it’s possible.
 
 ## Supported tap-target routes
 
-* MySQL to Snowflake.
-* MySQL to Redshift
-* MySQL to Postgres
 
-* Postgres to Snowflake
-* Postgres to Redshift
-* Postgres to Postgres
-
-* S3 CSV to Snowflake
-* S3 CSV to Redshift
-* S3 CSV to Postgres
-
-* MongoDB to Snowflake
-* MongoDB to Postgres
+| Source | Destination |
+|---|---|
+|  MySQL/MariaDB | * BigQuery<br />* Snowflake<br />* Postgres<br />* Redshift  |
+|  Postgres | * BigQuery<br />* Snowflake<br />* Postgres<br />* Redshift  |
+|  S3 CSV | * BigQuery<br />* Snowflake<br />* Postgres<br />* Redshift  |
+|  MongoDB | * BigQuery<br />* Snowflake<br />* Postgres<br /> |
diff --git a/tests/units/cli/test_cli.py b/tests/units/cli/test_cli.py
index e426edf9c..db02a4994 100644
--- a/tests/units/cli/test_cli.py
+++ b/tests/units/cli/test_cli.py
@@ -5,12 +5,15 @@
 import signal
 import psutil
 import pidfile
-from pathlib import Path
-
 import pytest
+
+from pathlib import Path
 from unittest.mock import patch
+
 from tests.units.cli.cli_args import CliArgs
 from pipelinewise import cli
+from pipelinewise.cli.constants import ConnectorType
+from pipelinewise.cli.config import Config
 from pipelinewise.cli.pipelinewise import PipelineWise
 
 RESOURCES_DIR = '{}/resources'.format(os.path.dirname(__file__))
@@ -21,6 +24,9 @@
 PROFILING_DIR = './profiling'
 
 
+# Can't inherit from unittest.TestCase because it breaks pytest fixture
+# https://github.com/pytest-dev/pytest/issues/2504#issuecomment-308828149
+
 # pylint: disable=no-self-use,too-many-public-methods,attribute-defined-outside-init,fixme
 class TestCli:
     """
@@ -64,22 +70,6 @@ def test_connector_bin(self):
             self.pipelinewise.get_connector_bin('dummy-type') == \
             '{}/dummy-type/bin/dummy-type'.format(VIRTUALENVS_DIR)
 
-    def test_connector_files(self):
-        """Every singer connector must have a list of JSON files at certain locations"""
-        # TODO: get_connector_files is duplicated in config.py and pipelinewise.py
-        #       Refactor to use only one
-        assert \
-            self.pipelinewise.get_connector_files('/var/singer-connector') == \
-            {
-                'config': '/var/singer-connector/config.json',
-                'inheritable_config': '/var/singer-connector/inheritable_config.json',
-                'properties': '/var/singer-connector/properties.json',
-                'state': '/var/singer-connector/state.json',
-                'transformation': '/var/singer-connector/transformation.json',
-                'selection': '/var/singer-connector/selection.json',
-                'pidfile': '/var/singer-connector/pipelinewise.pid'
-            }
-
     def test_not_existing_config_dir(self):
         """Test with not existing config dir"""
         # Create a new pipelinewise object pointing to a not existing config directory
@@ -103,8 +93,8 @@ def test_get_target(self):
         exp_target_two = next((item for item in targets if item['id'] == 'target_two'), False)
 
         # Append the connector file paths to the expected targets
-        exp_target_one['files'] = self.pipelinewise.get_connector_files('{}/target_one'.format(CONFIG_DIR))
-        exp_target_two['files'] = self.pipelinewise.get_connector_files('{}/target_two'.format(CONFIG_DIR))
+        exp_target_one['files'] = Config.get_connector_files('{}/target_one'.format(CONFIG_DIR))
+        exp_target_two['files'] = Config.get_connector_files('{}/target_two'.format(CONFIG_DIR))
 
         # Getting target by ID should match to original JSON and should contains the connector files list
         assert self.pipelinewise.get_target('target_one') == exp_target_one
@@ -140,7 +130,7 @@ def test_get_tap(self):
         # Append the tap status, files and target keys to the tap
         exp_tap_one = target_one['taps'][0]
         exp_tap_one['status'] = self.pipelinewise.detect_tap_status('target_one', exp_tap_one['id'])
-        exp_tap_one['files'] = self.pipelinewise.get_connector_files('{}/target_one/tap_one'.format(CONFIG_DIR))
+        exp_tap_one['files'] = Config.get_connector_files('{}/target_one/tap_one'.format(CONFIG_DIR))
         exp_tap_one['target'] = self.pipelinewise.get_target('target_one')
 
         # Getting tap by ID should match to original JSON and should contain  status, connector files and target props
@@ -175,16 +165,18 @@ def test_create_filtered_tap_props(self):
             tap_properties_singer,
             singer_stream_ids
         ) = self.pipelinewise.create_filtered_tap_properties(
-             target_type='target-snowflake',
-             tap_type='tap-mysql',
+            target_type=ConnectorType('target-snowflake'),
+            tap_type=ConnectorType('tap-mysql'),
              tap_properties='{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(
                  os.path.dirname(__file__)),
              tap_state='{}/resources/sample_json_config/target_one/tap_one/state.json'.format(
                  os.path.dirname(__file__)),
              filters={
                  'selected': True,
-                 'target_type': ['target-snowflake'],
-                 'tap_type': ['tap-mysql', 'tap-postgres'],
+                 'tap_target_pairs': {
+                     ConnectorType.TAP_MYSQL: {ConnectorType.TARGET_SNOWFLAKE},
+                     ConnectorType.TAP_POSTGRES: {ConnectorType.TARGET_SNOWFLAKE}
+                 },
                  'initial_sync_required': True
              },
              create_fallback=True)
@@ -201,6 +193,42 @@ def test_create_filtered_tap_props(self):
         assert fastsync_stream_ids == ['db_test_mysql-table_one', 'db_test_mysql-table_two']
         assert singer_stream_ids == ['db_test_mysql-table_one', 'db_test_mysql-table_two']
 
+    def test_create_filtered_tap_props_no_fastsync(self):
+        """Test creating only singer specific properties file"""
+        (
+            tap_properties_fastsync,
+            fastsync_stream_ids,
+            tap_properties_singer,
+            singer_stream_ids
+        ) = self.pipelinewise.create_filtered_tap_properties(
+            target_type=ConnectorType('target-snowflake'),
+            tap_type=ConnectorType('tap-mysql'),
+            tap_properties='{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(
+                os.path.dirname(__file__)),
+            tap_state='{}/resources/sample_json_config/target_one/tap_one/state.json'.format(
+                os.path.dirname(__file__)),
+            filters={
+                'selected': True,
+                'tap_target_pairs': {
+                    ConnectorType.TAP_MYSQL: {ConnectorType.TARGET_REDSHIFT},
+                    ConnectorType.TAP_POSTGRES: {ConnectorType.TARGET_SNOWFLAKE}
+                },
+                'initial_sync_required': True
+            },
+            create_fallback=True)
+
+        # fastsync and singer properties should be created
+        assert os.path.isfile(tap_properties_fastsync)
+        assert os.path.isfile(tap_properties_singer)
+
+        # Delete generated properties file
+        os.remove(tap_properties_fastsync)
+        os.remove(tap_properties_singer)
+
+        # only singer properties should be created
+        assert fastsync_stream_ids == []
+        assert singer_stream_ids == ['db_test_mysql-table_one', 'db_test_mysql-table_two']
+
     def test_merge_empty_catalog(self):
         """Merging two empty singer schemas should be another empty"""
         # TODO: Check if pipelinewise.merge_schemas is required at all or not
@@ -550,7 +578,7 @@ def test_post_import_checks(self):
             assert len(pipelinewise._run_post_import_tap_checks(tap_with_trans, tap_with_no_pk_incremental,
                                                                 'snowflake')) == 2
 
-            # mock successfull transformation validation command
+            # mock successful transformation validation command
             run_command_mock.return_value = (0, None, None)
 
             assert len(pipelinewise._run_post_import_tap_checks(tap_with_trans, tap_with_no_pk_not_selected,
diff --git a/tests/units/cli/test_config.py b/tests/units/cli/test_config.py
index 70b786c66..54a3ec98a 100644
--- a/tests/units/cli/test_config.py
+++ b/tests/units/cli/test_config.py
@@ -1,12 +1,13 @@
 import os
 import shutil
+import pytest
 
 from pipelinewise import cli
-import pytest
+from pipelinewise.cli.config import Config
 
 PIPELINEWISE_TEST_HOME = '/tmp/.pipelinewise'
 
-
+# Todo: Inherit from unittest.TestCase
 # pylint: disable=no-self-use,fixme
 class TestConfig:
     """
@@ -15,13 +16,26 @@ class TestConfig:
 
     def test_constructor(self):
         """Test Config construction functions"""
-        config = cli.config.Config(PIPELINEWISE_TEST_HOME)
+        config = Config(PIPELINEWISE_TEST_HOME)
 
         # config dir and path should be generated automatically
         assert config.config_dir == PIPELINEWISE_TEST_HOME
         assert config.config_path == '{}/config.json'.format(PIPELINEWISE_TEST_HOME)
         assert config.targets == {}
 
+    def test_connector_files(self):
+        """Every singer connector must have a list of JSON files at certain locations"""
+        assert Config.get_connector_files('/var/singer-connector') == \
+            {
+                'config': '/var/singer-connector/config.json',
+                'inheritable_config': '/var/singer-connector/inheritable_config.json',
+                'properties': '/var/singer-connector/properties.json',
+                'state': '/var/singer-connector/state.json',
+                'transformation': '/var/singer-connector/transformation.json',
+                'selection': '/var/singer-connector/selection.json',
+                'pidfile': '/var/singer-connector/pipelinewise.pid'
+            }
+
     def test_from_yamls(self):
         """Test creating Config object using YAML configuration directory as the input"""
 
@@ -30,7 +44,7 @@ def test_from_yamls(self):
         vault_secret = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__))
 
         # Parse YAML files and create the config object
-        config = cli.config.Config.from_yamls(PIPELINEWISE_TEST_HOME, yaml_config_dir, vault_secret)
+        config = Config.from_yamls(PIPELINEWISE_TEST_HOME, yaml_config_dir, vault_secret)
 
         # config dir and path should be generated automatically
         assert config.config_dir == PIPELINEWISE_TEST_HOME
@@ -73,7 +87,8 @@ def test_from_yamls(self):
                     'properties': '{}/test_snowflake_target/properties.json'.format(PIPELINEWISE_TEST_HOME),
                     'selection': '{}/test_snowflake_target/selection.json'.format(PIPELINEWISE_TEST_HOME),
                     'state': '{}/test_snowflake_target/state.json'.format(PIPELINEWISE_TEST_HOME),
-                    'transformation': '{}/test_snowflake_target/transformation.json'.format(PIPELINEWISE_TEST_HOME)
+                    'transformation': '{}/test_snowflake_target/transformation.json'.format(PIPELINEWISE_TEST_HOME),
+                    'pidfile': '{}/test_snowflake_target/pipelinewise.pid'.format(PIPELINEWISE_TEST_HOME),
                 },
                 'taps': [{
                     'id': 'mysql_sample',
@@ -106,7 +121,9 @@ def test_from_yamls(self):
                         'state':
                             '{}/test_snowflake_target/mysql_sample/state.json'.format(PIPELINEWISE_TEST_HOME),
                         'transformation':
-                            '{}/test_snowflake_target/mysql_sample/transformation.json'.format(PIPELINEWISE_TEST_HOME)
+                            '{}/test_snowflake_target/mysql_sample/transformation.json'.format(PIPELINEWISE_TEST_HOME),
+                        'pidfile': '{}/test_snowflake_target/mysql_sample/pipelinewise.pid'.format(
+                            PIPELINEWISE_TEST_HOME)
                     },
                     'schemas': [{
                         'source_schema': 'my_db',
@@ -133,7 +150,7 @@ def test_from_invalid_mongodb_yamls(self):
         vault_secret = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__))
         print(yaml_config_dir)
         with pytest.raises(SystemExit) as pytest_wrapped_e:
-            cli.config.Config.from_yamls(PIPELINEWISE_TEST_HOME, yaml_config_dir, vault_secret)
+            Config.from_yamls(PIPELINEWISE_TEST_HOME, yaml_config_dir, vault_secret)
 
         assert pytest_wrapped_e.type == SystemExit
         assert pytest_wrapped_e.value.code == 1
@@ -146,14 +163,14 @@ def test_from_invalid_yamls(self):
 
         # Initialising Config object with a not existing directory should raise an exception
         with pytest.raises(Exception):
-            cli.config.Config.from_yamls(PIPELINEWISE_TEST_HOME, 'not-existing-yaml-config-directory')
+            Config.from_yamls(PIPELINEWISE_TEST_HOME, 'not-existing-yaml-config-directory')
 
         # Initialising config object with a tap that's referencing an unknown target should exit
         yaml_config_dir = '{}/resources/test_invalid_yaml_config'.format(os.path.dirname(__file__))
         vault_secret = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__))
 
         with pytest.raises(SystemExit) as pytest_wrapped_e:
-            cli.config.Config.from_yamls(PIPELINEWISE_TEST_HOME, yaml_config_dir, vault_secret)
+            Config.from_yamls(PIPELINEWISE_TEST_HOME, yaml_config_dir, vault_secret)
         assert pytest_wrapped_e.type == SystemExit
         assert pytest_wrapped_e.value.code == 1
 
@@ -168,20 +185,20 @@ def test_from_invalid_yamls_fails(self):
 
         # Initialising Config object with a not existing directory should raise an exception
         with pytest.raises(Exception):
-            cli.config.Config.from_yamls(PIPELINEWISE_TEST_HOME, 'not-existing-yaml-config-directory')
+            Config.from_yamls(PIPELINEWISE_TEST_HOME, 'not-existing-yaml-config-directory')
 
         # Initialising config object with a tap that's referencing an unknown target should exit
         yaml_config_dir = f'{os.path.dirname(__file__)}/resources/test_invalid_yaml_config_with_duplicate_targets'
         vault_secret = f'{os.path.dirname(__file__)}/resources/vault-secret.txt'
 
         with pytest.raises(SystemExit) as pytest_wrapped_e:
-            cli.config.Config.from_yamls(PIPELINEWISE_TEST_HOME, yaml_config_dir, vault_secret)
+            Config.from_yamls(PIPELINEWISE_TEST_HOME, yaml_config_dir, vault_secret)
         assert pytest_wrapped_e.type == SystemExit
         assert pytest_wrapped_e.value.code == 1
 
     def test_getters(self):
         """Test Config getter functions"""
-        config = cli.config.Config(PIPELINEWISE_TEST_HOME)
+        config = Config(PIPELINEWISE_TEST_HOME)
 
         # Target and tap directory should be g
         assert config.get_temp_dir() == '{}/tmp'.format(PIPELINEWISE_TEST_HOME)
@@ -189,8 +206,6 @@ def test_getters(self):
         assert config.get_tap_dir('test-target-id',
                                   'test-tap-id') == '{}/test-target-id/test-tap-id'.format(PIPELINEWISE_TEST_HOME)
 
-        # TODO: get_connector_files is duplicated in config.py and pipelinewise.py
-        #       Refactor to use only one
         assert \
             config.get_connector_files('/var/singer-connector') == \
             {
@@ -199,7 +214,8 @@ def test_getters(self):
                 'properties': '/var/singer-connector/properties.json',
                 'state': '/var/singer-connector/state.json',
                 'transformation': '/var/singer-connector/transformation.json',
-                'selection': '/var/singer-connector/selection.json'
+                'selection': '/var/singer-connector/selection.json',
+                'pidfile': '/var/singer-connector/pipelinewise.pid'
             }
 
     def test_save_config(self):
@@ -210,7 +226,7 @@ def test_save_config(self):
         vault_secret = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__))
 
         json_config_dir = './pipelinewise-test-config'
-        config = cli.config.Config.from_yamls(json_config_dir, yaml_config_dir, vault_secret)
+        config = Config.from_yamls(json_config_dir, yaml_config_dir, vault_secret)
 
         # Save the config as singer compatible JSON files
         config.save()

From dc2363d370d219e4114b64a8d67d675c6029642d Mon Sep 17 00:00:00 2001
From: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
Date: Mon, 30 Aug 2021 13:11:28 +0300
Subject: [PATCH 16/78] AP-1010 Fix Decimal not JSON serializable in MongoDB
 FastSync (#786)

---
 .circleci/config.yml                          |  18 +-
 .github/workflows/connectors.yml              |  30 +++
 .github/workflows/dockerhub.yml               |   1 -
 .github/workflows/linter.yml                  |   5 +-
 ...linewise_unit_tests.yml => unit_tests.yml} |   6 +-
 dev-project/entrypoint.sh                     |  31 ++-
 .../tap_mongodb_to_pg.yaml                    |   3 +
 pipelinewise/fastsync/commons/errors.py       |   3 +
 pipelinewise/fastsync/commons/tap_mongodb.py  | 188 +++++++++++-------
 pylintrc                                      |   1 -
 setup.py                                      |   4 +-
 tests/db/mongodb_data/all_datatypes.bson.gz   | Bin 0 -> 10797 bytes
 tests/db/tap_mongodb.sh                       |   8 +
 .../tap_mongodb_to_bq.yml.template            |   3 +
 .../tap_mongodb_to_pg.yml.template            |   3 +
 .../tap_mongodb_to_sf.yml.template            |   3 +
 tests/end_to_end/test_target_bigquery.py      |   5 +
 tests/end_to_end/test_target_postgres.py      |   5 +
 tests/end_to_end/test_target_snowflake.py     |   5 +
 .../commons/test_fastsync_tap_mongodb.py      |  72 ++++++-
 20 files changed, 289 insertions(+), 105 deletions(-)
 create mode 100644 .github/workflows/connectors.yml
 rename .github/workflows/{pipelinewise_unit_tests.yml => unit_tests.yml} (88%)
 create mode 100644 tests/db/mongodb_data/all_datatypes.bson.gz

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 88a3d1a8f..db5de8242 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -97,10 +97,17 @@ jobs:
 
             if [[ $rc -eq 1 ]]
             then
+              sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 4B7C549A058F8B6B
+              echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb.list
               sudo apt-get update
-              sudo apt install mariadb-client postgresql-client mongo-tools mbuffer gettext-base
-              wget https://repo.mongodb.org/apt/ubuntu/dists/bionic/mongodb-org/4.2/multiverse/binary-amd64/mongodb-org-shell_4.2.7_amd64.deb
-              sudo dpkg -i ./mongodb-org-shell_4.2.7_amd64.deb && rm mongodb-org-shell_4.2.7_amd64.deb
+              sudo apt install --no-install-recommends \
+                  mariadb-client \
+                  postgresql-client \
+                  mongo-tools \
+                  mbuffer \
+                  gettext-base \
+                  mongodb-org-shell=4.2.7
+
               ./dev-project/mongo/init_rs.sh
 
               ./tests/db/tap_mysql_db.sh
@@ -108,11 +115,10 @@ jobs:
               ./tests/db/tap_mongodb.sh
               ./tests/db/target_postgres.sh
 
-              ./install.sh --acceptlicenses --connectors=all
-
+              ./install.sh --acceptlicenses --connectors=target-snowflake,target-postgres,target-bigquery,tap-mysql,tap-postgres,tap-mongodb,transform-field,tap-s3-csv
               . .virtualenvs/pipelinewise/bin/activate
               export PIPELINEWISE_HOME=$PWD
-              pytest tests/end_to_end -v
+              pytest tests/end_to_end -vx
             fi
           no_output_timeout: 30m
 
diff --git a/.github/workflows/connectors.yml b/.github/workflows/connectors.yml
new file mode 100644
index 000000000..e7838201a
--- /dev/null
+++ b/.github/workflows/connectors.yml
@@ -0,0 +1,30 @@
+# Workflow to check if all singer connectors are installable
+name: Singer connectors
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+  workflow_dispatch:
+
+jobs:
+  check:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [3.7]
+
+    steps:
+      - name: Checking out repo
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Check all connectors are installable
+        run: |
+          ./install.sh --acceptlicenses --connectors=all
diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml
index c0dcfe989..ae80a4a20 100644
--- a/.github/workflows/dockerhub.yml
+++ b/.github/workflows/dockerhub.yml
@@ -6,7 +6,6 @@ on:
       - published
 
 jobs:
-
   build:
     runs-on: ubuntu-latest
 
diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
index c179e29c8..568ad7e1b 100644
--- a/.github/workflows/linter.yml
+++ b/.github/workflows/linter.yml
@@ -9,8 +9,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
-
+  check:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
@@ -28,7 +27,7 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
 
-      - name: Set up Python ${{ matrix.container[1] }}
+      - name: Set up Python ${{ matrix.python-version }}
         if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v2
         with:
diff --git a/.github/workflows/pipelinewise_unit_tests.yml b/.github/workflows/unit_tests.yml
similarity index 88%
rename from .github/workflows/pipelinewise_unit_tests.yml
rename to .github/workflows/unit_tests.yml
index 6eb33100b..ed2f890ec 100644
--- a/.github/workflows/pipelinewise_unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -9,7 +9,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
+  test:
 
     runs-on: ubuntu-20.04
     strategy:
@@ -28,7 +28,7 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
 
-      - name: Set up Python ${{ matrix.container[1] }}
+      - name: Set up Python ${{ matrix.python-version }}
         if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v2
         with:
@@ -44,4 +44,4 @@ jobs:
         if: steps.check.outcome == 'failure'
         run: |
           export PIPELINEWISE_HOME=$PWD
-          pytest --cov=pipelinewise --cov-fail-under=69 -v tests/units
+          pytest --cov=pipelinewise --cov-fail-under=72 -v tests/units
diff --git a/dev-project/entrypoint.sh b/dev-project/entrypoint.sh
index a90b2a450..880b8350d 100755
--- a/dev-project/entrypoint.sh
+++ b/dev-project/entrypoint.sh
@@ -1,14 +1,24 @@
 #!/usr/bin/env bash
 
+set -e
+
+# Add Mongodb ppa
+apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 4B7C549A058F8B6B
+echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | tee /etc/apt/sources.list.d/mongodb.list
+
 # Install OS dependencies
 apt-get update
-apt-get install -y mariadb-client postgresql-client alien libaio1 mongo-tools mbuffer gettext-base
+apt-get install -y --no-install-recommends \
+  alien \
+  gettext-base \
+  libaio1 \
+  mariadb-client \
+  mbuffer \
+  mongo-tools \
+  mongodb-org-shell=4.2.7 \
+  postgresql-client
 
-wget https://repo.mongodb.org/apt/ubuntu/dists/bionic/mongodb-org/4.2/multiverse/binary-amd64/mongodb-org-shell_4.2.7_amd64.deb
-dpkg -i ./mongodb-org-shell_4.2.7_amd64.deb && rm mongodb-org-shell_4.2.7_amd64.deb
-
-# Change to dev-project folder
-cd dev-project
+rm -rf /var/lib/apt/lists/* \
 
 # Install Oracle Instant Client required for tap-oracle
 # ORA_INSTACLIENT_URL=https://download.oracle.com/otn_software/linux/instantclient/193000/oracle-instantclient19.3-basiclite-19.3.0.0.0-1.x86_64.rpm
@@ -17,16 +27,21 @@ cd dev-project
 # alien -i oracle-instantclient.rpm --scripts
 # rm -f oracle-instantclient.rpm
 
+
+# Change to dev-project folder
+cd dev-project
+
+# Install PipelineWise in the container
+
 # Build test databasese
 ../tests/db/tap_mysql_db.sh
 ../tests/db/tap_postgres_db.sh
 
 ./mongo/init_rs.sh
 ../tests/db/tap_mongodb.sh
-
 ../tests/db/target_postgres.sh
 
-# Install PipelineWise in the container
+# Install PipelineWise and connectors in the container
 ../install.sh --acceptlicenses --nousage --connectors=target-snowflake,target-postgres,target-bigquery,tap-mysql,tap-postgres,tap-mongodb,transform-field,tap-s3-csv
 if [[ $? != 0 ]]; then
     echo
diff --git a/dev-project/pipelinewise-config/tap_mongodb_to_pg.yaml b/dev-project/pipelinewise-config/tap_mongodb_to_pg.yaml
index 8aedf7761..8825b2d4a 100644
--- a/dev-project/pipelinewise-config/tap_mongodb_to_pg.yaml
+++ b/dev-project/pipelinewise-config/tap_mongodb_to_pg.yaml
@@ -42,3 +42,6 @@ schemas:
 
       - table_name: "my_collection"
         replication_method: "LOG_BASED"
+
+      - table_name: "all_datatypes"
+        replication_method: "LOG_BASED"
diff --git a/pipelinewise/fastsync/commons/errors.py b/pipelinewise/fastsync/commons/errors.py
index 405ff4a6e..f774367cd 100644
--- a/pipelinewise/fastsync/commons/errors.py
+++ b/pipelinewise/fastsync/commons/errors.py
@@ -6,3 +6,6 @@ class TableNotFoundError(Exception):
 
 class MongoDBInvalidDatetimeError(Exception):
     """Raised when a bson datetime is invalid and cannot be serialized"""
+
+class UnsupportedKeyTypeException(Exception):
+    """Raised if key type is unsupported"""
diff --git a/pipelinewise/fastsync/commons/tap_mongodb.py b/pipelinewise/fastsync/commons/tap_mongodb.py
index 5b908b144..f8ec7304a 100644
--- a/pipelinewise/fastsync/commons/tap_mongodb.py
+++ b/pipelinewise/fastsync/commons/tap_mongodb.py
@@ -2,7 +2,7 @@
 import csv
 import datetime
 import gzip
-import json
+import ujson
 import logging
 import os
 import ssl
@@ -12,87 +12,126 @@
 import pytz
 import tzlocal
 
-from typing import Tuple, Optional, Dict, Callable
+from typing import Tuple, Optional, Dict, Callable, Any
 from pymongo import MongoClient
 from pymongo.database import Database
 from singer.utils import strftime as singer_strftime
 
 from . import utils, split_gzip
-from .errors import ExportError, TableNotFoundError, MongoDBInvalidDatetimeError
+from .errors import ExportError, TableNotFoundError, MongoDBInvalidDatetimeError, UnsupportedKeyTypeException
 
 LOGGER = logging.getLogger(__name__)
 DEFAULT_WRITE_BATCH_ROWS = 50000
 
 
-class MongoDBJsonEncoder(json.JSONEncoder):
+def serialize_document(document: Dict) -> Dict:
     """
-    Custom JSON encoder to be used to serialize data from MongoDB
+    serialize mongodb Document into a json object
+
+    Args:
+        document: MongoDB document
+
+    Returns: Dict
     """
-    @staticmethod
-    def _serialize_datetime(val):
-        """
-        Serialize Bson and python datetime types
-        Args:
-            val: datetime value
+    return {key: transform_value(val, [key]) for key, val in document.items()
+            if not isinstance(val, (bson.min_key.MinKey, bson.max_key.MaxKey))}
 
-        Returns: serialized datetime value
 
-        """
-        if isinstance(val, bson.datetime.datetime):
-            timezone = tzlocal.get_localzone()
-            try:
-                local_datetime = timezone.localize(val)
-                utc_datetime = local_datetime.astimezone(pytz.UTC)
-            except Exception as exc:
-                if str(exc) == 'year is out of range' and val.year == 0:
-                    # NB: Since datetimes are persisted as strings, it doesn't
-                    # make sense to blow up on invalid Python datetimes (e.g.,
-                    # year=0). In this case we're formatting it as a string and
-                    # passing it along down the pipeline.
-                    return '{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}.{:06d}Z'.format(val.year,
-                                                                                      val.month,
-                                                                                      val.day,
-                                                                                      val.hour,
-                                                                                      val.minute,
-                                                                                      val.second,
-                                                                                      val.microsecond)
-                raise MongoDBInvalidDatetimeError('Found invalid datetime {}'.format(val)) from exc
-
-            return singer_strftime(utc_datetime)
-
-        if isinstance(val, datetime.datetime):
+def class_to_string(key_value: Any, key_type: str) -> str:
+    """
+    Converts specific types to string equivalent
+    The supported types are: datetime, bson Timestamp, bytes, int, Int64, float, ObjectId, str and UUID
+    Args:
+        key_value: The value to convert to string
+        key_type: the value type
+
+    Returns: string equivalent of key value
+    Raises: UnsupportedKeyTypeException if key_type is not supported
+    """
+    if key_type == 'datetime':
+        if key_value.tzinfo is None:
             timezone = tzlocal.get_localzone()
-            local_datetime = timezone.localize(val)
+            local_datetime = timezone.localize(key_value)
             utc_datetime = local_datetime.astimezone(pytz.UTC)
-            return singer_strftime(utc_datetime)
-        return None
+        else:
+            utc_datetime = key_value.astimezone(pytz.UTC)
 
-    def default(self, o): # false positive complaint -> pylint: disable=E0202
-        """
-        Custom function to serialize several sort of BSON and Python types
-        Args:
-            obj: Object to serialize
+        return singer_strftime(utc_datetime)
 
-        Returns: Serialized value
-        """
-        encoding_map = {
-            bson.objectid.ObjectId: str,
-            uuid.UUID: str,
-            bson.int64.Int64: str,
-            bson.timestamp.Timestamp: lambda value: singer_strftime(value.as_datetime()),
-            bytes: lambda value: base64.b64encode(value).decode('utf-8'),
-            bson.decimal128.Decimal128: lambda val: val.to_decimal(),
-            bson.regex.Regex: lambda val: dict(pattern=val.pattern, flags=val.flags),
-            bson.code.Code: lambda val: dict(value=str(val), scope=str(val.scope)) if val.scope else str(val),
-            bson.dbref.DBRef: lambda val: dict(id=str(val.id), collection=val.collection, database=val.database),
-            datetime.datetime: self._serialize_datetime,
-            bson.datetime.datetime: self._serialize_datetime
-        }
+    if key_type == 'Timestamp':
+        return '{}.{}'.format(key_value.time, key_value.inc)
+
+    if key_type == 'bytes':
+        return base64.b64encode(key_value).decode('utf-8')
+
+    if key_type in ['int', 'Int64', 'float', 'ObjectId', 'str', 'UUID']:
+        return str(key_value)
+
+    raise UnsupportedKeyTypeException('{} is not a supported key type'.format(key_type))
+
+
+def safe_transform_datetime(value: datetime.datetime, path) -> str:
+    """
+    Safely transform datetime from local tz to UTC if applicable
+    Args:
+        value: datetime value to transform
+        path:
+
+    Returns: utc datetime as string
+
+    """
+    timezone = tzlocal.get_localzone()
+    try:
+        local_datetime = timezone.localize(value)
+        utc_datetime = local_datetime.astimezone(pytz.UTC)
+    except Exception as ex:
+        if str(ex) == 'year is out of range' and value.year == 0:
+            # NB: Since datetimes are persisted as strings, it doesn't
+            # make sense to blow up on invalid Python datetimes (e.g.,
+            # year=0). In this case we're formatting it as a string and
+            # passing it along down the pipeline.
+            return '{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}.{:06d}Z'.format(value.year,
+                                                                              value.month,
+                                                                              value.day,
+                                                                              value.hour,
+                                                                              value.minute,
+                                                                              value.second,
+                                                                              value.microsecond)
+        raise MongoDBInvalidDatetimeError('Found invalid datetime at [{}]: {}'.format('.'.join(map(str, path)),
+                                                                                      value)) from ex
+    return singer_strftime(utc_datetime)
+
+
+def transform_value(value: Any, path) -> Any:
+    """
+    transform values to json friendly ones
+    Args:
+        value: value to transform
+        path:
 
-        if o.__class__ in encoding_map:
-            return encoding_map[o.__class__](o)
+    Returns: transformed value
 
-        return super().default(o)
+    """
+    conversion = {
+        list: lambda val, pat: list(map(lambda v: transform_value(v[1], pat + [v[0]]), enumerate(val))),
+        dict: lambda val, pat: {k: transform_value(v, pat + [k]) for k, v in val.items()},
+        uuid.UUID: lambda val, _: class_to_string(val, 'UUID'),
+        bson.objectid.ObjectId: lambda val, _: class_to_string(val, 'ObjectId'),
+        bson.datetime.datetime: safe_transform_datetime,
+        bson.timestamp.Timestamp: lambda val, _: singer_strftime(val.as_datetime()),
+        bson.int64.Int64: lambda val, _: class_to_string(val, 'Int64'),
+        bytes: lambda val, _: class_to_string(val, 'bytes'),
+        datetime.datetime: lambda val, _: class_to_string(val, 'datetime'),
+        bson.decimal128.Decimal128: lambda val, _: val.to_decimal(),
+        bson.regex.Regex: lambda val, _: dict(pattern=val.pattern, flags=val.flags),
+        bson.code.Code: lambda val, _: dict(value=str(val), scope=str(val.scope)) if val.scope else str(val),
+        bson.dbref.DBRef: lambda val, _: dict(id=str(val.id), collection=val.collection, database=val.database),
+    }
+
+    if isinstance(value, tuple(conversion.keys())):
+        return conversion[type(value)](value, path)
+
+    return value
 
 
 class FastSyncTapMongoDB:
@@ -160,6 +199,7 @@ def copy_table(self,
                                with -partXYZ postfix in the filename. (Default: False)
             split_file_chunk_size_mb: File chunk sizes if `split_large_files` enabled. (Default: 1000)
             split_file_max_chunks: Max number of chunks if `split_large_files` enabled. (Default: 20)
+            compress: Flag to indicate whether to compress export files
         """
         table_dict = utils.tablename_to_dict(table_name, '.')
 
@@ -192,13 +232,17 @@ def copy_table(self,
 
                 # bson.decode_file_iter will generate one document at a time from the exported file
                 for document in bson.decode_file_iter(export_file):
-                    rows.append({
-                        '_ID': str(document['_id']),
-                        'DOCUMENT': json.dumps(document, cls=MongoDBJsonEncoder, separators=(',', ':')),
-                        utils.SDC_EXTRACTED_AT: extracted_at,
-                        utils.SDC_BATCHED_AT: datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f'),
-                        utils.SDC_DELETED_AT: None
-                    })
+                    try:
+                        rows.append({
+                            '_ID': str(document['_id']),
+                            'DOCUMENT': ujson.dumps(serialize_document(document)),
+                            utils.SDC_EXTRACTED_AT: extracted_at,
+                            utils.SDC_BATCHED_AT: datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f'),
+                            utils.SDC_DELETED_AT: None
+                        })
+                    except TypeError:
+                        LOGGER.error('TypeError encountered when processing document ID: %s', document['_id'])
+                        raise
 
                     exported_rows += 1
 
@@ -239,7 +283,7 @@ def _get_collection_columns() -> Tuple:
             (utils.SDC_DELETED_AT, 'string'),
         )
 
-    def fetch_current_log_pos(self)->Dict:
+    def fetch_current_log_pos(self) -> Dict:
         """
         Find and returns the latest ChangeStream token.
         LOG_BASED method uses changes streams.
@@ -300,7 +344,7 @@ def map_column_types_to_target(self):
             'primary_key': ['_ID']
         }
 
-    def _export_collection(self, export_dir: str, collection_name)->str:
+    def _export_collection(self, export_dir: str, collection_name) -> str:
         """
         Dump a collection data into a compressed bson file and returns the path
         Args:
@@ -337,7 +381,7 @@ def _export_collection(self, export_dir: str, collection_name)->str:
         if return_code != 0:
             raise ExportError(f'Export failed with code {return_code}')
 
-        #mongodump creates two files "{collection_name}.metadata.json.gz" & "{collection_name}.bson.gz"
+        # mongodump creates two files "{collection_name}.metadata.json.gz" & "{collection_name}.bson.gz"
         # we are only interested in the latter so we delete the former.
         os.remove(os.path.join(export_dir, self.connection_config['database'], f'{collection_name}.metadata.json.gz'))
         return os.path.join(export_dir, self.connection_config['database'], f'{collection_name}.bson.gz')
diff --git a/pylintrc b/pylintrc
index 67e5096d5..a290fcb40 100644
--- a/pylintrc
+++ b/pylintrc
@@ -162,7 +162,6 @@ enable=import-error,
        nonzero-method,
        t-method,
        setslice-method,
-       old-division,
        logging-format-truncated,
        logging-too-few-args,
        logging-too-many-args,
diff --git a/setup.py b/setup.py
index bb0090e14..9d23d4026 100644
--- a/setup.py
+++ b/setup.py
@@ -35,7 +35,8 @@
           'pymongo>=3.10,<3.13',
           'tzlocal>=2.0,<2.2',
           'slackclient>=2.7,<2.10',
-          'psutil==5.8.0'
+          'psutil==5.8.0',
+          'ujson==4.1.*'
       ],
       extras_require={
           'test': [
@@ -43,7 +44,6 @@
               'pytest-dependency==0.4.0',
               'pytest-cov==2.12.1',
               'python-dotenv==0.19.0',
-              'mock==4.0.3',
               'pylint==2.10.2',
               'unify==0.5'
           ]
diff --git a/tests/db/mongodb_data/all_datatypes.bson.gz b/tests/db/mongodb_data/all_datatypes.bson.gz
new file mode 100644
index 0000000000000000000000000000000000000000..a59b54a3e9ecb52da24debc6d2df43612f1fdb46
GIT binary patch
literal 10797
zcmYjXWk8ef+XV?}gDwH-?k)jAKsp`W2#oF+h>`+=bR#HTQlq<TbdF91HxOh9bMQXD
z|EG7KwoiNR`?}7#&N<h^lt6&>zt8;v2C^zR4to4ILX>6vASsG1g^6XH;XO-AN<&9S
zb?3+Il)96;x;kXlNfq*>j)x{jRB%W`Bu$Fe?o(9JMPtHyzG~)#Aw2ArjG^d?wh+~>
z7hy^PmX<;w%AsgW%J&t_FCXkW)(blyX_C!cbr;IrC=@QNgx$-|Tuueoodqk*$Omrw
zZwF!~;bgJpo0vXGY;}DQLh#@G=Jsuu;*IFp-$31tGk$CDnY^=J^dwW;{q!~C)+O?l
z^~~ST@GaEX;3K&|F{e!3_g+$~dIvA7{4IFxCciygrJ9d}M$*@Jus{nEc#Ac!%%cY-
zjyST)qSRK^Mis!t2|jims*P3`91be)jL$yWUC58j?oXQq1YMXTgZDAl3O*Qk?%=K$
z{J)>C?t=mYuJe$+0YS)JufMCw-I$X+MO4BLtu^lbBsHVL;mP15>+S%|8S+u=plIK&
z#@epz-LJ3pO!vE-gM;M?x1`&CZ3r)^E{s<jzoOiwmsHrF-jJQoXeK$iUa79*%`3E(
zY0uI5+)BpvvqGO4oJW?Bzw4XVmXL#!t6sEpa`?$fMsoPx#H)Muq5w>Ryw~=X;B9tM
zzu;E*jQQ5Z5xQQ%4`ZksatkqK3jW1;@oIZJ;>uLvr&rL3iuLa!N$dMnQKp`I((B&+
zTP9@K;qLhc!xm%)CwT+Yr;@yJAA@G<xy#{=9o$`b7QEAs-bRfrJ=_k#e5jXUAnQF}
zH?@FRf%1PB$=!Bfyrksrd(dZ)o&M@%$lw`f#MxMozTP5YMG$W$e1ECcy5J1*H?ma$
z)f9sIdhcY))N}Dku+0ZST=(zu3Pa(gu=PbOQ_#VqEetet`(k-?%P3rm0V%~mW_?Hb
zXcIk#;~D&80tfhZj(N0oKg+s%sc=7GY85<MZFQB)YwaJjyz3Q&lqJ&*`IC2t`~Wn-
zj~P@$MCQZ#o_+hfAgM`I?|l%Q39@;2yX}?-`*KwIGe6y;w!n%v7d}gHD!d&e{@xZB
zW3wKz{$NB)4O^YbuD`q`Ez(xyg~aIy)?U!74oV|_+84=LJ*^3~6Yk51XW-t8la+s%
zSjBr`_znd$;%7GEOtAq$Z83y8Rrp6epP+aDw{t5NaUYb47^L;i6KU%YbCWDWSMfxr
zZI*B_yJ>#OX>CEY;`!mM8!+&=RcOG<`D00NGgwuEUuwUQXLe<gP&d$}I4!RISmp4#
zzHQSY&=9PMk|{!5;g}*j_ALdaT{i9>je>ixhoD6vPvF*9lZq(pOZGw79;!Y+BW)uf
z#@*yoQ_+Jnkg~Vin~9$^*rSwizD+Y23)hN{smm14jPLYFWI0iC+pCHwB5{KQsrsXu
zUj*6Fc}?76&&k!ylQrk4GdN_wW6}pld#nPV!h6G}P}Z^y?qN^h|Dua-^ez$g+c^kp
zdCv_beCm%xVrf7RK{$8KS!wASvsr9eF8Dil&{<pG5V6d$$!D1tCK?MX6*Z5arBYfp
zJzaZ)XGi`vrR7mEPxX{I^j%whfDIhYiOKQ2pWlv{Z^IOQTD$8fhlh_uUm%>rSF9su
zKInGd)mfvqAggz?qj1#y;Z-ryT{qO&J+Hya$SsVG0m^24=)k4U4<_gB3^sp2IIb3>
zx03!v(4y=~AvPWUcMT)w1)It2^?^ikJjBx#ycoI_8iL8zD)VJ|xz{20w^{eY5vZxO
zBY1BBa8RI}odR~~=)mrSO;5<#0@HmGP*7_{UJu3V#J29$(vr<peoLHIwJ$GT#cEqW
zv2tzVpLb@S#H)ZW6MgtrVZKt*#SHHdFO-V+&(Y^?ZTMs7Dj5@NTN^*Yy4~C`_t<Qb
zakYnqQ#u}T4_xfFR0HDJY6ztZZH+ikxq*tpd%aD1FnP{Hi2mi5VlE<vgcR0ZJ&ykF
z)gtNMsjq(uJ?{{*VNLasYQvtURPsL2NGqpm%;1i*w~8BCaCc}E6KZU8TBLm9iR(7+
z89JunQIVF<#(8rt43D@Rf;J_r-d%VGFLZ`K*y<I9Gf9hb-js-*z{ec)TcV^WXv`ur
z-tDuG_vLw{Wc&o>jjSC?EYutrQ;xC)OG}F3ac4;fKhMQpB_s1=cY9<wKH+dFOe62_
zCLVlp#5jbQ9McHD%tl;q5=oCID<EVe5V6R+U!&Vs3gCMb2&(86<Ib8ZEj*~nu<ZWW
zJi2SVWhQl8#i*-nEtUBRd7r|no*=DQIErM<PP^8RwP9KJC1@J&y#wF*OSiXgEFaP4
zjhDGdMlr62!`K%!!cVgiz5aIaVO4N1g7dC<k_lh73DfDkj5Q<Gxu%&Z(z(RNTEZ8H
zYMGp(8+%M_C}!6n_S&3G7QT3@oQ^FmX}RE4Xy-}28nXX3vtD&2MH^0~Rc8Sag>@u3
zfz06G>di>RWCiveOc<`{X9u6@e6tb0uXxvBjY)?rY1GO8oTA&=y<X>d-QF(1Nx(Wc
zwj?-QrViqw_Et}8Wn8oaUp4M@{H!7%;=HY_x*95{u<ZWxfeh<Y5_jG*v5Ee2-PRu*
z#-PgPnYd+-hfvQ7?84`q?Ez9wXwRZmQ~`BE?fpt%7;KkG9^wow3Nz8UaB*JLuqeqt
z3VNJDb3SyH`#U-d)LJ%s;LA%|pF~v0Z<zTp`<TZ*MU$BWdD49DVA@8w9`~|o{#)B`
zO)B0Wm57*1W|ej^he!UY&iOq$J_x8iG~(|Nv>&*^%i5Wcm4^|xzi&|rKB;cH(w;hY
zEY*@z=CrM@Q9UsmgPEscFX`efsGt5AS`N@5zQXFP!tPIKNMN6#;gFlpoGKv#W%-!R
zQ7ONk)g)S6NEv7(dXloPRB#X@dY!g<mk#{Zfys{ygVMVUL3FO*mT3<J;c`5!!L=Uq
zqHW-Vx51&uM=lC;Xw53mD{_GtHEda#77y~=VQ?2q^QWaQA>+I-eyt-0yat9PlU84L
z-(9zYl<&oj`{~ph?|{~OEtS=UTC|7W!{Xf)5znEXQ$r|Q^iSutyF;M8pRKcvECyl|
zXjbPZ`I85KOiS#3(>P0Q5X)hsm&PtF;%6XPDao#7cF<D)cG&+Vd}_a<G08{YPbJPs
zR_fr*C-$VuEv1c~sHQx(R;DJ*S57D<n7RSepM}}<?7-Z#p7C)8AAxRAhOn{L_-Eur
z&f8AUs;dZ+)G9rTCDO)N)1wpPO)V|SOaxVrKSzdRZ4&tDn~j|MT)5;;2250szfV2(
zH}MBm=o8@WSG>JpK+W=AAp`G)rSJNPb#v3?_JLLpw1ZpwH9T9nzoxeSjS<`iZG@wO
zXFTYPFKv+tF{p0mAl04*+<^Uw^5RyLPVS$$uM`?c60-Gs`aG0<HLx{4Ym+BaWJsIT
z&b|EYVv)BP`1G*uB+r)1@#OcjJyfTnhs16vX9K1NxWhZ(4u3v}D*kKQ&-ynwbMt;w
zv;hL3F2@<^yviXlMkW(0egxtKeW)qpU-nV`C?t_Va*|>=$l&Eg!^W={8{Eg|?`txp
z^w|6pQR~zJ{t8ul2X&K4GdZqj@bhtJzlS#{QmAgKXT%8rtpGc?sh}OY9EmpHzBxJW
zadq}aFTnCUrCB-kZO!}_#KJ1n9|%y1*0qYK&${nJJsj&n-G?e_v0XWG^~RN73l=u?
z@a!LH_e*`vP@?iM1O-kX(96G;KPg;cpRy=``#*#V?*2Mj_f1~Ai)~c~8Ztb!U^V<M
z7>#9_nbmT_o`6$A&mF5)-2arz`3x}t)$t(>T~Dhi@$)C}7w4z1_+8fc=8<>0{P<TA
zj}JCfk+ZhA9UqgBS_|o}g}*J3g8xzZF=G>EpA+h?qldGhp>y#QXzYumpBxwQ-oMOL
zgFltK6M239g7>0wmXx;kxks?26W&Cy8(Xt~?OfuzLfe$5bd_T(?pbUA-iJ)NS90cW
ztLOLlGe1doeuvyTk5e-=VR{QzQCXHlsEoj#d(fl%Qb|idXKiV0gzpB2mY&3rt&^Es
zn#3*XtgYj)_{rAmlj*6043omf7ll-@9Go07ogF2jC2*7%YX;gpA-K=Q)Gi2l2TfM4
z(sLf*bW+)>h6A_s75^{oDER&i=<bgRtKsMMTUWXY`^gcv`}eKuv4|NtPtp{s8$CyR
zRhi=W+LO=xl@bF|saaLS)!tT6X35f-c5UBP3uG-Nt5WU5x}x=%Wl9)@MS?3*UH14;
zBAA!nSheFtZq#^SiYJpmKkEK08xa8zt6*;&qCOO6YK0C78--mUW@LD`!~1EWlew=6
z;Qem0ste%g%H#grB>21-<Z)?Z4QLD+_XR7b-y$2Y!^fbc_;4slH;Um%ygg(fc!k8V
zJCUGI$Vd2V$;S_jsM&`>gc$gWmEROfQK11b(;kO7?m;H$vY)+59!3CtFdlr$_TrZ4
zvBnqIxiTzK`FCpc;%a)_TwN)m$jbC5dPi!)iO;^4N>(U$@!DGs+Gyujev=vX&1^}P
zqFlG|3;3x|#Yv8NhDzJ8L9@-I3Mx3Ei}F`d#JXMr?P>W3pG46bu=VaVhgyatutjl7
z!q94*yH|^<cVa<9h@)YzCVnICXm}lO(TZ7Sld|!y6px;IN8qHzsh$9_D)Rsomo|4F
zIzeDcy!-5gEo)BqVjjTf3?Pu^qQMa}kRP8rtvZ|X51MB3JUhJ$5Z`v@QReDynsN)-
z0?p%pzYpn)&JuT$hG(BFW*CdcH)eEzN5B7(XY>;0_`dYKL5NzUS}jv;qlB5R{>Z37
z!9f7oha#1)<7|K&W+RH)#GNKkQhDdn&M3_Sea`WaDziMdgw^+Y42|x@X(H9EujTcu
z2aNZ3!d5i1k7xTWOx$svSd>ZeQN#*4Xb$wpD$l+|2YdIcN=+=A&9-&@6!GzOid3xh
z9Zx%e3(7j7Id#EkcFqcH<Pd6d2F*u{t;VJWDU_@|>7JmQ%v)^nm59;Lg3j-k_*7`j
zuz0w3yE<0j;}pVONG$$_tXt%^lZn!;1e0WwIaw&biUQ}CEt++GXoC=5rmlo<3&ZdK
z3_*_p#Z#?#7ej4<zUrRl=-TV@hK~<NEt$#Zl$tE<IB4hesqplznn-`|c_T8G64it}
z!u}mnr<A$-ea<3;G}oX01B>I5yE!!u1^asqbIYMdf5_$n9>O+uko^}<7pfi5;`b&$
zIM<x-QO8Bszc?h?1hJ?^n8b<()S&H>kU7D$1gS-O-rQW6o5~B0hck|Bg|dqc%|`ZF
z=wOF0gPTUn_SU{>Bl)uPx9z(P3!nK6hcDz8H3V2aBDRF#On^Sr?TkZM-^kuugdHbS
zB;Tu@4Wg=HjTlc@w)Uz6k@rzeUCEme`X$_u+}~wJ^4sThq#Ev6Jr>G`@?aHd4eqJO
zvC`s|R^{9z9Oj)8M?A3$G_n3hG<fo-8O{T#lz$|TK^I3Wt=2cz3labnQTqXw$YE6V
z47@j~De`1#MS*zMtg=MZJrfUyqqCy;S#1B{)9T@lA9Ae?Zn812>2{Puy2NX9s|bb3
ziJSb@of~t?!6pqVZwI^fBt{w6BN6ns8ahu~dv~%C_oqY9ETAA|vfBXLqZ<V`auC}Q
zTb5Z=1Vk@vF>R4}2Wv+LV=Q3hT>>0(=<GtVNdVU|%NN@?%L2V)?mTVk&$L0dlh{=I
zLlUtp+FVV=b8o;N`46FoUquGVz}5rZgeyon0D%Bo2E5OA;JgV4u!4uMmT~93c%ADR
zP{1iIRqmRgw3$~*Us2R(vMxX|<UDO4z$M<rcpk%PN+L>wt2VgfAYIpF7Uv_*z3`lz
zmb!@uY({qd-Tdw;Qo9Ww@F&Nu1AOPlJ*TJJXo&T-s%%q1QJ^AWy8ZJf?}AT{23~HL
z^(@l6X0x?WEf7?2aSb#v2p5)#yV-YM(Yl4qzOqOs*A1h4RWxtX|AO9RTKiotq0Grs
zaY}scP6V@uwOTSSZ~|+9L)<xk;B3UZ`)!E^eh}7l94p^QbB7ir4s!^t(V6Q)*siiA
zg{(PQj8rA-Wmo$S@&fWR=Q2*ak_wXcMJP??-ZEGAzx7Yh$Xw!@s2%CJ9wysEIWB5^
z1eDmx5cCnCUj*;`??@H@eCD){SP8B4G<6<9h=L^>lC@ehb{<@P7y%<4S~&7~T$e-C
z^{H-F&FUS?j`XLD#EB81CE7IxhF=}OOw_++8DV`OE56t6HAQwh-(tM!XKB2hFMX|h
zB2b>TDgaE~pXBf!G_F4|bvc}2=>@j*eEn3-W&R9YIm*dHYAX%aP}ilJEsAn!`?$T-
z9BD1pdIpUfX!A1`Lv`x8ozN~y^-~MSwkPxCL)Bs(-c}DOWp(hI0GQJ{BmV*;B-jp)
zB=qGRYgx%}?U`2Y!4+7OFc#n2UmX-Cn)_OlFI#0q-r7t8d%8<8ePRps7%xb$09!7z
z<FS&L@XMw}hJ9@DqHB^lrPGm(wG^7C`ge4|R<yV_tpSeM1@wup9sC+FXp$Q@v0MM#
zw*yxIr|mXF-)d^GQkftf9l!ftJ9w`O=_dKTa-8Rh&l|}wTWtNqf2%SyYEH8RdQD>X
zmKi?2P&<CQ&1RA~uLW>LW0;1p;ChMRL!4`&<8a!U)Cd3?5rCDMRt0b0^axsnBPR>*
zgE*j%0L5noA30Fvmc)*70<(Q-iT}fDK4(&4m*LJ1n$BR<I{Ga(Ks)4vO(mqyvo4+G
ziRs&*3eK`MSi<ddqHoK}DSFds*O42EEbf8O$}(65PND80c5p;=h(+(SuB3>oEhqAm
zOW5X+R<Y7bkHPw}mzx$T=o6v?D_^Vi;8|IplwIF=;_0&$V-sRkH{~?vyc~-(_yD!j
zq{LVixW5c{A*QOBfyl*4G7Wh&l@77HLG3>ENnHUnauc8;)&Ug(8=X!^uzNIJqS~Nb
z-myml1B9IK%b%LB2if*B+bwDF2`-beiBX$xe|w2JvEO$*QKvF(Kuyj!w{r`5`^<V-
zX{J$Ap4wD25KB9f1ujgl?ViUEBPM^~d;_G4A}7=ms10Lp%jRXyc6aMq6l%(OT=Z50
zUw){jUiwnJuLSm@^{0ajHu+EQUpYmcFR0dt3v&Y2c<4`;4MemEJz_HH-PSEcidstN
zCRhKi(_rbDW92s1xH?NoN;<<Z<O%RBeH<_V`W4_je*o%gug8o@Ze2E)Bc}oax2(8D
zsk#fPw@ie;O+EaaZHC{x`;8%AP3mA_b!c`tRmZ+A=hu1_vp~)hi5n4dB>~arBBz8*
z#wIoz%xX%Hg|nYG$+%G${0IAyCN@PU^!5hP_3OKlnefZdo>R|o_*GzcmhbpoO}D{}
zEGuVD5Qz_C5O>+qr^}Q%N^epnubg@L^yUZcrS`JV<#v^!I3|mv;eC~(^4^BuM2)J<
z_JrzeyzHOQ$(0avy|YUb*C+3i#=wi-Z1g0eJ%)e*i85)kzQB`|bNn@k@;U+!&|V>e
z8|b}^Gt<bXz{ZxK0gaTBDA0!T=Wm1zxec*gm*lld;=GF_{*s5Zurxs_i}Nb=PhDIj
z@AAx6VEXIHqB&CUNo#K9H}bq_uiS&DP&0s+O#tM*`C@m(U3tXu+W+PkRK}CZhTBA?
z8&6}H<M=CLvb3U=1jhtLmma;)pcjlpe6%cj+1`$KBR!MNCAw&<H*bW)O!X?A@NuzB
zN|mSoVdcbob4Tn1?}y%KkrTFkVR)|>@6?0d^SX$uoNJT%El|M_5QfR^C*}{me`1V7
zMYwYLK;!Mv8;7d5W@0t=-OZ$XYgi`f1G9-=nTU);`}Ut7`KK9-xf+rdyDKJ%yLk7G
zH24PC5O`t@$OsjW=B0T?*g*yDldA!F_n~zl2TgnyGTym;yES^Y1J~GqIv>HE^Edwt
ze66W}k(Y~osA5$!rT5afbY%Kaq4gULr01`T&B*c4W0!SRzZkC8c`&gYQw+;a<P`YV
z;}~|;t#;MEITM50K9=vXpq<GdoH;<p4V$xgocU4(DR^#-Y=GGhQ}sH|rIR;>+HiSG
zN^-^YRpsaA*}pqxujR&Bt>ui2OWV=bSBIcBH&Qc&JhO_UoN_kjwaSPu_a;W9?W5nx
zj<pKfTg?MT<xB@0+8u|W|IgH+jtjJfQne7-PjlR8<q!G%@zs|H=A|>Jzt4l&7iEXh
z9Y4Xx7`dbBM4JY~DKFy@Z<~sVe=kl;$XhryhAP#9ak;Ckv@7>z<KS7gq0&z1ERX25
zC`e(u=W!CeJBK};)stJzgRFbpSszVvBGsvb)8NhgoLnC9(YR79p@9L^s6by!(tJ`D
zey&oZUf8^p$nS8{Pxr+2MqNV8*uM@1%v}+uiRJrT=vn*fr@ia@cF((;?#*Dv8QDCc
zsNhYBCG;@Ao9CNK7E8)n&9j;X<Cb9k7Anr!ruQRTEAZyO29&DvZ8i>+mB^T2a~rD$
z4jhMz@TV?(PSLvzqFnSHu`=V1Q=3tMtj5%Oc@8L?`P}Y_o^>g(dN}VlPtk+*nj6KE
ze9Gvg-e9NaB&yFB7ikrP`Vq|zB~8y#WL7Fl+33nI48DB9Oyxi3(0u!Nmv9BNFPfWS
zRtebaJYUa&X~irpfU9E3;ZW)m&fxOUD@n-c89WEk34`-4B^gUB#kgXtDM^${NJ_#2
z8WS4@-rL-is2ZluwN|++r+t0f&Z}$!#SxsW6I^lV*b*C58>KV?Jy1)t<?|+t&8@@B
zzZS~#8ho|_#4M@6G4^ZE`PyefCy)wfFfpWNlMJC*k_zlW-X!!_NQ|cCI-B{ks>DYS
z$rp-v`R;9$aW~PGMl0_`K2s;@`+jz>o~INq&c(ykFw5oqMm;G)zxV8j-Lr-!e*l&B
ztO+yOI$%{9*mDIsyWP<}b1(aKlH-ev-FEBT6ZjLn?EBf!nv~$h0{=k2+1dcHgoGQr
z`SNpVKT&ljqOaPkID7hP&Ahi{CS{~}hA(tqXoZ+lu_V4Koz5U8{yo2yPq)K)#TT3!
zhMol4Ul$-;c4~Mp<Qk%Q?wR+$6mdIr%Xz%x4VRnETxnQb+Rx7hG*7npI>2wCEqUkT
zGsZ_k&mo=Tsgz^v*4GtS*(M2*(J=K3p){A%++m4owkWmbDxP7B>}<rY!Zm=h_Ty}X
zbvI>*g=gCggcEwEc(z%oHWMs76pcITxczeG$6}h=5u`jF&NfN)#HGlYN7~kx)?apw
z%eBzhPVb%nWY=Q2y7EQ*ckG{~huxKuR_`(zzU0j#yy_P<_D~R@=<D(Brqm6a-RI@u
zWnKF{1#97h!Dl++hYoP#lEN#|D}4TbvJ_HzZf9#Pd$24$KOSG|Vp2dx^K;{~^KVXt
z^;)rW6zo=}+tmJ%sgJ*zpMH!^GLL<{flt#Rzabb*>z^tJ3=Ngm_Xzme3v2YAb=VvX
ztmw6+&`0(#FSnLf2*=9U5{vN?y2Lt<-|(d36HzktuvBJYD#yI0vQTEh!@TU%o&;Y{
z#yyokU-o<_!rC)2HeveC3=`OnM@K^l-9J;9KNxt#IUwD`csBwsySM+=oH*`hoSuaU
z2A22y>=JdnnceYleg73Z+Pi8M?Be9shdbd=xy(XOp>8aGfemI$x2(0na`By((FVnt
zb4aKxYm;yMk@~d2C%e|n{MFw|%YSXa_0e^@rCJYW1kk3KA*gl46ZrNu5^j})K?p+r
zNro^%ndHuLLTj-|l`;oEhV&3nSg;4p9CgL278LU&i)woyiYwx4pdTas>5jR?-=G!p
z67Z?t!%{S*2I2>5Ei7s$18NP0J=|GbHnsm_s?VOiH`$0s+eE%dCZ^VFJ2d}s_Xpj2
z6$OtHfxhLJ%zpb*i&vLy%*riGxYOU>zjp1cAq2C?7=e;q^JbRmR~~|9`8{73ifgQ>
zK+Ql)67zK@6xBXkD>x5a{+mdsG&vw|@Z2D}`G2{7#E&V_Yf;yCg%u_ImYr>CIvKoQ
z&1IR)6)ycxQu3d^<O($J=M-~dezD-lyd1`-HKtd*2qxMw+L+2$I^n>#Jp4LXxYV*S
zkm}g7wOhapM0Lx+Z2_77VYKVJv%foxihn;OW30{`C3fA93$pUnpMRf)ydk!~i35#1
zEHBscPx_JsgXR)vg{Y3Ectx>}GcZ|bA?%>I#}EFn@C{vb{j!KLoU-*PxC}VGbS)6)
zy3s-6DL|4>2S>pX2f((;=lxn#f!yox;o#$qR=d0^Vbiicl`r4$Qg{logs&?t4PI%L
zgIq#N==!sU=tZbuuL>wicz3_MDQEhp5E%a!nv867AaTP$jW|x_I9ZEBE3mCbkOi**
zsMng?t{f`(rYeRVH674OYfPHVty0;x(?$AEZI<B1+fU(fWyu9)EVT2rZWu1U!nbEv
z`SX>oQ>0GalN{9`VHwr9|MF5iXu?a<OPn`N?AN3Wd{Cip^2c^K8c+s+;BB=AU&E|#
z3?p``nN~p?Jpo&7(@r(onr^T-1HudO*SWB#^{=U@mf{mHO_+Z^C5aT(a6#%lRV9<0
zk$HgsMn*{6YRJF8V#J0+8~RvJg1AP`aemD|wy6)O!Y7~#$3xJwuGwzFYs^tW{SA5O
zX~ylu<rZ;`FPlMglA~kd<eNAWEjRuIolj*yPwFU}Q_4(V*th0XBGC=g2~Xy+j7DY4
zpwppnits9<1sU-G?$bnTG*6cC@(;N=`Hh0x-}MWNt@mTVVM6WT^<jHcn_)}A?oGyS
zLrxCnk^~<^-JG>ZTYXXp;UJST4s+w+Dbr|)=?%T&kH%LPkxPp-c^X$@<Sblq>Y1s9
zv1QK{9NNps4Avj9ZUU(`3H%H&oad3|Um`+_AnzjnOg&p^I`YVOuW)zht_p~e3vm3=
z7}nfD@9bI12&NP=yP)x5cHqJ#>F|=fBylZ~jn`uIVJhLW-%JIWExNW7Ijtzg*A*o6
zHv+kQxy5_LJ+Lj+m1#f-hVA{*LZ66+mclBqQ(>^O?<Kah-|+ZjKN_)uUOJHyR`5M}
zdt1xFZ?@5vsK#fm_91X+D2B(9&^Bda-L46<C@JF7ie<&W9#APEYSd2m{=ZDVSjXon
zJmOb20<Hr_D?1}4e{FBYBkJWvIi&{!bW^UH#KJ!G2=b@F&Z`<#Ewgxu+ikm&1QNVn
zC(%UuoDxCZTOL;hRD9$)i1U2Gm18e^8b_ZM`lOTFKzgLz)J>Cr3s?fJ@?MFLf)#Iu
zpn`saLicT<6R3IKtL=PA50x!r2Y>N*FJmZE92XRl+I8(!xF5m#@S|ERZ9pnZe)!aL
zD(^sVZhaU}C^{C8Sh*!EVTP`+YAY==EmXRFU~*W+5hXB=uub5Gq#F>tMgL&U-3M^%
zh%H$qnmj+dfd0)ri@j3`)1-&YEBhZW*GRui4o&D;Y5=h-Xv80!xvK4+>hvpQ?0m<!
zFOeJ+ANymXnM6eJ)tQs?dJ?|rMt*3dqA)yb4tQK(dM}d0?-n_ANY8ciTZ_KoE?#+z
z2e0Yo3LwTLzw+j1>g(FNttE;z`kj9F`ZBZM*KJNr0n%9yXqpnUo2sDuB}P`tXlWmJ
zF<$Ibc@k37hWy19V1pJZWWWClJilxMf*7%V*Uov*^|#$~6J9szDR)K|J3MXh4oo*l
zXQJ^Pi&Z3UM4bQiTc)MWFXPb$KR<Is+R(nTuDSXWc37?<zE%RUx~f&Gu+A7eWlreW
zu!H>(qKO&E_ki6YbSF^2OqFD4f}rEwt=~%QjtJxeE8()sBRF0jRv{Jj2_5C6jfjkg
z-FvRTT^8~NYs~s%Z%mT|l)jV_`a>jy3TE}l{Hv=ZhP~etL2JJH7~R;wGnAAJtf=Ka
zT)o=_(jzsU3(n-V@Y`BF&h|?P(@p37pX14bL-=;s79$xg-lE-VQusIrRzmZ64<C&W
zTkL`z6X_n$I#y;`(!e!I%!BIOY1k*scZz8D{-Ln6GFpu|<6^NxH6C?W{kL=4<{)Ho
z(SgVwDgtofqCoJe+(cN9K7aI2Ui~^V?O)|3QY!&m!8OY^`Rh$*zX08(U8b9!2jBEi
z{#%z8o!}My=~MlbIsAD)d4s~^kqH?V?2`JCzWuT{v+E0e%dk~fwuHQKa3-)^PP|1$
zL`<QY06>fbDbLAuH}d8z_&D%?DGzq|WR5pF(OE<9ko9V~2BgxQRdXq7tZmw$yrwa0
z=d1mQ_yqrec6HsQlt4<l<td!qWzO%E;_;s@akp`{dU^Uyx)4)*!_02Yp>K>xZ+VT4
zy9EFn=pm@$>2v5=$hW)P`$0vhpogG;s-iCtU5yi)^3LLj2n)}Em4k&fKXa5Xn@QxA
zI_CK=fJI|@%lCMcIQ;}Pe|n{fr{DOC;kpa(K_s;6xUgPZ_Bu5TxMA}#rECBo%+~%3
z3_Qgvlc3ChZi;uPmGIvgR}F35K*$~;b=LLmB%9Z@Cm^3>|AFKZzTC=Ni$+kn-cwMO
zK)Eu31&w5G&kr+4b^%RhDJ=12ybmh!O~T3+2{~7vz)MvObhmb-sj<=sU<^3|F$P>1
zJ|o@W?(+oB^sf*-%Np1n;0&B5bgR?Qa%or1n8MHR`-IJ$-}L*#?|jb%7#DAziY8w^
zp$wRc1e=W{J#O@+)3;*BY6>3on!vHFF5GF1pImj$KlDP?oyc3~h5x4!VC=n^Y{YTl
zDk>TX+1pJ|ECDC>h0myzJZAKXqc4xd+N!$+_403?I_-<H?_cSnD3<#J3L3P%nRC)Y
zlnY}NmL5>$(4!r>AHy)|T_;Ic*^vr5VMH_=`QfQ~y`jglpVq>+0Hpj7D3{&7R=H6O
z;z9uFPtaCasQGIO%fz)z(!&u(i%fi5g@YXYK^Xy&au)>da+*$}U&ceOW}l6ZZ=B+_
zh9%m<GVGmFU$GY8nfX}Jh=yidi!wZQMm%#O7uW|~%m8-sA30nh|I4U9C$#r;Dh!<{
z6E93d{>AZIchc<f09rpwU>Qc;h~(9mVM;%F{A}4MV1zDmibkZdvgC7)h4aj*e<a~2
z=qUH4eBZm;$||`it0vdg;qJaH@1wf{KziK9yZ8Qh4z>PWCyPYy_WoNeSkmZ0cXb}p
zKb%#*8heYE$U`tAR=7!^g~RF~dPK&i^)}Bfh56jhT}X}fgE)^7UNx8fSjPq^_jp=;
zLcm=8+a^U<I<$~z^ApRjK2#E%M8_6A10+_ofZ>?ugyo>G3(-CF23k`BwYj{-D`leL
zk$E6$QQt=HWZUwWrZL~Bac?Q0FNHyGbsh@d2(8VD^NiSjAe`a<;xxiu#Xa=-JwB<c
z=+{Do0Jx|}=RzJ>5d8Sxl70JocL1k1Izr~m<3>)YJ!-(<XHd~)K-1Gkyyb$5+yRp^
zH=b@hD;x@F$(+BF@k0ZCJHZ_@k{0)O$&<}Fc6^!79L17_Sw3P}isG}>zJ~b1HWVvm
z#x=nA(e_a5FgrN)I)B7q!HR4HMzN@U^eWM+vc?!>3;F5P>avH6{HTVvU6#9ADhir#
z(lTI>C_S~9;6L~Addo|rq?(XK1Kmo(iVZiiFY_+*-hR_*MCIq*xO`o)J?L@c3{3If
zpefXAfVS3j^DVQs1GnVsGDkn7`8+>Vd({tjNOX9oz)Y$sA6rthzcKBFD&-hlz{Vn3
zx1ovG?__blEDH<1)?dsMz-1I8;!ZIB<wbT>{A8l`KM|+~K?^cAK}P?h_1Hniv#_5F
zJs8;N86Ggluk%|+TLk6kV!~)6ox`-h@xU`~$|Rvq2|{H|%gt|jaSkf_Aa;>Syd;i>
z)X~b{sf0YMDl%HU9M&o%TWzgx#zUk@NiV()h5pFsNNmF(0RjP-DuEN$esJ%zRp26*
z&aM;uBl?G90rql6v{PXepGvV>UQuT@6;2o8<C=Oi<+K!*G<+kIdCW1Y{b`DSh1-Zy
zIViTp;+5${<%!Lo54ktbkldY#;i><<CN-%8mX-4kDGlG*K(`te=Fyb(<c<!E=8r%Z
zSC7SiW#p1P7%1%kD^n(VxvDbK$E*`5?gSKfDD^w$jB7KAT5)K+#@CD~W0qV#YW(_L
z`0yFndITc54UqYyXEg9;##z@L?=d7Z|KlfK&$p6w!3Q6n-n@vj%$wL9NeVd_Pok{6
zkp4ZN3DNh`;0E*cSCYfkhn!m2TOQi;ks@3D-)h7PziW4Q77^8>4sE-<6#<Ncyc3#|
p6WR^D!3JzJn7Zyf|Dl}a@9I3|;rG_SyP@~@!zk0Db4F~e{{u3@F`@tf

literal 0
HcmV?d00001

diff --git a/tests/db/tap_mongodb.sh b/tests/db/tap_mongodb.sh
index 17952b3c2..3bd2410ca 100755
--- a/tests/db/tap_mongodb.sh
+++ b/tests/db/tap_mongodb.sh
@@ -7,6 +7,7 @@ PWD="$(dirname "$0")"
 
 TEST_DB_DATA_1=${PWD}/mongodb_data/listings.csv
 TEST_DB_DATA_2=${PWD}/mongodb_data/my_collection.bson.gz
+TEST_DB_DATA_3=${PWD}/mongodb_data/all_datatypes.bson.gz
 echo "Building test Mongodb database..."
 
 # To run this script some environment variables must be set.
@@ -35,3 +36,10 @@ mongorestore --uri ${URL} \
   --drop \
   --gzip \
   ${TEST_DB_DATA_2}
+
+mongorestore --uri ${URL} \
+  --db ${TAP_MONGODB_DB} \
+  --collection all_datatypes \
+  --drop \
+  --gzip \
+  ${TEST_DB_DATA_3}
diff --git a/tests/end_to_end/test-project/tap_mongodb_to_bq.yml.template b/tests/end_to_end/test-project/tap_mongodb_to_bq.yml.template
index f449df856..f514bdd92 100644
--- a/tests/end_to_end/test-project/tap_mongodb_to_bq.yml.template
+++ b/tests/end_to_end/test-project/tap_mongodb_to_bq.yml.template
@@ -44,3 +44,6 @@ schemas:
 
       - table_name: "my_collection"
         replication_method: "LOG_BASED"
+
+      - table_name: "all_datatypes"
+        replication_method: "LOG_BASED"
diff --git a/tests/end_to_end/test-project/tap_mongodb_to_pg.yml.template b/tests/end_to_end/test-project/tap_mongodb_to_pg.yml.template
index 92156e082..eb677feb3 100644
--- a/tests/end_to_end/test-project/tap_mongodb_to_pg.yml.template
+++ b/tests/end_to_end/test-project/tap_mongodb_to_pg.yml.template
@@ -45,3 +45,6 @@ schemas:
 
       - table_name: "my_collection"
         replication_method: "LOG_BASED"
+
+      - table_name: "all_datatypes"
+        replication_method: "LOG_BASED"
diff --git a/tests/end_to_end/test-project/tap_mongodb_to_sf.yml.template b/tests/end_to_end/test-project/tap_mongodb_to_sf.yml.template
index 2d308cc55..5f9384c68 100644
--- a/tests/end_to_end/test-project/tap_mongodb_to_sf.yml.template
+++ b/tests/end_to_end/test-project/tap_mongodb_to_sf.yml.template
@@ -44,3 +44,6 @@ schemas:
 
       - table_name: "my_collection"
         replication_method: "LOG_BASED"
+
+      - table_name: "all_datatypes"
+        replication_method: "LOG_BASED"
diff --git a/tests/end_to_end/test_target_bigquery.py b/tests/end_to_end/test_target_bigquery.py
index 2081afd44..cd3d462b5 100644
--- a/tests/end_to_end/test_target_bigquery.py
+++ b/tests/end_to_end/test_target_bigquery.py
@@ -1,3 +1,4 @@
+import decimal
 import os
 import uuid
 from datetime import datetime, timezone
@@ -265,12 +266,15 @@ def assert_row_counts_equal(target_schema, table, count_in_source):
         assertions.assert_run_tap_success(TAP_MONGODB_ID, TARGET_ID, ['fastsync', 'singer'])
         assert_columns_exist('listings')
         assert_columns_exist('my_collection')
+        assert_columns_exist('all_datatypes')
 
         listing_count = self.mongodb_con['listings'].count_documents({})
         my_coll_count = self.mongodb_con['my_collection'].count_documents({})
+        all_datatypes_count = self.mongodb_con['all_datatypes'].count_documents({})
 
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'listings', listing_count)
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'my_collection', my_coll_count)
+        assert_row_counts_equal('ppw_e2e_tap_mongodb', 'all_datatypes', all_datatypes_count)
 
         result_insert = self.mongodb_con.my_collection.insert_many([
             {
@@ -288,6 +292,7 @@ def assert_row_counts_equal(target_schema, table, count_in_source):
             {
                 'uuid': uuid.uuid4(),
                 'id': 1003,
+                'decimal': bson.Decimal128(decimal.Decimal('5.64547548425446546546644')),
                 'nested_json': {'a': 1, 'b': 3, 'c': {'key': bson.datetime.datetime(2020, 5, 3, 10, 0, 0)}}
             }
         ])
diff --git a/tests/end_to_end/test_target_postgres.py b/tests/end_to_end/test_target_postgres.py
index dde857ecb..6c641329d 100644
--- a/tests/end_to_end/test_target_postgres.py
+++ b/tests/end_to_end/test_target_postgres.py
@@ -1,3 +1,4 @@
+import decimal
 import os
 import uuid
 from datetime import datetime
@@ -272,12 +273,15 @@ def assert_row_counts_equal(target_schema, table, count_in_source):
         assertions.assert_run_tap_success(TAP_MONGODB_ID, TARGET_ID, ['fastsync', 'singer'])
         assert_columns_exist('listings')
         assert_columns_exist('my_collection')
+        assert_columns_exist('all_datatypes')
 
         listing_count = self.mongodb_con['listings'].count_documents({})
         my_coll_count = self.mongodb_con['my_collection'].count_documents({})
+        all_datatypes_count = self.mongodb_con['all_datatypes'].count_documents({})
 
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'listings', listing_count)
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'my_collection', my_coll_count)
+        assert_row_counts_equal('ppw_e2e_tap_mongodb', 'all_datatypes', all_datatypes_count)
 
         result_insert = self.mongodb_con.my_collection.insert_many([
             {
@@ -295,6 +299,7 @@ def assert_row_counts_equal(target_schema, table, count_in_source):
             {
                 'uuid': uuid.uuid4(),
                 'id': 1003,
+                'decimal': bson.Decimal128(decimal.Decimal('5.64547548425446546546644')),
                 'nested_json': {'a': 1, 'b': 3, 'c': {'key': bson.datetime.datetime(2020, 5, 3, 10, 0, 0)}}
             }
         ])
diff --git a/tests/end_to_end/test_target_snowflake.py b/tests/end_to_end/test_target_snowflake.py
index 0d0015832..0beedc288 100644
--- a/tests/end_to_end/test_target_snowflake.py
+++ b/tests/end_to_end/test_target_snowflake.py
@@ -1,3 +1,4 @@
+import decimal
 import gzip
 import os
 import tempfile
@@ -344,12 +345,15 @@ def assert_row_counts_equal(target_schema, table, count_in_source):
         assertions.assert_run_tap_success(TAP_MONGODB_ID, TARGET_ID, ['fastsync', 'singer'])
         assert_columns_exist('listings')
         assert_columns_exist('my_collection')
+        assert_columns_exist('all_datatypes')
 
         listing_count = self.mongodb_con['listings'].count_documents({})
         my_coll_count = self.mongodb_con['my_collection'].count_documents({})
+        all_datatypes_count = self.mongodb_con['all_datatypes'].count_documents({})
 
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'listings', listing_count)
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'my_collection', my_coll_count)
+        assert_row_counts_equal('ppw_e2e_tap_mongodb', 'all_datatypes', all_datatypes_count)
 
         result_insert = self.mongodb_con.my_collection.insert_many([
             {
@@ -367,6 +371,7 @@ def assert_row_counts_equal(target_schema, table, count_in_source):
             {
                 'uuid': uuid.uuid4(),
                 'id': 1003,
+                'decimal': bson.Decimal128(decimal.Decimal('5.64547548425446546546644')),
                 'nested_json': {'a': 1, 'b': 3, 'c': {'key': bson.datetime.datetime(2020, 5, 3, 10, 0, 0)}}
             }
         ])
diff --git a/tests/units/fastsync/commons/test_fastsync_tap_mongodb.py b/tests/units/fastsync/commons/test_fastsync_tap_mongodb.py
index fb1d9311b..8b5afaf30 100644
--- a/tests/units/fastsync/commons/test_fastsync_tap_mongodb.py
+++ b/tests/units/fastsync/commons/test_fastsync_tap_mongodb.py
@@ -1,10 +1,14 @@
+import datetime
+import decimal
+import re
+import string
 import time
 import os
-from unittest import TestCase
-from unittest.mock import patch, Mock
+import random
+import bson
 
-from bson import ObjectId, Timestamp
-from mock import PropertyMock, call
+from unittest import TestCase
+from unittest.mock import patch, Mock, PropertyMock, call
 from pymongo import MongoClient
 from pymongo.change_stream import DatabaseChangeStream
 from pymongo.database import Database
@@ -15,6 +19,59 @@
 TEST_EXPORT_FILE = 'file.csv.gzip'
 
 
+def random_string_generator(size=6, chars=string.ascii_uppercase + string.digits):
+    """
+    Generate random string
+    Args:
+        size: length of the string
+        chars: sequence of chars to use
+
+    Returns: random string of length "size"
+    """
+    return ''.join(random.choice(chars) for _ in range(size))
+
+
+def generate_all_datatypes_doc():
+    """
+    Generates random dictionary suitable to be a mongodb Document with all supported datatypes
+    Returns: document
+    """
+    pattern = re.compile('.*')
+    regex = bson.Regex.from_native(pattern)
+    regex.flags ^= re.UNICODE
+
+    return {
+        '_id': bson.ObjectId(),
+        'double_field': random.randrange(-15, 15) / 3,
+        'string_field': random_string_generator(100),
+        'object_field': {
+            'obj_field_1_key': 'obj_field_1_val',
+            'obj_field_2_key': 'obj_field_2_val'
+        },
+        'array_field': [
+            None,
+            random.randrange(-1, 1) / 5,
+            {'k': 'v'},
+            'array_item',
+            bson.Decimal128(decimal.Decimal(f'{random.randrange(-10, 10) / 6}')),
+        ],
+        'binary_data_field': b'a binary string',
+        'object_id_field': bson.objectid.ObjectId(),
+        'boolean_field': True,
+        'date_field': datetime.datetime.now(),
+        'null_field': None,
+        'regex_field': regex,
+        '32_bit_integer_field': 32,
+        'timestamp_field': bson.timestamp.Timestamp(int(time.time()), random.randint(0, 100)),
+        '64_bit_integer_field': 34359738368,
+        'decimal_field': bson.Decimal128(decimal.Decimal(f'{random.randrange(-100, 100) / 33}')),
+        'javaScript_field': bson.code.Code('var x, y, z;'),
+        'javaScript_with_scope_field': bson.code.Code('function incrementX() { x++; }', scope={'x': 1}),
+        'min_key_field': bson.min_key.MinKey(),
+        'max_key_field': bson.max_key.MaxKey()
+    }
+
+
 # pylint: disable=invalid-name,no-self-use
 class TestFastSyncTapMongoDB(TestCase):
     """
@@ -129,11 +186,8 @@ def test_copy_table_with_collection_found_success(self):
                     with patch('pipelinewise.fastsync.commons.tap_mongodb.bson.decode_file_iter') as \
                             bson_decode_iter_mock:
 
-                        bson_decode_iter_mock.return_value = [
-                            {'_id': ObjectId('0123456789ab0123456789aa'), 'key1': 1, 'key2': time.time()},
-                            {'_id': ObjectId('0123456789ab0123456789ab'), 'key1': 2},
-                            {'_id': ObjectId('0123456789ab0123456789ac'), 'key3': Timestamp(10000, 50)},
-                        ]
+                        # generate 10 documents
+                        bson_decode_iter_mock.return_value = (generate_all_datatypes_doc() for _ in range(10))
 
                         mock_enter.return_value.open.return_value = Mock()
 

From 473229668f29e1c7103e4be5ae3df23f8989263f Mon Sep 17 00:00:00 2001
From: Amir Mofakhar <moftakhar@gmail.com>
Date: Thu, 9 Sep 2021 16:13:16 +0100
Subject: [PATCH 17/78] [AP-1059] downgrade setuptools to support use_2to3
 (#796)

* downgrade setuptools to support use_2to3

* fixed install method
---
 install.sh                                         | 3 +++
 singer-connectors/tap-adwords/pre_requirements.txt | 3 +++
 2 files changed, 6 insertions(+)
 create mode 100644 singer-connectors/tap-adwords/pre_requirements.txt

diff --git a/install.sh b/install.sh
index 36b25d2a4..6c671573a 100755
--- a/install.sh
+++ b/install.sh
@@ -61,6 +61,9 @@ make_virtualenv() {
     source $VENV_DIR/$1/bin/activate
     python3 -m pip install --upgrade pip setuptools wheel
 
+    if [ -f "pre_requirements.txt" ]; then
+        python3 -m pip install --upgrade -r pre_requirements.txt
+    fi
     if [ -f "requirements.txt" ]; then
         python3 -m pip install --upgrade -r requirements.txt
     fi
diff --git a/singer-connectors/tap-adwords/pre_requirements.txt b/singer-connectors/tap-adwords/pre_requirements.txt
new file mode 100644
index 000000000..3258b5277
--- /dev/null
+++ b/singer-connectors/tap-adwords/pre_requirements.txt
@@ -0,0 +1,3 @@
+# setuptools>58.0.0 is not compatible with googleads==17.0.0
+# remove this file whenever tap-adwards upgrades googleads version
+setuptools<=57.0.5
\ No newline at end of file

From acde3c18f4dd116113f47f3dc50d31fdcf59a1d6 Mon Sep 17 00:00:00 2001
From: Amir Mofakhar <moftakhar@gmail.com>
Date: Tue, 14 Sep 2021 08:08:40 +0100
Subject: [PATCH 18/78] [AP-1054] fixed pep8 and added github workflow for pep8
 check (#793)

---
 .github/workflows/linter.yml                  |   7 +
 pipelinewise/cli/__init__.py                  |  95 ++-
 .../cli/alert_handlers/base_alert_handler.py  |   1 +
 pipelinewise/cli/alert_handlers/errors.py     |   3 +
 .../cli/alert_handlers/slack_alert_handler.py |  29 +-
 .../alert_handlers/victorops_alert_handler.py |  38 +-
 pipelinewise/cli/alert_sender.py              |  45 +-
 pipelinewise/cli/commands.py                  | 148 ++--
 pipelinewise/cli/config.py                    | 232 +++--
 pipelinewise/cli/constants.py                 |   1 +
 pipelinewise/cli/errors.py                    |   6 +-
 pipelinewise/cli/pipelinewise.py              | 776 +++++++++++------
 pipelinewise/cli/tap_properties.py            |  69 +-
 pipelinewise/cli/utils.py                     |  60 +-
 pipelinewise/fastsync/commons/errors.py       |   3 +
 pipelinewise/fastsync/commons/split_gzip.py   |  49 +-
 pipelinewise/fastsync/commons/tap_mongodb.py  | 210 +++--
 pipelinewise/fastsync/commons/tap_mysql.py    | 184 ++--
 pipelinewise/fastsync/commons/tap_postgres.py | 132 ++-
 pipelinewise/fastsync/commons/tap_s3_csv.py   | 198 +++--
 .../fastsync/commons/target_bigquery.py       | 139 ++-
 .../fastsync/commons/target_postgres.py       | 141 ++-
 .../fastsync/commons/target_redshift.py       | 195 +++--
 .../fastsync/commons/target_snowflake.py      | 309 ++++---
 .../fastsync/commons/transform_utils.py       | 149 ++--
 pipelinewise/fastsync/commons/utils.py        | 111 ++-
 pipelinewise/fastsync/mongodb_to_bigquery.py  |  48 +-
 pipelinewise/fastsync/mongodb_to_postgres.py  |  55 +-
 pipelinewise/fastsync/mongodb_to_snowflake.py |  48 +-
 pipelinewise/fastsync/mysql_to_bigquery.py    | 111 +--
 pipelinewise/fastsync/mysql_to_postgres.py    |  57 +-
 pipelinewise/fastsync/mysql_to_redshift.py    |  61 +-
 pipelinewise/fastsync/mysql_to_snowflake.py   |  71 +-
 pipelinewise/fastsync/postgres_to_bigquery.py | 118 +--
 pipelinewise/fastsync/postgres_to_postgres.py |  66 +-
 pipelinewise/fastsync/postgres_to_redshift.py |  59 +-
 .../fastsync/postgres_to_snowflake.py         |  65 +-
 pipelinewise/fastsync/s3_csv_to_bigquery.py   |  63 +-
 pipelinewise/fastsync/s3_csv_to_postgres.py   |  69 +-
 pipelinewise/fastsync/s3_csv_to_redshift.py   |  71 +-
 pipelinewise/fastsync/s3_csv_to_snowflake.py  |  70 +-
 pipelinewise/logger.py                        |   1 +
 pipelinewise/utils.py                         |   4 +-
 setup.py                                      |   1 +
 tests/end_to_end/helpers/assertions.py        | 143 +++-
 tests/end_to_end/helpers/db.py                |  83 +-
 tests/end_to_end/helpers/env.py               | 446 ++++++----
 tests/end_to_end/helpers/tasks.py             |  12 +-
 tests/end_to_end/test_target_bigquery.py      | 345 +++++---
 tests/end_to_end/test_target_postgres.py      | 352 +++++---
 tests/end_to_end/test_target_redshift.py      | 152 +++-
 tests/end_to_end/test_target_snowflake.py     | 427 +++++++---
 tests/units/cli/cli_args.py                   |  27 +-
 tests/units/cli/test_alert_sender.py          | 109 ++-
 tests/units/cli/test_cli.py                   | 481 ++++++++---
 tests/units/cli/test_cli_utils.py             |   5 +-
 tests/units/cli/test_cli_utils_tap_github.py  |   8 +-
 tests/units/cli/test_commands.py              | 802 +++++++++++-------
 tests/units/cli/test_config.py                | 263 +++---
 tests/units/fastsync/assertions.py            | 177 ++--
 .../commons/test_fastsync_tap_mongodb.py      | 208 +++--
 .../commons/test_fastsync_tap_mysql.py        |  40 +-
 .../commons/test_fastsync_tap_postgres.py     | 101 ++-
 .../commons/test_fastsync_tap_s3_csv.py       | 318 ++++---
 .../commons/test_fastsync_target_bigquery.py  | 223 +++--
 .../commons/test_fastsync_target_postgres.py  | 233 ++---
 .../commons/test_fastsync_target_snowflake.py | 368 ++++----
 .../fastsync/commons/test_fastsync_utils.py   | 457 +++++-----
 .../units/fastsync/commons/test_split_gzip.py |  32 +-
 .../fastsync/commons/test_transform_utils.py  | 274 +++---
 .../fastsync/test_mongodb_to_bigquery.py      |  27 +-
 .../fastsync/test_mongodb_to_postgres.py      |  31 +-
 .../fastsync/test_mongodb_to_snowflake.py     |  31 +-
 .../units/fastsync/test_mysql_to_bigquery.py  |  31 +-
 .../units/fastsync/test_mysql_to_postgres.py  |  35 +-
 .../units/fastsync/test_mysql_to_redshift.py  |  75 +-
 .../units/fastsync/test_mysql_to_snowflake.py |  35 +-
 .../fastsync/test_postgres_to_bigquery.py     |  27 +-
 .../fastsync/test_postgres_to_postgres.py     |  31 +-
 .../fastsync/test_postgres_to_redshift.py     |  35 +-
 .../fastsync/test_postgres_to_snowflake.py    |  31 +-
 .../units/fastsync/test_s3_csv_to_bigquery.py |  27 +-
 .../units/fastsync/test_s3_csv_to_postgres.py |  31 +-
 .../units/fastsync/test_s3_csv_to_redshift.py |  31 +-
 .../fastsync/test_s3_csv_to_snowflake.py      |  31 +-
 tests/units/test_logger.py                    |   9 +-
 tests/units/test_utils.py                     |   2 +
 87 files changed, 7043 insertions(+), 3931 deletions(-)

diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
index 568ad7e1b..e287748ca 100644
--- a/.github/workflows/linter.yml
+++ b/.github/workflows/linter.yml
@@ -46,3 +46,10 @@ jobs:
       - name: Pylinting
         if: steps.check.outcome == 'failure'
         run: pylint pipelinewise tests
+
+      - name: Pep8
+        if: steps.check.outcome == 'failure'
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          flake8 . --count --max-complexity=15 --max-line-length=120 --statistics
diff --git a/pipelinewise/cli/__init__.py b/pipelinewise/cli/__init__.py
index fd2af924d..221205993 100644
--- a/pipelinewise/cli/__init__.py
+++ b/pipelinewise/cli/__init__.py
@@ -22,7 +22,9 @@
 CONFIG_DIR = os.path.join(USER_HOME, '.pipelinewise')
 PROFILING_DIR = os.path.join(CONFIG_DIR, 'profiling')
 PIPELINEWISE_DEFAULT_HOME = os.path.join(USER_HOME, 'pipelinewise')
-PIPELINEWISE_HOME = os.path.abspath(os.environ.setdefault('PIPELINEWISE_HOME', PIPELINEWISE_DEFAULT_HOME))
+PIPELINEWISE_HOME = os.path.abspath(
+    os.environ.setdefault('PIPELINEWISE_HOME', PIPELINEWISE_DEFAULT_HOME)
+)
 VENV_DIR = os.path.join(PIPELINEWISE_HOME, '.virtualenvs')
 COMMANDS = [
     'init',
@@ -61,7 +63,9 @@ def __init_logger(log_file=None, debug=False):
     return logger
 
 
-def __init_profiler(profiler_arg: bool, logger: logging.Logger) -> Tuple[Optional[Profile], Optional[str]]:
+def __init_profiler(
+    profiler_arg: bool, logger: logging.Logger
+) -> Tuple[Optional[Profile], Optional[str]]:
     """
     Initialise profiling environment by creating a cprofile.Profiler instance, a folder where pstats can be dumped
     Args:
@@ -82,9 +86,10 @@ def __init_profiler(profiler_arg: bool, logger: logging.Logger) -> Tuple[Optiona
 
         logger.debug('Profiler created.')
 
-        profiling_dir = os.path.join(PROFILING_DIR,
-                                     f'{datetime.utcnow().strftime("%Y%m%d_%H%M%S_%f")}_{generate_random_string(10)}'
-                                     )
+        profiling_dir = os.path.join(
+            PROFILING_DIR,
+            f'{datetime.utcnow().strftime("%Y%m%d_%H%M%S_%f")}_{generate_random_string(10)}',
+        )
 
         try:
             os.makedirs(profiling_dir)
@@ -103,10 +108,12 @@ def __init_profiler(profiler_arg: bool, logger: logging.Logger) -> Tuple[Optiona
     return None, None
 
 
-def __disable_profiler(profiler: Optional[Profile],
-                       profiling_dir: Optional[str],
-                       pstat_filename: Optional[str],
-                       logger: logging.Logger):
+def __disable_profiler(
+    profiler: Optional[Profile],
+    profiling_dir: Optional[str],
+    pstat_filename: Optional[str],
+    logger: logging.Logger,
+):
     """
     Disable given profiler and dump pipelinewise stats into a pStat file
     Args:
@@ -145,32 +152,42 @@ def main():
     parser.add_argument('--target', type=str, default='*', help='"Name of the target')
     parser.add_argument('--tap', type=str, default='*', help='Name of the tap')
     parser.add_argument('--tables', type=str, help='List of tables to sync')
-    parser.add_argument('--dir', type=str, default='*', help='Path to directory with config')
+    parser.add_argument(
+        '--dir', type=str, default='*', help='Path to directory with config'
+    )
     parser.add_argument('--name', type=str, default='*', help='Name of the project')
     parser.add_argument('--secret', type=str, help='Path to vault password file')
     parser.add_argument('--string', type=str)
-    parser.add_argument('--version',
-                        action='version',
-                        help='Displays the installed versions',
-                        version='PipelineWise {} - Command Line Interface'.format(__version__))
+    parser.add_argument(
+        '--version',
+        action='version',
+        help='Displays the installed versions',
+        version='PipelineWise {} - Command Line Interface'.format(__version__),
+    )
     parser.add_argument('--log', type=str, default='*', help='File to log into')
-    parser.add_argument('--extra_log',
-                        default=False,
-                        required=False,
-                        help='Copy singer and fastsync logging into PipelineWise logger',
-                        action='store_true')
-    parser.add_argument('--debug',
-                        default=False,
-                        required=False,
-                        help='Forces the debug mode with logging on stdout and log level debug',
-                        action='store_true')
-    parser.add_argument('--profiler', '-p',
-                        default=False,
-                        required=False,
-                        help='Enables code profiling mode using Python builtin profiler cProfile. '
-                             'The stats will be dumped into a folder in .pipelinewise/profiling',
-                        action='store_true'
-                        )
+    parser.add_argument(
+        '--extra_log',
+        default=False,
+        required=False,
+        help='Copy singer and fastsync logging into PipelineWise logger',
+        action='store_true',
+    )
+    parser.add_argument(
+        '--debug',
+        default=False,
+        required=False,
+        help='Forces the debug mode with logging on stdout and log level debug',
+        action='store_true',
+    )
+    parser.add_argument(
+        '--profiler',
+        '-p',
+        default=False,
+        required=False,
+        help='Enables code profiling mode using Python builtin profiler cProfile. '
+        'The stats will be dumped into a folder in .pipelinewise/profiling',
+        action='store_true',
+    )
 
     args = parser.parse_args()
 
@@ -201,7 +218,9 @@ def main():
     # import_config : this is for backward compatibility; use 'import' instead from CLI
     if args.command == 'import' or args.command == 'import_config':
         if args.dir == '*':
-            print('You must specify a directory path with config YAML files using the argument --dir')
+            print(
+                'You must specify a directory path with config YAML files using the argument --dir'
+            )
             sys.exit(1)
 
         # Every command argument is mapped to a python function with the same name, but 'import' is a
@@ -209,12 +228,16 @@ def main():
         args.command = 'import_project'
 
     if args.command == 'validate' and args.dir == '*':
-        print('You must specify a directory path with config YAML files using the argument --dir')
+        print(
+            'You must specify a directory path with config YAML files using the argument --dir'
+        )
         sys.exit(1)
 
     if args.command == 'encrypt_string':
         if not args.secret:
-            print('You must specify a path to a file with vault secret using the argument --secret')
+            print(
+                'You must specify a path to a file with vault secret using the argument --secret'
+            )
             sys.exit(1)
         if not args.string:
             print('You must specify a string to encrypt using the argument --string')
@@ -229,7 +252,9 @@ def main():
     try:
         getattr(ppw_instance, args.command)()
     finally:
-        __disable_profiler(profiler, profiling_dir, f'pipelinewise_{args.command}', logger)
+        __disable_profiler(
+            profiler, profiling_dir, f'pipelinewise_{args.command}', logger
+        )
 
 
 if __name__ == '__main__':
diff --git a/pipelinewise/cli/alert_handlers/base_alert_handler.py b/pipelinewise/cli/alert_handlers/base_alert_handler.py
index 91bdf6033..2e54d7a83 100644
--- a/pipelinewise/cli/alert_handlers/base_alert_handler.py
+++ b/pipelinewise/cli/alert_handlers/base_alert_handler.py
@@ -9,6 +9,7 @@ class BaseAlertHandler(ABC):
     """
     Abstract base class for alert handlers
     """
+
     LOG = 'log'
     INFO = 'info'
     WARNING = 'warning'
diff --git a/pipelinewise/cli/alert_handlers/errors.py b/pipelinewise/cli/alert_handlers/errors.py
index fdc1841d9..15c2d7a11 100644
--- a/pipelinewise/cli/alert_handlers/errors.py
+++ b/pipelinewise/cli/alert_handlers/errors.py
@@ -7,6 +7,7 @@ class NotImplementedAlertHandlerException(Exception):
     """
     Exception to raise when attempted to use a not implemented alert handler class
     """
+
     def __init__(self, *args, **kwargs):
         super().__init__(self, *args, **kwargs)
 
@@ -15,6 +16,7 @@ class NotConfiguredAlertHandlerException(Exception):
     """
     Exception to raise when attempted to use a not configured alert handler
     """
+
     def __init__(self, *args, **kwargs):
         super().__init__(self, *args, **kwargs)
 
@@ -23,5 +25,6 @@ class InvalidAlertHandlerException(Exception):
     """
     Exception to raise when alert handler not configured correctly
     """
+
     def __init__(self, *args, **kwargs):
         super().__init__(self, *args, **kwargs)
diff --git a/pipelinewise/cli/alert_handlers/slack_alert_handler.py b/pipelinewise/cli/alert_handlers/slack_alert_handler.py
index f15be1eec..81c604a26 100644
--- a/pipelinewise/cli/alert_handlers/slack_alert_handler.py
+++ b/pipelinewise/cli/alert_handlers/slack_alert_handler.py
@@ -11,7 +11,7 @@
     BaseAlertHandler.LOG: '36C5F0',
     BaseAlertHandler.INFO: 'good',
     BaseAlertHandler.WARNING: 'warning',
-    BaseAlertHandler.ERROR: 'danger'
+    BaseAlertHandler.ERROR: 'danger',
 }
 
 
@@ -20,6 +20,7 @@ class SlackAlertHandler(BaseAlertHandler):
     """
     Slack Alert Handler class
     """
+
     def __init__(self, config: dict) -> None:
         if config is not None:
             if 'token' not in config:
@@ -27,7 +28,9 @@ def __init__(self, config: dict) -> None:
             self.token = config['token']
 
             if 'channel' not in config:
-                raise InvalidAlertHandlerException('Missing channel in Slack connection')
+                raise InvalidAlertHandlerException(
+                    'Missing channel in Slack connection'
+                )
             self.channel = config['channel']
 
         else:
@@ -35,7 +38,9 @@ def __init__(self, config: dict) -> None:
 
         self.client = WebClient(self.token)
 
-    def send(self, message: str, level: str = BaseAlertHandler.ERROR, exc: Exception = None) -> None:
+    def send(
+        self, message: str, level: str = BaseAlertHandler.ERROR, exc: Exception = None
+    ) -> None:
         """
         Send alert
 
@@ -47,9 +52,15 @@ def send(self, message: str, level: str = BaseAlertHandler.ERROR, exc: Exception
         Returns:
             Initialised alert handler object
         """
-        self.client.chat_postMessage(channel=self.channel,
-                                     text=f'```{exc}```' if exc else None,
-                                     attachments=[{
-                                         'color': ALERT_LEVEL_SLACK_COLORS.get(level, BaseAlertHandler.ERROR),
-                                         'title': message
-                                     }])
+        self.client.chat_postMessage(
+            channel=self.channel,
+            text=f'```{exc}```' if exc else None,
+            attachments=[
+                {
+                    'color': ALERT_LEVEL_SLACK_COLORS.get(
+                        level, BaseAlertHandler.ERROR
+                    ),
+                    'title': message,
+                }
+            ],
+        )
diff --git a/pipelinewise/cli/alert_handlers/victorops_alert_handler.py b/pipelinewise/cli/alert_handlers/victorops_alert_handler.py
index 0713bff60..92ba39fee 100644
--- a/pipelinewise/cli/alert_handlers/victorops_alert_handler.py
+++ b/pipelinewise/cli/alert_handlers/victorops_alert_handler.py
@@ -12,7 +12,7 @@
     BaseAlertHandler.LOG: 'INFO',
     BaseAlertHandler.INFO: 'INFO',
     BaseAlertHandler.WARNING: 'WARNING',
-    BaseAlertHandler.ERROR: 'CRITICAL'
+    BaseAlertHandler.ERROR: 'CRITICAL',
 }
 
 
@@ -21,20 +21,27 @@ class VictoropsAlertHandler(BaseAlertHandler):
     """
     VictorOps Alert Handler class
     """
+
     def __init__(self, config: dict) -> None:
         if config is not None:
             if 'base_url' not in config:
-                raise InvalidAlertHandlerException('Missing REST Endpoint URL in VictorOps connection')
+                raise InvalidAlertHandlerException(
+                    'Missing REST Endpoint URL in VictorOps connection'
+                )
             self.base_url = config['base_url']
 
             if 'routing_key' not in config:
-                raise InvalidAlertHandlerException('Missing routing key in VictorOps connection')
+                raise InvalidAlertHandlerException(
+                    'Missing routing key in VictorOps connection'
+                )
             self.routing_key = config['routing_key']
 
         else:
             raise InvalidAlertHandlerException('No valid VictorOps config supplied.')
 
-    def send(self, message: str, level: str = BaseAlertHandler.ERROR, exc: Exception = None) -> None:
+    def send(
+        self, message: str, level: str = BaseAlertHandler.ERROR, exc: Exception = None
+    ) -> None:
         """
         Send alert
 
@@ -49,13 +56,22 @@ def send(self, message: str, level: str = BaseAlertHandler.ERROR, exc: Exception
         # Send alert to VictorOps REST Endpoint as a HTTP post request
         response = requests.post(
             f'{self.base_url}/{self.routing_key}',
-            data=json.dumps({
-                'message_type': ALERT_LEVEL_MESSAGE_TYPES.get(level, BaseAlertHandler.ERROR),
-                'entity_display_name': message,
-                'state_message': exc}),
-            headers={'Content-Type': 'application/json'})
+            data=json.dumps(
+                {
+                    'message_type': ALERT_LEVEL_MESSAGE_TYPES.get(
+                        level, BaseAlertHandler.ERROR
+                    ),
+                    'entity_display_name': message,
+                    'state_message': exc,
+                }
+            ),
+            headers={'Content-Type': 'application/json'},
+        )
 
         # Success victorops message should return 200
         if response.status_code != 200:
-            raise ValueError('Request to victorops returned an error {}. {}'.format(response.status_code,
-                                                                                    response.text))
+            raise ValueError(
+                'Request to victorops returned an error {}. {}'.format(
+                    response.status_code, response.text
+                )
+            )
diff --git a/pipelinewise/cli/alert_sender.py b/pipelinewise/cli/alert_sender.py
index 88a4ce7e9..847f3f232 100644
--- a/pipelinewise/cli/alert_sender.py
+++ b/pipelinewise/cli/alert_sender.py
@@ -23,7 +23,7 @@
 # Every alert handler class needs to implement the BaseAlertHandler base class
 ALERT_HANDLER_TYPES_TO_CLASS = {
     'slack': SlackAlertHandler,
-    'victorops': VictoropsAlertHandler
+    'victorops': VictoropsAlertHandler,
 }
 
 
@@ -45,7 +45,9 @@ def __init__(self, alert_handlers: Dict = None) -> None:
 
         # Raise an exception if alert_handlers is not a dictionary
         if not isinstance(self.alert_handlers, dict):
-            raise InvalidAlertHandlerException('alert_handlers needs to be a dictionary')
+            raise InvalidAlertHandlerException(
+                'alert_handlers needs to be a dictionary'
+            )
 
     @staticmethod
     def __init_handler_class(alert_handler: AlertHandler) -> BaseAlertHandler:
@@ -63,8 +65,9 @@ def __init_handler_class(alert_handler: AlertHandler) -> BaseAlertHandler:
             alert_handler_class = ALERT_HANDLER_TYPES_TO_CLASS[alert_handler.type]
             handler = alert_handler_class(alert_handler.config)
         except KeyError as key_error:
-            raise NotImplementedAlertHandlerException(f'Alert handler type not implemented: {alert_handler.type}') \
-                from key_error
+            raise NotImplementedAlertHandlerException(
+                f'Alert handler type not implemented: {alert_handler.type}'
+            ) from key_error
 
         return handler
 
@@ -80,16 +83,22 @@ def __get_alert_handler(self, alert_handler_type: str) -> AlertHandler:
         """
         if alert_handler_type in self.alert_handlers:
             alert_handler_config = self.alert_handlers[alert_handler_type]
-            alert_handler = AlertHandler(type=alert_handler_type, config=alert_handler_config)
+            alert_handler = AlertHandler(
+                type=alert_handler_type, config=alert_handler_config
+            )
             return alert_handler
 
-        raise NotConfiguredAlertHandlerException(f'Alert handler type not configured: {alert_handler_type}')
-
-    def send_to_handler(self,
-                        alert_handler_type: str,
-                        message: str,
-                        level: str = BaseAlertHandler.ERROR,
-                        exc: Exception = None) -> bool:
+        raise NotConfiguredAlertHandlerException(
+            f'Alert handler type not configured: {alert_handler_type}'
+        )
+
+    def send_to_handler(
+        self,
+        alert_handler_type: str,
+        message: str,
+        level: str = BaseAlertHandler.ERROR,
+        exc: Exception = None,
+    ) -> bool:
         """
         Sends an alert message to a specific alert handler type
 
@@ -112,10 +121,9 @@ def send_to_handler(self,
         # Alert sent successfully
         return True
 
-    def send_to_all_handlers(self,
-                             message: str,
-                             level: str = BaseAlertHandler.ERROR,
-                             exc: Exception = None) -> dict:
+    def send_to_all_handlers(
+        self, message: str, level: str = BaseAlertHandler.ERROR, exc: Exception = None
+    ) -> dict:
         """
         Get all the configured alert handlers and send alert
         message to all of them
@@ -128,5 +136,8 @@ def send_to_all_handlers(self,
         Returns:
             Dictionary with number of successfully sent alerts
         """
-        sents = [self.send_to_handler(handler_type, message, level, exc) for handler_type in self.alert_handlers]
+        sents = [
+            self.send_to_handler(handler_type, message, level, exc)
+            for handler_type in self.alert_handlers
+        ]
         return {'sent': len(sents)}
diff --git a/pipelinewise/cli/commands.py b/pipelinewise/cli/commands.py
index e824987cb..9537eb7c0 100644
--- a/pipelinewise/cli/commands.py
+++ b/pipelinewise/cli/commands.py
@@ -20,15 +20,20 @@
 STATUS_FAILED = 'failed'
 STATUS_SUCCESS = 'success'
 
-TapParams = namedtuple('TapParams', ['id', 'type', 'bin', 'python_bin', 'config', 'properties', 'state'])
+TapParams = namedtuple(
+    'TapParams', ['id', 'type', 'bin', 'python_bin', 'config', 'properties', 'state']
+)
 TargetParams = namedtuple('TargetParams', ['id', 'type', 'bin', 'python_bin', 'config'])
-TransformParams = namedtuple('TransformParams', ['bin', 'python_bin', 'config', 'tap_id', 'target_id'])
+TransformParams = namedtuple(
+    'TransformParams', ['bin', 'python_bin', 'config', 'tap_id', 'target_id']
+)
 
 
 class RunCommandException(Exception):
     """
     Custom exception to raise when run command fails
     """
+
     def __init__(self, *args, **kwargs):
         Exception.__init__(self, *args, **kwargs)
 
@@ -56,9 +61,9 @@ def exists_and_executable(bin_path: str) -> bool:
     return True
 
 
-def build_tap_command(tap: TapParams,
-                      profiling_mode: bool = False,
-                      profiling_dir: str = None) -> str:
+def build_tap_command(
+    tap: TapParams, profiling_mode: bool = False, profiling_dir: str = None
+) -> str:
     """
     Builds a command that starts a singer tap connector with the
     required command line arguments
@@ -73,7 +78,9 @@ def build_tap_command(tap: TapParams,
     # Following the singer spec the catalog JSON file needs to be passed by the --catalog argument
     # However some tap (i.e. tap-mysql and tap-postgres) requires it as --properties
     # This is probably for historical reasons and need to clarify on Singer slack channels
-    catalog_argument = utils.get_tap_property_by_tap_type(tap.type, 'tap_catalog_argument')
+    catalog_argument = utils.get_tap_property_by_tap_type(
+        tap.type, 'tap_catalog_argument'
+    )
 
     state_arg = ''
     if tap.state and os.path.isfile(tap.state):
@@ -88,9 +95,9 @@ def build_tap_command(tap: TapParams,
     return tap_command
 
 
-def build_target_command(target: TargetParams,
-                         profiling_mode: bool = False,
-                         profiling_dir: str = None) -> str:
+def build_target_command(
+    target: TargetParams, profiling_mode: bool = False, profiling_dir: str = None
+) -> str:
     """
     Builds a command that starts a singer target connector with the
     required command line arguments
@@ -107,14 +114,16 @@ def build_target_command(target: TargetParams,
 
     if profiling_mode:
         dump_file = os.path.join(profiling_dir, f'target_{target.id}.pstat')
-        target_command = f'{target.python_bin} -m cProfile -o {dump_file} {target_command}'
+        target_command = (
+            f'{target.python_bin} -m cProfile -o {dump_file} {target_command}'
+        )
 
     return target_command
 
 
-def build_transformation_command(transform: TransformParams,
-                                 profiling_mode: bool = False,
-                                 profiling_dir: str = None) -> str:
+def build_transformation_command(
+    transform: TransformParams, profiling_mode: bool = False, profiling_dir: str = None
+) -> str:
     """
     Builds a command that starts a singer transformation connector
     with the required command line arguments
@@ -138,16 +147,21 @@ def build_transformation_command(transform: TransformParams,
             if profiling_mode:
                 dump_file = os.path.join(
                     profiling_dir,
-                    f'transformation_{transform.tap_id}_{transform.target_id}.pstat')
+                    f'transformation_{transform.tap_id}_{transform.target_id}.pstat',
+                )
 
-                trans_command = f'{transform.python_bin} -m cProfile -o {dump_file} {trans_command}'
+                trans_command = (
+                    f'{transform.python_bin} -m cProfile -o {dump_file} {trans_command}'
+                )
 
     return trans_command
 
 
-def build_stream_buffer_command(buffer_size: int = 0,
-                                log_file: str = None,
-                                stream_buffer_bin: str = DEFAULT_STREAM_BUFFER_BIN) -> str:
+def build_stream_buffer_command(
+    buffer_size: int = 0,
+    log_file: str = None,
+    stream_buffer_bin: str = DEFAULT_STREAM_BUFFER_BIN,
+) -> str:
     """
     Builds a command that buffers data between tap and target
     connectors to stream data asynchronously. Buffering streams
@@ -188,11 +202,15 @@ def build_stream_buffer_command(buffer_size: int = 0,
     return buffer_command
 
 
-def build_singer_command(tap: TapParams, target: TargetParams, transform: TransformParams,
-                         stream_buffer_size: int = 0,
-                         stream_buffer_log_file: str = None,
-                         profiling_mode: bool = False,
-                         profiling_dir: str = None) -> str:
+def build_singer_command(
+    tap: TapParams,
+    target: TargetParams,
+    transform: TransformParams,
+    stream_buffer_size: int = 0,
+    stream_buffer_log_file: str = None,
+    profiling_mode: bool = False,
+    profiling_dir: str = None,
+) -> str:
     """
     Builds a command that starts a full singer command with tap,
     target and optional transformation connectors. The connectors are
@@ -211,46 +229,49 @@ def build_singer_command(tap: TapParams, target: TargetParams, transform: Transf
     Returns:
         string of command line executable
     """
-    tap_command = build_tap_command(tap,
-                                    profiling_mode,
-                                    profiling_dir)
+    tap_command = build_tap_command(tap, profiling_mode, profiling_dir)
 
     LOGGER.debug('Tap command: %s', tap_command)
 
-    target_command = build_target_command(target,
-                                          profiling_mode,
-                                          profiling_dir)
+    target_command = build_target_command(target, profiling_mode, profiling_dir)
 
     LOGGER.debug('Target command: %s', target_command)
 
-    transformation_command = build_transformation_command(transform,
-                                                          profiling_mode,
-                                                          profiling_dir)
+    transformation_command = build_transformation_command(
+        transform, profiling_mode, profiling_dir
+    )
     LOGGER.debug('Transformation command: %s', transformation_command)
 
-    stream_buffer_command = build_stream_buffer_command(stream_buffer_size,
-                                                        stream_buffer_log_file)
+    stream_buffer_command = build_stream_buffer_command(
+        stream_buffer_size, stream_buffer_log_file
+    )
 
     LOGGER.debug('Buffer command: %s', stream_buffer_command)
 
     # Generate the final piped command with all the required components
-    sub_commands = [tap_command, transformation_command, stream_buffer_command, target_command]
+    sub_commands = [
+        tap_command,
+        transformation_command,
+        stream_buffer_command,
+        target_command,
+    ]
     command = ' | '.join(list(filter(None, sub_commands)))
 
     return command
 
 
 # pylint: disable=too-many-arguments
-def build_fastsync_command(tap: TapParams,
-                           target: TargetParams,
-                           transform: TransformParams,
-                           venv_dir: str,
-                           temp_dir: str,
-                           tables: str = None,
-                           profiling_mode: bool = False,
-                           profiling_dir: str = None,
-                           drop_pg_slot: bool = False
-                           ) -> str:
+def build_fastsync_command(
+    tap: TapParams,
+    target: TargetParams,
+    transform: TransformParams,
+    venv_dir: str,
+    temp_dir: str,
+    tables: str = None,
+    profiling_mode: bool = False,
+    profiling_dir: str = None,
+    drop_pg_slot: bool = False,
+) -> str:
     """
     Builds a command that starts fastsync from a given tap to a
     given target with optional transformations.
@@ -273,16 +294,25 @@ def build_fastsync_command(tap: TapParams,
     fastsync_bin = utils.get_fastsync_bin(venv_dir, tap.type, target.type)
     ppw_python_bin = utils.get_pipelinewise_python_bin(venv_dir)
 
-    command_args = ' '.join(list(filter(None, [
-        f'--tap {tap.config}',
-        f'--properties {tap.properties}',
-        f'--state {tap.state}',
-        f'--target {target.config}',
-        f'--temp_dir {temp_dir}',
-        f'--transform {transform.config}' if transform.config and os.path.isfile(transform.config) else '',
-        f'--tables {tables}' if tables else '',
-        '--drop_pg_slot' if drop_pg_slot else '',
-    ])))
+    command_args = ' '.join(
+        list(
+            filter(
+                None,
+                [
+                    f'--tap {tap.config}',
+                    f'--properties {tap.properties}',
+                    f'--state {tap.state}',
+                    f'--target {target.config}',
+                    f'--temp_dir {temp_dir}',
+                    f'--transform {transform.config}'
+                    if transform.config and os.path.isfile(transform.config)
+                    else '',
+                    f'--tables {tables}' if tables else '',
+                    '--drop_pg_slot' if drop_pg_slot else '',
+                ],
+            )
+        )
+    )
 
     command = f'{fastsync_bin} {command_args}'
 
@@ -362,9 +392,11 @@ def run_command(command: str, log_file: str = None, line_callback: callable = No
 
             # Raise run command exception
             errors = ''.join(utils.find_errors_in_log_file(log_file_failed))
-            raise RunCommandException(f'Command failed. Return code: {proc_rc}\n'
-                                      f'Error(s) found:\n{errors}\n'
-                                      f'Full log: {log_file_failed}')
+            raise RunCommandException(
+                f'Command failed. Return code: {proc_rc}\n'
+                f'Error(s) found:\n{errors}\n'
+                f'Full log: {log_file_failed}'
+            )
 
         # Add success status to the log file name
         os.rename(log_file_running, log_file_success)
diff --git a/pipelinewise/cli/config.py b/pipelinewise/cli/config.py
index bd1f3ea0e..0530aa5fb 100644
--- a/pipelinewise/cli/config.py
+++ b/pipelinewise/cli/config.py
@@ -60,7 +60,9 @@ def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None):
         # Load every target yaml into targets dictionary
         for yaml_file in target_yamls:
             config.logger.info('LOADING TARGET: %s', yaml_file)
-            target_data = utils.load_yaml(os.path.join(yaml_dir, yaml_file), vault_secret)
+            target_data = utils.load_yaml(
+                os.path.join(yaml_dir, yaml_file), vault_secret
+            )
             utils.validate(instance=target_data, schema=target_schema)
 
             # Add generated extra keys that not available in the YAML
@@ -72,7 +74,9 @@ def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None):
                 config.logger.error('Duplicate target found "%s"', target_id)
                 sys.exit(1)
 
-            target_data['files'] = config.get_connector_files(config.get_target_dir(target_id))
+            target_data['files'] = config.get_connector_files(
+                config.get_target_dir(target_id)
+            )
             target_data['taps'] = []
 
             # Add target to list
@@ -94,19 +98,26 @@ def from_yamls(cls, config_dir, yaml_dir='.', vault_secret=None):
 
             target_id = tap_data['target']
             if target_id not in targets:
-                config.logger.error("Can't find the target with the ID \"%s\" but it's referenced in %s", target_id,
-                                    yaml_file)
+                config.logger.error(
+                    "Can't find the target with the ID \"%s\" but it's referenced in %s",
+                    target_id,
+                    yaml_file,
+                )
                 sys.exit(1)
 
             # Add generated extra keys that not available in the YAML
-            tap_data['files'] = config.get_connector_files(config.get_tap_dir(target_id, tap_id))
+            tap_data['files'] = config.get_connector_files(
+                config.get_tap_dir(target_id, tap_id)
+            )
 
             # Add tap to list
             taps[tap_id] = tap_data
 
         # Link taps to targets
         for target_key, target in targets.items():
-            target['taps'] = [tap for tap in taps.values() if tap['target'] == target_key]
+            target['taps'] = [
+                tap for tap in taps.values() if tap['target'] == target_key
+            ]
 
         # Final structure is ready
         config.targets = targets
@@ -138,12 +149,14 @@ def get_connector_files(connector_dir: str) -> Dict:
         """
         return {
             'config': os.path.join(connector_dir, 'config.json'),
-            'inheritable_config': os.path.join(connector_dir, 'inheritable_config.json'),
+            'inheritable_config': os.path.join(
+                connector_dir, 'inheritable_config.json'
+            ),
             'properties': os.path.join(connector_dir, 'properties.json'),
             'state': os.path.join(connector_dir, 'state.json'),
             'transformation': os.path.join(connector_dir, 'transformation.json'),
             'selection': os.path.join(connector_dir, 'selection.json'),
-            'pidfile': os.path.join(connector_dir, 'pipelinewise.pid')
+            'pidfile': os.path.join(connector_dir, 'pipelinewise.pid'),
         }
 
     def save(self):
@@ -163,7 +176,9 @@ def save(self):
 
             # Save every tap JSON files
             for tap in target['taps']:
-                extra_config_keys = utils.get_tap_extra_config_keys(tap, self.get_temp_dir())
+                extra_config_keys = utils.get_tap_extra_config_keys(
+                    tap, self.get_temp_dir()
+                )
                 self.save_tap_jsons(target, tap, extra_config_keys)
 
     def save_main_config_json(self):
@@ -181,23 +196,27 @@ def save_main_config_json(self):
             target = target_tuple[1]
             taps = []
             for tap in target.get('taps'):
-                taps.append({
-                    'id': tap.get('id'),
-                    'name': tap.get('name'),
-                    'type': tap.get('type'),
-                    'owner': tap.get('owner'),
-                    'stream_buffer_size': tap.get('stream_buffer_size'),
-                    'send_alert': tap.get('send_alert', True),
-                    'enabled': True
-                })
-
-            targets.append({
-                'id': target.get('id'),
-                'name': target.get('name'),
-                'status': 'ready',
-                'type': target.get('type'),
-                'taps': taps
-            })
+                taps.append(
+                    {
+                        'id': tap.get('id'),
+                        'name': tap.get('name'),
+                        'type': tap.get('type'),
+                        'owner': tap.get('owner'),
+                        'stream_buffer_size': tap.get('stream_buffer_size'),
+                        'send_alert': tap.get('send_alert', True),
+                        'enabled': True,
+                    }
+                )
+
+            targets.append(
+                {
+                    'id': target.get('id'),
+                    'name': target.get('name'),
+                    'status': 'ready',
+                    'type': target.get('type'),
+                    'taps': taps,
+                }
+            )
         main_config = {**self.global_config, **{'targets': targets}}
 
         # Create config dir if not exists
@@ -266,14 +285,23 @@ def save_tap_jsons(self, target, tap, extra_config_keys=None):
             schema_name = schema.get('source_schema')
             for table in schema.get('tables', []):
                 table_name = table.get('table_name')
-                replication_method = table.get('replication_method', utils.get_tap_default_replication_method(tap))
-                selection.append(utils.delete_empty_keys({
-                    'tap_stream_id': utils.get_tap_stream_id(tap, tap_dbname, schema_name, table_name),
-                    'replication_method': replication_method,
-
-                    # Add replication_key only if replication_method is INCREMENTAL
-                    'replication_key': table.get('replication_key') if replication_method == 'INCREMENTAL' else None
-                }))
+                replication_method = table.get(
+                    'replication_method', utils.get_tap_default_replication_method(tap)
+                )
+                selection.append(
+                    utils.delete_empty_keys(
+                        {
+                            'tap_stream_id': utils.get_tap_stream_id(
+                                tap, tap_dbname, schema_name, table_name
+                            ),
+                            'replication_method': replication_method,
+                            # Add replication_key only if replication_method is INCREMENTAL
+                            'replication_key': table.get('replication_key')
+                            if replication_method == 'INCREMENTAL'
+                            else None,
+                        }
+                    )
+                )
         tap_selection = {'selection': selection}
 
         # Generate tap transformation
@@ -283,29 +311,33 @@ def save_tap_jsons(self, target, tap, extra_config_keys=None):
             for table in schema.get('tables', []):
                 table_name = table.get('table_name')
                 for trans in table.get('transformations', []):
-                    transformations.append({
-                        'tap_stream_name': utils.get_tap_stream_name(tap, tap_dbname, schema_name, table_name),
-                        'field_id': trans['column'],
-                        # Make column name safe by wrapping it in quotes, it's useful when a field_id is a reserved word
-                        # to be used by target snowflake in fastsync
-                        'safe_field_id': safe_column_name(trans['column']),
-                        'type': trans['type'],
-                        'when': trans.get('when')
-                    })
-        tap_transformation = {
-            'transformations': transformations
-        }
+                    transformations.append(
+                        {
+                            'tap_stream_name': utils.get_tap_stream_name(
+                                tap, tap_dbname, schema_name, table_name
+                            ),
+                            'field_id': trans['column'],
+                            # Make column name safe by wrapping it in quotes, it's useful when a field_id is a reserved
+                            # word to be used by target snowflake in fastsync
+                            'safe_field_id': safe_column_name(trans['column']),
+                            'type': trans['type'],
+                            'when': trans.get('when'),
+                        }
+                    )
+        tap_transformation = {'transformations': transformations}
 
         # Generate stream to schema mapping
         schema_mapping = {}
         for schema in tap.get('schemas', []):
             source_schema = schema.get('source_schema')
             target_schema = schema.get('target_schema')
-            target_schema_select_perms = schema.get('target_schema_select_permissions', [])
+            target_schema_select_perms = schema.get(
+                'target_schema_select_permissions', []
+            )
 
             schema_mapping[source_schema] = {
                 'target_schema': target_schema,
-                'target_schema_select_permissions': target_schema_select_perms
+                'target_schema_select_permissions': target_schema_select_perms,
             }
 
             # Schema mapping can include list of indices to create. Some target components
@@ -322,53 +354,65 @@ def save_tap_jsons(self, target, tap, extra_config_keys=None):
                 schema_mapping[source_schema]['indices'] = indices
 
         # Generate tap inheritable_config dict
-        tap_inheritable_config = utils.delete_empty_keys({
-            'temp_dir': self.get_temp_dir(),
-            'tap_id': tap.get('id'),
-            'query_tag': json.dumps({
-                'ppw_component': tap.get('type'),
+        tap_inheritable_config = utils.delete_empty_keys(
+            {
+                'temp_dir': self.get_temp_dir(),
                 'tap_id': tap.get('id'),
-                'database': '{{database}}',
-                'schema': '{{schema}}',
-                'table': '{{table}}'
-            }),
-            'batch_size_rows': tap.get('batch_size_rows', 20000),
-            'batch_wait_limit_seconds': tap.get('batch_wait_limit_seconds', None),
-            'parallelism': tap.get('parallelism', 0),
-            'parallelism_max': tap.get('parallelism_max', 4),
-            'hard_delete': tap.get('hard_delete', True),
-            'flush_all_streams': tap.get('flush_all_streams', False),
-            'primary_key_required': tap.get('primary_key_required', True),
-            'default_target_schema': tap.get('default_target_schema'),
-            'default_target_schema_select_permissions': tap.get('default_target_schema_select_permissions'),
-            'schema_mapping': schema_mapping,
-
-            # data_flattening_max_level
-            # -------------------------
-            #
-            # 'data_flattening_max_level' is an optional parameter in some target connectors that specifies
-            # how to load nested object into destination.
-            #
-            # We can load the original object represented as JSON or string (data flattening off) or we can
-            # flatten the schema and data by creating columns automatically. When 'data_flattening_max_level'
-            # is set to 0 then flattening functionality is turned off.
-            #
-            #  The value can be set in mutliple place and evaluated in the following order:
-            # ------------
-            #   1: First we try to find it in the tap YAML
-            #   2: Second we try to get the tap type specific default value
-            #   3: Otherwise we set flattening level to 0 (disabled)
-            'data_flattening_max_level': tap.get('data_flattening_max_level',
-                                                 utils.get_tap_property(tap, 'default_data_flattening_max_level') or 0),
-            'validate_records': tap.get('validate_records', False),
-            'add_metadata_columns': tap.get('add_metadata_columns', False),
-            'split_large_files': tap.get('split_large_files', False),
-            'split_file_chunk_size_mb': tap.get('split_file_chunk_size_mb', 1000),
-            'split_file_max_chunks': tap.get('split_file_max_chunks', 20),
-            'archive_load_files': tap.get('archive_load_files', False),
-            'archive_load_files_s3_bucket': tap.get('archive_load_files_s3_bucket', None),
-            'archive_load_files_s3_prefix': tap.get('archive_load_files_s3_prefix', None)
-        })
+                'query_tag': json.dumps(
+                    {
+                        'ppw_component': tap.get('type'),
+                        'tap_id': tap.get('id'),
+                        'database': '{{database}}',
+                        'schema': '{{schema}}',
+                        'table': '{{table}}',
+                    }
+                ),
+                'batch_size_rows': tap.get('batch_size_rows', 20000),
+                'batch_wait_limit_seconds': tap.get('batch_wait_limit_seconds', None),
+                'parallelism': tap.get('parallelism', 0),
+                'parallelism_max': tap.get('parallelism_max', 4),
+                'hard_delete': tap.get('hard_delete', True),
+                'flush_all_streams': tap.get('flush_all_streams', False),
+                'primary_key_required': tap.get('primary_key_required', True),
+                'default_target_schema': tap.get('default_target_schema'),
+                'default_target_schema_select_permissions': tap.get(
+                    'default_target_schema_select_permissions'
+                ),
+                'schema_mapping': schema_mapping,
+                # data_flattening_max_level
+                # -------------------------
+                #
+                # 'data_flattening_max_level' is an optional parameter in some target connectors that specifies
+                # how to load nested object into destination.
+                #
+                # We can load the original object represented as JSON or string (data flattening off) or we can
+                # flatten the schema and data by creating columns automatically. When 'data_flattening_max_level'
+                # is set to 0 then flattening functionality is turned off.
+                #
+                #  The value can be set in mutliple place and evaluated in the following order:
+                # ------------
+                #   1: First we try to find it in the tap YAML
+                #   2: Second we try to get the tap type specific default value
+                #   3: Otherwise we set flattening level to 0 (disabled)
+                'data_flattening_max_level': tap.get(
+                    'data_flattening_max_level',
+                    utils.get_tap_property(tap, 'default_data_flattening_max_level')
+                    or 0,
+                ),
+                'validate_records': tap.get('validate_records', False),
+                'add_metadata_columns': tap.get('add_metadata_columns', False),
+                'split_large_files': tap.get('split_large_files', False),
+                'split_file_chunk_size_mb': tap.get('split_file_chunk_size_mb', 1000),
+                'split_file_max_chunks': tap.get('split_file_max_chunks', 20),
+                'archive_load_files': tap.get('archive_load_files', False),
+                'archive_load_files_s3_bucket': tap.get(
+                    'archive_load_files_s3_bucket', None
+                ),
+                'archive_load_files_s3_prefix': tap.get(
+                    'archive_load_files_s3_prefix', None
+                ),
+            }
+        )
 
         # Save the generated JSON files
         utils.save_json(tap_config, tap_config_path)
diff --git a/pipelinewise/cli/constants.py b/pipelinewise/cli/constants.py
index b676c81ec..6593d31d2 100644
--- a/pipelinewise/cli/constants.py
+++ b/pipelinewise/cli/constants.py
@@ -6,6 +6,7 @@ class ConnectorType(enum.Enum):
     Enums for various Singer connector type names
     Todo: add more
     """
+
     TAP_MYSQL = 'tap-mysql'
     TAP_POSTGRES = 'tap-postgres'
     TAP_MONGODB = 'tap-mongodb'
diff --git a/pipelinewise/cli/errors.py b/pipelinewise/cli/errors.py
index 88a93d623..e9a0f4872 100644
--- a/pipelinewise/cli/errors.py
+++ b/pipelinewise/cli/errors.py
@@ -10,6 +10,8 @@ class StreamBufferTooLargeException(Exception):
     """Raised if stream buffer size is greater than the max allowed size"""
 
     def __init__(self, buffer_size, max_buffer_size):
-        msg = f'{buffer_size}M buffer size is too large. The maximum allowed stream buffer size is ' \
-              f'{max_buffer_size}M'
+        msg = (
+            f'{buffer_size}M buffer size is too large. The maximum allowed stream buffer size is '
+            f'{max_buffer_size}M'
+        )
         super().__init__(msg)
diff --git a/pipelinewise/cli/pipelinewise.py b/pipelinewise/cli/pipelinewise.py
index 6ffdcbbfb..0216efa7a 100644
--- a/pipelinewise/cli/pipelinewise.py
+++ b/pipelinewise/cli/pipelinewise.py
@@ -30,24 +30,24 @@
         ConnectorType.TARGET_SNOWFLAKE,
         ConnectorType.TARGET_REDSHIFT,
         ConnectorType.TARGET_POSTGRES,
-        ConnectorType.TARGET_BIGQUERY
+        ConnectorType.TARGET_BIGQUERY,
     },
     ConnectorType.TAP_POSTGRES: {
         ConnectorType.TARGET_SNOWFLAKE,
         ConnectorType.TARGET_REDSHIFT,
         ConnectorType.TARGET_POSTGRES,
-        ConnectorType.TARGET_BIGQUERY
+        ConnectorType.TARGET_BIGQUERY,
     },
     ConnectorType.TAP_S3_CSV: {
         ConnectorType.TARGET_SNOWFLAKE,
         ConnectorType.TARGET_REDSHIFT,
         ConnectorType.TARGET_POSTGRES,
-        ConnectorType.TARGET_BIGQUERY
+        ConnectorType.TARGET_BIGQUERY,
     },
     ConnectorType.TAP_MONGODB: {
         ConnectorType.TARGET_SNOWFLAKE,
         ConnectorType.TARGET_POSTGRES,
-        ConnectorType.TARGET_BIGQUERY
+        ConnectorType.TARGET_BIGQUERY,
     },
 }
 
@@ -73,7 +73,9 @@ def __init__(self, args, config_dir, venv_dir, profiling_dir=None):
         self.config_dir = config_dir
         self.venv_dir = venv_dir
         self.extra_log = args.extra_log
-        self.pipelinewise_bin = os.path.join(self.venv_dir, 'cli', 'bin', 'pipelinewise')
+        self.pipelinewise_bin = os.path.join(
+            self.venv_dir, 'cli', 'bin', 'pipelinewise'
+        )
         self.config_path = os.path.join(self.config_dir, 'config.json')
         self.load_config()
         self.alert_sender = AlertSender(self.config.get('alert_handlers'))
@@ -88,18 +90,21 @@ def __init__(self, args, config_dir, venv_dir, profiling_dir=None):
             self.target_bin = self.get_connector_bin(self.target['type'])
             self.target_python_bin = self.get_connector_python_bin(self.target['type'])
 
-        self.transform_field_bin = self.get_connector_bin(self.TRANSFORM_FIELD_CONNECTOR_NAME)
-        self.transform_field_python_bin = self.get_connector_python_bin(self.TRANSFORM_FIELD_CONNECTOR_NAME)
+        self.transform_field_bin = self.get_connector_bin(
+            self.TRANSFORM_FIELD_CONNECTOR_NAME
+        )
+        self.transform_field_python_bin = self.get_connector_python_bin(
+            self.TRANSFORM_FIELD_CONNECTOR_NAME
+        )
         self.tap_run_log_file = None
 
         # Catch SIGINT and SIGTERM to exit gracefully
         for sig in [signal.SIGINT, signal.SIGTERM]:
             signal.signal(sig, self._exit_gracefully)
 
-    def send_alert(self,
-                   message: str,
-                   level: str = BaseAlertHandler.ERROR,
-                   exc: Exception = None) -> dict:
+    def send_alert(
+        self, message: str, level: str = BaseAlertHandler.ERROR, exc: Exception = None
+    ) -> dict:
         """
         Send alert messages to every alert handler if sender is not disabled for the tap
 
@@ -115,7 +120,9 @@ def send_alert(self,
 
         send_alert = self.tap.get('send_alert', True)
         if send_alert:
-            stats = self.alert_sender.send_to_all_handlers(message=message, level=level, exc=exc)
+            stats = self.alert_sender.send_to_all_handlers(
+                message=message, level=level, exc=exc
+            )
 
         return stats
 
@@ -132,23 +139,27 @@ def create_consumable_target_config(self, target_config, tap_inheritable_config)
             dict_a.update(dict_b)
 
             # Save the new dict as JSON into a temp file
-            tempfile_path = utils.create_temp_file(dir=self.get_temp_dir(),
-                                                   prefix='target_config_',
-                                                   suffix='.json')[1]
+            tempfile_path = utils.create_temp_file(
+                dir=self.get_temp_dir(), prefix='target_config_', suffix='.json'
+            )[1]
             utils.save_json(dict_a, tempfile_path)
 
             return tempfile_path
         except Exception as exc:
-            raise Exception(f'Cannot merge JSON files {dict_a} {dict_b} - {exc}') from exc
+            raise Exception(
+                f'Cannot merge JSON files {dict_a} {dict_b} - {exc}'
+            ) from exc
 
     # pylint: disable=too-many-statements,too-many-branches,too-many-nested-blocks,too-many-locals,too-many-arguments
-    def create_filtered_tap_properties(self,
-                                       target_type: ConnectorType,
-                                       tap_type: ConnectorType,
-                                       tap_properties: str,
-                                       tap_state: str,
-                                       filters: Dict[str, Any],
-                                       create_fallback=False):
+    def create_filtered_tap_properties(
+        self,
+        target_type: ConnectorType,
+        tap_type: ConnectorType,
+        tap_properties: str,
+        tap_state: str,
+        filters: Dict[str, Any],
+        create_fallback=False,
+    ):
         """
         Create a filtered version of tap properties file based on specific filter conditions.
 
@@ -183,7 +194,9 @@ def create_filtered_tap_properties(self,
             fallback_properties = copy.deepcopy(properties) if create_fallback else {}
 
             # Foreach stream (table) in the original properties
-            for stream_idx, stream in enumerate(properties.get('streams', tap_properties)):
+            for stream_idx, stream in enumerate(
+                properties.get('streams', tap_properties)
+            ):
                 initial_sync_required = False
 
                 # Collect required properties from the properties file
@@ -202,7 +215,11 @@ def create_filtered_tap_properties(self,
                 # Can we make sure that the stream has the right metadata?
                 # To be safe, check if no right metadata has been found, then throw an exception.
                 if not table_meta:
-                    self.logger.error('Stream %s has no metadata with no breadcrumbs: %s.', tap_stream_id, metadata)
+                    self.logger.error(
+                        'Stream %s has no metadata with no breadcrumbs: %s.',
+                        tap_stream_id,
+                        metadata,
+                    )
                     raise Exception(f'Missing metadata in stream {tap_stream_id}')
 
                 selected = table_meta.get('selected', False)
@@ -211,7 +228,9 @@ def create_filtered_tap_properties(self,
                 # Detect if initial sync is required. Look into the state file, get the bookmark
                 # for the current stream (table) and if valid bookmark doesn't exist then
                 # initial sync is required
-                bookmarks = state.get('bookmarks', {}) if isinstance(state, dict) else {}
+                bookmarks = (
+                    state.get('bookmarks', {}) if isinstance(state, dict) else {}
+                )
 
                 new_stream = False
 
@@ -222,7 +241,9 @@ def create_filtered_tap_properties(self,
                 else:
                     stream_bookmark = bookmarks[tap_stream_id]
 
-                    if self._is_initial_sync_required(replication_method, stream_bookmark):
+                    if self._is_initial_sync_required(
+                        replication_method, stream_bookmark
+                    ):
                         initial_sync_required = True
 
                 # Compare actual values to the filter conditions.
@@ -230,21 +251,39 @@ def create_filtered_tap_properties(self,
                 # Set the "selected" key to False if the actual values don't meet the filter criteria
                 # pylint: disable=too-many-boolean-expressions
                 if (
-                        (f_selected is None or selected == f_selected) and
-                        (f_tap_target_pairs is None or target_type in f_tap_target_pairs.get(tap_type, set())) and
-                        (f_replication_method is None or replication_method in f_replication_method) and
-                        (f_initial_sync_required is None or initial_sync_required == f_initial_sync_required)
+                    (f_selected is None or selected == f_selected)
+                    and (
+                        f_tap_target_pairs is None
+                        or target_type in f_tap_target_pairs.get(tap_type, set())
+                    )
+                    and (
+                        f_replication_method is None
+                        or replication_method in f_replication_method
+                    )
+                    and (
+                        f_initial_sync_required is None
+                        or initial_sync_required == f_initial_sync_required
+                    )
                 ):
-                    self.logger.debug("""Filter condition(s) matched:
+                    self.logger.debug(
+                        """Filter condition(s) matched:
                         Table              : %s
                         Tap Stream ID      : %s
                         Selected           : %s
                         Replication Method : %s
                         Init Sync Required : %s
-                    """, table_name, tap_stream_id, selected, replication_method, initial_sync_required)
+                    """,
+                        table_name,
+                        tap_stream_id,
+                        selected,
+                        replication_method,
+                        initial_sync_required,
+                    )
 
                     # Filter condition matched: mark table as selected to sync
-                    properties['streams'][stream_idx]['metadata'][meta_idx]['metadata']['selected'] = True
+                    properties['streams'][stream_idx]['metadata'][meta_idx]['metadata'][
+                        'selected'
+                    ] = True
                     filtered_tap_stream_ids.append(tap_stream_id)
 
                     # Filter condition matched:
@@ -252,47 +291,58 @@ def create_filtered_tap_properties(self,
                     # the fallback properties as well if the table is selected in the original properties.
                     # Otherwise, mark it as not selected
                     if create_fallback:
-                        if new_stream and replication_method in [self.INCREMENTAL, self.LOG_BASED]:
-                            fallback_properties['streams'][stream_idx]['metadata'][meta_idx]['metadata'][
-                                'selected'] = True
+                        if new_stream and replication_method in [
+                            self.INCREMENTAL,
+                            self.LOG_BASED,
+                        ]:
+                            fallback_properties['streams'][stream_idx]['metadata'][
+                                meta_idx
+                            ]['metadata']['selected'] = True
                             if selected:
                                 fallback_filtered_stream_ids.append(tap_stream_id)
                         else:
-                            fallback_properties['streams'][stream_idx]['metadata'][meta_idx]['metadata'][
-                                'selected'] = False
+                            fallback_properties['streams'][stream_idx]['metadata'][
+                                meta_idx
+                            ]['metadata']['selected'] = False
                 else:
                     # Filter condition didn't match: mark table as not selected to sync
-                    properties['streams'][stream_idx]['metadata'][meta_idx]['metadata']['selected'] = False
+                    properties['streams'][stream_idx]['metadata'][meta_idx]['metadata'][
+                        'selected'
+                    ] = False
 
                     # Filter condition didn't match: mark table as selected to sync in the fallback properties
                     # Fallback only if the table is selected in the original properties
                     if create_fallback and selected is True:
-                        fallback_properties['streams'][stream_idx]['metadata'][meta_idx]['metadata']['selected'] = True
+                        fallback_properties['streams'][stream_idx]['metadata'][
+                            meta_idx
+                        ]['metadata']['selected'] = True
                         fallback_filtered_stream_ids.append(tap_stream_id)
 
             # Save the generated properties file(s) and return
             # Fallback required: Save filtered and fallback properties JSON
             if create_fallback:
                 # Save to files: filtered and fallback properties
-                temp_properties_path = utils.create_temp_file(dir=self.get_temp_dir(),
-                                                              prefix='properties_',
-                                                              suffix='.json')[1]
+                temp_properties_path = utils.create_temp_file(
+                    dir=self.get_temp_dir(), prefix='properties_', suffix='.json'
+                )[1]
                 utils.save_json(properties, temp_properties_path)
 
-                temp_fallback_properties_path = utils.create_temp_file(dir=self.get_temp_dir(),
-                                                                       prefix='properties_',
-                                                                       suffix='.json')[1]
+                temp_fallback_properties_path = utils.create_temp_file(
+                    dir=self.get_temp_dir(), prefix='properties_', suffix='.json'
+                )[1]
                 utils.save_json(fallback_properties, temp_fallback_properties_path)
 
-                return (temp_properties_path,
-                        filtered_tap_stream_ids,
-                        temp_fallback_properties_path,
-                        fallback_filtered_stream_ids)
+                return (
+                    temp_properties_path,
+                    filtered_tap_stream_ids,
+                    temp_fallback_properties_path,
+                    fallback_filtered_stream_ids,
+                )
 
             # Fallback not required: Save only the filtered properties JSON
-            temp_properties_path = utils.create_temp_file(dir=self.get_temp_dir(),
-                                                          prefix='properties_',
-                                                          suffix='.json')[1]
+            temp_properties_path = utils.create_temp_file(
+                dir=self.get_temp_dir(), prefix='properties_', suffix='.json'
+            )[1]
             utils.save_json(properties, temp_properties_path)
 
             return temp_properties_path, filtered_tap_stream_ids
@@ -424,8 +474,9 @@ def get_tap(self, target_id: str, tap_id: str) -> Dict:
 
         return tap
 
+    # TODO: This method is too complex! make its complexity less than 15!
     # pylint: disable=too-many-branches,too-many-statements,too-many-nested-blocks,too-many-locals
-    def merge_schemas(self, old_schema, new_schema):
+    def merge_schemas(self, old_schema, new_schema):  # noqa: C901
         """
         Merge two schemas
         """
@@ -439,7 +490,14 @@ def merge_schemas(self, old_schema, new_schema):
             for new_stream_idx, new_stream in enumerate(new_streams):
                 new_tap_stream_id = new_stream['tap_stream_id']
 
-                old_stream = next((item for item in old_streams if item['tap_stream_id'] == new_tap_stream_id), None)
+                old_stream = next(
+                    (
+                        item
+                        for item in old_streams
+                        if item['tap_stream_id'] == new_tap_stream_id
+                    ),
+                    None,
+                )
 
                 # Is this a new stream?
                 if not old_stream:
@@ -451,54 +509,87 @@ def merge_schemas(self, old_schema, new_schema):
                     new_stream_table_mdata_idx = 0
                     old_stream_table_mdata_idx = 0
                     try:
-                        new_stream_table_mdata_idx = \
-                            [i for i, md in enumerate(new_stream['metadata']) if md['breadcrumb'] == []][0]
-                        old_stream_table_mdata_idx = \
-                            [i for i, md in enumerate(old_stream['metadata']) if md['breadcrumb'] == []][0]
+                        new_stream_table_mdata_idx = [
+                            i
+                            for i, md in enumerate(new_stream['metadata'])
+                            if md['breadcrumb'] == []
+                        ][0]
+                        old_stream_table_mdata_idx = [
+                            i
+                            for i, md in enumerate(old_stream['metadata'])
+                            if md['breadcrumb'] == []
+                        ][0]
                     except Exception:
                         pass
 
                     # Copy is-new flag from the old stream
                     try:
-                        new_schema['streams'][new_stream_idx]['is-new'] = old_stream['is-new']
+                        new_schema['streams'][new_stream_idx]['is-new'] = old_stream[
+                            'is-new'
+                        ]
                     except Exception:
                         pass
 
                     # Copy selected from the old stream
                     try:
-                        new_schema['streams'][new_stream_idx]['metadata'][new_stream_table_mdata_idx]['metadata'][
-                            'selected'] = old_stream['metadata'][old_stream_table_mdata_idx]['metadata']['selected']
+                        new_schema['streams'][new_stream_idx]['metadata'][
+                            new_stream_table_mdata_idx
+                        ]['metadata']['selected'] = old_stream['metadata'][
+                            old_stream_table_mdata_idx
+                        ][
+                            'metadata'
+                        ][
+                            'selected'
+                        ]
                     except Exception:
                         pass
 
                     # Copy replication method from the old stream
                     try:
-                        new_schema['streams'][new_stream_idx]['metadata'] \
-                            [new_stream_table_mdata_idx]['metadata']['replication-method'] = \
-                            old_stream['metadata'][old_stream_table_mdata_idx]['metadata']['replication-method']
+                        new_schema['streams'][new_stream_idx]['metadata'][
+                            new_stream_table_mdata_idx
+                        ]['metadata']['replication-method'] = old_stream['metadata'][
+                            old_stream_table_mdata_idx
+                        ][
+                            'metadata'
+                        ][
+                            'replication-method'
+                        ]
                     except Exception:
                         pass
 
                     # Copy replication key from the old stream
                     try:
-                        new_schema['streams'][new_stream_idx]['metadata'][new_stream_table_mdata_idx] \
-                            ['metadata']['replication-key'] = \
-                            old_stream['metadata'][old_stream_table_mdata_idx]['metadata'][
-                                'replication-key']
+                        new_schema['streams'][new_stream_idx]['metadata'][
+                            new_stream_table_mdata_idx
+                        ]['metadata']['replication-key'] = old_stream['metadata'][
+                            old_stream_table_mdata_idx
+                        ][
+                            'metadata'
+                        ][
+                            'replication-key'
+                        ]
                     except Exception:
                         pass
 
                     # Is this new or modified field?
-                    new_fields = new_schema['streams'][new_stream_idx]['schema']['properties']
+                    new_fields = new_schema['streams'][new_stream_idx]['schema'][
+                        'properties'
+                    ]
                     old_fields = old_stream['schema']['properties']
                     for new_field_key in new_fields:
                         new_field = new_fields[new_field_key]
                         new_field_mdata_idx = -1
 
                         # Find new field metadata index
-                        for i, mdata in enumerate(new_schema['streams'][new_stream_idx]['metadata']):
-                            if len(mdata['breadcrumb']) == 2 and mdata['breadcrumb'][0] == 'properties' and \
-                                    mdata['breadcrumb'][1] == new_field_key:
+                        for i, mdata in enumerate(
+                            new_schema['streams'][new_stream_idx]['metadata']
+                        ):
+                            if (
+                                len(mdata['breadcrumb']) == 2
+                                and mdata['breadcrumb'][0] == 'properties'
+                                and mdata['breadcrumb'][1] == new_field_key
+                            ):
                                 new_field_mdata_idx = i
 
                         # Field exists
@@ -508,13 +599,19 @@ def merge_schemas(self, old_schema, new_schema):
 
                             # Find old field metadata index
                             for i, mdata in enumerate(old_stream['metadata']):
-                                if len(mdata['breadcrumb']) == 2 and mdata['breadcrumb'][0] == 'properties' and \
-                                        mdata['breadcrumb'][1] == new_field_key:
+                                if (
+                                    len(mdata['breadcrumb']) == 2
+                                    and mdata['breadcrumb'][0] == 'properties'
+                                    and mdata['breadcrumb'][1] == new_field_key
+                                ):
                                     old_field_mdata_idx = i
 
-                            new_mdata = new_schema['streams'][new_stream_idx]['metadata'][new_field_mdata_idx][
-                                'metadata']
-                            old_mdata = old_stream['metadata'][old_field_mdata_idx]['metadata']
+                            new_mdata = new_schema['streams'][new_stream_idx][
+                                'metadata'
+                            ][new_field_mdata_idx]['metadata']
+                            old_mdata = old_stream['metadata'][old_field_mdata_idx][
+                                'metadata'
+                            ]
 
                             # Copy is-new flag from the old properties
                             try:
@@ -536,28 +633,43 @@ def merge_schemas(self, old_schema, new_schema):
 
                             # Field exists and type is the same - Do nothing more in the schema
                             if new_field == old_field:
-                                self.logger.debug('Field exists in %s stream with the same type: %s: %s',
-                                                  new_tap_stream_id, new_field_key, new_field)
+                                self.logger.debug(
+                                    'Field exists in %s stream with the same type: %s: %s',
+                                    new_tap_stream_id,
+                                    new_field_key,
+                                    new_field,
+                                )
 
                             # Field exists but types are different - Mark the field as modified in the metadata
                             else:
-                                self.logger.debug('Field exists in %s stream but types are different: %s: %s}',
-                                                  new_tap_stream_id, new_field_key, new_field)
+                                self.logger.debug(
+                                    'Field exists in %s stream but types are different: %s: %s}',
+                                    new_tap_stream_id,
+                                    new_field_key,
+                                    new_field,
+                                )
                                 try:
-                                    new_schema['streams'][new_stream_idx]['metadata'][new_field_mdata_idx]['metadata'][
-                                        'is-modified'] = True
-                                    new_schema['streams'][new_stream_idx]['metadata'][new_field_mdata_idx]['metadata'][
-                                        'is-new'] = False
+                                    new_schema['streams'][new_stream_idx]['metadata'][
+                                        new_field_mdata_idx
+                                    ]['metadata']['is-modified'] = True
+                                    new_schema['streams'][new_stream_idx]['metadata'][
+                                        new_field_mdata_idx
+                                    ]['metadata']['is-new'] = False
                                 except Exception:
                                     pass
 
                         # New field - Mark the field as new in the metadata
                         else:
-                            self.logger.debug('New field in stream %s: %s: %s', new_tap_stream_id, new_field_key,
-                                              new_field)
+                            self.logger.debug(
+                                'New field in stream %s: %s: %s',
+                                new_tap_stream_id,
+                                new_field_key,
+                                new_field,
+                            )
                             try:
-                                new_schema['streams'][new_stream_idx]['metadata'][new_field_mdata_idx]['metadata'][
-                                    'is-new'] = True
+                                new_schema['streams'][new_stream_idx]['metadata'][
+                                    new_field_mdata_idx
+                                ]['metadata']['is-new'] = True
                             except Exception:
                                 pass
 
@@ -579,28 +691,52 @@ def make_default_selection(self, schema, selection_file):
                 tap_stream_id = stream.get('tap_stream_id')
                 tap_stream_sel = None
                 for sel in selection:
-                    if 'tap_stream_id' in sel and tap_stream_id.lower() == sel['tap_stream_id'].lower():
+                    if (
+                        'tap_stream_id' in sel
+                        and tap_stream_id.lower() == sel['tap_stream_id'].lower()
+                    ):
                         tap_stream_sel = sel
 
                 # Find table specific metadata entries in the old and new streams
                 try:
-                    stream_table_mdata_idx = [i for i, md in enumerate(stream['metadata']) if md['breadcrumb'] == []][0]
+                    stream_table_mdata_idx = [
+                        i
+                        for i, md in enumerate(stream['metadata'])
+                        if md['breadcrumb'] == []
+                    ][0]
                 except Exception as exc:
-                    raise Exception(f'Metadata of stream {tap_stream_id} doesn\'t have an empty breadcrumb') from exc
+                    raise Exception(
+                        f'Metadata of stream {tap_stream_id} doesn\'t have an empty breadcrumb'
+                    ) from exc
 
                 if tap_stream_sel:
-                    self.logger.debug('Mark %s tap_stream_id as selected with properties %s', tap_stream_id,
-                                      tap_stream_sel)
-                    schema['streams'][stream_idx]['metadata'][stream_table_mdata_idx]['metadata']['selected'] = True
+                    self.logger.debug(
+                        'Mark %s tap_stream_id as selected with properties %s',
+                        tap_stream_id,
+                        tap_stream_sel,
+                    )
+                    schema['streams'][stream_idx]['metadata'][stream_table_mdata_idx][
+                        'metadata'
+                    ]['selected'] = True
                     if 'replication_method' in tap_stream_sel:
-                        schema['streams'][stream_idx]['metadata'][stream_table_mdata_idx]['metadata'][
-                            'replication-method'] = tap_stream_sel['replication_method']
+                        schema['streams'][stream_idx]['metadata'][
+                            stream_table_mdata_idx
+                        ]['metadata']['replication-method'] = tap_stream_sel[
+                            'replication_method'
+                        ]
                     if 'replication_key' in tap_stream_sel:
-                        schema['streams'][stream_idx]['metadata'][stream_table_mdata_idx]['metadata'][
-                            'replication-key'] = tap_stream_sel['replication_key']
+                        schema['streams'][stream_idx]['metadata'][
+                            stream_table_mdata_idx
+                        ]['metadata']['replication-key'] = tap_stream_sel[
+                            'replication_key'
+                        ]
                 else:
-                    self.logger.debug('Mark %s tap_stream_id as not selected', tap_stream_id)
-                    schema['streams'][stream_idx]['metadata'][stream_table_mdata_idx]['metadata']['selected'] = False
+                    self.logger.debug(
+                        'Mark %s tap_stream_id as not selected', tap_stream_id
+                    )
+                    schema['streams'][stream_idx]['metadata'][stream_table_mdata_idx][
+                        'metadata'
+                    ]['selected'] = False
 
         return schema
 
@@ -614,7 +750,9 @@ def init(self):
 
         # Create project dir if not exists
         if os.path.exists(project_dir):
-            self.logger.error('Directory exists and cannot create new project: %s', self.args.name)
+            self.logger.error(
+                'Directory exists and cannot create new project: %s', self.args.name
+            )
             sys.exit(1)
         else:
             os.mkdir(project_dir)
@@ -636,7 +774,13 @@ def test_tap_connection(self):
         target_id = self.target['id']
         target_type = self.target['type']
 
-        self.logger.info('Testing %s (%s) tap connection in %s (%s) target', tap_id, tap_type, target_id, target_type)
+        self.logger.info(
+            'Testing %s (%s) tap connection in %s (%s) target',
+            tap_id,
+            tap_type,
+            target_id,
+            target_type,
+        )
 
         # Generate and run the command to run the tap directly
         # We will use the discover option to test connection
@@ -654,15 +798,21 @@ def test_tap_connection(self):
         returncode, new_schema, tap_output = result
 
         if returncode != 0:
-            self.logger.error('Testing tap connection (%s - %s) FAILED', target_id, tap_id)
+            self.logger.error(
+                'Testing tap connection (%s - %s) FAILED', target_id, tap_id
+            )
             sys.exit(1)
 
         # If the connection success then the response needs to be a valid JSON string
         if not utils.is_json(new_schema):
-            self.logger.error('Schema discovered by %s (%s) is not a valid JSON.', tap_id, tap_type)
+            self.logger.error(
+                'Schema discovered by %s (%s) is not a valid JSON.', tap_id, tap_type
+            )
             sys.exit(1)
         else:
-            self.logger.info('Testing tap connection (%s - %s) PASSED', target_id, tap_id)
+            self.logger.info(
+                'Testing tap connection (%s - %s) PASSED', target_id, tap_id
+            )
 
     # pylint: disable=too-many-locals,inconsistent-return-statements
     def discover_tap(self, tap=None, target=None):
@@ -688,7 +838,13 @@ def discover_tap(self, tap=None, target=None):
         target_id = target.get('id')
         target_type = target.get('type')
 
-        self.logger.info('Discovering %s (%s) tap in %s (%s) target...', tap_id, tap_type, target_id, target_type)
+        self.logger.info(
+            'Discovering %s (%s) tap in %s (%s) target...',
+            tap_id,
+            tap_type,
+            target_id,
+            target_type,
+        )
 
         # Generate and run the command to run the tap directly
         command = f'{tap_bin} --config {tap_config_file} --discover'
@@ -724,25 +880,33 @@ def discover_tap(self, tap=None, target=None):
 
         # Make selection from selection.json if exists
         try:
-            schema_with_diff = self.make_default_selection(schema_with_diff, tap_selection_file)
+            schema_with_diff = self.make_default_selection(
+                schema_with_diff, tap_selection_file
+            )
             schema_with_diff = utils.delete_keys_from_dict(
                 self.make_default_selection(schema_with_diff, tap_selection_file),
-
                 # Removing multipleOf json schema validations from properties.json,
                 # that's causing run time issues
-                ['multipleOf'])
+                ['multipleOf'],
+            )
 
         except Exception as exc:
             return f'Cannot load selection JSON at {tap_selection_file}. {str(exc)}'
 
         # Post import checks
-        post_import_errors = self._run_post_import_tap_checks(tap, schema_with_diff, target_id)
+        post_import_errors = self._run_post_import_tap_checks(
+            tap, schema_with_diff, target_id
+        )
         if len(post_import_errors) > 0:
-            return f'Post import tap checks failed in tap {tap_id}: {post_import_errors}'
+            return (
+                f'Post import tap checks failed in tap {tap_id}: {post_import_errors}'
+            )
 
         # Save the new catalog into the tap
         try:
-            self.logger.info('Writing new properties file with changes into %s', tap_properties_file)
+            self.logger.info(
+                'Writing new properties file with changes into %s', tap_properties_file
+            )
             utils.save_json(schema_with_diff, tap_properties_file)
         except Exception as exc:
             return f'Cannot save file. {str(exc)}'
@@ -758,7 +922,7 @@ def detect_tap_status(self, target_id, tap_id):
         status = {
             'currentStatus': 'unknown',
             'lastStatus': 'unknown',
-            'lastTimestamp': None
+            'lastTimestamp': None,
         }
 
         # Tap exists but configuration not completed
@@ -766,7 +930,10 @@ def detect_tap_status(self, target_id, tap_id):
             status['currentStatus'] = 'not-configured'
 
         # Tap exists and has log in running status
-        elif os.path.isdir(log_dir) and len(utils.search_files(log_dir, patterns=['*.log.running'])) > 0:
+        elif (
+            os.path.isdir(log_dir)
+            and len(utils.search_files(log_dir, patterns=['*.log.running'])) > 0
+        ):
             status['currentStatus'] = 'running'
 
         # Configured and not running
@@ -775,7 +942,9 @@ def detect_tap_status(self, target_id, tap_id):
 
         # Get last run instance
         if os.path.isdir(log_dir):
-            log_files = utils.search_files(log_dir, patterns=['*.log.success', '*.log.failed'], sort=True)
+            log_files = utils.search_files(
+                log_dir, patterns=['*.log.success', '*.log.failed'], sort=True
+            )
             if len(log_files) > 0:
                 last_log_file = log_files[0]
                 log_attr = utils.extract_log_attributes(last_log_file)
@@ -798,7 +967,7 @@ def status(self):
             'Enabled',
             'Status',
             'Last Sync',
-            'Last Sync Result'
+            'Last Sync Result',
         ]
         tab_body = []
         pipelines = 0
@@ -806,44 +975,55 @@ def status(self):
             taps = self.get_taps(target['id'])
 
             for tap in taps:
-                tab_body.append([
-                    tap.get('id', '<Unknown>'),
-                    tap.get('type', '<Unknown>'),
-                    target.get('id', '<Unknown>'),
-                    target.get('type', '<Unknown>'),
-                    tap.get('enabled', '<Unknown>'),
-                    tap.get('status', {}).get('currentStatus', '<Unknown>'),
-                    tap.get('status', {}).get('lastTimestamp', '<Unknown>'),
-                    tap.get('status', {}).get('lastStatus', '<Unknown>')
-                ])
+                tab_body.append(
+                    [
+                        tap.get('id', '<Unknown>'),
+                        tap.get('type', '<Unknown>'),
+                        target.get('id', '<Unknown>'),
+                        target.get('type', '<Unknown>'),
+                        tap.get('enabled', '<Unknown>'),
+                        tap.get('status', {}).get('currentStatus', '<Unknown>'),
+                        tap.get('status', {}).get('lastTimestamp', '<Unknown>'),
+                        tap.get('status', {}).get('lastStatus', '<Unknown>'),
+                    ]
+                )
                 pipelines += 1
 
         print(tabulate(tab_body, headers=tab_headers, tablefmt='simple'))
         print(f'{pipelines} pipeline(s)')
 
-    def run_tap_singer(self,
-                       tap: TapParams,
-                       target: TargetParams,
-                       transform: TransformParams,
-                       stream_buffer_size: int = 0) -> str:
+    def run_tap_singer(
+        self,
+        tap: TapParams,
+        target: TargetParams,
+        transform: TransformParams,
+        stream_buffer_size: int = 0,
+    ) -> str:
         """
         Generate and run piped shell command to sync tables using singer taps and targets
         """
         # Build the piped executable command
-        command = commands.build_singer_command(tap=tap,
-                                                target=target,
-                                                transform=transform,
-                                                stream_buffer_size=stream_buffer_size,
-                                                stream_buffer_log_file=self.tap_run_log_file,
-                                                profiling_mode=self.profiling_mode,
-                                                profiling_dir=self.profiling_dir)
+        command = commands.build_singer_command(
+            tap=tap,
+            target=target,
+            transform=transform,
+            stream_buffer_size=stream_buffer_size,
+            stream_buffer_log_file=self.tap_run_log_file,
+            profiling_mode=self.profiling_mode,
+            profiling_dir=self.profiling_dir,
+        )
 
         # Do not run if another instance is already running
         log_dir = os.path.dirname(self.tap_run_log_file)
-        if os.path.isdir(log_dir) and len(utils.search_files(log_dir, patterns=['*.log.running'])) > 0:
+        if (
+            os.path.isdir(log_dir)
+            and len(utils.search_files(log_dir, patterns=['*.log.running'])) > 0
+        ):
             self.logger.info(
                 'Failed to run. Another instance of the same tap is already running. '
-                'Log file detected in running status at %s', log_dir)
+                'Log file detected in running status at %s',
+                log_dir,
+            )
             sys.exit(1)
 
         start = None
@@ -879,7 +1059,9 @@ def update_state_file_with_extra_log(line: str) -> str:
 
         # Run command with update_state_file as a callback to call for every stdout line
         if self.extra_log:
-            commands.run_command(command, self.tap_run_log_file, update_state_file_with_extra_log)
+            commands.run_command(
+                command, self.tap_run_log_file, update_state_file_with_extra_log
+            )
         else:
             commands.run_command(command, self.tap_run_log_file, update_state_file)
 
@@ -888,27 +1070,36 @@ def update_state_file_with_extra_log(line: str) -> str:
             with open(tap.state, 'w', encoding='utf-8') as statefile:
                 statefile.write(state)
 
-    def run_tap_fastsync(self, tap: TapParams, target: TargetParams, transform: TransformParams):
+    def run_tap_fastsync(
+        self, tap: TapParams, target: TargetParams, transform: TransformParams
+    ):
         """
         Generating and running shell command to sync tables using the native fastsync components
         """
         # Build the fastsync executable command
-        command = commands.build_fastsync_command(tap=tap,
-                                                  target=target,
-                                                  transform=transform,
-                                                  venv_dir=self.venv_dir,
-                                                  temp_dir=self.get_temp_dir(),
-                                                  tables=self.args.tables,
-                                                  profiling_mode=self.profiling_mode,
-                                                  profiling_dir=self.profiling_dir,
-                                                  drop_pg_slot=self.drop_pg_slot)
+        command = commands.build_fastsync_command(
+            tap=tap,
+            target=target,
+            transform=transform,
+            venv_dir=self.venv_dir,
+            temp_dir=self.get_temp_dir(),
+            tables=self.args.tables,
+            profiling_mode=self.profiling_mode,
+            profiling_dir=self.profiling_dir,
+            drop_pg_slot=self.drop_pg_slot,
+        )
 
         # Do not run if another instance is already running
         log_dir = os.path.dirname(self.tap_run_log_file)
-        if os.path.isdir(log_dir) and len(utils.search_files(log_dir, patterns=['*.log.running'])) > 0:
+        if (
+            os.path.isdir(log_dir)
+            and len(utils.search_files(log_dir, patterns=['*.log.running'])) > 0
+        ):
             self.logger.info(
                 'Failed to run. Another instance of the same tap is already running. '
-                'Log file detected in running status at %s', log_dir)
+                'Log file detected in running status at %s',
+                log_dir,
+            )
             sys.exit(1)
 
         # Fastsync is running in subprocess.
@@ -920,7 +1111,9 @@ def add_fastsync_output_to_main_logger(line: str) -> str:
 
         if self.extra_log:
             # Run command and copy fastsync output to main logger
-            commands.run_command(command, self.tap_run_log_file, add_fastsync_output_to_main_logger)
+            commands.run_command(
+                command, self.tap_run_log_file, add_fastsync_output_to_main_logger
+            )
         else:
             # Run command
             commands.run_command(command, self.tap_run_log_file)
@@ -949,7 +1142,9 @@ def run_tap(self):
         tap_type = self.tap['type']
         target_id = self.target['id']
         target_type = self.target['type']
-        stream_buffer_size = self.tap.get('stream_buffer_size', commands.DEFAULT_STREAM_BUFFER_SIZE)
+        stream_buffer_size = self.tap.get(
+            'stream_buffer_size', commands.DEFAULT_STREAM_BUFFER_SIZE
+        )
 
         self.logger.info('Running %s tap in %s target', tap_id, target_id)
 
@@ -974,7 +1169,9 @@ def run_tap(self):
 
         # Some target attributes can be passed and override by tap (aka. inheritable config)
         # We merge the two configs and use that with the target
-        cons_target_config = self.create_consumable_target_config(target_config, tap_inheritable_config)
+        cons_target_config = self.create_consumable_target_config(
+            target_config, tap_inheritable_config
+        )
 
         # Output will be redirected into target and tap specific log directory
         log_dir = self.get_tap_log_dir(target_id, tap_id)
@@ -986,7 +1183,7 @@ def run_tap(self):
             tap_properties_fastsync,
             fastsync_stream_ids,
             tap_properties_singer,
-            singer_stream_ids
+            singer_stream_ids,
         ) = self.create_filtered_tap_properties(
             ConnectorType(target_type),
             ConnectorType(tap_type),
@@ -995,61 +1192,84 @@ def run_tap(self):
             {
                 'selected': True,
                 'tap_target_pairs': FASTSYNC_PAIRS,
-                'initial_sync_required': True
+                'initial_sync_required': True,
             },
-            create_fallback=True)
+            create_fallback=True,
+        )
 
         start_time = datetime.now()
         try:
             with pidfile.PIDFile(self.tap['files']['pidfile']):
-                target_params = TargetParams(id=target_id,
-                                             type=target_type,
-                                             bin=self.target_bin,
-                                             python_bin=self.target_python_bin,
-                                             config=cons_target_config)
-
-                transform_params = TransformParams(bin=self.transform_field_bin,
-                                                   python_bin=self.transform_field_python_bin,
-                                                   config=tap_transformation,
-                                                   tap_id=tap_id,
-                                                   target_id=target_id)
+                target_params = TargetParams(
+                    id=target_id,
+                    type=target_type,
+                    bin=self.target_bin,
+                    python_bin=self.target_python_bin,
+                    config=cons_target_config,
+                )
+
+                transform_params = TransformParams(
+                    bin=self.transform_field_bin,
+                    python_bin=self.transform_field_python_bin,
+                    config=tap_transformation,
+                    tap_id=tap_id,
+                    target_id=target_id,
+                )
 
                 # Run fastsync for FULL_TABLE replication method
                 if len(fastsync_stream_ids) > 0:
-                    self.logger.info('Table(s) selected to sync by fastsync: %s', fastsync_stream_ids)
-                    self.tap_run_log_file = os.path.join(log_dir, f'{target_id}-{tap_id}-{current_time}.fastsync.log')
-                    tap_params = TapParams(id=tap_id,
-                                           type=tap_type,
-                                           bin=self.tap_bin,
-                                           python_bin=self.tap_python_bin,
-                                           config=tap_config,
-                                           properties=tap_properties_fastsync,
-                                           state=tap_state)
-
-                    self.run_tap_fastsync(tap=tap_params,
-                                          target=target_params,
-                                          transform=transform_params)
+                    self.logger.info(
+                        'Table(s) selected to sync by fastsync: %s', fastsync_stream_ids
+                    )
+                    self.tap_run_log_file = os.path.join(
+                        log_dir, f'{target_id}-{tap_id}-{current_time}.fastsync.log'
+                    )
+                    tap_params = TapParams(
+                        id=tap_id,
+                        type=tap_type,
+                        bin=self.tap_bin,
+                        python_bin=self.tap_python_bin,
+                        config=tap_config,
+                        properties=tap_properties_fastsync,
+                        state=tap_state,
+                    )
+
+                    self.run_tap_fastsync(
+                        tap=tap_params, target=target_params, transform=transform_params
+                    )
                 else:
-                    self.logger.info('No table available that needs to be sync by fastsync')
+                    self.logger.info(
+                        'No table available that needs to be sync by fastsync'
+                    )
 
                 # Run singer tap for INCREMENTAL and LOG_BASED replication methods
                 if len(singer_stream_ids) > 0:
-                    self.logger.info('Table(s) selected to sync by singer: %s', singer_stream_ids)
-                    self.tap_run_log_file = os.path.join(log_dir, f'{target_id}-{tap_id}-{current_time}.singer.log')
-                    tap_params = TapParams(id=tap_id,
-                                           type=tap_type,
-                                           bin=self.tap_bin,
-                                           python_bin=self.tap_python_bin,
-                                           config=tap_config,
-                                           properties=tap_properties_singer,
-                                           state=tap_state)
-
-                    self.run_tap_singer(tap=tap_params,
-                                        target=target_params,
-                                        transform=transform_params,
-                                        stream_buffer_size=stream_buffer_size)
+                    self.logger.info(
+                        'Table(s) selected to sync by singer: %s', singer_stream_ids
+                    )
+                    self.tap_run_log_file = os.path.join(
+                        log_dir, f'{target_id}-{tap_id}-{current_time}.singer.log'
+                    )
+                    tap_params = TapParams(
+                        id=tap_id,
+                        type=tap_type,
+                        bin=self.tap_bin,
+                        python_bin=self.tap_python_bin,
+                        config=tap_config,
+                        properties=tap_properties_singer,
+                        state=tap_state,
+                    )
+
+                    self.run_tap_singer(
+                        tap=tap_params,
+                        target=target_params,
+                        transform=transform_params,
+                        stream_buffer_size=stream_buffer_size,
+                    )
                 else:
-                    self.logger.info('No table available that needs to be sync by singer')
+                    self.logger.info(
+                        'No table available that needs to be sync by singer'
+                    )
 
         except pidfile.AlreadyRunningError:
             self.logger.error('Another instance of the tap is already running.')
@@ -1102,11 +1322,16 @@ def stop_tap(self):
                 self.logger.info('Sending SIGINT to main pid %s...', parent.pid)
                 parent.send_signal(signal.SIGINT)
         except ProcessLookupError:
-            self.logger.error('Pid %s not found. Is the tap running on this machine? '
-                              'Stopping taps remotely is not supported.', pid)
+            self.logger.error(
+                'Pid %s not found. Is the tap running on this machine? '
+                'Stopping taps remotely is not supported.',
+                pid,
+            )
             sys.exit(1)
         except FileNotFoundError:
-            self.logger.error('No pidfile found at %s. Tap does not seem to be running.', pidfile_path)
+            self.logger.error(
+                'No pidfile found at %s. Tap does not seem to be running.', pidfile_path
+            )
             sys.exit(1)
 
     # pylint: disable=too-many-locals
@@ -1125,7 +1350,13 @@ def sync_tables(self):
         target_type = self.target['type']
         fastsync_bin = utils.get_fastsync_bin(self.venv_dir, tap_type, target_type)
 
-        self.logger.info('Syncing tables from %s (%s) to %s (%s)...', tap_id, tap_type, target_id, target_type)
+        self.logger.info(
+            'Syncing tables from %s (%s) to %s (%s)...',
+            tap_id,
+            tap_type,
+            target_id,
+            target_type,
+        )
 
         # Run only if tap enabled
         if not self.tap.get('enabled', False):
@@ -1135,14 +1366,19 @@ def sync_tables(self):
         # Run only if tap not running
         tap_status = self.detect_tap_status(target_id, tap_id)
         if tap_status['currentStatus'] == 'running':
-            self.logger.info('Tap %s is currently running and cannot sync. Stop the tap and try again.',
-                             self.tap['name'])
+            self.logger.info(
+                'Tap %s is currently running and cannot sync. Stop the tap and try again.',
+                self.tap['name'],
+            )
             sys.exit(1)
 
         # Tap exists but configuration not completed
         if not os.path.isfile(fastsync_bin):
-            self.logger.error('Table sync function is not implemented from %s datasources to %s type of targets',
-                              tap_type, target_type)
+            self.logger.error(
+                'Table sync function is not implemented from %s datasources to %s type of targets',
+                tap_type,
+                target_type,
+            )
             sys.exit(1)
 
         # Generate and run the command to run the tap directly
@@ -1159,7 +1395,9 @@ def sync_tables(self):
 
         # Some target attributes can be passed and override by tap (aka. inheritable config)
         # We merge the two configs and use that with the target
-        cons_target_config = self.create_consumable_target_config(target_config, tap_inheritable_config)
+        cons_target_config = self.create_consumable_target_config(
+            target_config, tap_inheritable_config
+        )
 
         # Output will be redirected into target and tap specific log directory
         log_dir = self.get_tap_log_dir(target_id, tap_id)
@@ -1168,7 +1406,9 @@ def sync_tables(self):
         # sync_tables command always using fastsync
         try:
             with pidfile.PIDFile(self.tap['files']['pidfile']):
-                self.tap_run_log_file = os.path.join(log_dir, f'{target_id}-{tap_id}-{current_time}.fastsync.log')
+                self.tap_run_log_file = os.path.join(
+                    log_dir, f'{target_id}-{tap_id}-{current_time}.fastsync.log'
+                )
 
                 # Create parameters as NamedTuples
                 tap_params = TapParams(
@@ -1178,14 +1418,15 @@ def sync_tables(self):
                     python_bin=self.tap_python_bin,
                     config=tap_config,
                     properties=tap_properties,
-                    state=tap_state)
+                    state=tap_state,
+                )
 
                 target_params = TargetParams(
                     id=target_id,
                     type=target_type,
                     bin=self.target_bin,
                     python_bin=self.target_python_bin,
-                    config=cons_target_config
+                    config=cons_target_config,
                 )
 
                 transform_params = TransformParams(
@@ -1193,12 +1434,12 @@ def sync_tables(self):
                     config=tap_transformation,
                     python_bin=self.transform_field_python_bin,
                     tap_id=tap_id,
-                    target_id=target_id
+                    target_id=target_id,
                 )
 
-                self.run_tap_fastsync(tap=tap_params,
-                                      target=target_params,
-                                      transform=transform_params)
+                self.run_tap_fastsync(
+                    tap=tap_params, target=target_params, transform=transform_params
+                )
 
         except pidfile.AlreadyRunningError:
             self.logger.error('Another instance of the tap is already running.')
@@ -1237,7 +1478,9 @@ def validate(self):
         # Validate target json schemas and that no duplicate IDs exist
         for yaml_file in target_yamls:
             self.logger.info('Started validating %s', yaml_file)
-            loaded_yaml = utils.load_yaml(os.path.join(yaml_dir, yaml_file), vault_secret)
+            loaded_yaml = utils.load_yaml(
+                os.path.join(yaml_dir, yaml_file), vault_secret
+            )
             utils.validate(loaded_yaml, target_schema)
 
             if loaded_yaml['id'] in target_ids:
@@ -1252,7 +1495,9 @@ def validate(self):
         # Validate tap json schemas, check that every tap has valid 'target' and that no duplicate IDs exist
         for yaml_file in tap_yamls:
             self.logger.info('Started validating %s', yaml_file)
-            loaded_yaml = utils.load_yaml(os.path.join(yaml_dir, yaml_file), vault_secret)
+            loaded_yaml = utils.load_yaml(
+                os.path.join(yaml_dir, yaml_file), vault_secret
+            )
             utils.validate(loaded_yaml, tap_schema)
 
             if loaded_yaml['id'] in tap_ids:
@@ -1260,8 +1505,12 @@ def validate(self):
                 sys.exit(1)
 
             if loaded_yaml['target'] not in target_ids:
-                self.logger.error("Can'f find the target with the ID '%s' referenced in '%s'. Available target IDs: %s",
-                                  loaded_yaml['target'], yaml_file, target_ids)
+                self.logger.error(
+                    "Can'f find the target with the ID '%s' referenced in '%s'. Available target IDs: %s",
+                    loaded_yaml['target'],
+                    yaml_file,
+                    target_ids,
+                )
                 sys.exit(1)
 
             tap_ids.add(loaded_yaml['id'])
@@ -1302,11 +1551,17 @@ def import_project(self):
 
             with parallel_backend('threading', n_jobs=-1):
                 # Discover taps in parallel and return the list of exception of the failed ones
-                discover_excs.extend(list(filter(None,
-                                                 Parallel(verbose=100)(delayed(self.discover_tap)(
-                                                     tap=tap,
-                                                     target=target
-                                                 ) for tap in target.get('taps')))))
+                discover_excs.extend(
+                    list(
+                        filter(
+                            None,
+                            Parallel(verbose=100)(
+                                delayed(self.discover_tap)(tap=tap, target=target)
+                                for tap in target.get('taps')
+                            ),
+                        )
+                    )
+                )
 
         # Log summary
         end_time = datetime.now()
@@ -1327,7 +1582,7 @@ def import_project(self):
             total_taps,
             total_taps - len(discover_excs),
             str(discover_excs),
-            end_time - start_time
+            end_time - start_time,
         )
         if len(discover_excs) > 0:
             sys.exit(1)
@@ -1342,7 +1597,9 @@ def encrypt_string(self):
         print(yaml_text)
         print('Encryption successful')
 
-    def _is_initial_sync_required(self, replication_method: str, stream_bookmark: Dict) -> bool:
+    def _is_initial_sync_required(
+        self, replication_method: str, stream_bookmark: Dict
+    ) -> bool:
         """
             Detects if a stream needs initial sync or not.
             Initial sync is required for INCREMENTAL and LOG_BASED tables
@@ -1360,14 +1617,20 @@ def _is_initial_sync_required(self, replication_method: str, stream_bookmark: Di
         :param stream_bookmark: stream state bookmark
         :return: Boolean, True if needs initial sync, False otherwise
         """
-        return replication_method == self.FULL_TABLE \
-               or (replication_method == self.INCREMENTAL and
-                   'replication_key_value' not in stream_bookmark and
-                   'modified_since' not in stream_bookmark) \
-               or (replication_method == self.LOG_BASED and
-                   'lsn' not in stream_bookmark and
-                   'log_pos' not in stream_bookmark and
-                   'token' not in stream_bookmark)
+        return (
+            replication_method == self.FULL_TABLE
+            or (
+                replication_method == self.INCREMENTAL
+                and 'replication_key_value' not in stream_bookmark
+                and 'modified_since' not in stream_bookmark
+            )
+            or (
+                replication_method == self.LOG_BASED
+                and 'lsn' not in stream_bookmark
+                and 'log_pos' not in stream_bookmark
+                and 'token' not in stream_bookmark
+            )
+        )
 
     # pylint: disable=unused-argument
     def _exit_gracefully(self, sig, frame, exit_code=1):
@@ -1414,7 +1677,9 @@ def _print_tap_run_summary(self, status, start_time, end_time):
                     logfile.write(summary)
 
     # pylint: disable=unused-variable
-    def _run_post_import_tap_checks(self, tap: Dict, catalog: Dict, target_id: str) -> List:
+    def _run_post_import_tap_checks(
+        self, tap: Dict, catalog: Dict, target_id: str
+    ) -> List:
         """
         Run post import checks on a tap.
 
@@ -1426,10 +1691,8 @@ def _run_post_import_tap_checks(self, tap: Dict, catalog: Dict, target_id: str)
         errors = []
 
         error = self.__validate_transformations(
-            tap.get('files', {}).get('transformation'),
-            catalog,
-            tap['id'],
-            target_id)
+            tap.get('files', {}).get('transformation'), catalog, tap['id'], target_id
+        )
 
         if error:
             errors.append(error)
@@ -1453,19 +1716,22 @@ def _run_post_import_tap_checks(self, tap: Dict, catalog: Dict, target_id: str)
             primary_key_required = tap.get('primary_key_required', True)
 
             # Check if primary key is set for INCREMENTAL and LOG_BASED replications
-            if (selected and replication_method in [self.INCREMENTAL, self.LOG_BASED] and
-                    len(table_key_properties) == 0 and primary_key_required):
-                errors.append(f'No primary key set for {tap_stream_id} stream ({replication_method})')
+            if (
+                selected
+                and replication_method in [self.INCREMENTAL, self.LOG_BASED]
+                and len(table_key_properties) == 0
+                and primary_key_required
+            ):
+                errors.append(
+                    f'No primary key set for {tap_stream_id} stream ({replication_method})'
+                )
                 break
 
         return errors
 
     def __validate_transformations(
-            self,
-            transformation_file: str,
-            catalog: Dict,
-            tap_id: str,
-            target_id: str) -> Optional[str]:
+        self, transformation_file: str, catalog: Dict, tap_id: str, target_id: str
+    ) -> Optional[str]:
         """
         Run validation of transformation config
         Args:
@@ -1480,9 +1746,9 @@ def __validate_transformations(
 
             # create a temp file with the content being the given catalog object
             # we need this file to execute the validation cli command
-            temp_catalog_file = utils.create_temp_file(dir=self.get_temp_dir(),
-                                                       prefix='properties_',
-                                                       suffix='.json')[1]
+            temp_catalog_file = utils.create_temp_file(
+                dir=self.get_temp_dir(), prefix='properties_', suffix='.json'
+            )[1]
 
             utils.save_json(catalog, temp_catalog_file)
 
@@ -1491,7 +1757,9 @@ def __validate_transformations(
                 """
 
             if self.profiling_mode:
-                dump_file = os.path.join(self.profiling_dir, f'transformation_{tap_id}_{target_id}.pstat')
+                dump_file = os.path.join(
+                    self.profiling_dir, f'transformation_{tap_id}_{target_id}.pstat'
+                )
                 command = f'{self.transform_field_python_bin} -m cProfile -o {dump_file} {command}'
 
             self.logger.debug('Transformation validation command: %s', command)
diff --git a/pipelinewise/cli/tap_properties.py b/pipelinewise/cli/tap_properties.py
index b90c7114e..81cae13b6 100644
--- a/pipelinewise/cli/tap_properties.py
+++ b/pipelinewise/cli/tap_properties.py
@@ -105,31 +105,33 @@ def get_tap_properties(tap=None, temp_dir=None):
             'tap_stream_name_pattern': '{{schema_name}}-{{table_name}}',
             'tap_catalog_argument': '--properties',
             'default_replication_method': 'LOG_BASED',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-postgres': {
             'tap_config_extras': {
                 # Set tap_id to locate the corresponding replication slot
-                'tap_id': tap['id'] if tap else None,
+                'tap_id': tap['id']
+                if tap
+                else None,
             },
             'tap_stream_id_pattern': '{{schema_name}}-{{table_name}}',
             'tap_stream_name_pattern': '{{schema_name}}-{{table_name}}',
             'tap_catalog_argument': '--properties',
             'default_replication_method': 'LOG_BASED',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-zuora': {
             'tap_config_extras': {
                 'username': tap.get('db_conn', {}).get('username') if tap else None,
                 'password': tap.get('db_conn', {}).get('password') if tap else None,
                 'start_date': tap.get('db_conn', {}).get('start_date') if tap else None,
-                'api_type': tap.get('db_conn', {}).get('api_type') if tap else None
+                'api_type': tap.get('db_conn', {}).get('api_type') if tap else None,
             },
             'tap_stream_id_pattern': '{{table_name}}',
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--catalog',
             'default_replication_method': 'FULL_TABLE',
-            'default_data_flattening_max_level': 10
+            'default_data_flattening_max_level': 10,
         },
         'tap-oracle': {
             'tap_config_extras': {},
@@ -137,18 +139,15 @@ def get_tap_properties(tap=None, temp_dir=None):
             'tap_stream_name_pattern': '{{schema_name}}-{{table_name}}',
             'tap_catalog_argument': '--catalog',
             'default_replication_method': 'LOG_BASED',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-kafka': {
-            'tap_config_extras': {
-                'local_store_dir': temp_dir,
-                'encoding': 'utf-8'
-            },
+            'tap_config_extras': {'local_store_dir': temp_dir, 'encoding': 'utf-8'},
             'tap_stream_id_pattern': '{{table_name}}',
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--properties',
             'default_replication_method': 'LOG_BASED',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-zendesk': {
             'tap_config_extras': {},
@@ -156,7 +155,7 @@ def get_tap_properties(tap=None, temp_dir=None):
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--catalog',
             'default_replication_method': 'INCREMENTAL',
-            'default_data_flattening_max_level': 10
+            'default_data_flattening_max_level': 10,
         },
         'tap-adwords': {
             'tap_config_extras': {},
@@ -164,27 +163,23 @@ def get_tap_properties(tap=None, temp_dir=None):
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--catalog',
             'default_replication_method': 'INCREMENTAL',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-jira': {
-            'tap_config_extras': {
-                'user_agent': 'PipelineWise - Tap Jira'
-            },
+            'tap_config_extras': {'user_agent': 'PipelineWise - Tap Jira'},
             'tap_stream_id_pattern': '{{table_name}}',
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--properties',
             'default_replication_method': 'INCREMENTAL',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-s3-csv': {
-            'tap_config_extras': {
-                'tables': generate_tap_s3_csv_to_table_mappings(tap)
-            },
+            'tap_config_extras': {'tables': generate_tap_s3_csv_to_table_mappings(tap)},
             'tap_stream_id_pattern': '{{table_name}}',
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--properties',
             'default_replication_method': 'INCREMENTAL',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-snowflake': {
             'tap_config_extras': {
@@ -195,28 +190,26 @@ def get_tap_properties(tap=None, temp_dir=None):
             'tap_stream_name_pattern': '{{schema_name}}-{{table_name}}',
             'tap_catalog_argument': '--properties',
             'default_replication_method': 'INCREMENTAL',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-salesforce': {
-            'tap_config_extras': {
-                'select_fields_by_default': True
-            },
+            'tap_config_extras': {'select_fields_by_default': True},
             'tap_stream_id_pattern': '{{table_name}}',
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--properties',
             'default_replication_method': 'INCREMENTAL',
-            'default_data_flattening_max_level': 10
+            'default_data_flattening_max_level': 10,
         },
         'tap-mongodb': {
             'tap_config_extras': {
                 'database': tap.get('db_conn', {}).get('dbname') if tap else None,
-                'include_schemas_in_destination_stream_name': 'true'
+                'include_schemas_in_destination_stream_name': 'true',
             },
             'tap_stream_id_pattern': '{{database_name}}-{{table_name}}',
             'tap_stream_name_pattern': '{{database_name}}-{{table_name}}',
             'tap_catalog_argument': '--catalog',
             'default_replication_method': 'LOG_BASED',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-google-analytics': {
             'tap_config_extras': {},
@@ -224,7 +217,7 @@ def get_tap_properties(tap=None, temp_dir=None):
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--catalog',
             'default_replication_method': 'INCREMENTAL',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-github': {
             'tap_config_extras': {
@@ -236,7 +229,7 @@ def get_tap_properties(tap=None, temp_dir=None):
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--properties',
             'default_replication_method': 'LOG_BASED',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-shopify': {
             'tap_config_extras': {},
@@ -244,7 +237,7 @@ def get_tap_properties(tap=None, temp_dir=None):
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--catalog',
             'default_replication_method': 'INCREMENTAL',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-slack': {
             'tap_config_extras': {},
@@ -252,19 +245,23 @@ def get_tap_properties(tap=None, temp_dir=None):
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--catalog',
             'default_replication_method': 'LOG_BASED',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-mixpanel': {
             'tap_config_extras': {
                 'user_agent': 'PipelineWise - Tap Mixpanel',
                 # Do not denest properties by default
-                'denest_properties': tap.get('db_conn', {}).get('denest_properties', 'false') if tap else None
+                'denest_properties': tap.get('db_conn', {}).get(
+                    'denest_properties', 'false'
+                )
+                if tap
+                else None,
             },
             'tap_stream_id_pattern': '{{table_name}}',
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--catalog',
             'default_replication_method': 'LOG_BASED',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         'tap-twilio': {
             'tap_config_extras': {},
@@ -272,7 +269,7 @@ def get_tap_properties(tap=None, temp_dir=None):
             'tap_stream_name_pattern': '{{table_name}}',
             'tap_catalog_argument': '--catalog',
             'default_replication_method': 'INCREMENTAL',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
         # Default values to use as a fallback method
         'DEFAULT': {
@@ -281,6 +278,6 @@ def get_tap_properties(tap=None, temp_dir=None):
             'tap_stream_name_pattern': '{{schema_name}}-{{table_name}}',
             'tap_catalog_argument': '--catalog',
             'default_replication_method': 'LOG_BASED',
-            'default_data_flattening_max_level': 0
+            'default_data_flattening_max_level': 0,
         },
     }
diff --git a/pipelinewise/cli/utils.py b/pipelinewise/cli/utils.py
index 1d3dc4ff0..9015b4dcf 100644
--- a/pipelinewise/cli/utils.py
+++ b/pipelinewise/cli/utils.py
@@ -23,7 +23,7 @@
 from ansible.module_utils._text import to_text
 from ansible.module_utils.common._collections_compat import Mapping
 from ansible.parsing.dataloader import DataLoader
-from ansible.parsing.vault import (VaultLib, get_file_vault_secret, is_encrypted_file)
+from ansible.parsing.vault import VaultLib, get_file_vault_secret, is_encrypted_file
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleVaultEncryptedUnicode
 
@@ -116,7 +116,9 @@ def save_json(data, path):
     try:
         LOGGER.debug('Saving JSON %s', path)
         with open(path, 'w', encoding='utf-8') as jsonfile:
-            return json.dump(data, jsonfile, cls=AnsibleJSONEncoder, indent=4, sort_keys=True)
+            return json.dump(
+                data, jsonfile, cls=AnsibleJSONEncoder, indent=4, sort_keys=True
+            )
     except Exception as exc:
         raise Exception(f'Cannot save JSON {path} {exc}') from exc
 
@@ -156,8 +158,12 @@ def get_tap_target_names(yaml_dir):
         (tap_yamls, target_yamls): tap_yamls is a list of names inside yaml_dir with "tap_*.y(a)ml" pattern.
                                    target_yamls is a list of names inside yaml_dir with "target_*.y(a)ml" pattern.
     """
-    yamls = [f for f in os.listdir(yaml_dir) if os.path.isfile(os.path.join(yaml_dir, f))
-             and (f.endswith('.yml') or f.endswith('.yaml'))]
+    yamls = [
+        f
+        for f in os.listdir(yaml_dir)
+        if os.path.isfile(os.path.join(yaml_dir, f))
+        and (f.endswith('.yml') or f.endswith('.yaml'))
+    ]
     target_yamls = set(filter(lambda y: y.startswith('target_'), yamls))
     tap_yamls = set(filter(lambda y: y.startswith('tap_'), yamls))
 
@@ -196,11 +202,15 @@ def load_yaml(yaml_file, vault_secret=None):
                     try:
                         data = loader.get_single_data()
                     except Exception as exc:
-                        raise Exception(f'Error when loading YAML config at {yaml_file} {exc}') from exc
+                        raise Exception(
+                            f'Error when loading YAML config at {yaml_file} {exc}'
+                        ) from exc
                     finally:
                         loader.dispose()
             except yaml.YAMLError as exc:
-                raise Exception(f'Error when loading YAML config at {yaml_file} {exc}') from exc
+                raise Exception(
+                    f'Error when loading YAML config at {yaml_file} {exc}'
+                ) from exc
     else:
         LOGGER.debug('No file at %s', yaml_file)
 
@@ -267,7 +277,9 @@ def get_sample_file_paths():
     Get list of every available sample files (YAML, etc.) with absolute paths
     """
     samples_dir = os.path.join(os.path.dirname(__file__), 'samples')
-    return search_files(samples_dir, patterns=['config.yml', '*.yml.sample', 'README.md'], abs_path=True)
+    return search_files(
+        samples_dir, patterns=['config.yml', '*.yml.sample', 'README.md'], abs_path=True
+    )
 
 
 def validate(instance, schema):
@@ -299,7 +311,11 @@ def delete_keys_from_dict(dic, keys):
     if isinstance(dic, list):
         return [v for v in (delete_keys_from_dict(v, keys) for v in dic) if v]
     # pylint: disable=C0325  # False positive on tuples
-    return {k: v for k, v in ((k, delete_keys_from_dict(v, keys)) for k, v in dic.items()) if k not in keys}
+    return {
+        k: v
+        for k, v in ((k, delete_keys_from_dict(v, keys)) for k, v in dic.items())
+        if k not in keys
+    }
 
 
 def silentremove(path):
@@ -327,7 +343,9 @@ def search_files(search_dir, patterns=None, sort=False, abs_path=False):
         # Search files and sort if required
         p_files = []
         for pattern in patterns:
-            p_files.extend(filter(os.path.isfile, glob.glob(os.path.join(search_dir, pattern))))
+            p_files.extend(
+                filter(os.path.isfile, glob.glob(os.path.join(search_dir, pattern)))
+            )
         if sort:
             p_files.sort(key=os.path.getmtime, reverse=True)
 
@@ -368,7 +386,7 @@ def extract_log_attributes(log_file):
         'tap_id': tap_id,
         'timestamp': timestamp,
         'sync_engine': sync_engine,
-        'status': status
+        'status': status,
     }
 
 
@@ -412,10 +430,11 @@ def get_tap_stream_id(tap, database_name, schema_name, table_name):
     """
     pattern = get_tap_property(tap, 'tap_stream_id_pattern')
 
-    return pattern \
-        .replace('{{database_name}}', f'{database_name}') \
-        .replace('{{schema_name}}', f'{schema_name}') \
+    return (
+        pattern.replace('{{database_name}}', f'{database_name}')
+        .replace('{{schema_name}}', f'{schema_name}')
         .replace('{{table_name}}', f'{table_name}')
+    )
 
 
 def get_tap_stream_name(tap, database_name, schema_name, table_name):
@@ -428,10 +447,11 @@ def get_tap_stream_name(tap, database_name, schema_name, table_name):
     """
     pattern = get_tap_property(tap, 'tap_stream_name_pattern')
 
-    return pattern \
-        .replace('{{database_name}}', f'{database_name}') \
-        .replace('{{schema_name}}', f'{schema_name}') \
+    return (
+        pattern.replace('{{database_name}}', f'{database_name}')
+        .replace('{{schema_name}}', f'{schema_name}')
         .replace('{{table_name}}', f'{table_name}')
+    )
 
 
 def get_tap_default_replication_method(tap):
@@ -498,7 +518,8 @@ def find_errors_in_log_file(file, max_errors=10, error_pattern=None):
         r'botocore\.exceptions\.|'
         # Generic python exceptions
         r'\.[E|e]xception|'
-        r'\.[E|e]rror')
+        r'\.[E|e]rror'
+    )
 
     # Use known error patterns by default
     if not error_pattern:
@@ -533,5 +554,6 @@ def generate_random_string(length: int = 8) -> str:
     if 0 < length < 8:
         warnings.warn('Length is too small! consider 8 or more characters')
 
-    return ''.join(secrets.choice(string.ascii_uppercase + string.digits)
-                   for _ in range(length))
+    return ''.join(
+        secrets.choice(string.ascii_uppercase + string.digits) for _ in range(length)
+    )
diff --git a/pipelinewise/fastsync/commons/errors.py b/pipelinewise/fastsync/commons/errors.py
index f774367cd..c21e69796 100644
--- a/pipelinewise/fastsync/commons/errors.py
+++ b/pipelinewise/fastsync/commons/errors.py
@@ -1,11 +1,14 @@
 class ExportError(Exception):
     """Raised when export fails"""
 
+
 class TableNotFoundError(Exception):
     """Raised when configured table doesn't exist in source"""
 
+
 class MongoDBInvalidDatetimeError(Exception):
     """Raised when a bson datetime is invalid and cannot be serialized"""
 
+
 class UnsupportedKeyTypeException(Exception):
     """Raised if key type is unsupported"""
diff --git a/pipelinewise/fastsync/commons/split_gzip.py b/pipelinewise/fastsync/commons/split_gzip.py
index 8d7838915..3f49684fd 100644
--- a/pipelinewise/fastsync/commons/split_gzip.py
+++ b/pipelinewise/fastsync/commons/split_gzip.py
@@ -16,7 +16,14 @@
 
 
 # pylint: disable=W0622,R1732
-def open(base_filename, mode='wb', chunk_size_mb=None, max_chunks=None, est_compr_rate=None, compress=True):
+def open(
+    base_filename,
+    mode='wb',
+    chunk_size_mb=None,
+    max_chunks=None,
+    est_compr_rate=None,
+    compress=True,
+):
     """Open a gzip-compressed file in binary or text mode.
 
     Args:
@@ -40,7 +47,9 @@ def open(base_filename, mode='wb', chunk_size_mb=None, max_chunks=None, est_comp
         raise ValueError('Invalid chunk_size_mb: %d' % (chunk_size_mb,))
     if max_chunks is not None and max_chunks < 0:
         raise ValueError('Invalid max_chunks: %d' % (max_chunks,))
-    return SplitGzipFile(base_filename, mode, chunk_size_mb, max_chunks, est_compr_rate, compress)
+    return SplitGzipFile(
+        base_filename, mode, chunk_size_mb, max_chunks, est_compr_rate, compress
+    )
 
 
 # pylint: disable=R0902
@@ -49,22 +58,27 @@ class SplitGzipFile(io.BufferedIOBase):
 
     This class only supports writing files in binary mode.
     """
-    def __init__(self,
-                 base_filename,
-                 mode: str = None,
-                 chunk_size_mb: int = None,
-                 max_chunks: int = None,
-                 est_compr_rate: float = None,
-                 compress=True):
+
+    def __init__(
+        self,
+        base_filename,
+        mode: str = None,
+        chunk_size_mb: int = None,
+        max_chunks: int = None,
+        est_compr_rate: float = None,
+        compress=True,
+    ):
         super().__init__()
 
         self.base_filename = base_filename
         self.mode = mode
         self.chunk_size_mb = chunk_size_mb or DEFAULT_CHUNK_SIZE_MB
         self.max_chunks = max_chunks if max_chunks is not None else DEFAULT_MAX_CHUNKS
-        self.compress= compress
+        self.compress = compress
         if compress:
-            self.est_compr_rate = est_compr_rate if est_compr_rate is not None else EST_COMPR_RATE
+            self.est_compr_rate = (
+                est_compr_rate if est_compr_rate is not None else EST_COMPR_RATE
+            )
         else:
             self.est_compr_rate = 1.0
         self.chunk_seq = 1
@@ -85,7 +99,10 @@ def _gen_chunk_filename(self) -> str:
         if self.max_chunks == 0:
             chunk_filename = self.base_filename
         else:
-            if self.current_chunk_size_mb >= self.chunk_size_mb and self.chunk_seq < self.max_chunks:
+            if (
+                self.current_chunk_size_mb >= self.chunk_size_mb
+                and self.chunk_seq < self.max_chunks
+            ):
                 # Increase the chunk sequence and reset size to zero
                 self.chunk_seq += 1
                 self.current_chunk_size_mb = 0
@@ -109,7 +126,9 @@ def _activate_chunk_file(self):
             if self.compress:
                 self.chunk_file = gzip.open(self.chunk_filename, self.mode)
             else:
-                self.chunk_file = builtins.open(self.chunk_filename, self.mode, encoding='utf-8')
+                self.chunk_file = builtins.open(
+                    self.chunk_filename, self.mode, encoding='utf-8'
+                )
 
     @staticmethod
     def _bytes_to_megabytes(size: int) -> float:
@@ -136,7 +155,9 @@ def write(self, _bytes):
         self._activate_chunk_file()
 
         self.chunk_file.write(_bytes)
-        self.current_chunk_size_mb = SplitGzipFile._bytes_to_megabytes(self.chunk_file.tell() * self.est_compr_rate)
+        self.current_chunk_size_mb = SplitGzipFile._bytes_to_megabytes(
+            self.chunk_file.tell() * self.est_compr_rate
+        )
 
     def close(self):
         """
diff --git a/pipelinewise/fastsync/commons/tap_mongodb.py b/pipelinewise/fastsync/commons/tap_mongodb.py
index f8ec7304a..ee4498a97 100644
--- a/pipelinewise/fastsync/commons/tap_mongodb.py
+++ b/pipelinewise/fastsync/commons/tap_mongodb.py
@@ -18,7 +18,12 @@
 from singer.utils import strftime as singer_strftime
 
 from . import utils, split_gzip
-from .errors import ExportError, TableNotFoundError, MongoDBInvalidDatetimeError, UnsupportedKeyTypeException
+from .errors import (
+    ExportError,
+    TableNotFoundError,
+    MongoDBInvalidDatetimeError,
+    UnsupportedKeyTypeException,
+)
 
 LOGGER = logging.getLogger(__name__)
 DEFAULT_WRITE_BATCH_ROWS = 50000
@@ -33,8 +38,11 @@ def serialize_document(document: Dict) -> Dict:
 
     Returns: Dict
     """
-    return {key: transform_value(val, [key]) for key, val in document.items()
-            if not isinstance(val, (bson.min_key.MinKey, bson.max_key.MaxKey))}
+    return {
+        key: transform_value(val, [key])
+        for key, val in document.items()
+        if not isinstance(val, (bson.min_key.MinKey, bson.max_key.MaxKey))
+    }
 
 
 def class_to_string(key_value: Any, key_type: str) -> str:
@@ -90,15 +98,18 @@ def safe_transform_datetime(value: datetime.datetime, path) -> str:
             # make sense to blow up on invalid Python datetimes (e.g.,
             # year=0). In this case we're formatting it as a string and
             # passing it along down the pipeline.
-            return '{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}.{:06d}Z'.format(value.year,
-                                                                              value.month,
-                                                                              value.day,
-                                                                              value.hour,
-                                                                              value.minute,
-                                                                              value.second,
-                                                                              value.microsecond)
-        raise MongoDBInvalidDatetimeError('Found invalid datetime at [{}]: {}'.format('.'.join(map(str, path)),
-                                                                                      value)) from ex
+            return '{:04d}-{:02d}-{:02d}T{:02d}:{:02d}:{:02d}.{:06d}Z'.format(
+                value.year,
+                value.month,
+                value.day,
+                value.hour,
+                value.minute,
+                value.second,
+                value.microsecond,
+            )
+        raise MongoDBInvalidDatetimeError(
+            'Found invalid datetime at [{}]: {}'.format('.'.join(map(str, path)), value)
+        ) from ex
     return singer_strftime(utc_datetime)
 
 
@@ -113,8 +124,12 @@ def transform_value(value: Any, path) -> Any:
 
     """
     conversion = {
-        list: lambda val, pat: list(map(lambda v: transform_value(v[1], pat + [v[0]]), enumerate(val))),
-        dict: lambda val, pat: {k: transform_value(v, pat + [k]) for k, v in val.items()},
+        list: lambda val, pat: list(
+            map(lambda v: transform_value(v[1], pat + [v[0]]), enumerate(val))
+        ),
+        dict: lambda val, pat: {
+            k: transform_value(v, pat + [k]) for k, v in val.items()
+        },
         uuid.UUID: lambda val, _: class_to_string(val, 'UUID'),
         bson.objectid.ObjectId: lambda val, _: class_to_string(val, 'ObjectId'),
         bson.datetime.datetime: safe_transform_datetime,
@@ -124,8 +139,12 @@ def transform_value(value: Any, path) -> Any:
         datetime.datetime: lambda val, _: class_to_string(val, 'datetime'),
         bson.decimal128.Decimal128: lambda val, _: val.to_decimal(),
         bson.regex.Regex: lambda val, _: dict(pattern=val.pattern, flags=val.flags),
-        bson.code.Code: lambda val, _: dict(value=str(val), scope=str(val.scope)) if val.scope else str(val),
-        bson.dbref.DBRef: lambda val, _: dict(id=str(val.id), collection=val.collection, database=val.database),
+        bson.code.Code: lambda val, _: dict(value=str(val), scope=str(val.scope))
+        if val.scope
+        else str(val),
+        bson.dbref.DBRef: lambda val, _: dict(
+            id=str(val.id), collection=val.collection, database=val.database
+        ),
     }
 
     if isinstance(value, tuple(conversion.keys())):
@@ -147,8 +166,9 @@ def __init__(self, connection_config: Dict, tap_type_to_target_type: Callable):
             tap_type_to_target_type: Function that maps tap types to target ones
         """
         self.connection_config = connection_config
-        self.connection_config['write_batch_rows'] = connection_config.get('write_batch_rows',
-                                                                           DEFAULT_WRITE_BATCH_ROWS)
+        self.connection_config['write_batch_rows'] = connection_config.get(
+            'write_batch_rows', DEFAULT_WRITE_BATCH_ROWS
+        )
 
         self.tap_type_to_target_type = tap_type_to_target_type
         self.database: Optional[Database] = None
@@ -161,17 +181,24 @@ def open_connection(self):
         verify_mode = self.connection_config.get('verify_mode', 'true') == 'true'
         use_ssl = self.connection_config.get('ssl') == 'true'
 
-        connection_params = dict(host=self.connection_config['host'], port=int(self.connection_config['port']),
-                                 username=self.connection_config['user'], password=self.connection_config['password'],
-                                 authSource=self.connection_config['auth_database'], ssl=use_ssl,
-                                 replicaSet=self.connection_config.get('replica_set', None),
-                                 readPreference='secondaryPreferred')
+        connection_params = dict(
+            host=self.connection_config['host'],
+            port=int(self.connection_config['port']),
+            username=self.connection_config['user'],
+            password=self.connection_config['password'],
+            authSource=self.connection_config['auth_database'],
+            ssl=use_ssl,
+            replicaSet=self.connection_config.get('replica_set', None),
+            readPreference='secondaryPreferred',
+        )
 
         # NB: "ssl_cert_reqs" must ONLY be supplied if `SSL` is true.
         if not verify_mode and use_ssl:
             connection_params['ssl_cert_reqs'] = ssl.CERT_NONE
 
-        self.database = MongoClient(**connection_params)[self.connection_config['database']]
+        self.database = MongoClient(**connection_params)[
+            self.connection_config['database']
+        ]
 
     def close_connection(self):
         """
@@ -180,15 +207,16 @@ def close_connection(self):
         self.database.client.close()
 
     # pylint: disable=R0914,R0913
-    def copy_table(self,
-                   table_name: str,
-                   filepath: str,
-                   temp_dir: str,
-                   split_large_files=False,
-                   split_file_chunk_size_mb=1000,
-                   split_file_max_chunks=20,
-                   compress=True
-                   ):
+    def copy_table(
+        self,
+        table_name: str,
+        filepath: str,
+        temp_dir: str,
+        split_large_files=False,
+        split_file_chunk_size_mb=1000,
+        split_file_max_chunks=20,
+        compress=True,
+    ):
         """
         Export data from table to a zipped csv
         Args:
@@ -213,17 +241,23 @@ def copy_table(self,
         exported_rows = 0
 
         try:
-            gzip_splitter = split_gzip.open(filepath,
-                                            mode='wt',
-                                            chunk_size_mb=split_file_chunk_size_mb,
-                                            max_chunks=split_file_max_chunks if split_large_files else 0,
-                                            compress=compress)
-            with gzip.open(export_file_path, 'rb') as export_file, gzip_splitter as gzfile:
-                writer = csv.DictWriter(gzfile,
-                                        fieldnames=[elem[0] for elem in self._get_collection_columns()],
-                                        delimiter=',',
-                                        quotechar='"',
-                                        quoting=csv.QUOTE_MINIMAL)
+            gzip_splitter = split_gzip.open(
+                filepath,
+                mode='wt',
+                chunk_size_mb=split_file_chunk_size_mb,
+                max_chunks=split_file_max_chunks if split_large_files else 0,
+                compress=compress,
+            )
+            with gzip.open(
+                export_file_path, 'rb'
+            ) as export_file, gzip_splitter as gzfile:
+                writer = csv.DictWriter(
+                    gzfile,
+                    fieldnames=[elem[0] for elem in self._get_collection_columns()],
+                    delimiter=',',
+                    quotechar='"',
+                    quoting=csv.QUOTE_MINIMAL,
+                )
 
                 writer.writeheader()
                 rows = []
@@ -233,15 +267,22 @@ def copy_table(self,
                 # bson.decode_file_iter will generate one document at a time from the exported file
                 for document in bson.decode_file_iter(export_file):
                     try:
-                        rows.append({
-                            '_ID': str(document['_id']),
-                            'DOCUMENT': ujson.dumps(serialize_document(document)),
-                            utils.SDC_EXTRACTED_AT: extracted_at,
-                            utils.SDC_BATCHED_AT: datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S.%f'),
-                            utils.SDC_DELETED_AT: None
-                        })
+                        rows.append(
+                            {
+                                '_ID': str(document['_id']),
+                                'DOCUMENT': ujson.dumps(serialize_document(document)),
+                                utils.SDC_EXTRACTED_AT: extracted_at,
+                                utils.SDC_BATCHED_AT: datetime.datetime.utcnow().strftime(
+                                    '%Y-%m-%d %H:%M:%S.%f'
+                                ),
+                                utils.SDC_DELETED_AT: None,
+                            }
+                        )
                     except TypeError:
-                        LOGGER.error('TypeError encountered when processing document ID: %s', document['_id'])
+                        LOGGER.error(
+                            'TypeError encountered when processing document ID: %s',
+                            document['_id'],
+                        )
                         raise
 
                     exported_rows += 1
@@ -251,7 +292,8 @@ def copy_table(self,
                         LOGGER.info(
                             'Exporting batch from %s to %s rows from %s...',
                             (exported_rows - write_batch_rows),
-                            exported_rows, table_name
+                            exported_rows,
+                            table_name,
                         )
 
                         writer.writerows(rows)
@@ -312,14 +354,12 @@ def fetch_current_log_pos(self) -> Dict:
         # token can contain a property '_typeBits' of type bytes which cannot be json
         # serialized when saving the state in the function 'utils.save_state_file'.
         # '_data' is enough to resume LOG_BASED Singer replication after FastSync
-        return {
-            'token': {
-                '_data': token['_data']
-            }
-        }
+        return {'token': {'_data': token['_data']}}
 
     # pylint: disable=invalid-name
-    def fetch_current_incremental_key_pos(self, fully_qualified_table_name: str, replication_key: str):
+    def fetch_current_incremental_key_pos(
+        self, fully_qualified_table_name: str, replication_key: str
+    ):
         """
         No Implemented
         Args:
@@ -337,12 +377,11 @@ def map_column_types_to_target(self):
         mapped_columns = []
 
         for column_name, column_type in self._get_collection_columns():
-            mapped_columns.append(f'{column_name} {self.tap_type_to_target_type(column_type)}')
+            mapped_columns.append(
+                f'{column_name} {self.tap_type_to_target_type(column_type)}'
+            )
 
-        return {
-            'columns': mapped_columns,
-            'primary_key': ['_ID']
-        }
+        return {'columns': mapped_columns, 'primary_key': ['_ID']}
 
     def _export_collection(self, export_dir: str, collection_name) -> str:
         """
@@ -356,10 +395,12 @@ def _export_collection(self, export_dir: str, collection_name) -> str:
         """
         LOGGER.info('Starting export of table "%s"', collection_name)
 
-        url = f'mongodb://{self.connection_config["user"]}:{self.connection_config["password"]}' \
-              f'@{self.connection_config["host"]}:{self.connection_config["port"]}/' \
-              f'{self.connection_config["database"]}?authSource={self.connection_config["auth_database"]}' \
-              f'&readPreference=secondaryPreferred'
+        url = (
+            f'mongodb://{self.connection_config["user"]}:{self.connection_config["password"]}'
+            f'@{self.connection_config["host"]}:{self.connection_config["port"]}/'
+            f'{self.connection_config["database"]}?authSource={self.connection_config["auth_database"]}'
+            f'&readPreference=secondaryPreferred'
+        )
 
         if self.connection_config.get('replica_set', None) is not None:
             url += f'&replicaSet={self.connection_config["replica_set"]}'
@@ -367,14 +408,19 @@ def _export_collection(self, export_dir: str, collection_name) -> str:
         if self.connection_config.get('ssl', None) is not None:
             url += f'&ssl={self.connection_config["ssl"]}'
 
-        return_code = subprocess.call([
-            'mongodump',
-            '--uri', f'"{url}"',
-            '--forceTableScan',
-            '--gzip',
-            '-c', collection_name,
-            '-o', export_dir
-        ])
+        return_code = subprocess.call(
+            [
+                'mongodump',
+                '--uri',
+                f'"{url}"',
+                '--forceTableScan',
+                '--gzip',
+                '-c',
+                collection_name,
+                '-o',
+                export_dir,
+            ]
+        )
 
         LOGGER.debug('Export command return code %s', return_code)
 
@@ -383,5 +429,13 @@ def _export_collection(self, export_dir: str, collection_name) -> str:
 
         # mongodump creates two files "{collection_name}.metadata.json.gz" & "{collection_name}.bson.gz"
         # we are only interested in the latter so we delete the former.
-        os.remove(os.path.join(export_dir, self.connection_config['database'], f'{collection_name}.metadata.json.gz'))
-        return os.path.join(export_dir, self.connection_config['database'], f'{collection_name}.bson.gz')
+        os.remove(
+            os.path.join(
+                export_dir,
+                self.connection_config['database'],
+                f'{collection_name}.metadata.json.gz',
+            )
+        )
+        return os.path.join(
+            export_dir, self.connection_config['database'], f'{collection_name}.bson.gz'
+        )
diff --git a/pipelinewise/fastsync/commons/tap_mysql.py b/pipelinewise/fastsync/commons/tap_mysql.py
index ac9628646..f8c2ab344 100644
--- a/pipelinewise/fastsync/commons/tap_mysql.py
+++ b/pipelinewise/fastsync/commons/tap_mysql.py
@@ -13,10 +13,12 @@
 
 DEFAULT_CHARSET = 'utf8'
 DEFAULT_EXPORT_BATCH_ROWS = 50000
-DEFAULT_SESSION_SQLS = ['SET @@session.time_zone="+0:00"',
-                        'SET @@session.wait_timeout=28800',
-                        'SET @@session.net_read_timeout=3600',
-                        'SET @@session.innodb_lock_wait_timeout=3600']
+DEFAULT_SESSION_SQLS = [
+    'SET @@session.time_zone="+0:00"',
+    'SET @@session.wait_timeout=28800',
+    'SET @@session.net_read_timeout=3600',
+    'SET @@session.innodb_lock_wait_timeout=3600',
+]
 
 
 class FastSyncTapMySql:
@@ -26,10 +28,15 @@ class FastSyncTapMySql:
 
     def __init__(self, connection_config, tap_type_to_target_type, target_quote=None):
         self.connection_config = connection_config
-        self.connection_config['charset'] = connection_config.get('charset', DEFAULT_CHARSET)
-        self.connection_config['export_batch_rows'] = connection_config.get('export_batch_rows',
-                                                                            DEFAULT_EXPORT_BATCH_ROWS)
-        self.connection_config['session_sqls'] = connection_config.get('session_sqls', DEFAULT_SESSION_SQLS)
+        self.connection_config['charset'] = connection_config.get(
+            'charset', DEFAULT_CHARSET
+        )
+        self.connection_config['export_batch_rows'] = connection_config.get(
+            'export_batch_rows', DEFAULT_EXPORT_BATCH_ROWS
+        )
+        self.connection_config['session_sqls'] = connection_config.get(
+            'session_sqls', DEFAULT_SESSION_SQLS
+        )
         self.tap_type_to_target_type = tap_type_to_target_type
         self.target_quote = target_quote
         self.conn = None
@@ -45,24 +52,46 @@ def open_connections(self):
             #
             # If bulk_sync_{host|port|user|password} values are not defined in the config then it's
             # using the normal credentials to connect
-            host=self.connection_config.get('bulk_sync_host', self.connection_config['host']),
-            port=int(self.connection_config.get('bulk_sync_port', self.connection_config['port'])),
-            user=self.connection_config.get('bulk_sync_user', self.connection_config['user']),
-            password=self.connection_config.get('bulk_sync_password', self.connection_config['password']),
+            host=self.connection_config.get(
+                'bulk_sync_host', self.connection_config['host']
+            ),
+            port=int(
+                self.connection_config.get(
+                    'bulk_sync_port', self.connection_config['port']
+                )
+            ),
+            user=self.connection_config.get(
+                'bulk_sync_user', self.connection_config['user']
+            ),
+            password=self.connection_config.get(
+                'bulk_sync_password', self.connection_config['password']
+            ),
             charset=self.connection_config['charset'],
-            cursorclass=pymysql.cursors.DictCursor)
+            cursorclass=pymysql.cursors.DictCursor,
+        )
         self.conn_unbuffered = pymysql.connect(
             # Fastsync is using bulk_sync_{host|port|user|password} values from the config by default
             # to avoid making heavy load on the primary source database when syncing large tables
             #
             # If bulk_sync_{host|port|user|password} values are not defined in the config then it's
             # using the normal credentials to connect
-            host=self.connection_config.get('bulk_sync_host', self.connection_config['host']),
-            port=int(self.connection_config.get('bulk_sync_port', self.connection_config['port'])),
-            user=self.connection_config.get('bulk_sync_user', self.connection_config['user']),
-            password=self.connection_config.get('bulk_sync_password', self.connection_config['password']),
+            host=self.connection_config.get(
+                'bulk_sync_host', self.connection_config['host']
+            ),
+            port=int(
+                self.connection_config.get(
+                    'bulk_sync_port', self.connection_config['port']
+                )
+            ),
+            user=self.connection_config.get(
+                'bulk_sync_user', self.connection_config['user']
+            ),
+            password=self.connection_config.get(
+                'bulk_sync_password', self.connection_config['password']
+            ),
             charset=self.connection_config['charset'],
-            cursorclass=pymysql.cursors.SSCursor)
+            cursorclass=pymysql.cursors.SSCursor,
+        )
 
         # Set session variables by running a list of SQLs which is defined
         # in the optional session_sqls connection parameters
@@ -84,7 +113,9 @@ def run_session_sqls(self):
                     warnings.append(f'Could not set session variable: {sql}')
 
         if warnings:
-            LOGGER.warning('Encountered non-fatal errors when configuring session that could impact performance:')
+            LOGGER.warning(
+                'Encountered non-fatal errors when configuring session that could impact performance:'
+            )
         for warning in warnings:
             LOGGER.warning(warning)
 
@@ -121,16 +152,22 @@ def query(self, query, conn=None, params=None, return_as_cursor=False, n_retry=1
 
                 return []
         except (InterfaceError, OperationalError) as exc:
-            LOGGER.exception('Exception happened during running a query. Number of retries: %s. %s', n_retry, exc)
+            LOGGER.exception(
+                'Exception happened during running a query. Number of retries: %s. %s',
+                n_retry,
+                exc,
+            )
             if n_retry > 0:
                 LOGGER.info('Reopening the connections.')
                 self.close_connections(silent=True)
                 self.open_connections()
                 LOGGER.info('Retrying to run a query.')
-                return self.query(query,
-                                  params=params,
-                                  return_as_cursor=return_as_cursor,
-                                  n_retry=n_retry - 1)
+                return self.query(
+                    query,
+                    params=params,
+                    return_as_cursor=return_as_cursor,
+                    n_retry=n_retry - 1,
+                )
 
             raise exc
 
@@ -147,7 +184,7 @@ def fetch_current_log_pos(self):
         return {
             'log_file': binlog_pos.get('File'),
             'log_pos': binlog_pos.get('Position'),
-            'version': binlog_pos.get('version', 1)
+            'version': binlog_pos.get('version', 1),
         }
 
     # pylint: disable=invalid-name
@@ -155,9 +192,13 @@ def fetch_current_incremental_key_pos(self, table, replication_key):
         """
         Get the actual incremental key position in the table
         """
-        result = self.query('SELECT MAX({}) AS key_value FROM {}'.format(replication_key, table))
+        result = self.query(
+            'SELECT MAX({}) AS key_value FROM {}'.format(replication_key, table)
+        )
         if len(result) == 0:
-            raise Exception('Cannot get replication key value for table: {}'.format(table))
+            raise Exception(
+                'Cannot get replication key value for table: {}'.format(table)
+            )
 
         mysql_key_value = result[0].get('key_value')
         key_value = mysql_key_value
@@ -175,7 +216,7 @@ def fetch_current_incremental_key_pos(self, table, replication_key):
         return {
             'replication_key': replication_key,
             'replication_key_value': key_value,
-            'version': 1
+            'version': 1,
         }
 
     def get_primary_keys(self, table_name):
@@ -183,11 +224,15 @@ def get_primary_keys(self, table_name):
         Get the primary key of a table
         """
         table_dict = utils.tablename_to_dict(table_name)
-        sql = "SHOW KEYS FROM `{}`.`{}` WHERE Key_name = 'PRIMARY'".format(table_dict['schema_name'],
-                                                                           table_dict['table_name'])
+        sql = "SHOW KEYS FROM `{}`.`{}` WHERE Key_name = 'PRIMARY'".format(
+            table_dict['schema_name'], table_dict['table_name']
+        )
         pk_specs = self.query(sql)
         if len(pk_specs) > 0:
-            return [safe_column_name(k.get('Column_name'), self.target_quote) for k in pk_specs]
+            return [
+                safe_column_name(k.get('Column_name'), self.target_quote)
+                for k in pk_specs
+            ]
 
         return None
 
@@ -251,7 +296,7 @@ def get_table_columns(self, table_name, max_num=None, date_type='date'):
                         AND table_name = '{table_name}') x
                 ORDER BY
                         ordinal_position
-            """
+            """  # noqa: E501
         return self.query(sql)
 
     def map_column_types_to_target(self, table_name):
@@ -260,26 +305,32 @@ def map_column_types_to_target(self, table_name):
         """
         mysql_columns = self.get_table_columns(table_name)
         mapped_columns = [
-            '{} {}'.format(safe_column_name(pc.get('column_name'), self.target_quote),
-                           self.tap_type_to_target_type(pc.get('data_type'), pc.get('column_type')))
-            for pc in mysql_columns]
+            '{} {}'.format(
+                safe_column_name(pc.get('column_name'), self.target_quote),
+                self.tap_type_to_target_type(
+                    pc.get('data_type'), pc.get('column_type')
+                ),
+            )
+            for pc in mysql_columns
+        ]
 
         return {
             'columns': mapped_columns,
-            'primary_key': self.get_primary_keys(table_name)
+            'primary_key': self.get_primary_keys(table_name),
         }
 
     # pylint: disable=too-many-locals
-    def copy_table(self,
-                   table_name,
-                   path,
-                   max_num=None,
-                   date_type='date',
-                   split_large_files=False,
-                   split_file_chunk_size_mb=1000,
-                   split_file_max_chunks=20,
-                   compress=True,
-                   ):
+    def copy_table(
+        self,
+        table_name,
+        path,
+        max_num=None,
+        date_type='date',
+        split_large_files=False,
+        split_file_chunk_size_mb=1000,
+        split_file_max_chunks=20,
+        compress=True,
+    ):
         """
         Export data from table to a zipped csv
         Args:
@@ -303,24 +354,30 @@ def copy_table(self,
         ,CONVERT_TZ( NOW(),@@session.time_zone,'+00:00') AS _SDC_BATCHED_AT
         ,null AS _SDC_DELETED_AT
         FROM `{}`.`{}`
-        """.format(','.join(column_safe_sql_values),
-                   table_dict['schema_name'],
-                   table_dict['table_name'])
+        """.format(
+            ','.join(column_safe_sql_values),
+            table_dict['schema_name'],
+            table_dict['table_name'],
+        )
         export_batch_rows = self.connection_config['export_batch_rows']
         exported_rows = 0
         with self.conn_unbuffered as cur:
             cur.execute(sql)
-            gzip_splitter = split_gzip.open(path,
-                                            mode='wt',
-                                            chunk_size_mb=split_file_chunk_size_mb,
-                                            max_chunks=split_file_max_chunks if split_large_files else 0,
-                                            compress=compress)
+            gzip_splitter = split_gzip.open(
+                path,
+                mode='wt',
+                chunk_size_mb=split_file_chunk_size_mb,
+                max_chunks=split_file_max_chunks if split_large_files else 0,
+                compress=compress,
+            )
 
             with gzip_splitter as split_gzip_files:
-                writer = csv.writer(split_gzip_files,
-                                    delimiter=',',
-                                    quotechar='"',
-                                    quoting=csv.QUOTE_MINIMAL)
+                writer = csv.writer(
+                    split_gzip_files,
+                    delimiter=',',
+                    quotechar='"',
+                    quoting=csv.QUOTE_MINIMAL,
+                )
 
                 while True:
                     rows = cur.fetchmany(export_batch_rows)
@@ -335,9 +392,14 @@ def copy_table(self,
                         # Then we believe this to be just an interim batch and not the final one so report on progress
 
                         LOGGER.info(
-                            'Exporting batch from %s to %s rows from %s...', (exported_rows - export_batch_rows),
-                            exported_rows, table_name)
+                            'Exporting batch from %s to %s rows from %s...',
+                            (exported_rows - export_batch_rows),
+                            exported_rows,
+                            table_name,
+                        )
                     # Write rows to file in one go
                     writer.writerows(rows)
 
-                LOGGER.info('Exported total of %s rows from %s...', exported_rows, table_name)
+                LOGGER.info(
+                    'Exported total of %s rows from %s...', exported_rows, table_name
+                )
diff --git a/pipelinewise/fastsync/commons/tap_postgres.py b/pipelinewise/fastsync/commons/tap_postgres.py
index b5e7285d1..ddfcc40af 100644
--- a/pipelinewise/fastsync/commons/tap_postgres.py
+++ b/pipelinewise/fastsync/commons/tap_postgres.py
@@ -52,7 +52,12 @@ def generate_replication_slot_name(dbname, tap_id=None, prefix='pipelinewise'):
         return re.sub('[^a-z0-9_]', '_', slot_name)
 
     @classmethod
-    def __get_slot_name(cls, connection, dbname: str, tap_id: str,) -> str:
+    def __get_slot_name(
+        cls,
+        connection,
+        dbname: str,
+        tap_id: str,
+    ) -> str:
         """
         Finds the right slot name to use and returns it
 
@@ -74,7 +79,9 @@ def __get_slot_name(cls, connection, dbname: str, tap_id: str,) -> str:
         try:
             # Backward compatibility: try to locate existing v15 slot first. PPW <= 0.15.0
             with connection.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
-                cur.execute(f"SELECT * FROM pg_replication_slots WHERE slot_name = '{slot_name_v15}';")
+                cur.execute(
+                    f"SELECT * FROM pg_replication_slots WHERE slot_name = '{slot_name_v15}';"
+                )
                 v15_slots_count = cur.rowcount
 
         except psycopg2.Error:
@@ -102,13 +109,17 @@ def drop_slot(cls, connection_config: Dict) -> None:
         LOGGER.debug('Connection to Primary server created.')
 
         try:
-            slot_name = cls.__get_slot_name(connection, connection_config['dbname'], connection_config['tap_id'])
+            slot_name = cls.__get_slot_name(
+                connection, connection_config['dbname'], connection_config['tap_id']
+            )
 
             LOGGER.info('Dropping the slot "%s"', slot_name)
             # drop the replication host
             with connection.cursor() as cur:
-                cur.execute(f'SELECT pg_drop_replication_slot(slot_name) '
-                            f"FROM pg_replication_slots WHERE slot_name = '{slot_name}';")
+                cur.execute(
+                    f'SELECT pg_drop_replication_slot(slot_name) '
+                    f"FROM pg_replication_slots WHERE slot_name = '{slot_name}';"
+                )
                 LOGGER.info('Number of dropped slots: %s', cur.rowcount)
 
         finally:
@@ -135,7 +146,8 @@ def get_connection(cls, connection_config: Dict, prioritize_primary: bool = Fals
                 connection_config['port'],
                 connection_config['user'],
                 connection_config['password'],
-                connection_config['dbname'])
+                connection_config['dbname'],
+            )
         else:
             LOGGER.info('Connecting to replica')
             conn_string = template.format(
@@ -147,8 +159,11 @@ def get_connection(cls, connection_config: Dict, prioritize_primary: bool = Fals
                 connection_config.get('replica_host', connection_config['host']),
                 connection_config.get('replica_port', connection_config['port']),
                 connection_config.get('replica_user', connection_config['user']),
-                connection_config.get('replica_password', connection_config['password']),
-                connection_config['dbname'])
+                connection_config.get(
+                    'replica_password', connection_config['password']
+                ),
+                connection_config['dbname'],
+            )
 
         if 'ssl' in connection_config and connection_config['ssl'] == 'true':
             conn_string += " sslmode='require'"
@@ -166,7 +181,9 @@ def open_connection(self):
         """
         Open connection
         """
-        self.conn = self.get_connection(self.connection_config, prioritize_primary=False)
+        self.conn = self.get_connection(
+            self.connection_config, prioritize_primary=False
+        )
         self.curr = self.conn.cursor()
 
     def close_connection(self):
@@ -221,12 +238,16 @@ def create_replication_slot(self):
         replication slot and full-resync the new taps.
         """
         try:
-            slot_name = self.__get_slot_name(self.primary_host_conn,
-                                             self.connection_config['dbname'],
-                                             self.connection_config['tap_id'])
+            slot_name = self.__get_slot_name(
+                self.primary_host_conn,
+                self.connection_config['dbname'],
+                self.connection_config['tap_id'],
+            )
 
             # Create the replication host
-            self.primary_host_query(f"SELECT * FROM pg_create_logical_replication_slot('{slot_name}', 'wal2json')")
+            self.primary_host_query(
+                f"SELECT * FROM pg_create_logical_replication_slot('{slot_name}', 'wal2json')"
+            )
         except Exception as exc:
             # ERROR: replication slot already exists SQL state: 42710
             if hasattr(exc, 'pgcode') and exc.pgcode == '42710':
@@ -241,12 +262,15 @@ def fetch_current_log_pos(self):
         """
         # Create replication slot dedicated connection
         # Always use Primary server for creating replication_slot
-        self.primary_host_conn = self.get_connection(self.connection_config, prioritize_primary=True)
+        self.primary_host_conn = self.get_connection(
+            self.connection_config, prioritize_primary=True
+        )
         self.primary_host_curr = self.primary_host_conn.cursor()
 
         # Make sure PostgreSQL version is 9.4 or higher
         result = self.primary_host_query(
-            "SELECT setting::int AS version FROM pg_settings WHERE name='server_version_num'")
+            "SELECT setting::int AS version FROM pg_settings WHERE name='server_version_num'"
+        )
         version = result[0].get('version')
 
         # Do not allow minor versions with PostgreSQL BUG #15114
@@ -275,9 +299,13 @@ def fetch_current_log_pos(self):
             if version >= 100000:
                 result = self.query('SELECT pg_last_wal_replay_lsn() AS current_lsn')
             elif version >= 90400:
-                result = self.query('SELECT pg_last_xlog_replay_location() AS current_lsn')
+                result = self.query(
+                    'SELECT pg_last_xlog_replay_location() AS current_lsn'
+                )
             else:
-                raise Exception('Logical replication not supported before PostgreSQL 9.4')
+                raise Exception(
+                    'Logical replication not supported before PostgreSQL 9.4'
+                )
         else:
             # Get current lsn from primary host
             if version >= 100000:
@@ -285,16 +313,15 @@ def fetch_current_log_pos(self):
             elif version >= 90400:
                 result = self.query('SELECT pg_current_xlog_location() AS current_lsn')
             else:
-                raise Exception('Logical replication not supported before PostgreSQL 9.4')
+                raise Exception(
+                    'Logical replication not supported before PostgreSQL 9.4'
+                )
 
         current_lsn = result[0].get('current_lsn')
         file, index = current_lsn.split('/')
         lsn = (int(file, 16) << 32) + int(index, 16)
 
-        return {
-            'lsn': lsn,
-            'version': 1
-        }
+        return {'lsn': lsn, 'version': 1}
 
     # pylint: disable=invalid-name
     def fetch_current_incremental_key_pos(self, table, replication_key):
@@ -302,9 +329,13 @@ def fetch_current_incremental_key_pos(self, table, replication_key):
         Get the actual incremental key position in the table
         """
         schema_name, table_name = table.split('.')
-        result = self.query(f'SELECT MAX({replication_key}) AS key_value FROM {schema_name}."{table_name}"')
+        result = self.query(
+            f'SELECT MAX({replication_key}) AS key_value FROM {schema_name}."{table_name}"'
+        )
         if len(result) == 0:
-            raise Exception('Cannot get replication key value for table: {}'.format(table))
+            raise Exception(
+                'Cannot get replication key value for table: {}'.format(table)
+            )
 
         postgres_key_value = result[0].get('key_value')
         key_value = postgres_key_value
@@ -322,7 +353,7 @@ def fetch_current_incremental_key_pos(self, table, replication_key):
         return {
             'replication_key': replication_key,
             'replication_key_value': key_value,
-            'version': 1
+            'version': 1,
         }
 
     def get_primary_keys(self, table):
@@ -339,7 +370,9 @@ def get_primary_keys(self, table):
                         pg_class.relnamespace = pg_namespace.oid AND
                         pg_attribute.attrelid = pg_class.oid AND
                         pg_attribute.attnum = any(pg_index.indkey)
-                    AND indisprimary""".format(schema_name, table_name)
+                    AND indisprimary""".format(
+            schema_name, table_name
+        )
         pk_specs = self.query(sql)
         if len(pk_specs) > 0:
             return [safe_column_name(k[0], self.target_quote) for k in pk_specs]
@@ -355,7 +388,7 @@ def get_table_columns(self, table_name, max_num=None, date_type='date'):
         if max_num:
             decimals = len(max_num.split('.')[1]) if '.' in max_num else 0
             decimal_format = f"""
-              'CASE WHEN "' || column_name || '" IS NULL THEN NULL ELSE GREATEST(LEAST({max_num}, ROUND("' || column_name || '"::numeric , {decimals})), -{max_num}) END'
+              'CASE WHEN "' || column_name || '" IS NULL THEN NULL ELSE GREATEST(LEAST({max_num}, ROUND("' || column_name || '"::numeric , {decimals})), -{max_num}) END'  # noqa E501
             """
             integer_format = """
               '"' || column_name || '"'
@@ -397,7 +430,7 @@ def get_table_columns(self, table_name, max_num=None, date_type='date'):
                     AND table_name = '{table_name}'
                 ORDER BY ordinal_position
                 ) AS x
-            """
+            """  # noqa: E501
         return self.query(sql)
 
     def map_column_types_to_target(self, table_name):
@@ -412,25 +445,28 @@ def map_column_types_to_target(self, table_name):
             # most targets would want to map length 1 to boolean and the rest to number
             if isinstance(column_type, list):
                 column_type = column_type[1 if pc[3] > 1 else 0]
-            mapping = '{} {}'.format(safe_column_name(pc[0], self.target_quote), column_type)
+            mapping = '{} {}'.format(
+                safe_column_name(pc[0], self.target_quote), column_type
+            )
             mapped_columns.append(mapping)
 
         return {
             'columns': mapped_columns,
-            'primary_key': self.get_primary_keys(table_name)
+            'primary_key': self.get_primary_keys(table_name),
         }
 
     # pylint: disable=too-many-arguments
-    def copy_table(self,
-                   table_name,
-                   path,
-                   max_num=None,
-                   date_type='date',
-                   split_large_files=False,
-                   split_file_chunk_size_mb=1000,
-                   split_file_max_chunks=20,
-                   compress=True
-                   ):
+    def copy_table(
+        self,
+        table_name,
+        path,
+        max_num=None,
+        date_type='date',
+        split_large_files=False,
+        split_file_chunk_size_mb=1000,
+        split_file_max_chunks=20,
+        compress=True,
+    ):
         """
         Export data from table to a zipped csv
         Args:
@@ -455,14 +491,18 @@ def copy_table(self,
         ,now() AT TIME ZONE 'UTC'
         ,null
         FROM {}."{}") TO STDOUT with CSV DELIMITER ','
-        """.format(','.join(column_safe_sql_values), schema_name, table_name)
+        """.format(
+            ','.join(column_safe_sql_values), schema_name, table_name
+        )
         LOGGER.info('Exporting data: %s', sql)
 
-        gzip_splitter = split_gzip.open(path,
-                                        mode='wb',
-                                        chunk_size_mb=split_file_chunk_size_mb,
-                                        max_chunks=split_file_max_chunks if split_large_files else 0,
-                                        compress=compress)
+        gzip_splitter = split_gzip.open(
+            path,
+            mode='wb',
+            chunk_size_mb=split_file_chunk_size_mb,
+            max_chunks=split_file_max_chunks if split_large_files else 0,
+            compress=compress,
+        )
 
         with gzip_splitter as split_gzip_files:
             self.curr.copy_expert(sql, split_gzip_files, size=131072)
diff --git a/pipelinewise/fastsync/commons/tap_s3_csv.py b/pipelinewise/fastsync/commons/tap_s3_csv.py
index e23f60e42..772982a17 100644
--- a/pipelinewise/fastsync/commons/tap_s3_csv.py
+++ b/pipelinewise/fastsync/commons/tap_s3_csv.py
@@ -9,7 +9,14 @@
 from datetime import datetime
 from time import struct_time
 from typing import Callable, Dict, List, Optional, Set
-from messytables import (CSVTableSet, headers_guess, headers_processor, jts, offset_processor, type_guess)
+from messytables import (
+    CSVTableSet,
+    headers_guess,
+    headers_processor,
+    jts,
+    offset_processor,
+    type_guess,
+)
 from singer.utils import strptime_with_tz
 from singer_encodings import csv as singer_encodings_csv
 
@@ -26,7 +33,12 @@ class FastSyncTapS3Csv:
     """
 
     # pylint: disable=bare-except
-    def __init__(self, connection_config: Dict, tap_type_to_target_type: Callable, target_quote=None):
+    def __init__(
+        self,
+        connection_config: Dict,
+        tap_type_to_target_type: Callable,
+        target_quote=None,
+    ):
         """
         Constructor
         :param connection_config: tap connection config
@@ -34,10 +46,16 @@ def __init__(self, connection_config: Dict, tap_type_to_target_type: Callable, t
         """
         try:
             # Check if bucket can be accessed without credentials/assuming role
-            list(S3Helper.list_files_in_bucket(connection_config['bucket'],
-                                               connection_config.get('aws_endpoint_url', None)))
-            LOGGER.info('I have direct access to the bucket without assuming the configured role.')
-        except:
+            list(
+                S3Helper.list_files_in_bucket(
+                    connection_config['bucket'],
+                    connection_config.get('aws_endpoint_url', None),
+                )
+            )
+            LOGGER.info(
+                'I have direct access to the bucket without assuming the configured role.'
+            )
+        except Exception:
             # Setup AWS session
             S3Helper.setup_aws_client(connection_config)
 
@@ -48,7 +66,12 @@ def __init__(self, connection_config: Dict, tap_type_to_target_type: Callable, t
 
     def _find_table_spec_by_name(self, table_name: str) -> Dict:
         # look in tables array for the full specs dict of given table
-        return next(filter(lambda x: x['table_name'] == table_name, self.connection_config['tables']))
+        return next(
+            filter(
+                lambda x: x['table_name'] == table_name,
+                self.connection_config['tables'],
+            )
+        )
 
     def copy_table(self, table_name: str, file_path: str) -> None:
         """
@@ -67,7 +90,9 @@ def copy_table(self, table_name: str, file_path: str) -> None:
         modified_since = strptime_with_tz(self.connection_config['start_date'])
 
         # get all the files in the bucket that match the criteria and were modified after start date
-        s3_files = S3Helper.get_input_files_for_table(self.connection_config, table_spec, modified_since)
+        s3_files = S3Helper.get_input_files_for_table(
+            self.connection_config, table_spec, modified_since
+        )
 
         # variable to hold all the records from all matching files
         records = []
@@ -84,7 +109,10 @@ def copy_table(self, table_name: str, file_path: str) -> None:
             self._get_file_records(s3_file['key'], table_spec, records, headers)
 
             # check if the current file has the most recent modification date
-            if max_last_modified is None or max_last_modified < s3_file['last_modified']:
+            if (
+                max_last_modified is None
+                or max_last_modified < s3_file['last_modified']
+            ):
                 max_last_modified = s3_file['last_modified']
 
         # add the found last modified date to the dictionary
@@ -93,19 +121,23 @@ def copy_table(self, table_name: str, file_path: str) -> None:
         # write to the given compressed csv file
         with gzip.open(file_path, 'wt') as gzfile:
 
-            writer = csv.DictWriter(gzfile,
-                                    fieldnames=sorted(list(headers)),
-                                    # we need to sort the headers so that copying into snowflake works
-                                    delimiter=',',
-                                    quotechar='"',
-                                    quoting=csv.QUOTE_MINIMAL)
+            writer = csv.DictWriter(
+                gzfile,
+                fieldnames=sorted(list(headers)),
+                # we need to sort the headers so that copying into snowflake works
+                delimiter=',',
+                quotechar='"',
+                quoting=csv.QUOTE_MINIMAL,
+            )
             # write the header
             writer.writeheader()
             # write all records at once
             writer.writerows(records)
 
     # pylint: disable=too-many-locals
-    def _get_file_records(self, s3_path: str, table_spec: Dict, records: List[Dict], headers: Set) -> None:
+    def _get_file_records(
+        self, s3_path: str, table_spec: Dict, records: List[Dict], headers: Set
+    ) -> None:
         """
         Reads the file in s3_path and inserts the rows in records
         :param config: tap connection configuration
@@ -129,7 +161,9 @@ def _get_file_records(self, s3_path: str, table_spec: Dict, records: List[Dict],
         csv.field_size_limit(sys.maxsize)
 
         # pylint:disable=protected-access
-        iterator = singer_encodings_csv.get_row_iterator(s3_file_handle._raw_stream, table_spec)
+        iterator = singer_encodings_csv.get_row_iterator(
+            s3_file_handle._raw_stream, table_spec
+        )
 
         records_copied = len(records)
 
@@ -141,7 +175,7 @@ def _get_file_records(self, s3_path: str, table_spec: Dict, records: List[Dict],
                 S3Helper.SDC_SOURCE_LINENO_COLUMN: records_copied + 1,
                 '_SDC_EXTRACTED_AT': now_datetime,
                 '_SDC_BATCHED_AT': now_datetime,
-                '_SDC_DELETED_AT': None
+                '_SDC_DELETED_AT': None,
             }
 
             new_row = {}
@@ -171,20 +205,26 @@ def map_column_types_to_target(self, filepath: str, table: str):
 
         # use timestamp as a type instead if column is set in date_overrides configuration
         mapped_columns = []
-        date_overrides = None if 'date_overrides' not in specs \
-            else {safe_column_name(c, self.target_quote) for c in specs['date_overrides']}
+        date_overrides = (
+            None
+            if 'date_overrides' not in specs
+            else {
+                safe_column_name(c, self.target_quote) for c in specs['date_overrides']
+            }
+        )
 
         for column_name, column_type in csv_columns:
 
             if date_overrides and column_name in date_overrides:
-                mapped_columns.append(f'{column_name} {self.tap_type_to_target_type("date_override")}')
+                mapped_columns.append(
+                    f'{column_name} {self.tap_type_to_target_type("date_override")}'
+                )
             else:
-                mapped_columns.append(f'{column_name} {self.tap_type_to_target_type(column_type)}')
+                mapped_columns.append(
+                    f'{column_name} {self.tap_type_to_target_type(column_type)}'
+                )
 
-        return {
-            'columns': mapped_columns,
-            'primary_key': self._get_primary_keys(specs)
-        }
+        return {'columns': mapped_columns, 'primary_key': self._get_primary_keys(specs)}
 
     def _get_table_columns(self, csv_file_path: str) -> zip:
         """
@@ -203,12 +243,15 @@ def _get_table_columns(self, csv_file_path: str) -> zip:
 
             row_set.register_processor(offset_processor(offset + 1))
 
-            types = list(map(jts.celltype_as_string, type_guess(row_set.sample, strict=True)))
+            types = list(
+                map(jts.celltype_as_string, type_guess(row_set.sample, strict=True))
+            )
             return zip(headers, types)
 
     # pylint: disable=invalid-name
-    def fetch_current_incremental_key_pos(self, table: str,
-                                          replication_key: Optional[str] = 'modified_since') -> Optional[Dict]:
+    def fetch_current_incremental_key_pos(
+        self, table: str, replication_key: Optional[str] = 'modified_since'
+    ) -> Optional[Dict]:
         """
         Returns the last time a the table has been modified in ISO format.
         :param table: table name
@@ -217,9 +260,11 @@ def fetch_current_incremental_key_pos(self, table: str,
         """
         replication_key = 'modified_since'
 
-        return {
-            replication_key: self.tables_last_modified[table].isoformat()
-        } if table in self.tables_last_modified else {}
+        return (
+            {replication_key: self.tables_last_modified[table].isoformat()}
+            if table in self.tables_last_modified
+            else {}
+        )
 
     def _get_primary_keys(self, table_specs: Dict) -> Optional[List]:
         """
@@ -229,7 +274,10 @@ def _get_primary_keys(self, table_specs: Dict) -> Optional[List]:
         :return: the keys concatenated and separated by comma if keys are given, otherwise None
         """
         if table_specs.get('key_properties', False):
-            return [safe_column_name(k, self.target_quote) for k in table_specs['key_properties']]
+            return [
+                safe_column_name(k, self.target_quote)
+                for k in table_specs['key_properties']
+            ]
 
         return None
 
@@ -240,6 +288,7 @@ class S3Helper:
     """
     S3 helper methods
     """
+
     SDC_SOURCE_BUCKET_COLUMN = '_sdc_source_bucket'
     SDC_SOURCE_FILE_COLUMN = '_sdc_source_file'
     SDC_SOURCE_LINENO_COLUMN = '_sdc_source_lineno'
@@ -254,9 +303,15 @@ def setup_aws_client(cls, config: Dict) -> None:
         LOGGER.info('Attempting to create AWS session')
 
         # Get the required parameters from config file and/or environment variables
-        aws_access_key_id = config.get('aws_access_key_id') or os.environ.get('AWS_ACCESS_KEY_ID')
-        aws_secret_access_key = config.get('aws_secret_access_key') or os.environ.get('AWS_SECRET_ACCESS_KEY')
-        aws_session_token = config.get('aws_session_token') or os.environ.get('AWS_SESSION_TOKEN')
+        aws_access_key_id = config.get('aws_access_key_id') or os.environ.get(
+            'AWS_ACCESS_KEY_ID'
+        )
+        aws_secret_access_key = config.get('aws_secret_access_key') or os.environ.get(
+            'AWS_SECRET_ACCESS_KEY'
+        )
+        aws_session_token = config.get('aws_session_token') or os.environ.get(
+            'AWS_SESSION_TOKEN'
+        )
         aws_profile = config.get('aws_profile') or os.environ.get('AWS_PROFILE')
 
         # AWS credentials based authentication
@@ -264,14 +319,16 @@ def setup_aws_client(cls, config: Dict) -> None:
             boto3.setup_default_session(
                 aws_access_key_id=aws_access_key_id,
                 aws_secret_access_key=aws_secret_access_key,
-                aws_session_token=aws_session_token
+                aws_session_token=aws_session_token,
             )
         # AWS Profile based authentication, will use IAM role if no profile is found
         else:
             boto3.setup_default_session(profile_name=aws_profile)
 
     @classmethod
-    def get_input_files_for_table(cls, config: Dict, table_spec: Dict, modified_since: struct_time = None):
+    def get_input_files_for_table(
+        cls, config: Dict, table_spec: Dict, modified_since: struct_time = None
+    ):
         bucket = config['bucket']
         prefix = table_spec.get('search_prefix')
         pattern = table_spec['search_pattern']
@@ -279,10 +336,14 @@ def get_input_files_for_table(cls, config: Dict, table_spec: Dict, modified_sinc
         try:
             matcher = re.compile(pattern)
         except re.error as exc:
-            raise ValueError((f'search_pattern for table `{table_spec["table_name"]}` is not a valid regular '
-                              'expression. See '
-                              'https://docs.python.org/3.5/library/re.html#regular-expression-syntax'),
-                             pattern) from exc
+            raise ValueError(
+                (
+                    f'search_pattern for table `{table_spec["table_name"]}` is not a valid regular '
+                    'expression. See '
+                    'https://docs.python.org/3.5/library/re.html#regular-expression-syntax'
+                ),
+                pattern,
+            ) from exc
 
         LOGGER.info('Checking bucket "%s" for keys matching "%s"', bucket, pattern)
 
@@ -290,7 +351,9 @@ def get_input_files_for_table(cls, config: Dict, table_spec: Dict, modified_sinc
         unmatched_files_count = 0
         max_files_before_log = 30000
 
-        for s3_object in cls.list_files_in_bucket(bucket, prefix, aws_endpoint_url=config.get('aws_endpoint_url')):
+        for s3_object in cls.list_files_in_bucket(
+            bucket, prefix, aws_endpoint_url=config.get('aws_endpoint_url')
+        ):
             key = s3_object['Key']
             last_modified = s3_object['LastModified']
 
@@ -302,29 +365,48 @@ def get_input_files_for_table(cls, config: Dict, table_spec: Dict, modified_sinc
             if matcher.search(key):
                 matched_files_count += 1
                 if modified_since is None or modified_since < last_modified:
-                    LOGGER.info('Will download key "%s" as it was last modified %s', key, last_modified)
+                    LOGGER.info(
+                        'Will download key "%s" as it was last modified %s',
+                        key,
+                        last_modified,
+                    )
                     yield {'key': key, 'last_modified': last_modified}
             else:
                 unmatched_files_count += 1
 
-            if (unmatched_files_count + matched_files_count) % max_files_before_log == 0:
+            if (
+                unmatched_files_count + matched_files_count
+            ) % max_files_before_log == 0:
                 # Are we skipping greater than 50% of the files?
                 # pylint: disable=old-division
-                if (unmatched_files_count / (matched_files_count + unmatched_files_count)) > 0.5:
-                    LOGGER.info('Found %s matching files and %s non-matching files. '
-                                'You should consider adding a `search_prefix` to the config '
-                                'or removing non-matching files from the bucket.',
-                                matched_files_count, unmatched_files_count)
+                if (
+                    unmatched_files_count
+                    / (matched_files_count + unmatched_files_count)
+                ) > 0.5:
+                    LOGGER.info(
+                        'Found %s matching files and %s non-matching files. '
+                        'You should consider adding a `search_prefix` to the config '
+                        'or removing non-matching files from the bucket.',
+                        matched_files_count,
+                        unmatched_files_count,
+                    )
                 else:
-                    LOGGER.info('Found %s matching files and %s non-matching files',
-                                matched_files_count, unmatched_files_count)
+                    LOGGER.info(
+                        'Found %s matching files and %s non-matching files',
+                        matched_files_count,
+                        unmatched_files_count,
+                    )
 
         if matched_files_count == 0:
             if prefix:
-                raise Exception(f'No files found in bucket "{bucket}" '
-                                f'that matches prefix "{prefix}" and pattern "{pattern}"')
+                raise Exception(
+                    f'No files found in bucket "{bucket}" '
+                    f'that matches prefix "{prefix}" and pattern "{pattern}"'
+                )
 
-            raise Exception(f'No files found in bucket "{bucket}" that matches pattern "{pattern}"')
+            raise Exception(
+                f'No files found in bucket "{bucket}" that matches pattern "{pattern}"'
+            )
 
     @classmethod
     @retry_pattern()
@@ -356,7 +438,11 @@ def list_files_in_bucket(cls, bucket, search_prefix=None, aws_endpoint_url=None)
         if s3_object_count > 0:
             LOGGER.info('Found %s files.', s3_object_count)
         else:
-            LOGGER.info('Found no files for bucket "%s" that match prefix "%s"', bucket, search_prefix)
+            LOGGER.info(
+                'Found no files for bucket "%s" that match prefix "%s"',
+                bucket,
+                search_prefix,
+            )
 
     @classmethod
     @retry_pattern()
diff --git a/pipelinewise/fastsync/commons/target_bigquery.py b/pipelinewise/fastsync/commons/target_bigquery.py
index 60ceb226d..aa5a10a1e 100644
--- a/pipelinewise/fastsync/commons/target_bigquery.py
+++ b/pipelinewise/fastsync/commons/target_bigquery.py
@@ -33,6 +33,7 @@ class FastSyncTargetBigquery:
     """
     Common functions for fastsync to BigQuery
     """
+
     def __init__(self, connection_config, transformation_config=None):
         self.connection_config = connection_config
         self.transformation_config = transformation_config
@@ -48,7 +49,7 @@ def to_query_parameter(value):
                 value_type = 'INT64'
             elif isinstance(value, float):
                 value_type = 'NUMERIC'
-            #TODO: repeated float here and in target
+            # TODO: repeated float here and in target
             elif isinstance(value, float):
                 value_type = 'FLOAT64'
             elif isinstance(value, bool):
@@ -91,27 +92,45 @@ def create_schema(self, schema_name):
 
     def drop_table(self, target_schema, table_name, is_temporary=False):
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = safe_name(table_dict.get('table_name' if not is_temporary else 'temp_table_name'))
+        target_table = safe_name(
+            table_dict.get('table_name' if not is_temporary else 'temp_table_name')
+        )
 
         sql = 'DROP TABLE IF EXISTS {}.{}'.format(target_schema, target_table.lower())
         self.query(sql)
 
-    def create_table(self, target_schema: str, table_name: str, columns: List[str],
-                     is_temporary: bool = False, sort_columns=False):
+    def create_table(
+        self,
+        target_schema: str,
+        table_name: str,
+        columns: List[str],
+        is_temporary: bool = False,
+        sort_columns=False,
+    ):
 
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = safe_name(table_dict.get('table_name' if not is_temporary else 'temp_table_name').lower())
+        target_table = safe_name(
+            table_dict.get(
+                'table_name' if not is_temporary else 'temp_table_name'
+            ).lower()
+        )
 
         # skip the EXTRACTED, BATCHED and DELETED columns in case they exist because they gonna be added later
-        columns = [c for c in columns if not (
-                                              c.upper().startswith(utils.SDC_EXTRACTED_AT.upper()) or
-                                              c.upper().startswith(utils.SDC_BATCHED_AT.upper()) or
-                                              c.upper().startswith(utils.SDC_DELETED_AT.upper()))]
-
-        columns += [f'{utils.SDC_EXTRACTED_AT} TIMESTAMP',
-                    f'{utils.SDC_BATCHED_AT} TIMESTAMP',
-                    f'{utils.SDC_DELETED_AT} TIMESTAMP'
-                    ]
+        columns = [
+            c
+            for c in columns
+            if not (
+                c.upper().startswith(utils.SDC_EXTRACTED_AT.upper())
+                or c.upper().startswith(utils.SDC_BATCHED_AT.upper())
+                or c.upper().startswith(utils.SDC_DELETED_AT.upper())
+            )
+        ]
+
+        columns += [
+            f'{utils.SDC_EXTRACTED_AT} TIMESTAMP',
+            f'{utils.SDC_BATCHED_AT} TIMESTAMP',
+            f'{utils.SDC_DELETED_AT} TIMESTAMP',
+        ]
 
         # We need the sort the columns for some taps( for now tap-s3-csv)
         # because later on when copying a csv file into Snowflake
@@ -121,18 +140,33 @@ def create_table(self, target_schema: str, table_name: str, columns: List[str],
 
         columns = [c.lower() for c in columns]
 
-        sql = f'CREATE OR REPLACE TABLE {target_schema}.{target_table} (' \
-              f'{",".join(columns)})'
+        sql = (
+            f'CREATE OR REPLACE TABLE {target_schema}.{target_table} ('
+            f'{",".join(columns)})'
+        )
 
         self.query(sql)
 
     # pylint: disable=R0913,R0914
-    def copy_to_table(self, filepath, target_schema, table_name, size_bytes, is_temporary,
-                      skip_csv_header=False, allow_quoted_newlines=True, write_truncate=True):
+    def copy_to_table(
+        self,
+        filepath,
+        target_schema,
+        table_name,
+        size_bytes,
+        is_temporary,
+        skip_csv_header=False,
+        allow_quoted_newlines=True,
+        write_truncate=True,
+    ):
         LOGGER.info('BIGQUERY - Loading %s into Bigquery...', filepath)
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = safe_name(table_dict.get('table_name' if not is_temporary else 'temp_table_name').lower(),
-                                 quotes=False)
+        target_table = safe_name(
+            table_dict.get(
+                'table_name' if not is_temporary else 'temp_table_name'
+            ).lower(),
+            quotes=False,
+        )
 
         client = self.open_connection()
         dataset_ref = client.dataset(target_schema)
@@ -141,11 +175,15 @@ def copy_to_table(self, filepath, target_schema, table_name, size_bytes, is_temp
         job_config = bigquery.LoadJobConfig()
         job_config.source_format = bigquery.SourceFormat.CSV
         job_config.schema = table_schema
-        job_config.write_disposition = 'WRITE_TRUNCATE' if write_truncate else 'WRITE_APPEND'
+        job_config.write_disposition = (
+            'WRITE_TRUNCATE' if write_truncate else 'WRITE_APPEND'
+        )
         job_config.allow_quoted_newlines = allow_quoted_newlines
         job_config.skip_leading_rows = 1 if skip_csv_header else 0
         with open(filepath, 'rb') as exported_data:
-            job = client.load_table_from_file(exported_data, table_ref, job_config=job_config)
+            job = client.load_table_from_file(
+                exported_data, table_ref, job_config=job_config
+            )
         try:
             job.result()
         except exceptions.BadRequest as exc:
@@ -156,10 +194,12 @@ def copy_to_table(self, filepath, target_schema, table_name, size_bytes, is_temp
         LOGGER.info('Job %s', job)
         LOGGER.info('Job.output_rows %s', job.output_rows)
         inserts = job.output_rows
-        LOGGER.info('Loading into %s."%s": %s',
-                    target_schema,
-                    target_table,
-                    json.dumps({'inserts': inserts, 'updates': 0, 'size_bytes': size_bytes}))
+        LOGGER.info(
+            'Loading into %s."%s": %s',
+            target_schema,
+            target_table,
+            json.dumps({'inserts': inserts, 'updates': 0, 'size_bytes': size_bytes}),
+        )
 
         LOGGER.info(job.errors)
 
@@ -167,12 +207,18 @@ def copy_to_table(self, filepath, target_schema, table_name, size_bytes, is_temp
     # "to_group" is not used here but exists for compatibility reasons with other database types
     # "to_group" is for databases that can grant to users and groups separately like Amazon Redshift
     # pylint: disable=unused-argument
-    def grant_select_on_table(self, target_schema, table_name, role, is_temporary, to_group=False):
+    def grant_select_on_table(
+        self, target_schema, table_name, role, is_temporary, to_group=False
+    ):
         # Grant role is not mandatory parameter, do nothing if not specified
         if role:
             table_dict = utils.tablename_to_dict(table_name)
-            target_table = safe_name(table_dict.get('table_name' if not is_temporary else 'temp_table_name'))
-            sql = 'GRANT SELECT ON {}.{} TO ROLE {}'.format(target_schema, target_table, role)
+            target_table = safe_name(
+                table_dict.get('table_name' if not is_temporary else 'temp_table_name')
+            )
+            sql = 'GRANT SELECT ON {}.{} TO ROLE {}'.format(
+                target_schema, target_table, role
+            )
             self.query(sql)
 
     # pylint: disable=unused-argument
@@ -186,7 +232,9 @@ def grant_usage_on_schema(self, target_schema, role, to_group=False):
     def grant_select_on_schema(self, target_schema, role, to_group=False):
         # Grant role is not mandatory parameter, do nothing if not specified
         if role:
-            sql = 'GRANT SELECT ON ALL TABLES IN SCHEMA {} TO ROLE {}'.format(target_schema, role)
+            sql = 'GRANT SELECT ON ALL TABLES IN SCHEMA {} TO ROLE {}'.format(
+                target_schema, role
+            )
             self.query(sql)
 
     def obfuscate_columns(self, target_schema: str, table_name: str):
@@ -207,21 +255,22 @@ def obfuscate_columns(self, target_schema: str, table_name: str):
         #
         # We need to convert to the same format to find the transformation
         # has that has to be applied
-        tap_stream_name_by_table_name = '{}-{}'.format(table_dict['schema_name'], table_dict['table_name']) \
-            if table_dict['schema_name'] is not None else table_dict['table_name']
+        tap_stream_name_by_table_name = (
+            '{}-{}'.format(table_dict['schema_name'], table_dict['table_name'])
+            if table_dict['schema_name'] is not None
+            else table_dict['table_name']
+        )
 
         # Find obfuscation rules for the current table
         # trans_map = self.__get_stream_transformation_map(tap_stream_name_by_table_name, transformations)
         trans_map = TransformationHelper.get_trans_in_sql_flavor(
-            tap_stream_name_by_table_name,
-            transformations,
-            SQLFlavor('bigquery'))
+            tap_stream_name_by_table_name, transformations, SQLFlavor('bigquery')
+        )
 
         self.__apply_transformations(trans_map, target_schema, temp_table)
 
         LOGGER.info('Obfuscation rules applied.')
 
-
     def swap_tables(self, schema, table_name):
         project_id = self.connection_config['project_id']
         table_dict = utils.tablename_to_dict(table_name)
@@ -243,7 +292,9 @@ def swap_tables(self, schema, table_name):
         # delete the temp table
         client.delete_table(temp_table_id)
 
-    def __apply_transformations(self, transformations: List[Dict], target_schema: str, table_name: str) -> None:
+    def __apply_transformations(
+        self, transformations: List[Dict], target_schema: str, table_name: str
+    ) -> None:
         """
         Generate and execute the SQL queries based on the given transformations.
         Args:
@@ -251,7 +302,9 @@ def __apply_transformations(self, transformations: List[Dict], target_schema: st
             target_schema: name of the target schema where the table lives
             table_name: the table name on which we want to apply the transformations
         """
-        full_qual_table_name = '{}.{}'.format(safe_name(target_schema), safe_name(table_name))
+        full_qual_table_name = '{}.{}'.format(
+            safe_name(target_schema), safe_name(table_name)
+        )
 
         if transformations:
             all_cols_update_sql = ''
@@ -263,8 +316,10 @@ def __apply_transformations(self, transformations: List[Dict], target_schema: st
                 # If we have conditions, then we need to construct the query and execute it to transform the
                 # single column conditionally
                 if trans_item['conditions']:
-                    sql = f'UPDATE {full_qual_table_name} ' \
-                          f'SET {trans_item["trans"]} WHERE {trans_item["conditions"]};'
+                    sql = (
+                        f'UPDATE {full_qual_table_name} '
+                        f'SET {trans_item["trans"]} WHERE {trans_item["conditions"]};'
+                    )
 
                     self.query(sql)
 
@@ -276,7 +331,9 @@ def __apply_transformations(self, transformations: List[Dict], target_schema: st
                     if not all_cols_update_sql:
                         all_cols_update_sql = trans_item['trans']
                     else:
-                        all_cols_update_sql = f'{all_cols_update_sql}, {trans_item["trans"]}'
+                        all_cols_update_sql = (
+                            f'{all_cols_update_sql}, {trans_item["trans"]}'
+                        )
 
             # If we have some non-conditional transformations then construct and execute a query
             if all_cols_update_sql:
diff --git a/pipelinewise/fastsync/commons/target_postgres.py b/pipelinewise/fastsync/commons/target_postgres.py
index 0b7f201ae..c8b025b87 100644
--- a/pipelinewise/fastsync/commons/target_postgres.py
+++ b/pipelinewise/fastsync/commons/target_postgres.py
@@ -32,7 +32,7 @@ def open_connection(self):
             self.connection_config['dbname'],
             self.connection_config['user'],
             self.connection_config['password'],
-            self.connection_config['port']
+            self.connection_config['port'],
         )
 
         if 'ssl' in self.connection_config and self.connection_config['ssl'] == 'true':
@@ -62,25 +62,48 @@ def create_schemas(self, tables):
 
     def drop_table(self, target_schema, table_name, is_temporary=False):
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
+        target_table = (
+            table_dict.get('table_name')
+            if not is_temporary
+            else table_dict.get('temp_table_name')
+        )
 
         sql = 'DROP TABLE IF EXISTS {}."{}"'.format(target_schema, target_table.lower())
         self.query(sql)
 
-    def create_table(self, target_schema: str, table_name: str, columns: List[str], primary_key: List[str],
-                     is_temporary: bool = False, sort_columns=False):
+    def create_table(
+        self,
+        target_schema: str,
+        table_name: str,
+        columns: List[str],
+        primary_key: List[str],
+        is_temporary: bool = False,
+        sort_columns=False,
+    ):
 
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
+        target_table = (
+            table_dict.get('table_name')
+            if not is_temporary
+            else table_dict.get('temp_table_name')
+        )
 
         # skip the EXTRACTED, BATCHED and DELETED columns in case they exist because they gonna be added later
-        columns = [c for c in columns if not (c.startswith(self.EXTRACTED_AT_COLUMN) or
-                                              c.startswith(self.BATCHED_AT_COLUMN) or
-                                              c.startswith(self.DELETED_AT_COLUMN))]
-
-        columns += [f'{self.EXTRACTED_AT_COLUMN} TIMESTAMP WITHOUT TIME ZONE',
-                    f'{self.BATCHED_AT_COLUMN} TIMESTAMP WITHOUT TIME ZONE',
-                    f'{self.DELETED_AT_COLUMN} CHARACTER VARYING']
+        columns = [
+            c
+            for c in columns
+            if not (
+                c.startswith(self.EXTRACTED_AT_COLUMN)
+                or c.startswith(self.BATCHED_AT_COLUMN)
+                or c.startswith(self.DELETED_AT_COLUMN)
+            )
+        ]
+
+        columns += [
+            f'{self.EXTRACTED_AT_COLUMN} TIMESTAMP WITHOUT TIME ZONE',
+            f'{self.BATCHED_AT_COLUMN} TIMESTAMP WITHOUT TIME ZONE',
+            f'{self.DELETED_AT_COLUMN} CHARACTER VARYING',
+        ]
 
         # We need the sort the columns for some taps( for now tap-s3-csv)
         # because later on when copying a csv file into Snowflake
@@ -90,17 +113,30 @@ def create_table(self, target_schema: str, table_name: str, columns: List[str],
 
         sql_columns = ','.join(columns).lower()
         sql_primary_keys = ','.join(primary_key).lower() if primary_key else None
-        sql = f'CREATE TABLE IF NOT EXISTS {target_schema}."{target_table.lower()}" (' \
-              f'{sql_columns}' \
-              f'{f", PRIMARY KEY ({sql_primary_keys}))" if primary_key else ")"}'
+        sql = (
+            f'CREATE TABLE IF NOT EXISTS {target_schema}."{target_table.lower()}" ('
+            f'{sql_columns}'
+            f'{f", PRIMARY KEY ({sql_primary_keys}))" if primary_key else ")"}'
+        )
 
         self.query(sql)
 
-    def copy_to_table(self, filepath, target_schema: str, table_name: str, size_bytes: int,
-                      is_temporary: bool = False, skip_csv_header: bool = False):
+    def copy_to_table(
+        self,
+        filepath,
+        target_schema: str,
+        table_name: str,
+        size_bytes: int,
+        is_temporary: bool = False,
+        skip_csv_header: bool = False,
+    ):
         LOGGER.info('Loading %s into Postgres...', filepath)
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
+        target_table = (
+            table_dict.get('table_name')
+            if not is_temporary
+            else table_dict.get('temp_table_name')
+        )
 
         with self.open_connection() as connection:
             with connection.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
@@ -114,21 +150,33 @@ def copy_to_table(self, filepath, target_schema: str, table_name: str, size_byte
                     cur.copy_expert(copy_sql, file)
 
                 inserts = cur.rowcount
-                LOGGER.info('Loading into %s."%s": %s',
-                            target_schema,
-                            target_table.lower(),
-                            json.dumps({'inserts': inserts, 'updates': 0, 'size_bytes': size_bytes}))
+                LOGGER.info(
+                    'Loading into %s."%s": %s',
+                    target_schema,
+                    target_table.lower(),
+                    json.dumps(
+                        {'inserts': inserts, 'updates': 0, 'size_bytes': size_bytes}
+                    ),
+                )
 
     # grant_... functions are common functions called by utils.py: grant_privilege function
     # "to_group" is not used here but exists for compatibility reasons with other database types
     # "to_group" is for databases that can grant to users and groups separately like Amazon Redshift
     # pylint: disable=unused-argument
-    def grant_select_on_table(self, target_schema, table_name, role, is_temporary, to_group=False):
+    def grant_select_on_table(
+        self, target_schema, table_name, role, is_temporary, to_group=False
+    ):
         # Grant role is not mandatory parameter, do nothing if not specified
         if role:
             table_dict = utils.tablename_to_dict(table_name)
-            target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
-            sql = 'GRANT SELECT ON {}."{}" TO GROUP {}'.format(target_schema, target_table.lower(), role)
+            target_table = (
+                table_dict.get('table_name')
+                if not is_temporary
+                else table_dict.get('temp_table_name')
+            )
+            sql = 'GRANT SELECT ON {}."{}" TO GROUP {}'.format(
+                target_schema, target_table.lower(), role
+            )
             self.query(sql)
 
     # pylint: disable=unused-argument
@@ -142,10 +190,14 @@ def grant_usage_on_schema(self, target_schema, role, to_group=False):
     def grant_select_on_schema(self, target_schema, role, to_group=False):
         # Grant role is not mandatory parameter, do nothing if not specified
         if role:
-            sql = 'GRANT SELECT ON ALL TABLES IN SCHEMA {} TO GROUP {}'.format(target_schema, role)
+            sql = 'GRANT SELECT ON ALL TABLES IN SCHEMA {} TO GROUP {}'.format(
+                target_schema, role
+            )
             self.query(sql)
 
-    def obfuscate_columns(self, target_schema: str, table_name: str, is_temporary: bool = False):
+    def obfuscate_columns(
+        self, target_schema: str, table_name: str, is_temporary: bool = False
+    ):
         """
         Apply any configured transformations to the given table
         Args:
@@ -155,7 +207,11 @@ def obfuscate_columns(self, target_schema: str, table_name: str, is_temporary: b
         LOGGER.info('Starting obfuscation rules...')
 
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
+        target_table = (
+            table_dict.get('table_name')
+            if not is_temporary
+            else table_dict.get('temp_table_name')
+        )
         transformations = self.transformation_config.get('transformations', [])
 
         # Input table_name is formatted as {{schema}}.{{table}}
@@ -163,12 +219,13 @@ def obfuscate_columns(self, target_schema: str, table_name: str, is_temporary: b
         #
         # We need to convert to the same format to find the transformation
         # has that has to be applied
-        tap_stream_name_by_table_name = '{}-{}'.format(table_dict.get('schema_name'), table_dict.get('table_name'))
+        tap_stream_name_by_table_name = '{}-{}'.format(
+            table_dict.get('schema_name'), table_dict.get('table_name')
+        )
 
         trans_cols = TransformationHelper.get_trans_in_sql_flavor(
-            tap_stream_name_by_table_name,
-            transformations,
-            SQLFlavor('postgres'))
+            tap_stream_name_by_table_name, transformations, SQLFlavor('postgres')
+        )
 
         self.__apply_transformations(trans_cols, target_schema, target_table)
 
@@ -181,7 +238,11 @@ def swap_tables(self, schema, table_name):
 
         # Swap tables and drop the temp tamp
         self.query('DROP TABLE IF EXISTS {}."{}"'.format(schema, target_table.lower()))
-        self.query('ALTER TABLE {}."{}" RENAME TO "{}"'.format(schema, temp_table.lower(), target_table.lower()))
+        self.query(
+            'ALTER TABLE {}."{}" RENAME TO "{}"'.format(
+                schema, temp_table.lower(), target_table.lower()
+            )
+        )
 
     def __apply_transformations(self, transformations, target_schema, table_name):
         """
@@ -203,8 +264,10 @@ def __apply_transformations(self, transformations, target_schema, table_name):
                 # If we have conditions, then we need to construct the query and execute it to transform the
                 # single column conditionally
                 if trans_item['conditions']:
-                    sql = f'UPDATE {full_qual_table_name} ' \
-                          f'SET {trans_item["trans"]} WHERE {trans_item["conditions"]};'
+                    sql = (
+                        f'UPDATE {full_qual_table_name} '
+                        f'SET {trans_item["trans"]} WHERE {trans_item["conditions"]};'
+                    )
 
                     self.query(sql)
 
@@ -216,10 +279,14 @@ def __apply_transformations(self, transformations, target_schema, table_name):
                     if not all_cols_update_sql:
                         all_cols_update_sql = trans_item['trans']
                     else:
-                        all_cols_update_sql = f'{all_cols_update_sql}, {trans_item["trans"]}'
+                        all_cols_update_sql = (
+                            f'{all_cols_update_sql}, {trans_item["trans"]}'
+                        )
 
             # If we have some non-conditional transformations then construct and execute a query
             if all_cols_update_sql:
-                all_cols_update_sql = f'UPDATE {full_qual_table_name} SET {all_cols_update_sql};'
+                all_cols_update_sql = (
+                    f'UPDATE {full_qual_table_name} SET {all_cols_update_sql};'
+                )
 
                 self.query(all_cols_update_sql)
diff --git a/pipelinewise/fastsync/commons/target_redshift.py b/pipelinewise/fastsync/commons/target_redshift.py
index ff71fb58a..f4190a41b 100644
--- a/pipelinewise/fastsync/commons/target_redshift.py
+++ b/pipelinewise/fastsync/commons/target_redshift.py
@@ -28,11 +28,18 @@ def __init__(self, connection_config, transformation_config=None):
         self.transformation_config = transformation_config
 
         # Get the required parameters from config file and/or environment variables
-        aws_profile = self.connection_config.get('aws_profile') or os.environ.get('AWS_PROFILE')
-        aws_access_key_id = self.connection_config.get('aws_access_key_id') or os.environ.get('AWS_ACCESS_KEY_ID')
-        aws_secret_access_key = self.connection_config.get('aws_secret_access_key') or \
-                                os.environ.get('AWS_SECRET_ACCESS_KEY')
-        aws_session_token = self.connection_config.get('aws_session_token') or os.environ.get('AWS_SESSION_TOKEN')
+        aws_profile = self.connection_config.get('aws_profile') or os.environ.get(
+            'AWS_PROFILE'
+        )
+        aws_access_key_id = self.connection_config.get(
+            'aws_access_key_id'
+        ) or os.environ.get('AWS_ACCESS_KEY_ID')
+        aws_secret_access_key = self.connection_config.get(
+            'aws_secret_access_key'
+        ) or os.environ.get('AWS_SECRET_ACCESS_KEY')
+        aws_session_token = self.connection_config.get(
+            'aws_session_token'
+        ) or os.environ.get('AWS_SESSION_TOKEN')
 
         # Init S3 client
         # Conditionally pass keys as this seems to affect whether instance credentials
@@ -41,7 +48,7 @@ def __init__(self, connection_config, transformation_config=None):
             aws_session = boto3.session.Session(
                 aws_access_key_id=aws_access_key_id,
                 aws_secret_access_key=aws_secret_access_key,
-                aws_session_token=aws_session_token
+                aws_session_token=aws_session_token,
             )
             credentials = aws_session.get_credentials().get_frozen_credentials()
 
@@ -60,7 +67,7 @@ def open_connection(self):
             self.connection_config['dbname'],
             self.connection_config['user'],
             self.connection_config['password'],
-            self.connection_config['port']
+            self.connection_config['port'],
         )
 
         return psycopg2.connect(conn_string)
@@ -84,7 +91,12 @@ def upload_to_s3(self, file):
 
         extra_args = {'ACL': s3_acl} if s3_acl else None
 
-        LOGGER.info('Uploading to S3 bucket: %s, local file: %s, S3 key: %s', bucket, file, s3_key)
+        LOGGER.info(
+            'Uploading to S3 bucket: %s, local file: %s, S3 key: %s',
+            bucket,
+            file,
+            s3_key,
+        )
 
         self.s3.upload_file(file, bucket, s3_key, ExtraArgs=extra_args)
 
@@ -101,25 +113,48 @@ def create_schemas(self, tables):
 
     def drop_table(self, target_schema, table_name, is_temporary=False):
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
+        target_table = (
+            table_dict.get('table_name')
+            if not is_temporary
+            else table_dict.get('temp_table_name')
+        )
 
         sql = 'DROP TABLE IF EXISTS {}."{}"'.format(target_schema, target_table.upper())
         self.query(sql)
 
-    def create_table(self, target_schema: str, table_name: str, columns: List[str], primary_key: List[str],
-                     is_temporary: bool = False, sort_columns=False):
+    def create_table(
+        self,
+        target_schema: str,
+        table_name: str,
+        columns: List[str],
+        primary_key: List[str],
+        is_temporary: bool = False,
+        sort_columns=False,
+    ):
 
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
+        target_table = (
+            table_dict.get('table_name')
+            if not is_temporary
+            else table_dict.get('temp_table_name')
+        )
 
         # skip the EXTRACTED, BATCHED and DELETED columns in case they exist because they gonna be added later
-        columns = [c for c in columns if not (c.startswith(self.EXTRACTED_AT_COLUMN) or
-                                              c.startswith(self.BATCHED_AT_COLUMN) or
-                                              c.startswith(self.DELETED_AT_COLUMN))]
+        columns = [
+            c
+            for c in columns
+            if not (
+                c.startswith(self.EXTRACTED_AT_COLUMN)
+                or c.startswith(self.BATCHED_AT_COLUMN)
+                or c.startswith(self.DELETED_AT_COLUMN)
+            )
+        ]
 
-        columns += [f'{self.EXTRACTED_AT_COLUMN} TIMESTAMP WITHOUT TIME ZONE',
-                    f'{self.BATCHED_AT_COLUMN} TIMESTAMP WITHOUT TIME ZONE',
-                    f'{self.DELETED_AT_COLUMN} CHARACTER VARYING']
+        columns += [
+            f'{self.EXTRACTED_AT_COLUMN} TIMESTAMP WITHOUT TIME ZONE',
+            f'{self.BATCHED_AT_COLUMN} TIMESTAMP WITHOUT TIME ZONE',
+            f'{self.DELETED_AT_COLUMN} CHARACTER VARYING',
+        ]
 
         # We need the sort the columns for some taps( for now tap-s3-csv)
         # because later on when copying a csv file into Snowflake
@@ -129,80 +164,121 @@ def create_table(self, target_schema: str, table_name: str, columns: List[str],
 
         sql_columns = ','.join(columns)
         sql_primary_keys = ','.join(primary_key) if primary_key else None
-        sql = f'CREATE TABLE IF NOT EXISTS {target_schema}."{target_table.upper()}" (' \
-              f'{sql_columns}' \
-              f'{f", PRIMARY KEY ({sql_primary_keys}))" if primary_key else ")"}'
+        sql = (
+            f'CREATE TABLE IF NOT EXISTS {target_schema}."{target_table.upper()}" ('
+            f'{sql_columns}'
+            f'{f", PRIMARY KEY ({sql_primary_keys}))" if primary_key else ")"}'
+        )
 
         self.query(sql)
 
-    def copy_to_table(self, s3_key, target_schema, table_name, size_bytes, is_temporary, skip_csv_header=False):
+    def copy_to_table(
+        self,
+        s3_key,
+        target_schema,
+        table_name,
+        size_bytes,
+        is_temporary,
+        skip_csv_header=False,
+    ):
         LOGGER.info('Loading %s into Redshift...', s3_key)
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
+        target_table = (
+            table_dict.get('table_name')
+            if not is_temporary
+            else table_dict.get('temp_table_name')
+        )
         inserts = 0
         bucket = self.connection_config['s3_bucket']
 
         # Step 1: Generate copy credentials - prefer role if provided, otherwise use access and secret keys
-        copy_credentials = """
+        copy_credentials = (
+            """
             iam_role '{aws_role_arn}'
-        """.format(aws_role_arn=self.connection_config['aws_redshift_copy_role_arn']) \
-            if self.connection_config.get('aws_redshift_copy_role_arn') else """
+        """.format(
+                aws_role_arn=self.connection_config['aws_redshift_copy_role_arn']
+            )
+            if self.connection_config.get('aws_redshift_copy_role_arn')
+            else """
             ACCESS_KEY_ID '{aws_access_key_id}'
             SECRET_ACCESS_KEY '{aws_secret_access_key}'
             {aws_session_token}
         """.format(
-            aws_access_key_id=self.connection_config['aws_access_key_id'],
-            aws_secret_access_key=self.connection_config['aws_secret_access_key'],
-            aws_session_token="SESSION_TOKEN '{}'".format(self.connection_config['aws_session_token']) \
-                if self.connection_config.get('aws_session_token') else '',
+                aws_access_key_id=self.connection_config['aws_access_key_id'],
+                aws_secret_access_key=self.connection_config['aws_secret_access_key'],
+                aws_session_token="SESSION_TOKEN '{}'".format(
+                    self.connection_config['aws_session_token']
+                )
+                if self.connection_config.get('aws_session_token')
+                else '',
+            )
         )
 
         # Step 2: Generate copy options - Override defaults from config.json if defined
-        copy_options = self.connection_config.get('copy_options', f"""
+        copy_options = self.connection_config.get(
+            'copy_options',
+            f"""
             EMPTYASNULL BLANKSASNULL TRIMBLANKS TRUNCATECOLUMNS IGNOREHEADER {int(skip_csv_header)}
             TIMEFORMAT 'auto'
-        """)
+        """,
+        )
 
         # Step3: Using the built-in CSV COPY option to load
-        copy_sql = f'COPY {target_schema}."{target_table.upper()}" FROM \'s3://{bucket}/{s3_key}\'' \
-                   f'{copy_credentials}' \
-                   f'{copy_options}' \
-                   f'CSV GZIP'
+        copy_sql = (
+            f'COPY {target_schema}."{target_table.upper()}" FROM \'s3://{bucket}/{s3_key}\''
+            f'{copy_credentials}'
+            f'{copy_options}'
+            f'CSV GZIP'
+        )
 
         # Get number of inserted records - COPY does insert only
         results = self.query(copy_sql)
         if len(results) > 0:
             inserts = results[0].get('rows_loaded', 0)
 
-        LOGGER.info('Loading into %s."%s": %s',
-                    target_schema,
-                    target_table.upper(),
-                    json.dumps({'inserts': inserts, 'updates': 0, 'size_bytes': size_bytes}))
+        LOGGER.info(
+            'Loading into %s."%s": %s',
+            target_schema,
+            target_table.upper(),
+            json.dumps({'inserts': inserts, 'updates': 0, 'size_bytes': size_bytes}),
+        )
 
         LOGGER.info('Deleting %s from S3...', s3_key)
         self.s3.delete_object(Bucket=bucket, Key=s3_key)
 
-    def grant_select_on_table(self, target_schema, table_name, grantee, is_temporary, to_group=False):
+    def grant_select_on_table(
+        self, target_schema, table_name, grantee, is_temporary, to_group=False
+    ):
         # Grant role is not mandatory parameter, do nothing if not specified
         if grantee:
             table_dict = utils.tablename_to_dict(table_name)
-            target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
-            sql = 'GRANT SELECT ON {}."{}" TO {} {}'.format(target_schema,
-                                                            target_table.upper(), 'GROUP' if to_group else '',
-                                                            grantee)
+            target_table = (
+                table_dict.get('table_name')
+                if not is_temporary
+                else table_dict.get('temp_table_name')
+            )
+            sql = 'GRANT SELECT ON {}."{}" TO {} {}'.format(
+                target_schema,
+                target_table.upper(),
+                'GROUP' if to_group else '',
+                grantee,
+            )
             self.query(sql)
 
     def grant_usage_on_schema(self, target_schema, grantee, to_group=False):
         # Grant role is not mandatory parameter, do nothing if not specified
         if grantee:
-            sql = 'GRANT USAGE ON SCHEMA {} TO {} {}'.format(target_schema, 'GROUP' if to_group else '', grantee)
+            sql = 'GRANT USAGE ON SCHEMA {} TO {} {}'.format(
+                target_schema, 'GROUP' if to_group else '', grantee
+            )
             self.query(sql)
 
     def grant_select_on_schema(self, target_schema, grantee, to_group=False):
         # Grant role is not mandatory parameter, do nothing if not specified
         if grantee:
-            sql = 'GRANT SELECT ON ALL TABLES IN SCHEMA {} TO {} {}'.format(target_schema, 'GROUP' if to_group else '',
-                                                                            grantee)
+            sql = 'GRANT SELECT ON ALL TABLES IN SCHEMA {} TO {} {}'.format(
+                target_schema, 'GROUP' if to_group else '', grantee
+            )
             self.query(sql)
 
     # pylint: disable=duplicate-string-formatting-argument
@@ -220,7 +296,9 @@ def obfuscate_columns(self, target_schema, table_name):
             #
             # We need to convert to the same format to find the transformation
             # has that has to be applied
-            tap_stream_name_by_table_name = '{}-{}'.format(table_dict.get('schema_name'), table_dict.get('table_name'))
+            tap_stream_name_by_table_name = '{}-{}'.format(
+                table_dict.get('schema_name'), table_dict.get('table_name')
+            )
             if trans.get('tap_stream_name') == tap_stream_name_by_table_name:
                 column = trans.get('field_id')
                 transform_type = trans.get('type')
@@ -230,10 +308,17 @@ def obfuscate_columns(self, target_schema, table_name):
                     trans_cols.append('"{}" = FUNC_SHA1("{}")'.format(column, column))
                 elif 'HASH-SKIP-FIRST' in transform_type:
                     skip_first_n = transform_type[-1]
-                    trans_cols.append('"{}" = CONCAT(SUBSTRING("{}", 1, {}), FUNC_SHA1(SUBSTRING("{}", {} + 1)))'
-                                      .format(column, column, skip_first_n, column, skip_first_n))
+                    trans_cols.append(
+                        '"{}" = CONCAT(SUBSTRING("{}", 1, {}), FUNC_SHA1(SUBSTRING("{}", {} + 1)))'.format(
+                            column, column, skip_first_n, column, skip_first_n
+                        )
+                    )
                 elif transform_type == 'MASK-DATE':
-                    trans_cols.append('"{}" = TO_CHAR("{}"::DATE, \'YYYY-01-01\')::DATE'.format(column, column))
+                    trans_cols.append(
+                        '"{}" = TO_CHAR("{}"::DATE, \'YYYY-01-01\')::DATE'.format(
+                            column, column
+                        )
+                    )
                 elif transform_type == 'MASK-NUMBER':
                     trans_cols.append('"{}" = 0'.format(column))
 
@@ -252,4 +337,8 @@ def swap_tables(self, schema, table_name):
 
         # Swap tables and drop the temp tamp
         self.query('DROP TABLE IF EXISTS {}."{}"'.format(schema, target_table.upper()))
-        self.query('ALTER TABLE {}."{}" RENAME TO "{}"'.format(schema, temp_table.upper(), target_table.upper()))
+        self.query(
+            'ALTER TABLE {}."{}" RENAME TO "{}"'.format(
+                schema, temp_table.upper(), target_table.upper()
+            )
+        )
diff --git a/pipelinewise/fastsync/commons/target_snowflake.py b/pipelinewise/fastsync/commons/target_snowflake.py
index 685490b46..d1f495087 100644
--- a/pipelinewise/fastsync/commons/target_snowflake.py
+++ b/pipelinewise/fastsync/commons/target_snowflake.py
@@ -6,8 +6,7 @@
 
 from typing import List, Dict
 from snowflake.connector.encryption_util import SnowflakeEncryptionUtil
-from snowflake.connector.remote_storage_util import \
-    SnowflakeFileEncryptionMaterial
+from snowflake.connector.remote_storage_util import SnowflakeFileEncryptionMaterial
 
 from . import utils
 from .transform_utils import TransformationHelper, SQLFlavor
@@ -30,27 +29,36 @@ def __init__(self, connection_config, transformation_config=None):
         self.transformation_config = transformation_config
 
         # Get the required parameters from config file and/or environment variables
-        aws_profile = self.connection_config.get('aws_profile') or os.environ.get('AWS_PROFILE')
-        aws_access_key_id = self.connection_config.get('aws_access_key_id') or os.environ.get('AWS_ACCESS_KEY_ID')
-        aws_secret_access_key = self.connection_config.get('aws_secret_access_key') or \
-                                os.environ.get('AWS_SECRET_ACCESS_KEY')
-        aws_session_token = self.connection_config.get('aws_session_token') or os.environ.get('AWS_SESSION_TOKEN')
+        aws_profile = self.connection_config.get('aws_profile') or os.environ.get(
+            'AWS_PROFILE'
+        )
+        aws_access_key_id = self.connection_config.get(
+            'aws_access_key_id'
+        ) or os.environ.get('AWS_ACCESS_KEY_ID')
+        aws_secret_access_key = self.connection_config.get(
+            'aws_secret_access_key'
+        ) or os.environ.get('AWS_SECRET_ACCESS_KEY')
+        aws_session_token = self.connection_config.get(
+            'aws_session_token'
+        ) or os.environ.get('AWS_SESSION_TOKEN')
 
         # AWS credentials based authentication
         if aws_access_key_id and aws_secret_access_key:
             aws_session = boto3.session.Session(
                 aws_access_key_id=aws_access_key_id,
                 aws_secret_access_key=aws_secret_access_key,
-                aws_session_token=aws_session_token
+                aws_session_token=aws_session_token,
             )
         # AWS Profile based authentication
         else:
             aws_session = boto3.session.Session(profile_name=aws_profile)
 
         # Create the s3 client
-        self.s3 = aws_session.client('s3',
-                                     region_name=self.connection_config.get('s3_region_name'),
-                                     endpoint_url=self.connection_config.get('s3_endpoint_url'))
+        self.s3 = aws_session.client(
+            's3',
+            region_name=self.connection_config.get('s3_region_name'),
+            endpoint_url=self.connection_config.get('s3_endpoint_url'),
+        )
 
     def create_query_tag(self, query_tag_props: dict = None) -> str:
         schema = None
@@ -60,24 +68,30 @@ def create_query_tag(self, query_tag_props: dict = None) -> str:
             schema = query_tag_props.get('schema')
             table = query_tag_props.get('table')
 
-        return json.dumps({'ppw_component': 'fastsync',
-                           'tap_id': self.connection_config.get('tap_id'),
-                           'database': self.connection_config['dbname'],
-                           'schema': schema,
-                           'table': table})
+        return json.dumps(
+            {
+                'ppw_component': 'fastsync',
+                'tap_id': self.connection_config.get('tap_id'),
+                'database': self.connection_config['dbname'],
+                'schema': schema,
+                'table': table,
+            }
+        )
 
     def open_connection(self, query_tag_props=None):
-        return snowflake.connector.connect(user=self.connection_config['user'],
-                                           password=self.connection_config['password'],
-                                           account=self.connection_config['account'],
-                                           database=self.connection_config['dbname'],
-                                           warehouse=self.connection_config['warehouse'],
-                                           autocommit=True,
-                                           session_parameters={
-                                               # Quoted identifiers should be case sensitive
-                                               'QUOTED_IDENTIFIERS_IGNORE_CASE': 'FALSE',
-                                               'QUERY_TAG': self.create_query_tag(query_tag_props)
-                                           })
+        return snowflake.connector.connect(
+            user=self.connection_config['user'],
+            password=self.connection_config['password'],
+            account=self.connection_config['account'],
+            database=self.connection_config['dbname'],
+            warehouse=self.connection_config['warehouse'],
+            autocommit=True,
+            session_parameters={
+                # Quoted identifiers should be case sensitive
+                'QUOTED_IDENTIFIERS_IGNORE_CASE': 'FALSE',
+                'QUERY_TAG': self.create_query_tag(query_tag_props),
+            },
+        )
 
     def query(self, query, params=None, query_tag_props=None):
         LOGGER.debug('Running query: %s', query)
@@ -96,7 +110,12 @@ def upload_to_s3(self, file, tmp_dir=None):
         s3_key_prefix = self.connection_config.get('s3_key_prefix', '')
         s3_key = '{}{}'.format(s3_key_prefix, os.path.basename(file))
 
-        LOGGER.info('Uploading to S3 bucket: %s, local file: %s, S3 key: %s', bucket, file, s3_key)
+        LOGGER.info(
+            'Uploading to S3 bucket: %s, local file: %s, S3 key: %s',
+            bucket,
+            file,
+            s3_key,
+        )
 
         # Encrypt csv if client side encryption enabled
         master_key = self.connection_config.get('client_side_encryption_master_key', '')
@@ -104,14 +123,10 @@ def upload_to_s3(self, file, tmp_dir=None):
             # Encrypt the file
             LOGGER.info('Encrypting file %s...', file)
             encryption_material = SnowflakeFileEncryptionMaterial(
-                query_stage_master_key=master_key,
-                query_id='',
-                smk_id=0
+                query_stage_master_key=master_key, query_id='', smk_id=0
             )
             encryption_metadata, encrypted_file = SnowflakeEncryptionUtil.encrypt_file(
-                encryption_material,
-                file,
-                tmp_dir=tmp_dir
+                encryption_material, file, tmp_dir=tmp_dir
             )
 
             # Upload to s3
@@ -120,7 +135,7 @@ def upload_to_s3(self, file, tmp_dir=None):
             # Send key and iv in the metadata, that will be required to decrypt and upload the encrypted file
             extra_args['Metadata'] = {
                 'x-amz-key': encryption_metadata.key,
-                'x-amz-iv': encryption_metadata.iv
+                'x-amz-iv': encryption_metadata.iv,
             }
             self.s3.upload_file(encrypted_file, bucket, s3_key, ExtraArgs=extra_args)
 
@@ -144,28 +159,43 @@ def copy_to_archive(self, source_s3_key, tap_id, table):
         archive_file_basename = os.path.basename(source_s3_key)
 
         # Get archive s3 prefix from config, defaulting to 'archive' if not specified
-        archive_s3_prefix = self.connection_config.get('archive_load_files_s3_prefix', 'archive')
+        archive_s3_prefix = self.connection_config.get(
+            'archive_load_files_s3_prefix', 'archive'
+        )
 
         source_s3_bucket = self.connection_config.get('s3_bucket')
 
         # Combine existing metadata with archive related headers
-        metadata = self.s3.head_object(Bucket=source_s3_bucket, Key=source_s3_key).get('Metadata', {})
-        metadata.update({
-            'tap': tap_id,
-            'schema': archive_schema,
-            'table': archive_table,
-            'archived-by': 'pipelinewise_fastsync_postgres_to_snowflake'
-        })
+        metadata = self.s3.head_object(Bucket=source_s3_bucket, Key=source_s3_key).get(
+            'Metadata', {}
+        )
+        metadata.update(
+            {
+                'tap': tap_id,
+                'schema': archive_schema,
+                'table': archive_table,
+                'archived-by': 'pipelinewise_fastsync_postgres_to_snowflake',
+            }
+        )
 
         # Get archive s3 bucket from config, defaulting to same bucket used for Snowflake imports if not specified
-        archive_s3_bucket = self.connection_config.get('archive_load_files_s3_bucket', source_s3_bucket)
+        archive_s3_bucket = self.connection_config.get(
+            'archive_load_files_s3_bucket', source_s3_bucket
+        )
 
-        archive_key = '{}/{}/{}/{}'.format(archive_s3_prefix, tap_id, archive_table, archive_file_basename)
+        archive_key = '{}/{}/{}/{}'.format(
+            archive_s3_prefix, tap_id, archive_table, archive_file_basename
+        )
         copy_source = '{}/{}'.format(source_s3_bucket, source_s3_key)
         LOGGER.info('Archiving %s to %s', copy_source, archive_key)
 
-        self.s3.copy_object(CopySource=copy_source, Bucket=archive_s3_bucket, Key=archive_key,
-                            Metadata=metadata, MetadataDirective='REPLACE')
+        self.s3.copy_object(
+            CopySource=copy_source,
+            Bucket=archive_s3_bucket,
+            Key=archive_key,
+            Metadata=metadata,
+            MetadataDirective='REPLACE',
+        )
 
     def create_schema(self, schema):
         sql = 'CREATE SCHEMA IF NOT EXISTS {}'.format(schema)
@@ -173,25 +203,48 @@ def create_schema(self, schema):
 
     def drop_table(self, target_schema, table_name, is_temporary=False):
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
+        target_table = (
+            table_dict.get('table_name')
+            if not is_temporary
+            else table_dict.get('temp_table_name')
+        )
 
         sql = 'DROP TABLE IF EXISTS {}."{}"'.format(target_schema, target_table.upper())
         self.query(sql, query_tag_props={'schema': target_schema, 'table': table_name})
 
-    def create_table(self, target_schema: str, table_name: str, columns: List[str], primary_key: List[str],
-                     is_temporary: bool = False, sort_columns=False):
+    def create_table(
+        self,
+        target_schema: str,
+        table_name: str,
+        columns: List[str],
+        primary_key: List[str],
+        is_temporary: bool = False,
+        sort_columns=False,
+    ):
 
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
+        target_table = (
+            table_dict.get('table_name')
+            if not is_temporary
+            else table_dict.get('temp_table_name')
+        )
 
         # skip the EXTRACTED, BATCHED and DELETED columns in case they exist because they gonna be added later
-        columns = [c for c in columns if not (c.startswith(utils.SDC_EXTRACTED_AT) or
-                                              c.startswith(utils.SDC_BATCHED_AT) or
-                                              c.startswith(utils.SDC_DELETED_AT))]
+        columns = [
+            c
+            for c in columns
+            if not (
+                c.startswith(utils.SDC_EXTRACTED_AT)
+                or c.startswith(utils.SDC_BATCHED_AT)
+                or c.startswith(utils.SDC_DELETED_AT)
+            )
+        ]
 
-        columns += [f'{utils.SDC_EXTRACTED_AT} TIMESTAMP_NTZ',
-                    f'{utils.SDC_BATCHED_AT} TIMESTAMP_NTZ',
-                    f'{utils.SDC_DELETED_AT} VARCHAR']
+        columns += [
+            f'{utils.SDC_EXTRACTED_AT} TIMESTAMP_NTZ',
+            f'{utils.SDC_BATCHED_AT} TIMESTAMP_NTZ',
+            f'{utils.SDC_DELETED_AT} VARCHAR',
+        ]
 
         # We need the sort the columns for some taps( for now tap-s3-csv)
         # because later on when copying a csv file into Snowflake
@@ -201,49 +254,85 @@ def create_table(self, target_schema: str, table_name: str, columns: List[str],
 
         sql_columns = ','.join(columns)
         sql_primary_keys = ','.join(primary_key) if primary_key else None
-        sql = f'CREATE OR REPLACE TABLE {target_schema}."{target_table.upper()}" (' \
-              f'{sql_columns}' \
-              f'{f", PRIMARY KEY ({sql_primary_keys}))" if primary_key else ")"}'
+        sql = (
+            f'CREATE OR REPLACE TABLE {target_schema}."{target_table.upper()}" ('
+            f'{sql_columns}'
+            f'{f", PRIMARY KEY ({sql_primary_keys}))" if primary_key else ")"}'
+        )
 
-        self.query(sql, query_tag_props={'schema': target_schema, 'table': target_table})
+        self.query(
+            sql, query_tag_props={'schema': target_schema, 'table': target_table}
+        )
 
     # pylint: disable=too-many-locals
-    def copy_to_table(self, s3_key, target_schema, table_name, size_bytes, is_temporary, skip_csv_header=False):
+    def copy_to_table(
+        self,
+        s3_key,
+        target_schema,
+        table_name,
+        size_bytes,
+        is_temporary,
+        skip_csv_header=False,
+    ):
         LOGGER.info('Loading %s into Snowflake...', s3_key)
         table_dict = utils.tablename_to_dict(table_name)
-        target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
+        target_table = (
+            table_dict.get('table_name')
+            if not is_temporary
+            else table_dict.get('temp_table_name')
+        )
         inserts = 0
 
         stage = self.connection_config['stage']
-        sql = f'COPY INTO {target_schema}."{target_table.upper()}" FROM \'@{stage}/{s3_key}\'' \
-              f' FILE_FORMAT = (type=CSV escape=\'\\x1e\' escape_unenclosed_field=\'\\x1e\'' \
-              f' field_optionally_enclosed_by=\'\"\' skip_header={int(skip_csv_header)}' \
-              f' compression=GZIP binary_format=HEX)'
+        sql = (
+            f'COPY INTO {target_schema}."{target_table.upper()}" FROM \'@{stage}/{s3_key}\''
+            f' FILE_FORMAT = (type=CSV escape=\'\\x1e\' escape_unenclosed_field=\'\\x1e\''
+            f' field_optionally_enclosed_by=\'\"\' skip_header={int(skip_csv_header)}'
+            f' compression=GZIP binary_format=HEX)'
+        )
 
         # Get number of inserted records - COPY does insert only
-        results = self.query(sql, query_tag_props={'schema': target_schema, 'table': target_table})
+        results = self.query(
+            sql, query_tag_props={'schema': target_schema, 'table': target_table}
+        )
         if len(results) > 0:
             inserts = sum([file_part.get('rows_loaded', 0) for file_part in results])
 
-        LOGGER.info('Loading into %s."%s": %s',
-                    target_schema,
-                    target_table.upper(),
-                    json.dumps({'inserts': inserts,
-                                'updates': 0,
-                                'file_parts': len(results),
-                                'size_bytes': size_bytes}))
+        LOGGER.info(
+            'Loading into %s."%s": %s',
+            target_schema,
+            target_table.upper(),
+            json.dumps(
+                {
+                    'inserts': inserts,
+                    'updates': 0,
+                    'file_parts': len(results),
+                    'size_bytes': size_bytes,
+                }
+            ),
+        )
 
     # grant_... functions are common functions called by utils.py: grant_privilege function
     # "to_group" is not used here but exists for compatibility reasons with other database types
     # "to_group" is for databases that can grant to users and groups separately like Amazon Redshift
     # pylint: disable=unused-argument
-    def grant_select_on_table(self, target_schema, table_name, role, is_temporary, to_group=False):
+    def grant_select_on_table(
+        self, target_schema, table_name, role, is_temporary, to_group=False
+    ):
         # Grant role is not mandatory parameter, do nothing if not specified
         if role:
             table_dict = utils.tablename_to_dict(table_name)
-            target_table = table_dict.get('table_name') if not is_temporary else table_dict.get('temp_table_name')
-            sql = 'GRANT SELECT ON {}."{}" TO ROLE {}'.format(target_schema, target_table.upper(), role)
-            self.query(sql, query_tag_props={'schema': target_schema, 'table': table_name})
+            target_table = (
+                table_dict.get('table_name')
+                if not is_temporary
+                else table_dict.get('temp_table_name')
+            )
+            sql = 'GRANT SELECT ON {}."{}" TO ROLE {}'.format(
+                target_schema, target_table.upper(), role
+            )
+            self.query(
+                sql, query_tag_props={'schema': target_schema, 'table': table_name}
+            )
 
     # pylint: disable=unused-argument
     def grant_usage_on_schema(self, target_schema, role, to_group=False):
@@ -256,7 +345,9 @@ def grant_usage_on_schema(self, target_schema, role, to_group=False):
     def grant_select_on_schema(self, target_schema, role, to_group=False):
         # Grant role is not mandatory parameter, do nothing if not specified
         if role:
-            sql = 'GRANT SELECT ON ALL TABLES IN SCHEMA {} TO ROLE {}'.format(target_schema, role)
+            sql = 'GRANT SELECT ON ALL TABLES IN SCHEMA {} TO ROLE {}'.format(
+                target_schema, role
+            )
             self.query(sql, query_tag_props={'schema': target_schema})
 
     def obfuscate_columns(self, target_schema: str, table_name: str):
@@ -277,15 +368,17 @@ def obfuscate_columns(self, target_schema: str, table_name: str):
         #
         # We need to convert to the same format to find the transformation
         # has that has to be applied
-        tap_stream_name_by_table_name = '{}-{}'.format(table_dict['schema_name'], table_dict['table_name']) \
-            if table_dict['schema_name'] is not None else table_dict['table_name']
+        tap_stream_name_by_table_name = (
+            '{}-{}'.format(table_dict['schema_name'], table_dict['table_name'])
+            if table_dict['schema_name'] is not None
+            else table_dict['table_name']
+        )
 
         # Find obfuscation rules for the current table
         # trans_map = self.__get_stream_transformation_map(tap_stream_name_by_table_name, transformations)
         trans_map = TransformationHelper.get_trans_in_sql_flavor(
-            tap_stream_name_by_table_name,
-            transformations,
-            SQLFlavor('snowflake'))
+            tap_stream_name_by_table_name, transformations, SQLFlavor('snowflake')
+        )
 
         self.__apply_transformations(trans_map, target_schema, temp_table)
 
@@ -304,13 +397,19 @@ def swap_tables(self, schema, table_name) -> None:
         temp_table = table_dict.get('temp_table_name')
 
         # Swap tables and drop the temp tamp
-        self.query(f'ALTER TABLE {schema}."{temp_table.upper()}" SWAP WITH {schema}."{target_table.upper()}"',
-                   query_tag_props={'schema': schema, 'table': target_table})
-
-        self.query(f'DROP TABLE IF EXISTS {schema}."{temp_table.upper()}"',
-                   query_tag_props={'schema': schema, 'table': temp_table})
-
-    def __apply_transformations(self, transformations: List[Dict], target_schema: str, table_name: str) -> None:
+        self.query(
+            f'ALTER TABLE {schema}."{temp_table.upper()}" SWAP WITH {schema}."{target_table.upper()}"',
+            query_tag_props={'schema': schema, 'table': target_table},
+        )
+
+        self.query(
+            f'DROP TABLE IF EXISTS {schema}."{temp_table.upper()}"',
+            query_tag_props={'schema': schema, 'table': temp_table},
+        )
+
+    def __apply_transformations(
+        self, transformations: List[Dict], target_schema: str, table_name: str
+    ) -> None:
         """
         Generate and execute the SQL queries based on the given transformations.
         Args:
@@ -330,10 +429,15 @@ def __apply_transformations(self, transformations: List[Dict], target_schema: st
                 # If we have conditions, then we need to construct the query and execute it to transform the
                 # single column conditionally
                 if trans_item['conditions']:
-                    sql = f'UPDATE {full_qual_table_name} ' \
-                          f'SET {trans_item["trans"]} WHERE {trans_item["conditions"]};'
+                    sql = (
+                        f'UPDATE {full_qual_table_name} '
+                        f'SET {trans_item["trans"]} WHERE {trans_item["conditions"]};'
+                    )
 
-                    self.query(sql, query_tag_props={'schema': target_schema, 'table': table_name})
+                    self.query(
+                        sql,
+                        query_tag_props={'schema': target_schema, 'table': table_name},
+                    )
 
                 # Otherwise, we can add this column to a general UPDATE query with no predicates
                 else:
@@ -343,10 +447,17 @@ def __apply_transformations(self, transformations: List[Dict], target_schema: st
                     if not all_cols_update_sql:
                         all_cols_update_sql = trans_item['trans']
                     else:
-                        all_cols_update_sql = f'{all_cols_update_sql}, {trans_item["trans"]}'
+                        all_cols_update_sql = (
+                            f'{all_cols_update_sql}, {trans_item["trans"]}'
+                        )
 
             # If we have some non-conditional transformations then construct and execute a query
             if all_cols_update_sql:
-                all_cols_update_sql = f'UPDATE {full_qual_table_name} SET {all_cols_update_sql};'
-
-                self.query(all_cols_update_sql, query_tag_props={'schema': target_schema, 'table': table_name})
+                all_cols_update_sql = (
+                    f'UPDATE {full_qual_table_name} SET {all_cols_update_sql};'
+                )
+
+                self.query(
+                    all_cols_update_sql,
+                    query_tag_props={'schema': target_schema, 'table': table_name},
+                )
diff --git a/pipelinewise/fastsync/commons/transform_utils.py b/pipelinewise/fastsync/commons/transform_utils.py
index 6dc9a300c..ef831f8a5 100644
--- a/pipelinewise/fastsync/commons/transform_utils.py
+++ b/pipelinewise/fastsync/commons/transform_utils.py
@@ -7,6 +7,7 @@ class TransformationType(Enum):
     """
     List of supported transformation types
     """
+
     SET_NULL = 'SET-NULL'
     MASK_HIDDEN = 'MASK-HIDDEN'
     MASK_DATE = 'MASK-DATE'
@@ -28,6 +29,7 @@ class SQLFlavor(Enum):
     """
     List of supported sql flavors
     """
+
     SNOWFLAKE = 'snowflake'
     POSTGRES = 'postgres'
     BIGQUERY = 'bigquery'
@@ -41,10 +43,8 @@ class TransformationHelper:
 
     @classmethod
     def get_trans_in_sql_flavor(
-            cls,
-            stream_name: str,
-            transformations: List[Dict],
-            sql_flavor: SQLFlavor) -> List[Dict]:
+        cls, stream_name: str, transformations: List[Dict], sql_flavor: SQLFlavor
+    ) -> List[Dict]:
 
         """
         Find the transformations to apply to the given stream and does proper formatting and mapping
@@ -78,52 +78,56 @@ def get_trans_in_sql_flavor(
                 conditions = cls.__conditions_to_sql(transform_conditions, sql_flavor)
 
                 if transform_type == TransformationType.SET_NULL:
-                    trans_map.append({
-                        'trans': f'{column} = NULL',
-                        'conditions': conditions
-                    })
+                    trans_map.append(
+                        {'trans': f'{column} = NULL', 'conditions': conditions}
+                    )
 
                 elif transform_type == TransformationType.HASH:
 
-                    trans_map.append({
-                        'trans': cls.__hash_to_sql(column, sql_flavor),
-                        'conditions': conditions
-                    })
+                    trans_map.append(
+                        {
+                            'trans': cls.__hash_to_sql(column, sql_flavor),
+                            'conditions': conditions,
+                        }
+                    )
 
                 elif transform_type.value.startswith('HASH-SKIP-FIRST-'):
 
-                    trans_map.append({
-                        'trans': cls.__hash_skip_first_to_sql(transform_type, column, sql_flavor),
-                        'conditions': conditions
-                    })
+                    trans_map.append(
+                        {
+                            'trans': cls.__hash_skip_first_to_sql(
+                                transform_type, column, sql_flavor
+                            ),
+                            'conditions': conditions,
+                        }
+                    )
 
                 elif transform_type == TransformationType.MASK_DATE:
 
-                    trans_map.append({
-                        'trans': cls.__mask_date_to_sql(column, sql_flavor),
-                        'conditions': conditions
-                    })
+                    trans_map.append(
+                        {
+                            'trans': cls.__mask_date_to_sql(column, sql_flavor),
+                            'conditions': conditions,
+                        }
+                    )
 
                 elif transform_type == TransformationType.MASK_NUMBER:
-                    trans_map.append({
-                        'trans': f'{column} = 0',
-                        'conditions': conditions
-                    })
+                    trans_map.append(
+                        {'trans': f'{column} = 0', 'conditions': conditions}
+                    )
 
                 elif transform_type == TransformationType.MASK_HIDDEN:
-                    trans_map.append({
-                        'trans': f"{column} = 'hidden'",
-                        'conditions': conditions
-                    })
+                    trans_map.append(
+                        {'trans': f"{column} = 'hidden'", 'conditions': conditions}
+                    )
 
         return trans_map
 
     @classmethod
     # pylint: disable=W0238  # False positive when it is used by another classmethod
     def __conditions_to_sql(
-            cls,
-            transform_conditions: List[Dict],
-            sql_flavor: SQLFlavor) -> Optional[str]:
+        cls, transform_conditions: List[Dict], sql_flavor: SQLFlavor
+    ) -> Optional[str]:
         """
         Convert the conditional transformations into equivalent form in SF SQL.
         Args:
@@ -160,7 +164,11 @@ def __conditions_to_sql(
 
                 else:
                     operator = '='
-                    value = f"'{condition['equals']}'" if isinstance(condition['equals'], str) else condition['equals']
+                    value = (
+                        f"'{condition['equals']}'"
+                        if isinstance(condition['equals'], str)
+                        else condition['equals']
+                    )
 
             elif 'regex_match' in condition:
 
@@ -173,17 +181,23 @@ def __conditions_to_sql(
                     operator = '~'
 
                 elif sql_flavor == SQLFlavor.BIGQUERY:
-                    conditions.append(f"REGEXP_CONTAINS({cls.__safe_column(condition['column'], sql_flavor)}, {value})")
+                    conditions.append(
+                        f"REGEXP_CONTAINS({cls.__safe_column(condition['column'], sql_flavor)}, {value})"
+                    )
                     continue
 
                 else:
-                    raise NotImplementedError(f'regex_match conditional transformation in {sql_flavor.value} SQL '
-                                              f'flavor not implemented!')
+                    raise NotImplementedError(
+                        f'regex_match conditional transformation in {sql_flavor.value} SQL '
+                        f'flavor not implemented!'
+                    )
 
             else:
                 continue
 
-            conditions.append(f"({cls.__safe_column(condition['column'], sql_flavor)} {operator} {value})")
+            conditions.append(
+                f"({cls.__safe_column(condition['column'], sql_flavor)} {operator} {value})"
+            )
 
         return ' AND '.join(conditions)
 
@@ -229,13 +243,16 @@ def __hash_to_sql(cls, column: str, sql_flavor: SQLFlavor) -> str:
 
         else:
             raise NotImplementedError(
-                f'HASH transformation in {sql_flavor.value} SQL flavor not implemented!')
+                f'HASH transformation in {sql_flavor.value} SQL flavor not implemented!'
+            )
 
         return trans
 
     @classmethod
     # pylint: disable=W0238  # False positive when it is used by another classmethod
-    def __hash_skip_first_to_sql(cls, transform_type: TransformationType, column: str, sql_flavor: SQLFlavor) -> str:
+    def __hash_skip_first_to_sql(
+        cls, transform_type: TransformationType, column: str, sql_flavor: SQLFlavor
+    ) -> str:
         """
         convert HASH-SKIP-FIRST-n transformation into the right sql string
         Args:
@@ -251,16 +268,22 @@ def __hash_skip_first_to_sql(cls, transform_type: TransformationType, column: st
 
         if sql_flavor == SQLFlavor.SNOWFLAKE:
             trans = '{0} = CONCAT(SUBSTRING({0}, 1, {1}), SHA2(SUBSTRING({0}, {1} + 1), 256))'.format(
-                column, skip_first_n)
+                column, skip_first_n
+            )
         elif sql_flavor == SQLFlavor.POSTGRES:
-            trans = '{0} = CONCAT(SUBSTRING({0}, 1, {1}), ENCODE(DIGEST(SUBSTRING({0}, {1} + 1), ' \
-                    '\'sha256\'), \'hex\'))'.format(column, skip_first_n)
+            trans = (
+                '{0} = CONCAT(SUBSTRING({0}, 1, {1}), ENCODE(DIGEST(SUBSTRING({0}, {1} + 1), '
+                '\'sha256\'), \'hex\'))'.format(column, skip_first_n)
+            )
         elif sql_flavor == SQLFlavor.BIGQUERY:
             trans = '{0} = CONCAT(SUBSTRING({0}, 1, {1}), TO_BASE64(SHA256(SUBSTRING({0}, {1} + 1))))'.format(
-                column, skip_first_n)
+                column, skip_first_n
+            )
         else:
-            raise NotImplementedError(f'HASH-SKIP-FIRST-{skip_first_n} transformation in {sql_flavor.value} SQL flavor '
-                                      f'not implemented!')
+            raise NotImplementedError(
+                f'HASH-SKIP-FIRST-{skip_first_n} transformation in {sql_flavor.value} SQL flavor '
+                f'not implemented!'
+            )
 
         return trans
 
@@ -278,24 +301,32 @@ def __mask_date_to_sql(cls, column: str, sql_flavor: SQLFlavor) -> str:
         Returns: sql string equivalent of the mask date
         """
         if sql_flavor == SQLFlavor.SNOWFLAKE:
-            trans = f'{column} = TIMESTAMP_NTZ_FROM_PARTS(' \
-                    f'DATE_FROM_PARTS(YEAR({column}), 1, 1),' \
-                    f'TO_TIME({column}))'
+            trans = (
+                f'{column} = TIMESTAMP_NTZ_FROM_PARTS('
+                f'DATE_FROM_PARTS(YEAR({column}), 1, 1),'
+                f'TO_TIME({column}))'
+            )
 
         elif sql_flavor == SQLFlavor.POSTGRES:
-            trans = '{0} = MAKE_TIMESTAMP(' \
-                    'DATE_PART(\'year\', {0})::int, ' \
-                    '1, ' \
-                    '1, ' \
-                    'DATE_PART(\'hour\', {0})::int, ' \
-                    'DATE_PART(\'minute\', {0})::int, ' \
-                    'DATE_PART(\'second\', {0})::double precision)'.format(column)
+            trans = (
+                '{0} = MAKE_TIMESTAMP('
+                'DATE_PART(\'year\', {0})::int, '
+                '1, '
+                '1, '
+                'DATE_PART(\'hour\', {0})::int, '
+                'DATE_PART(\'minute\', {0})::int, '
+                'DATE_PART(\'second\', {0})::double precision)'.format(column)
+            )
         elif sql_flavor == SQLFlavor.BIGQUERY:
-            trans = f'{column} = TIMESTAMP(DATETIME(' \
-                    f'DATE(EXTRACT(YEAR FROM {column}), 1, 1),' \
-                    f'TIME({column})))'
+            trans = (
+                f'{column} = TIMESTAMP(DATETIME('
+                f'DATE(EXTRACT(YEAR FROM {column}), 1, 1),'
+                f'TIME({column})))'
+            )
         else:
-            raise NotImplementedError(f'MASK-DATE transformation in {sql_flavor.value} SQL flavor '
-                                      f'not implemented!')
+            raise NotImplementedError(
+                f'MASK-DATE transformation in {sql_flavor.value} SQL flavor '
+                f'not implemented!'
+            )
 
         return trans
diff --git a/pipelinewise/fastsync/commons/utils.py b/pipelinewise/fastsync/commons/utils.py
index 3beb3efdb..761101c97 100644
--- a/pipelinewise/fastsync/commons/utils.py
+++ b/pipelinewise/fastsync/commons/utils.py
@@ -19,6 +19,7 @@ class NotSelectedTableException(Exception):
     """
     Exception to raise when a table is not selected for resync
     """
+
     def __init__(self, table_name, selected_tables):
         self.message = f'Cannot Resync unselected table "{table_name}"! Selected tables are: {selected_tables}'
         super().__init__(self, self.message)
@@ -70,7 +71,7 @@ def tablename_to_dict(table, separator='.'):
         'catalog_name': catalog_name,
         'schema_name': schema_name,
         'table_name': table_name,
-        'temp_table_name': '{}_temp'.format(table_name)
+        'temp_table_name': '{}_temp'.format(table_name),
     }
 
 
@@ -84,8 +85,14 @@ def get_tables_from_properties(properties: Dict) -> set:
         metadata = stream.get('metadata', [])
         table_name = stream.get('table_name', stream['stream'])
 
-        table_meta = next((i for i in metadata if isinstance(i, dict) and len(i.get('breadcrumb', [])) == 0),
-                          {}).get('metadata')
+        table_meta = next(
+            (
+                i
+                for i in metadata
+                if isinstance(i, dict) and len(i.get('breadcrumb', [])) == 0
+            ),
+            {},
+        ).get('metadata')
         selected = table_meta.get('selected', False)
         schema_name = table_meta.get('schema-name')
         db_name = table_meta.get('database-name')
@@ -100,11 +107,7 @@ def get_tables_from_properties(properties: Dict) -> set:
     return tables
 
 
-def get_bookmark_for_table(
-        table,
-        properties,
-        db_engine,
-        dbname=None):
+def get_bookmark_for_table(table, properties, db_engine, dbname=None):
     """Get actual bookmark for a specific table used for LOG_BASED or INCREMENTAL
     replications
     """
@@ -116,24 +119,37 @@ def get_bookmark_for_table(
         table_name = stream.get('table_name', stream['stream'])
 
         # Get table specific metadata i.e. replication method, replication key, etc.
-        table_meta = next((i for i in metadata if isinstance(i, dict) and len(i.get('breadcrumb', [])) == 0),
-                          {}).get('metadata')
+        table_meta = next(
+            (
+                i
+                for i in metadata
+                if isinstance(i, dict) and len(i.get('breadcrumb', [])) == 0
+            ),
+            {},
+        ).get('metadata')
         db_name = table_meta.get('database-name')
         schema_name = table_meta.get('schema-name')
         replication_method = table_meta.get('replication-method')
         replication_key = table_meta.get('replication-key')
 
-        fully_qualified_table_name = '{}.{}'.format(schema_name or db_name, table_name) \
-            if schema_name is not None or db_name is not None else table_name
+        fully_qualified_table_name = (
+            '{}.{}'.format(schema_name or db_name, table_name)
+            if schema_name is not None or db_name is not None
+            else table_name
+        )
 
-        if (dbname is None or db_name == dbname) and fully_qualified_table_name == table:
+        if (
+            dbname is None or db_name == dbname
+        ) and fully_qualified_table_name == table:
             # Log based replication: get mysql binlog position
             if replication_method == 'LOG_BASED':
                 bookmark = db_engine.fetch_current_log_pos()
 
             # Key based incremental replication: Get max replication key from source
             elif replication_method == 'INCREMENTAL':
-                bookmark = db_engine.fetch_current_incremental_key_pos(fully_qualified_table_name, replication_key)
+                bookmark = db_engine.fetch_current_incremental_key_pos(
+                    fully_qualified_table_name, replication_key
+                )
 
             break
 
@@ -155,7 +171,9 @@ def get_target_schema(target_config, table):
                                             }
     """
     target_schema = None
-    config_default_target_schema = target_config.get('default_target_schema', '').strip()
+    config_default_target_schema = target_config.get(
+        'default_target_schema', ''
+    ).strip()
     config_schema_mapping = target_config.get('schema_mapping', {})
 
     table_dict = tablename_to_dict(table)
@@ -168,7 +186,8 @@ def get_target_schema(target_config, table):
     if not target_schema:
         raise Exception(
             "Target schema name not defined in config. Neither 'default_target_schema' (string) nor 'schema_mapping' "
-            '(object) defines target schema for {} stream. '.format(table))
+            '(object) defines target schema for {} stream. '.format(table)
+        )
 
     return target_schema
 
@@ -202,13 +221,17 @@ def get_grantees(target_config, table):
                                                         }
     """
     grantees = []
-    config_default_target_schema_select_permissions = target_config.get('default_target_schema_select_permissions', [])
+    config_default_target_schema_select_permissions = target_config.get(
+        'default_target_schema_select_permissions', []
+    )
     config_schema_mapping = target_config.get('schema_mapping', {})
 
     table_dict = tablename_to_dict(table)
     table_schema = table_dict['schema_name']
     if config_schema_mapping and table_schema in config_schema_mapping:
-        grantees = config_schema_mapping[table_schema].get('target_schema_select_permissions', [])
+        grantees = config_schema_mapping[table_schema].get(
+            'target_schema_select_permissions', []
+        )
     elif config_default_target_schema_select_permissions:
         grantees = config_default_target_schema_select_permissions
 
@@ -248,9 +271,13 @@ def grant_privilege(schema, grantees, grant_method, to_group=False):
 def save_state_file(path, table, bookmark, dbname=None):
     table_dict = tablename_to_dict(table)
     if dbname:
-        stream_id = '{}-{}-{}'.format(dbname, table_dict.get('schema_name'), table_dict.get('table_name'))
+        stream_id = '{}-{}-{}'.format(
+            dbname, table_dict.get('schema_name'), table_dict.get('table_name')
+        )
     elif table_dict['schema_name']:
-        stream_id = '{}-{}'.format(table_dict['schema_name'], table_dict.get('table_name'))
+        stream_id = '{}-{}'.format(
+            table_dict['schema_name'], table_dict.get('table_name')
+        )
     else:
         stream_id = table_dict['table_name']
 
@@ -275,7 +302,6 @@ def save_state_file(path, table, bookmark, dbname=None):
     save_dict_to_json(path, state)
 
 
-
 def parse_args(required_config_keys: Dict) -> argparse.Namespace:
     """Parse standard command-line args.
 
@@ -299,8 +325,14 @@ def parse_args(required_config_keys: Dict) -> argparse.Namespace:
     parser.add_argument('--target', help='Target Config file', required=True)
     parser.add_argument('--transform', help='Transformations Config file')
     parser.add_argument('--tables', help='Sync only specific tables')
-    parser.add_argument('--temp_dir', help='Temporary directory required for CSV exports')
-    parser.add_argument('--drop_pg_slot', help='Drop pg replication slot before starting resync', action='store_true')
+    parser.add_argument(
+        '--temp_dir', help='Temporary directory required for CSV exports'
+    )
+    parser.add_argument(
+        '--drop_pg_slot',
+        help='Drop pg replication slot before starting resync',
+        action='store_true',
+    )
 
     args: argparse.Namespace = parser.parse_args()
 
@@ -348,15 +380,20 @@ def retry_pattern():
     import backoff
     from botocore.exceptions import ClientError
 
-    return backoff.on_exception(backoff.expo,
-                                ClientError,
-                                max_tries=5,
-                                on_backoff=log_backoff_attempt,
-                                factor=10)
+    return backoff.on_exception(
+        backoff.expo,
+        ClientError,
+        max_tries=5,
+        on_backoff=log_backoff_attempt,
+        factor=10,
+    )
 
 
 def log_backoff_attempt(details):
-    LOGGER.error('Error detected communicating with Amazon, triggering backoff: %s try', details.get('tries'))
+    LOGGER.error(
+        'Error detected communicating with Amazon, triggering backoff: %s try',
+        details.get('tries'),
+    )
 
 
 def get_pool_size(tap: Dict) -> int:
@@ -377,11 +414,9 @@ def get_pool_size(tap: Dict) -> int:
     return min(fastsync_parallelism, cpu_cores)
 
 
-def gen_export_filename(tap_id: str,
-                        table: str,
-                        suffix: str = None,
-                        postfix: str = None,
-                        ext: str = None) -> str:
+def gen_export_filename(
+    tap_id: str, table: str, suffix: str = None, postfix: str = None, ext: str = None
+) -> str:
     """
     Generates a unique filename used for exported fastsync data that avoids file name collision
 
@@ -407,8 +442,6 @@ def gen_export_filename(tap_id: str,
     if not ext:
         ext = 'csv.gz'
 
-    return 'pipelinewise_{}_{}_{}_fastsync_{}.{}'.format(tap_id,
-                                                         table,
-                                                         suffix,
-                                                         postfix,
-                                                         ext)
+    return 'pipelinewise_{}_{}_{}_fastsync_{}.{}'.format(
+        tap_id, table, suffix, postfix, ext
+    )
diff --git a/pipelinewise/fastsync/mongodb_to_bigquery.py b/pipelinewise/fastsync/mongodb_to_bigquery.py
index 9fc88b343..4e7b724ad 100644
--- a/pipelinewise/fastsync/mongodb_to_bigquery.py
+++ b/pipelinewise/fastsync/mongodb_to_bigquery.py
@@ -25,9 +25,7 @@
         'auth_database',
         'dbname',
     ],
-    'target': [
-        'project_id'
-    ]
+    'target': ['project_id'],
 }
 
 LOCK = multiprocessing.Lock()
@@ -53,7 +51,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
 
     try:
         dbname = args.tap.get('dbname')
-        filename = 'pipelinewise_fastsync_{}_{}_{}.csv'.format(dbname, table, time.strftime('%Y%m%d-%H%M%S'))
+        filename = 'pipelinewise_fastsync_{}_{}_{}.csv'.format(
+            dbname, table, time.strftime('%Y%m%d-%H%M%S')
+        )
         filepath = os.path.join(args.temp_dir, filename)
         target_schema = utils.get_target_schema(args.target, table)
 
@@ -61,7 +61,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
         mongodb.open_connection()
 
         # Get bookmark - LSN position or Incremental Key value
-        bookmark = utils.get_bookmark_for_table(table, args.properties, mongodb, dbname=dbname)
+        bookmark = utils.get_bookmark_for_table(
+            table, args.properties, mongodb, dbname=dbname
+        )
 
         # Exporting table data, get table definitions and close connection to avoid timeouts
         mongodb.copy_table(table, filepath, args.temp_dir)
@@ -75,7 +77,14 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
         bigquery.create_table(target_schema, table, bigquery_columns, is_temporary=True)
 
         # Load into Bigquery table
-        bigquery.copy_to_table(filepath, target_schema, table, size_bytes, is_temporary=True, skip_csv_header=True)
+        bigquery.copy_to_table(
+            filepath,
+            target_schema,
+            table,
+            size_bytes,
+            is_temporary=True,
+            skip_csv_header=True,
+        )
         os.remove(filepath)
 
         # Obfuscate columns
@@ -113,7 +122,8 @@ def main_impl():
     table_sync_excs = []
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -121,16 +131,25 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -141,8 +160,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs), str(table_sync_excs),
-                pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/mongodb_to_postgres.py b/pipelinewise/fastsync/mongodb_to_postgres.py
index 80003dde3..fe4643afb 100644
--- a/pipelinewise/fastsync/mongodb_to_postgres.py
+++ b/pipelinewise/fastsync/mongodb_to_postgres.py
@@ -24,12 +24,7 @@
         'auth_database',
         'dbname',
     ],
-    'target': [
-        'host',
-        'port',
-        'user',
-        'password'
-    ]
+    'target': ['host', 'port', 'user', 'password'],
 }
 
 LOCK = multiprocessing.Lock()
@@ -54,7 +49,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
     postgres = FastSyncTargetPostgres(args.target, args.transform)
 
     try:
-        filename = utils.gen_export_filename(tap_id=args.target.get('tap_id'), table=table)
+        filename = utils.gen_export_filename(
+            tap_id=args.target.get('tap_id'), table=table
+        )
         filepath = os.path.join(args.temp_dir, filename)
         target_schema = utils.get_target_schema(args.target, table)
 
@@ -62,7 +59,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
         mongodb.open_connection()
 
         # Get bookmark - token of the most recent ChangeStream for logbased
-        bookmark = utils.get_bookmark_for_table(table, args.properties, mongodb, dbname=args.tap.get('dbname'))
+        bookmark = utils.get_bookmark_for_table(
+            table, args.properties, mongodb, dbname=args.tap.get('dbname')
+        )
 
         # Exporting table data, get table definitions and close connection to avoid timeouts
         mongodb.copy_table(table, filepath, args.temp_dir)
@@ -74,10 +73,19 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
 
         # Creating temp table in Postgres
         postgres.drop_table(target_schema, table, is_temporary=True)
-        postgres.create_table(target_schema, table, postgres_columns, primary_key, is_temporary=True)
+        postgres.create_table(
+            target_schema, table, postgres_columns, primary_key, is_temporary=True
+        )
 
         # Load into Postgres table
-        postgres.copy_to_table(filepath, target_schema, table, size_bytes, is_temporary=True, skip_csv_header=True)
+        postgres.copy_to_table(
+            filepath,
+            target_schema,
+            table,
+            size_bytes,
+            is_temporary=True,
+            skip_csv_header=True,
+        )
         os.remove(filepath)
 
         # Obfuscate columns
@@ -114,7 +122,8 @@ def main_impl():
     table_sync_excs = []
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -122,7 +131,11 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # Create target schemas sequentially, Postgres doesn't like it running in parallel
     postgres_target = FastSyncTargetPostgres(args.target, args.transform)
@@ -131,11 +144,16 @@ def main_impl():
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -146,8 +164,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs), str(table_sync_excs),
-                pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/mongodb_to_snowflake.py b/pipelinewise/fastsync/mongodb_to_snowflake.py
index 2ffe06cc0..96fa606c1 100644
--- a/pipelinewise/fastsync/mongodb_to_snowflake.py
+++ b/pipelinewise/fastsync/mongodb_to_snowflake.py
@@ -32,8 +32,8 @@
         'warehouse',
         's3_bucket',
         'stage',
-        'file_format'
-    ]
+        'file_format',
+    ],
 }
 
 LOCK = multiprocessing.Lock()
@@ -69,7 +69,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
         mongodb.open_connection()
 
         # Get bookmark - LSN position or Incremental Key value
-        bookmark = utils.get_bookmark_for_table(table, args.properties, mongodb, dbname=dbname)
+        bookmark = utils.get_bookmark_for_table(
+            table, args.properties, mongodb, dbname=dbname
+        )
 
         # Exporting table data, get table definitions and close connection to avoid timeouts
         mongodb.copy_table(table, filepath, args.temp_dir)
@@ -85,10 +87,19 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
 
         # Creating temp table in Snowflake
         snowflake.create_schema(target_schema)
-        snowflake.create_table(target_schema, table, snowflake_columns, primary_key, is_temporary=True)
+        snowflake.create_table(
+            target_schema, table, snowflake_columns, primary_key, is_temporary=True
+        )
 
         # Load into Snowflake table
-        snowflake.copy_to_table(s3_key, target_schema, table, size_bytes, is_temporary=True, skip_csv_header=True)
+        snowflake.copy_to_table(
+            s3_key,
+            target_schema,
+            table,
+            size_bytes,
+            is_temporary=True,
+            skip_csv_header=True,
+        )
 
         if archive_load_files:
             # Copy load file to archive
@@ -132,7 +143,8 @@ def main_impl():
     table_sync_excs = []
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -140,16 +152,25 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -160,8 +181,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs), str(table_sync_excs),
-                pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/mysql_to_bigquery.py b/pipelinewise/fastsync/mysql_to_bigquery.py
index 4fa2e92ad..b8529f39d 100644
--- a/pipelinewise/fastsync/mysql_to_bigquery.py
+++ b/pipelinewise/fastsync/mysql_to_bigquery.py
@@ -14,20 +14,15 @@
 from .commons.tap_mysql import FastSyncTapMySql
 from .commons.target_bigquery import FastSyncTargetBigquery
 
-MAX_NUM='99999999999999999999999999999.999999999'
+MAX_NUM = '99999999999999999999999999999.999999999'
 
 LOGGER = logging.getLogger(__name__)
 
 REQUIRED_CONFIG_KEYS = {
-    'tap': [
-        'host',
-        'port',
-        'user',
-        'password'
-    ],
+    'tap': ['host', 'port', 'user', 'password'],
     'target': [
         'project_id',
-    ]
+    ],
 }
 
 LOCK = multiprocessing.Lock()
@@ -36,35 +31,35 @@
 def tap_type_to_target_type(mysql_type, mysql_column_type):
     """Data type mapping from MySQL to Bigquery"""
     return {
-        'char':'STRING',
-        'varchar':'STRING',
-        'binary':'STRING',
-        'varbinary':'STRING',
-        'blob':'STRING',
-        'tinyblob':'STRING',
-        'mediumblob':'STRING',
-        'longblob':'STRING',
-        'geometry':'STRING',
-        'text':'STRING',
-        'tinytext':'STRING',
-        'mediumtext':'STRING',
-        'longtext':'STRING',
-        'enum':'STRING',
-        'int':'INT64',
-        'tinyint':'BOOL' if mysql_column_type == 'tinyint(1)' else 'INT64',
-        'smallint':'INT64',
-        'mediumint':'INT64',
-        'bigint':'INT64',
-        'bit':'BOOL',
-        'decimal':'NUMERIC',
-        'double':'NUMERIC',
-        'float':'NUMERIC',
-        'bool':'BOOL',
-        'boolean':'BOOL',
-        'date':'TIMESTAMP',
-        'datetime':'TIMESTAMP',
-        'timestamp':'TIMESTAMP',
-        'time':'TIME'
+        'char': 'STRING',
+        'varchar': 'STRING',
+        'binary': 'STRING',
+        'varbinary': 'STRING',
+        'blob': 'STRING',
+        'tinyblob': 'STRING',
+        'mediumblob': 'STRING',
+        'longblob': 'STRING',
+        'geometry': 'STRING',
+        'text': 'STRING',
+        'tinytext': 'STRING',
+        'mediumtext': 'STRING',
+        'longtext': 'STRING',
+        'enum': 'STRING',
+        'int': 'INT64',
+        'tinyint': 'BOOL' if mysql_column_type == 'tinyint(1)' else 'INT64',
+        'smallint': 'INT64',
+        'mediumint': 'INT64',
+        'bigint': 'INT64',
+        'bit': 'BOOL',
+        'decimal': 'NUMERIC',
+        'double': 'NUMERIC',
+        'float': 'NUMERIC',
+        'bool': 'BOOL',
+        'boolean': 'BOOL',
+        'date': 'TIMESTAMP',
+        'datetime': 'TIMESTAMP',
+        'timestamp': 'TIMESTAMP',
+        'time': 'TIME',
     }.get(mysql_type, 'STRING')
 
 
@@ -75,7 +70,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
     bigquery = FastSyncTargetBigquery(args.target, args.transform)
 
     try:
-        filename = 'pipelinewise_fastsync_{}_{}.csv'.format(table, time.strftime('%Y%m%d-%H%M%S'))
+        filename = 'pipelinewise_fastsync_{}_{}.csv'.format(
+            table, time.strftime('%Y%m%d-%H%M%S')
+        )
         filepath = os.path.join(args.temp_dir, filename)
         target_schema = utils.get_target_schema(args.target, table)
 
@@ -86,11 +83,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
         bookmark = utils.get_bookmark_for_table(table, args.properties, mysql)
 
         # Exporting table data, get table definitions and close connection to avoid timeouts
-        mysql.copy_table(table,
-                         filepath,
-                         compress=False,
-                         max_num=MAX_NUM,
-                         date_type='datetime')
+        mysql.copy_table(
+            table, filepath, compress=False, max_num=MAX_NUM, date_type='datetime'
+        )
         file_parts = glob.glob(f'{filepath}*')
         size_bytes = sum([os.path.getsize(file_part) for file_part in file_parts])
         bigquery_types = mysql.map_column_types_to_target(table)
@@ -110,7 +105,8 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
                 table,
                 size_bytes,
                 is_temporary=True,
-                write_truncate=write_truncate)
+                write_truncate=write_truncate,
+            )
             os.remove(file_part)
 
         # Obfuscate columns
@@ -148,7 +144,8 @@ def main_impl():
     table_sync_excs = []
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -156,16 +153,25 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -176,8 +182,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs), str(table_sync_excs),
-                pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/mysql_to_postgres.py b/pipelinewise/fastsync/mysql_to_postgres.py
index 0821c98a6..20d8d2ecb 100644
--- a/pipelinewise/fastsync/mysql_to_postgres.py
+++ b/pipelinewise/fastsync/mysql_to_postgres.py
@@ -15,18 +15,8 @@
 LOGGER = Logger().get_logger(__name__)
 
 REQUIRED_CONFIG_KEYS = {
-    'tap': [
-        'host',
-        'port',
-        'user',
-        'password'
-    ],
-    'target': [
-        'host',
-        'port',
-        'user',
-        'password'
-    ]
+    'tap': ['host', 'port', 'user', 'password'],
+    'target': ['host', 'port', 'user', 'password'],
 }
 
 LOCK = multiprocessing.Lock()
@@ -57,7 +47,9 @@ def tap_type_to_target_type(mysql_type, mysql_column_type):
         'longtext': 'CHARACTER VARYING',
         'enum': 'CHARACTER VARYING',
         'int': 'INTEGER NULL',
-        'tinyint': 'BOOLEAN' if mysql_column_type and mysql_column_type.startswith('tinyint(1)') else 'SMALLINT NULL',
+        'tinyint': 'BOOLEAN'
+        if mysql_column_type and mysql_column_type.startswith('tinyint(1)')
+        else 'SMALLINT NULL',
         'smallint': 'SMALLINT NULL',
         'mediumint': 'INTEGER NULL',
         'bigint': 'BIGINT NULL',
@@ -84,7 +76,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
     postgres = FastSyncTargetPostgres(args.target, args.transform)
 
     try:
-        filename = utils.gen_export_filename(tap_id=args.target.get('tap_id'), table=table)
+        filename = utils.gen_export_filename(
+            tap_id=args.target.get('tap_id'), table=table
+        )
         filepath = os.path.join(args.temp_dir, filename)
         target_schema = utils.get_target_schema(args.target, table)
 
@@ -104,10 +98,14 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
 
         # Creating temp table in Postgres
         postgres.drop_table(target_schema, table, is_temporary=True)
-        postgres.create_table(target_schema, table, postgres_columns, primary_key, is_temporary=True)
+        postgres.create_table(
+            target_schema, table, postgres_columns, primary_key, is_temporary=True
+        )
 
         # Load into Postgres table
-        postgres.copy_to_table(filepath, target_schema, table, size_bytes, is_temporary=True)
+        postgres.copy_to_table(
+            filepath, target_schema, table, size_bytes, is_temporary=True
+        )
         os.remove(filepath)
 
         # Obfuscate columns
@@ -144,7 +142,8 @@ def main_impl():
     table_sync_excs = []
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -152,7 +151,11 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # Create target schemas sequentially, Postgres doesn't like it running in parallel
     postgres_target = FastSyncTargetPostgres(args.target, args.transform)
@@ -161,11 +164,16 @@ def main_impl():
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -176,8 +184,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs), str(table_sync_excs),
-                pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/mysql_to_redshift.py b/pipelinewise/fastsync/mysql_to_redshift.py
index 8bb948f32..94ac6afbd 100644
--- a/pipelinewise/fastsync/mysql_to_redshift.py
+++ b/pipelinewise/fastsync/mysql_to_redshift.py
@@ -15,20 +15,8 @@
 LOGGER = Logger().get_logger(__name__)
 
 REQUIRED_CONFIG_KEYS = {
-    'tap': [
-        'host',
-        'port',
-        'user',
-        'password'
-    ],
-    'target': [
-        'host',
-        'port',
-        'user',
-        'password',
-        'dbname',
-        's3_bucket'
-    ]
+    'tap': ['host', 'port', 'user', 'password'],
+    'target': ['host', 'port', 'user', 'password', 'dbname', 's3_bucket'],
 }
 
 DEFAULT_VARCHAR_LENGTH = 10000
@@ -63,7 +51,9 @@ def tap_type_to_target_type(mysql_type, mysql_column_type):
         'longtext': 'CHARACTER VARYING({})'.format(LONG_VARCHAR_LENGTH),
         'enum': 'CHARACTER VARYING({})'.format(DEFAULT_VARCHAR_LENGTH),
         'int': 'NUMERIC NULL',
-        'tinyint': 'BOOLEAN' if mysql_column_type and mysql_column_type.startswith('tinyint(1)') else 'NUMERIC NULL',
+        'tinyint': 'BOOLEAN'
+        if mysql_column_type and mysql_column_type.startswith('tinyint(1)')
+        else 'NUMERIC NULL',
         'smallint': 'NUMERIC NULL',
         'mediumint': 'NUMERIC NULL',
         'bigint': 'NUMERIC NULL',
@@ -76,7 +66,7 @@ def tap_type_to_target_type(mysql_type, mysql_column_type):
         'date': 'TIMESTAMP WITHOUT TIME ZONE',
         'datetime': 'TIMESTAMP WITHOUT TIME ZONE',
         'timestamp': 'TIMESTAMP WITHOUT TIME ZONE',
-        'json': 'CHARACTER VARYING({})'.format(LONG_VARCHAR_LENGTH)
+        'json': 'CHARACTER VARYING({})'.format(LONG_VARCHAR_LENGTH),
     }.get(
         mysql_type,
         'CHARACTER VARYING({})'.format(DEFAULT_VARCHAR_LENGTH),
@@ -89,7 +79,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
     redshift = FastSyncTargetRedshift(args.target, args.transform)
 
     try:
-        filename = utils.gen_export_filename(tap_id=args.target.get('tap_id'), table=table)
+        filename = utils.gen_export_filename(
+            tap_id=args.target.get('tap_id'), table=table
+        )
         filepath = os.path.join(args.temp_dir, filename)
         target_schema = utils.get_target_schema(args.target, table)
 
@@ -113,10 +105,14 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
 
         # Creating temp table in Redshift
         redshift.drop_table(target_schema, table, is_temporary=True)
-        redshift.create_table(target_schema, table, redshift_columns, primary_key, is_temporary=True)
+        redshift.create_table(
+            target_schema, table, redshift_columns, primary_key, is_temporary=True
+        )
 
         # Load into Redshift table
-        redshift.copy_to_table(s3_key, target_schema, table, size_bytes, is_temporary=True)
+        redshift.copy_to_table(
+            s3_key, target_schema, table, size_bytes, is_temporary=True
+        )
 
         # Obfuscate columns
         redshift.obfuscate_columns(target_schema, table)
@@ -152,7 +148,8 @@ def main_impl():
     table_sync_excs = []
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -160,7 +157,11 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # Create target schemas sequentially, Redshift doesn't like it running in parallel
     redshift = FastSyncTargetRedshift(args.target, args.transform)
@@ -169,11 +170,16 @@ def main_impl():
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -184,8 +190,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs), str(table_sync_excs),
-                pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/mysql_to_snowflake.py b/pipelinewise/fastsync/mysql_to_snowflake.py
index 2b37ffd4e..2eb722e17 100644
--- a/pipelinewise/fastsync/mysql_to_snowflake.py
+++ b/pipelinewise/fastsync/mysql_to_snowflake.py
@@ -17,12 +17,7 @@
 LOGGER = Logger().get_logger(__name__)
 
 REQUIRED_CONFIG_KEYS = {
-    'tap': [
-        'host',
-        'port',
-        'user',
-        'password'
-    ],
+    'tap': ['host', 'port', 'user', 'password'],
     'target': [
         'account',
         'dbname',
@@ -31,8 +26,8 @@
         'warehouse',
         's3_bucket',
         'stage',
-        'file_format'
-    ]
+        'file_format',
+    ],
 }
 
 LOCK = multiprocessing.Lock()
@@ -63,7 +58,9 @@ def tap_type_to_target_type(mysql_type, mysql_column_type):
         'longtext': 'VARCHAR',
         'enum': 'VARCHAR',
         'int': 'NUMBER',
-        'tinyint': 'BOOLEAN' if mysql_column_type and mysql_column_type.startswith('tinyint(1)') else 'NUMBER',
+        'tinyint': 'BOOLEAN'
+        if mysql_column_type and mysql_column_type.startswith('tinyint(1)')
+        else 'NUMBER',
         'smallint': 'NUMBER',
         'mediumint': 'NUMBER',
         'bigint': 'NUMBER',
@@ -77,7 +74,7 @@ def tap_type_to_target_type(mysql_type, mysql_column_type):
         'datetime': 'TIMESTAMP_NTZ',
         'timestamp': 'TIMESTAMP_NTZ',
         'time': 'TIME',
-        'json': 'VARIANT'
+        'json': 'VARIANT',
     }.get(mysql_type, 'VARCHAR')
 
 
@@ -89,7 +86,6 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
     tap_id = args.target.get('tap_id')
     archive_load_files = args.target.get('archive_load_files', False)
 
-
     try:
         filename = utils.gen_export_filename(tap_id=tap_id, table=table)
         filepath = os.path.join(args.temp_dir, filename)
@@ -102,11 +98,13 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
         bookmark = utils.get_bookmark_for_table(table, args.properties, mysql)
 
         # Exporting table data, get table definitions and close connection to avoid timeouts
-        mysql.copy_table(table,
-                         filepath,
-                         split_large_files=args.target.get('split_large_files'),
-                         split_file_chunk_size_mb=args.target.get('split_file_chunk_size_mb'),
-                         split_file_max_chunks=args.target.get('split_file_max_chunks'))
+        mysql.copy_table(
+            table,
+            filepath,
+            split_large_files=args.target.get('split_large_files'),
+            split_file_chunk_size_mb=args.target.get('split_file_chunk_size_mb'),
+            split_file_max_chunks=args.target.get('split_file_max_chunks'),
+        )
         file_parts = glob.glob(f'{filepath}*')
         size_bytes = sum([os.path.getsize(file_part) for file_part in file_parts])
         snowflake_types = mysql.map_column_types_to_target(table)
@@ -121,14 +119,22 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
             os.remove(file_part)
 
         # Create a pattern that match all file parts by removing multipart suffix
-        s3_key_pattern = re.sub(r'\.part\d*$', '', s3_keys[0]) if len(s3_keys) > 0 else 'NO_FILES_TO_LOAD'
+        s3_key_pattern = (
+            re.sub(r'\.part\d*$', '', s3_keys[0])
+            if len(s3_keys) > 0
+            else 'NO_FILES_TO_LOAD'
+        )
 
         # Creating temp table in Snowflake
         snowflake.create_schema(target_schema)
-        snowflake.create_table(target_schema, table, snowflake_columns, primary_key, is_temporary=True)
+        snowflake.create_table(
+            target_schema, table, snowflake_columns, primary_key, is_temporary=True
+        )
 
         # Load into Snowflake table
-        snowflake.copy_to_table(s3_key_pattern, target_schema, table, size_bytes, is_temporary=True)
+        snowflake.copy_to_table(
+            s3_key_pattern, target_schema, table, size_bytes, is_temporary=True
+        )
 
         for s3_key in s3_keys:
             if archive_load_files:
@@ -173,7 +179,8 @@ def main_impl():
     table_sync_excs = []
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -181,16 +188,25 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -201,8 +217,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs), str(table_sync_excs),
-                pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/postgres_to_bigquery.py b/pipelinewise/fastsync/postgres_to_bigquery.py
index 3e75ea5e5..8bc207886 100644
--- a/pipelinewise/fastsync/postgres_to_bigquery.py
+++ b/pipelinewise/fastsync/postgres_to_bigquery.py
@@ -14,58 +14,52 @@
 from .commons.tap_postgres import FastSyncTapPostgres
 from .commons.target_bigquery import FastSyncTargetBigquery
 
-MAX_NUM='99999999999999999999999999999.999999999'
+MAX_NUM = '99999999999999999999999999999.999999999'
 
 LOGGER = logging.getLogger(__name__)
 
 REQUIRED_CONFIG_KEYS = {
-    'tap': [
-        'host',
-        'port',
-        'user',
-        'password'
-    ],
-    'target': [
-        'project_id'
-    ]
+    'tap': ['host', 'port', 'user', 'password'],
+    'target': ['project_id'],
 }
 
 LOCK = multiprocessing.Lock()
 
+
 def tap_type_to_target_type(pg_type):
     """Data type mapping from Postgres to Bigquery"""
     return {
-        'char':'STRING',
-        'character':'STRING',
-        'varchar':'STRING',
-        'character varying':'STRING',
-        'text':'STRING',
+        'char': 'STRING',
+        'character': 'STRING',
+        'varchar': 'STRING',
+        'character varying': 'STRING',
+        'text': 'STRING',
         'bit': ['BOOL', 'NUMERIC'],
-        'varbit':'NUMERIC',
-        'bit varying':'NUMERIC',
-        'smallint':'INT64',
-        'int':'INT64',
-        'integer':'INT64',
-        'bigint':'INT64',
-        'smallserial':'INT64',
-        'serial':'INT64',
-        'bigserial':'INT64',
-        'numeric':'NUMERIC',
-        'double precision':'NUMERIC',
-        'real':'NUMERIC',
-        'bool':'BOOL',
-        'boolean':'BOOL',
-        'date':'TIMESTAMP',
-        'timestamp':'TIMESTAMP',
-        'timestamp without time zone':'TIMESTAMP',
-        'timestamp with time zone':'TIMESTAMP',
-        'time':'TIME',
-        'time without time zone':'TIME',
-        'time with time zone':'TIME',
+        'varbit': 'NUMERIC',
+        'bit varying': 'NUMERIC',
+        'smallint': 'INT64',
+        'int': 'INT64',
+        'integer': 'INT64',
+        'bigint': 'INT64',
+        'smallserial': 'INT64',
+        'serial': 'INT64',
+        'bigserial': 'INT64',
+        'numeric': 'NUMERIC',
+        'double precision': 'NUMERIC',
+        'real': 'NUMERIC',
+        'bool': 'BOOL',
+        'boolean': 'BOOL',
+        'date': 'TIMESTAMP',
+        'timestamp': 'TIMESTAMP',
+        'timestamp without time zone': 'TIMESTAMP',
+        'timestamp with time zone': 'TIMESTAMP',
+        'time': 'TIME',
+        'time without time zone': 'TIME',
+        'time with time zone': 'TIME',
         # This is all uppercase, because postgres stores it in this format in information_schema.columns.data_type
-        'ARRAY':'STRING',
-        'json':'STRING',
-        'jsonb':'STRING'
+        'ARRAY': 'STRING',
+        'json': 'STRING',
+        'jsonb': 'STRING',
     }.get(pg_type, 'STRING')
 
 
@@ -77,7 +71,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
 
     try:
         dbname = args.tap.get('dbname')
-        filename = 'pipelinewise_fastsync_{}_{}_{}.csv'.format(dbname, table, time.strftime('%Y%m%d-%H%M%S'))
+        filename = 'pipelinewise_fastsync_{}_{}_{}.csv'.format(
+            dbname, table, time.strftime('%Y%m%d-%H%M%S')
+        )
         filepath = os.path.join(args.temp_dir, filename)
         target_schema = utils.get_target_schema(args.target, table)
 
@@ -85,14 +81,14 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
         postgres.open_connection()
 
         # Get bookmark - LSN position or Incremental Key value
-        bookmark = utils.get_bookmark_for_table(table, args.properties, postgres, dbname=dbname)
+        bookmark = utils.get_bookmark_for_table(
+            table, args.properties, postgres, dbname=dbname
+        )
 
         # Exporting table data, get table definitions and close connection to avoid timeouts
-        postgres.copy_table(table,
-                            filepath,
-                            compress=False,
-                            max_num=MAX_NUM,
-                            date_type='timestamp')
+        postgres.copy_table(
+            table, filepath, compress=False, max_num=MAX_NUM, date_type='timestamp'
+        )
         file_parts = glob.glob(f'{filepath}*')
         size_bytes = sum([os.path.getsize(file_part) for file_part in file_parts])
 
@@ -113,7 +109,8 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
                 table,
                 size_bytes,
                 is_temporary=True,
-                write_truncate=write_truncate)
+                write_truncate=write_truncate,
+            )
             os.remove(file_part)
 
         # Obfuscate columns
@@ -151,7 +148,8 @@ def main_impl():
     table_sync_excs = []
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -159,7 +157,11 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # if internal arg drop_pg_slot is set to True, then we drop the slot before starting resync
     if args.drop_pg_slot:
@@ -168,11 +170,16 @@ def main_impl():
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -183,8 +190,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs), str(table_sync_excs),
-                pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/postgres_to_postgres.py b/pipelinewise/fastsync/postgres_to_postgres.py
index fd4e1e2a3..728d0c140 100644
--- a/pipelinewise/fastsync/postgres_to_postgres.py
+++ b/pipelinewise/fastsync/postgres_to_postgres.py
@@ -22,14 +22,9 @@
         'user',
         'password',
         'dbname',
-        'tap_id'  # tap_id is required to generate unique replication slot names
+        'tap_id',  # tap_id is required to generate unique replication slot names
     ],
-    'target': [
-        'host',
-        'port',
-        'user',
-        'password'
-    ]
+    'target': ['host', 'port', 'user', 'password'],
 }
 
 LOCK = multiprocessing.Lock()
@@ -68,7 +63,7 @@ def tap_type_to_target_type(pg_type):
         # ARRAY is uppercase, because postgres stores it in this format in information_schema.columns.data_type
         'ARRAY': 'JSONB',
         'json': 'JSONB',
-        'jsonb': 'JSONB'
+        'jsonb': 'JSONB',
     }.get(pg_type, 'CHARACTER VARYING')
 
 
@@ -79,7 +74,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
 
     try:
         dbname = args.tap.get('dbname')
-        filename = utils.gen_export_filename(tap_id=args.target.get('tap_id'), table=table)
+        filename = utils.gen_export_filename(
+            tap_id=args.target.get('tap_id'), table=table
+        )
         filepath = os.path.join(args.temp_dir, filename)
         target_schema = utils.get_target_schema(args.target, table)
 
@@ -87,7 +84,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
         postgres.open_connection()
 
         # Get bookmark - LSN position or Incremental Key value
-        bookmark = utils.get_bookmark_for_table(table, args.properties, postgres, dbname=dbname)
+        bookmark = utils.get_bookmark_for_table(
+            table, args.properties, postgres, dbname=dbname
+        )
 
         # Exporting table data, get table definitions and close connection to avoid timeouts
         postgres.copy_table(table, filepath)
@@ -99,10 +98,18 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
 
         # Creating temp table in Postgres
         postgres_target.drop_table(target_schema, table, is_temporary=True)
-        postgres_target.create_table(target_schema, table, postgres_target_columns, primary_key, is_temporary=True)
+        postgres_target.create_table(
+            target_schema,
+            table,
+            postgres_target_columns,
+            primary_key,
+            is_temporary=True,
+        )
 
         # Load into Postgres table
-        postgres_target.copy_to_table(filepath, target_schema, table, size_bytes, is_temporary=True)
+        postgres_target.copy_to_table(
+            filepath, target_schema, table, size_bytes, is_temporary=True
+        )
         os.remove(filepath)
 
         # Obfuscate columns
@@ -121,8 +128,12 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
 
         # Table loaded, grant select on all tables in target schema
         grantees = utils.get_grantees(args.target, table)
-        utils.grant_privilege(target_schema, grantees, postgres_target.grant_usage_on_schema)
-        utils.grant_privilege(target_schema, grantees, postgres_target.grant_select_on_schema)
+        utils.grant_privilege(
+            target_schema, grantees, postgres_target.grant_usage_on_schema
+        )
+        utils.grant_privilege(
+            target_schema, grantees, postgres_target.grant_select_on_schema
+        )
 
         return True
 
@@ -139,7 +150,8 @@ def main_impl():
     table_sync_excs = []
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -147,7 +159,11 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # if internal arg drop_pg_slot is set to True, then we drop the slot before starting resync
     if args.drop_pg_slot:
@@ -160,11 +176,16 @@ def main_impl():
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -175,8 +196,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs), str(table_sync_excs),
-                pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/postgres_to_redshift.py b/pipelinewise/fastsync/postgres_to_redshift.py
index f47a1dcba..ab45de889 100644
--- a/pipelinewise/fastsync/postgres_to_redshift.py
+++ b/pipelinewise/fastsync/postgres_to_redshift.py
@@ -16,20 +16,8 @@
 LOGGER = Logger().get_logger(__name__)
 
 REQUIRED_CONFIG_KEYS = {
-    'tap': [
-        'host',
-        'port',
-        'user',
-        'password'
-    ],
-    'target': [
-        'host',
-        'port',
-        'user',
-        'password',
-        'dbname',
-        's3_bucket'
-    ]
+    'tap': ['host', 'port', 'user', 'password'],
+    'target': ['host', 'port', 'user', 'password', 'dbname', 's3_bucket'],
 }
 
 DEFAULT_VARCHAR_LENGTH = 10000
@@ -84,7 +72,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
 
     try:
         dbname = args.tap.get('dbname')
-        filename = utils.gen_export_filename(tap_id=args.target.get('tap_id'), table=table)
+        filename = utils.gen_export_filename(
+            tap_id=args.target.get('tap_id'), table=table
+        )
         filepath = os.path.join(args.temp_dir, filename)
         target_schema = utils.get_target_schema(args.target, table)
 
@@ -92,7 +82,9 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
         postgres.open_connection()
 
         # Get bookmark - LSN position or Incremental Key value
-        bookmark = utils.get_bookmark_for_table(table, args.properties, postgres, dbname=dbname)
+        bookmark = utils.get_bookmark_for_table(
+            table, args.properties, postgres, dbname=dbname
+        )
 
         # Exporting table data, get table definitions and close connection to avoid timeouts
         postgres.copy_table(table, filepath)
@@ -108,10 +100,14 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
 
         # Creating temp table in Redshift
         redshift.drop_table(target_schema, table, is_temporary=True)
-        redshift.create_table(target_schema, table, redshift_columns, primary_key, is_temporary=True)
+        redshift.create_table(
+            target_schema, table, redshift_columns, primary_key, is_temporary=True
+        )
 
         # Load into Redshift table
-        redshift.copy_to_table(s3_key, target_schema, table, size_bytes, is_temporary=True)
+        redshift.copy_to_table(
+            s3_key, target_schema, table, size_bytes, is_temporary=True
+        )
 
         # Obfuscate columns
         redshift.obfuscate_columns(target_schema, table)
@@ -147,7 +143,8 @@ def main_impl():
     table_sync_excs = []
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -155,7 +152,11 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # if internal arg drop_pg_slot is set to True, then we drop the slot before starting resync
     if args.drop_pg_slot:
@@ -168,11 +169,16 @@ def main_impl():
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -183,8 +189,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs), str(table_sync_excs),
-                pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/postgres_to_snowflake.py b/pipelinewise/fastsync/postgres_to_snowflake.py
index 3d9d4eca2..867f2b96a 100644
--- a/pipelinewise/fastsync/postgres_to_snowflake.py
+++ b/pipelinewise/fastsync/postgres_to_snowflake.py
@@ -24,7 +24,7 @@
         'user',
         'password',
         'dbname',
-        'tap_id'    # tap_id is required to generate unique replication slot names
+        'tap_id',  # tap_id is required to generate unique replication slot names
     ],
     'target': [
         'account',
@@ -34,8 +34,8 @@
         'warehouse',
         's3_bucket',
         'stage',
-        'file_format'
-    ]
+        'file_format',
+    ],
 }
 
 LOCK = multiprocessing.Lock()
@@ -74,7 +74,7 @@ def tap_type_to_target_type(pg_type):
         # ARRAY is uppercase, because postgres stores it in this format in information_schema.columns.data_type
         'ARRAY': 'VARIANT',
         'json': 'VARIANT',
-        'jsonb': 'VARIANT'
+        'jsonb': 'VARIANT',
     }.get(pg_type, 'VARCHAR')
 
 
@@ -96,14 +96,18 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
         postgres.open_connection()
 
         # Get bookmark - LSN position or Incremental Key value
-        bookmark = utils.get_bookmark_for_table(table, args.properties, postgres, dbname=dbname)
+        bookmark = utils.get_bookmark_for_table(
+            table, args.properties, postgres, dbname=dbname
+        )
 
         # Exporting table data, get table definitions and close connection to avoid timeouts
-        postgres.copy_table(table,
-                            filepath,
-                            split_large_files=args.target.get('split_large_files'),
-                            split_file_chunk_size_mb=args.target.get('split_file_chunk_size_mb'),
-                            split_file_max_chunks=args.target.get('split_file_max_chunks'))
+        postgres.copy_table(
+            table,
+            filepath,
+            split_large_files=args.target.get('split_large_files'),
+            split_file_chunk_size_mb=args.target.get('split_file_chunk_size_mb'),
+            split_file_max_chunks=args.target.get('split_file_max_chunks'),
+        )
         file_parts = glob.glob(f'{filepath}*')
         size_bytes = sum([os.path.getsize(file_part) for file_part in file_parts])
         snowflake_types = postgres.map_column_types_to_target(table)
@@ -118,14 +122,22 @@ def sync_table(table: str, args: Namespace) -> Union[bool, str]:
             os.remove(file_part)
 
         # Create a pattern that match all file parts by removing multipart suffix
-        s3_key_pattern = re.sub(r'\.part\d*$', '', s3_keys[0]) if len(s3_keys) > 0 else 'NO_FILES_TO_LOAD'
+        s3_key_pattern = (
+            re.sub(r'\.part\d*$', '', s3_keys[0])
+            if len(s3_keys) > 0
+            else 'NO_FILES_TO_LOAD'
+        )
 
         # Creating temp table in Snowflake
         snowflake.create_schema(target_schema)
-        snowflake.create_table(target_schema, table, snowflake_columns, primary_key, is_temporary=True)
+        snowflake.create_table(
+            target_schema, table, snowflake_columns, primary_key, is_temporary=True
+        )
 
         # Load into Snowflake table
-        snowflake.copy_to_table(s3_key_pattern, target_schema, table, size_bytes, is_temporary=True)
+        snowflake.copy_to_table(
+            s3_key_pattern, target_schema, table, size_bytes, is_temporary=True
+        )
 
         for s3_key in s3_keys:
             if archive_load_files:
@@ -170,7 +182,8 @@ def main_impl():
     table_sync_excs = []
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -178,7 +191,11 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # if internal arg drop_pg_slot is set to True, then we drop the slot before starting resync
     if args.drop_pg_slot:
@@ -187,11 +204,16 @@ def main_impl():
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -202,8 +224,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs), str(table_sync_excs),
-                pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/s3_csv_to_bigquery.py b/pipelinewise/fastsync/s3_csv_to_bigquery.py
index 859f582d9..7c81c30fe 100644
--- a/pipelinewise/fastsync/s3_csv_to_bigquery.py
+++ b/pipelinewise/fastsync/s3_csv_to_bigquery.py
@@ -16,14 +16,10 @@
 
 
 REQUIRED_CONFIG_KEYS = {
-    'tap': [
-        'bucket',
-        'start_date'
-    ],
+    'tap': ['bucket', 'start_date'],
     'target': [
-
         'project_id',
-    ]
+    ],
 }
 
 LOCK = multiprocessing.Lock()
@@ -38,8 +34,7 @@ def tap_type_to_target_type(csv_type):
         'string': 'STRING',
         'boolean': 'STRING',  # The guess sometimes can be wrong, we'll use string for now.
         'date': 'STRING',  # The guess sometimes can be wrong, we'll use string for now.
-
-        'date_override': 'TIMESTAMP'  # Column type to use when date_override defined in YAML
+        'date_override': 'TIMESTAMP',  # Column type to use when date_override defined in YAML
     }.get(csv_type, 'STRING')
 
 
@@ -49,7 +44,9 @@ def sync_table(table_name: str, args: Namespace) -> Union[bool, str]:
     bigquery = FastSyncTargetBigquery(args.target, args.transform)
 
     try:
-        filename = utils.gen_export_filename(tap_id=args.target.get('tap_id'), table=table_name)
+        filename = utils.gen_export_filename(
+            tap_id=args.target.get('tap_id'), table=table_name
+        )
         filepath = os.path.join(args.temp_dir, filename)
 
         target_schema = utils.get_target_schema(args.target, table_name)
@@ -62,14 +59,23 @@ def sync_table(table_name: str, args: Namespace) -> Union[bool, str]:
 
         # Creating temp table in Bigquery
         bigquery.create_schema(target_schema)
-        bigquery.create_table(target_schema,
-                               table_name,
-                               bigquery_columns,
-                               is_temporary=True,
-                               sort_columns=True)
+        bigquery.create_table(
+            target_schema,
+            table_name,
+            bigquery_columns,
+            is_temporary=True,
+            sort_columns=True,
+        )
 
         # Load into Bigquery table
-        bigquery.copy_to_table(filepath, target_schema, table_name, size_bytes, is_temporary=True, skip_csv_header=True)
+        bigquery.copy_to_table(
+            filepath,
+            target_schema,
+            table_name,
+            size_bytes,
+            is_temporary=True,
+            skip_csv_header=True,
+        )
         os.remove(filepath)
 
         # Obfuscate columns
@@ -109,7 +115,8 @@ def main_impl():
     start_time = datetime.now()
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -117,17 +124,26 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # Start loading tables in parallel in spawning processes by
     # utilising all available Pool size
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -138,8 +154,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs),
-                str(table_sync_excs), pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/s3_csv_to_postgres.py b/pipelinewise/fastsync/s3_csv_to_postgres.py
index 4b8a9ed67..6ac9639f6 100644
--- a/pipelinewise/fastsync/s3_csv_to_postgres.py
+++ b/pipelinewise/fastsync/s3_csv_to_postgres.py
@@ -15,16 +15,8 @@
 LOGGER = Logger().get_logger(__name__)
 
 REQUIRED_CONFIG_KEYS = {
-    'tap': [
-        'bucket',
-        'start_date'
-    ],
-    'target': [
-        'host',
-        'port',
-        'user',
-        'password'
-    ]
+    'tap': ['bucket', 'start_date'],
+    'target': ['host', 'port', 'user', 'password'],
 }
 
 LOCK = multiprocessing.Lock()
@@ -39,8 +31,7 @@ def tap_type_to_target_type(csv_type):
         'string': 'CHARACTER VARYING',
         'boolean': 'CHARACTER VARYING',  # The guess sometimes can be wrong, we'll use varchar for now.
         'date': 'CHARACTER VARYING',  # The guess sometimes can be wrong, we'll use varchar for now.
-
-        'date_override': 'TIMESTAMP WITHOUT TIME ZONE'    # Column type to use when date_override defined in YAML
+        'date_override': 'TIMESTAMP WITHOUT TIME ZONE',  # Column type to use when date_override defined in YAML
     }.get(csv_type, 'CHARACTER VARYING')
 
 
@@ -50,7 +41,9 @@ def sync_table(table_name: str, args: Namespace) -> Union[bool, str]:
     postgres = FastSyncTargetPostgres(args.target, args.transform)
 
     try:
-        filename = utils.gen_export_filename(tap_id=args.target.get('tap_id'), table=table_name)
+        filename = utils.gen_export_filename(
+            tap_id=args.target.get('tap_id'), table=table_name
+        )
         filepath = os.path.join(args.temp_dir, filename)
 
         target_schema = utils.get_target_schema(args.target, table_name)
@@ -64,15 +57,24 @@ def sync_table(table_name: str, args: Namespace) -> Union[bool, str]:
 
         # Creating temp table in Postgres
         postgres.drop_table(target_schema, table_name, is_temporary=True)
-        postgres.create_table(target_schema,
-                              table_name,
-                              postgres_columns,
-                              primary_key,
-                              is_temporary=True,
-                              sort_columns=True)
+        postgres.create_table(
+            target_schema,
+            table_name,
+            postgres_columns,
+            primary_key,
+            is_temporary=True,
+            sort_columns=True,
+        )
 
         # Load into Postgres table
-        postgres.copy_to_table(filepath, target_schema, table_name, size_bytes, is_temporary=True, skip_csv_header=True)
+        postgres.copy_to_table(
+            filepath,
+            target_schema,
+            table_name,
+            size_bytes,
+            is_temporary=True,
+            skip_csv_header=True,
+        )
         os.remove(filepath)
 
         # Obfuscate columns
@@ -111,7 +113,8 @@ def main_impl():
     start_time = datetime.now()
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -119,7 +122,11 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # Create target schemas sequentially, Postgres doesn't like it running in parallel
     postgres_target = FastSyncTargetPostgres(args.target, args.transform)
@@ -128,11 +135,16 @@ def main_impl():
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -143,8 +155,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs),
-                str(table_sync_excs), pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/s3_csv_to_redshift.py b/pipelinewise/fastsync/s3_csv_to_redshift.py
index 0e41f7d93..b65ac6fe0 100644
--- a/pipelinewise/fastsync/s3_csv_to_redshift.py
+++ b/pipelinewise/fastsync/s3_csv_to_redshift.py
@@ -15,18 +15,8 @@
 LOGGER = Logger().get_logger(__name__)
 
 REQUIRED_CONFIG_KEYS = {
-    'tap': [
-        'bucket',
-        'start_date'
-    ],
-    'target': [
-        'host',
-        'port',
-        'user',
-        'password',
-        'dbname',
-        's3_bucket'
-    ]
+    'tap': ['bucket', 'start_date'],
+    'target': ['host', 'port', 'user', 'password', 'dbname', 's3_bucket'],
 }
 
 LOCK = multiprocessing.Lock()
@@ -41,8 +31,7 @@ def tap_type_to_target_type(csv_type):
         'string': 'CHARACTER VARYING',
         'boolean': 'CHARACTER VARYING',  # The guess sometimes can be wrong, we'll use varchar for now.
         'date': 'CHARACTER VARYING',  # The guess sometimes can be wrong, we'll use varchar for now.
-
-        'date_override': 'TIMESTAMP WITHOUT TIME ZONE'  # Column type to use when date_override defined in YAML
+        'date_override': 'TIMESTAMP WITHOUT TIME ZONE',  # Column type to use when date_override defined in YAML
     }.get(csv_type, 'CHARACTER VARYING')
 
 
@@ -52,7 +41,9 @@ def sync_table(table_name: str, args: Namespace) -> Union[bool, str]:
     redshift = FastSyncTargetRedshift(args.target, args.transform)
 
     try:
-        filename = utils.gen_export_filename(tap_id=args.target.get('tap_id'), table=table_name)
+        filename = utils.gen_export_filename(
+            tap_id=args.target.get('tap_id'), table=table_name
+        )
         filepath = os.path.join(args.temp_dir, filename)
 
         target_schema = utils.get_target_schema(args.target, table_name)
@@ -70,15 +61,24 @@ def sync_table(table_name: str, args: Namespace) -> Union[bool, str]:
 
         # Creating temp table in Redshift
         redshift.create_schema(target_schema)
-        redshift.create_table(target_schema,
-                              table_name,
-                              redshift_columns,
-                              primary_key,
-                              is_temporary=True,
-                              sort_columns=True)
+        redshift.create_table(
+            target_schema,
+            table_name,
+            redshift_columns,
+            primary_key,
+            is_temporary=True,
+            sort_columns=True,
+        )
 
         # Load into Redshift table
-        redshift.copy_to_table(s3_key, target_schema, table_name, size_bytes, is_temporary=True, skip_csv_header=True)
+        redshift.copy_to_table(
+            s3_key,
+            target_schema,
+            table_name,
+            size_bytes,
+            is_temporary=True,
+            skip_csv_header=True,
+        )
 
         # Obfuscate columns
         redshift.obfuscate_columns(target_schema, table_name)
@@ -116,7 +116,8 @@ def main_impl():
     start_time = datetime.now()
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -124,7 +125,11 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # Create target schemas sequentially, Redshift doesn't like it running in parallel
     redshift = FastSyncTargetRedshift(args.target, args.transform)
@@ -133,11 +138,16 @@ def main_impl():
     # Start loading tables in parallel in spawning processes
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -148,8 +158,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs),
-                str(table_sync_excs), pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/fastsync/s3_csv_to_snowflake.py b/pipelinewise/fastsync/s3_csv_to_snowflake.py
index 04f8a2b95..84c0268e4 100644
--- a/pipelinewise/fastsync/s3_csv_to_snowflake.py
+++ b/pipelinewise/fastsync/s3_csv_to_snowflake.py
@@ -16,10 +16,7 @@
 
 
 REQUIRED_CONFIG_KEYS = {
-    'tap': [
-        'bucket',
-        'start_date'
-    ],
+    'tap': ['bucket', 'start_date'],
     'target': [
         'account',
         'dbname',
@@ -28,8 +25,8 @@
         'warehouse',
         's3_bucket',
         'stage',
-        'file_format'
-    ]
+        'file_format',
+    ],
 }
 
 LOCK = multiprocessing.Lock()
@@ -44,8 +41,7 @@ def tap_type_to_target_type(csv_type):
         'string': 'VARCHAR',
         'boolean': 'VARCHAR',  # The guess sometimes can be wrong, we'll use varchar for now.
         'date': 'VARCHAR',  # The guess sometimes can be wrong, we'll use varchar for now.
-
-        'date_override': 'TIMESTAMP_NTZ'  # Column type to use when date_override defined in YAML
+        'date_override': 'TIMESTAMP_NTZ',  # Column type to use when date_override defined in YAML
     }.get(csv_type, 'VARCHAR')
 
 
@@ -55,7 +51,9 @@ def sync_table(table_name: str, args: Namespace) -> Union[bool, str]:
     snowflake = FastSyncTargetSnowflake(args.target, args.transform)
 
     try:
-        filename = utils.gen_export_filename(tap_id=args.target.get('tap_id'), table=table_name)
+        filename = utils.gen_export_filename(
+            tap_id=args.target.get('tap_id'), table=table_name
+        )
         filepath = os.path.join(args.temp_dir, filename)
 
         target_schema = utils.get_target_schema(args.target, table_name)
@@ -73,21 +71,32 @@ def sync_table(table_name: str, args: Namespace) -> Union[bool, str]:
 
         # Creating temp table in Snowflake
         snowflake.create_schema(target_schema)
-        snowflake.create_table(target_schema,
-                               table_name,
-                               snowflake_columns,
-                               primary_key,
-                               is_temporary=True,
-                               sort_columns=True)
+        snowflake.create_table(
+            target_schema,
+            table_name,
+            snowflake_columns,
+            primary_key,
+            is_temporary=True,
+            sort_columns=True,
+        )
 
         # Load into Snowflake table
-        snowflake.copy_to_table(s3_key, target_schema, table_name, size_bytes, is_temporary=True, skip_csv_header=True)
+        snowflake.copy_to_table(
+            s3_key,
+            target_schema,
+            table_name,
+            size_bytes,
+            is_temporary=True,
+            skip_csv_header=True,
+        )
 
         # Obfuscate columns
         snowflake.obfuscate_columns(target_schema, table_name)
 
         # Create target table and swap with the temp table in Snowflake
-        snowflake.create_table(target_schema, table_name, snowflake_columns, primary_key, sort_columns=True)
+        snowflake.create_table(
+            target_schema, table_name, snowflake_columns, primary_key, sort_columns=True
+        )
         snowflake.swap_tables(target_schema, table_name)
 
         # Get bookmark
@@ -120,7 +129,8 @@ def main_impl():
     start_time = datetime.now()
 
     # Log start info
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         STARTING SYNC
         -------------------------------------------------------
@@ -128,17 +138,26 @@ def main_impl():
             Total tables selected to sync  : %s
             Pool size                      : %s
         -------------------------------------------------------
-        """, args.tables, len(args.tables), pool_size)
+        """,
+        args.tables,
+        len(args.tables),
+        pool_size,
+    )
 
     # Start loading tables in parallel in spawning processes by
     # utilising all available Pool size
     with multiprocessing.Pool(pool_size) as proc:
         table_sync_excs = list(
-            filter(lambda x: not isinstance(x, bool), proc.map(partial(sync_table, args=args), args.tables)))
+            filter(
+                lambda x: not isinstance(x, bool),
+                proc.map(partial(sync_table, args=args), args.tables),
+            )
+        )
 
     # Log summary
     end_time = datetime.now()
-    LOGGER.info("""
+    LOGGER.info(
+        """
         -------------------------------------------------------
         SYNC FINISHED - SUMMARY
         -------------------------------------------------------
@@ -149,8 +168,13 @@ def main_impl():
             Pool size                      : %s
             Runtime                        : %s
         -------------------------------------------------------
-        """, len(args.tables), len(args.tables) - len(table_sync_excs),
-                str(table_sync_excs), pool_size, end_time - start_time)
+        """,
+        len(args.tables),
+        len(args.tables) - len(table_sync_excs),
+        str(table_sync_excs),
+        pool_size,
+        end_time - start_time,
+    )
 
     if len(table_sync_excs) > 0:
         sys.exit(1)
diff --git a/pipelinewise/logger.py b/pipelinewise/logger.py
index 2a1a95cb6..2dd32e64a 100644
--- a/pipelinewise/logger.py
+++ b/pipelinewise/logger.py
@@ -4,6 +4,7 @@
 from logging.config import fileConfig
 from pathlib import Path
 
+
 # pylint: disable=too-few-public-methods
 class Logger:
     """PipelineWise logger class"""
diff --git a/pipelinewise/utils.py b/pipelinewise/utils.py
index 3746d1d09..1f3a8d6e1 100644
--- a/pipelinewise/utils.py
+++ b/pipelinewise/utils.py
@@ -4,7 +4,9 @@
 from typing import Optional
 
 
-def safe_column_name(name: Optional[str], quote_character: Optional[str]=None) -> Optional[str]:
+def safe_column_name(
+    name: Optional[str], quote_character: Optional[str] = None
+) -> Optional[str]:
     """
     Makes column name safe by capitalizing and wrapping it in double quotes
     Args:
diff --git a/setup.py b/setup.py
index 9d23d4026..506f4482c 100644
--- a/setup.py
+++ b/setup.py
@@ -40,6 +40,7 @@
       ],
       extras_require={
           'test': [
+              'flake8==3.9.2',
               'pytest==6.2.4',
               'pytest-dependency==0.4.0',
               'pytest-cov==2.12.1',
diff --git a/tests/end_to_end/helpers/assertions.py b/tests/end_to_end/helpers/assertions.py
index 810129cde..470bf6a7a 100644
--- a/tests/end_to_end/helpers/assertions.py
+++ b/tests/end_to_end/helpers/assertions.py
@@ -26,7 +26,9 @@ def assert_run_tap_success(tap, target, sync_engines, profiling=False):
         assert_state_file_valid(target, tap, log_file)
 
     if profiling:
-        assert_profiling_stats_files_created(stdout, 'run_tap', sync_engines, tap, target)
+        assert_profiling_stats_files_created(
+            stdout, 'run_tap', sync_engines, tap, target
+        )
 
 
 def assert_resync_tables_success(tap, target, profiling=False):
@@ -45,7 +47,9 @@ def assert_resync_tables_success(tap, target, profiling=False):
     assert_state_file_valid(target, tap, log_file)
 
     if profiling:
-        assert_profiling_stats_files_created(stdout, 'sync_tables', ['fastsync'], tap, target)
+        assert_profiling_stats_files_created(
+            stdout, 'sync_tables', ['fastsync'], tap, target
+        )
 
 
 def assert_command_success(return_code, stdout, stderr, log_path=None):
@@ -74,7 +78,9 @@ def assert_command_success(return_code, stdout, stderr, log_path=None):
 def assert_state_file_valid(target_name, tap_name, log_path=None):
     """Assert helper function to check if state file exists for
     a certain tap for a certain target"""
-    state_file = Path(f'{Path.home()}/.pipelinewise/{target_name}/{tap_name}/state.json').resolve()
+    state_file = Path(
+        f'{Path.home()}/.pipelinewise/{target_name}/{tap_name}/state.json'
+    ).resolve()
     assert os.path.isfile(state_file)
 
     # Check if state file content equals to last emitted state in log
@@ -82,7 +88,10 @@ def assert_state_file_valid(target_name, tap_name, log_path=None):
         success_log_path = f'{log_path}.success'
         state_in_log = None
         with open(success_log_path, 'r', encoding='utf-8') as log_f:
-            state_log_pattern = re.search(r'\nINFO STATE emitted from target: (.+\n)', '\n'.join(log_f.readlines()))
+            state_log_pattern = re.search(
+                r'\nINFO STATE emitted from target: (.+\n)',
+                '\n'.join(log_f.readlines()),
+            )
             if state_log_pattern:
                 state_in_log = state_log_pattern.groups()[-1]
 
@@ -92,7 +101,9 @@ def assert_state_file_valid(target_name, tap_name, log_path=None):
                 assert state_in_log == ''.join(state_f.readlines())
 
 
-def assert_cols_in_table(query_runner_fn: callable, table_schema: str, table_name: str, columns: List[str]):
+def assert_cols_in_table(
+    query_runner_fn: callable, table_schema: str, table_name: str, columns: List[str]
+):
     """Fetches the given table's columns from information_schema and
     tests if every given column is in the result
 
@@ -102,14 +113,20 @@ def assert_cols_in_table(query_runner_fn: callable, table_schema: str, table_nam
     :param columns: list of columns to check if there are in the table's columns
     """
     funcs = _map_tap_to_target_functions(None, query_runner_fn)
-    sql_get_columns_for_table_fn = funcs.get('target_sql_get_table_cols_fn', db.sql_get_columns_for_table)
+    sql_get_columns_for_table_fn = funcs.get(
+        'target_sql_get_table_cols_fn', db.sql_get_columns_for_table
+    )
     sql = sql_get_columns_for_table_fn(table_schema, table_name)
     result = query_runner_fn(sql)
     cols = [res[0] for res in result]
     try:
         assert all(col in cols for col in columns)
     except AssertionError as ex:
-        ex.args += ('Error', columns, f'One ore more columns not found in target table {table_name}')
+        ex.args += (
+            'Error',
+            columns,
+            f'One ore more columns not found in target table {table_name}',
+        )
         raise
 
 
@@ -118,7 +135,9 @@ def _run_sql(query_runner_fn: callable, sql_query: str) -> List:
     return list(query_runner_fn(sql_query))
 
 
-def _map_tap_to_target_functions(tap_query_runner_fn: callable, target_query_runner_fn: callable) -> dict:
+def _map_tap_to_target_functions(
+    tap_query_runner_fn: callable, target_query_runner_fn: callable
+) -> dict:
     """Takes two query runner methods and creates a map with the compatible database
     specific functions that required to run assertions.
 
@@ -132,19 +151,19 @@ def _map_tap_to_target_functions(tap_query_runner_fn: callable, target_query_run
             'source_schemas': ['mysql_source_db'],
             'target_schemas': ['ppw_e2e_tap_mysql'],
             'source_sql_get_cols_fn': db.sql_get_columns_mysql,
-            'source_sql_dynamic_row_count_fn': db.sql_dynamic_row_count_mysql
+            'source_sql_dynamic_row_count_fn': db.sql_dynamic_row_count_mysql,
         },
         # tap-postgres specific attributes and functions
         'run_query_tap_postgres': {
             'source_schemas': ['public', 'public2'],
             'target_schemas': ['ppw_e2e_tap_postgres', 'ppw_e2e_tap_postgres_public2'],
             'source_sql_get_cols_fn': db.sql_get_columns_postgres,
-            'source_sql_dynamic_row_count_fn': db.sql_dynamic_row_count_postgres
+            'source_sql_dynamic_row_count_fn': db.sql_dynamic_row_count_postgres,
         },
         # target-postgres specific attributes and functions
         'run_query_target_postgres': {
             'target_sql_get_cols_fn': db.sql_get_columns_postgres,
-            'target_sql_dynamic_row_count_fn': db.sql_dynamic_row_count_postgres
+            'target_sql_dynamic_row_count_fn': db.sql_dynamic_row_count_postgres,
         },
         # target-snowflake specific attributes and functions
         'run_query_target_snowflake': {
@@ -162,16 +181,21 @@ def _map_tap_to_target_functions(tap_query_runner_fn: callable, target_query_run
         'run_query_target_redshift': {
             'target_sql_get_cols_fn': db.sql_get_columns_redshift,
             'target_sql_dynamic_row_count_fn': db.sql_dynamic_row_count_redshift,
-        }
+        },
     }
 
     # Merge the keys into one dict by tap and target query runner names
     if tap_query_runner_fn:
-        return {**f_map[tap_query_runner_fn.__name__], **f_map[target_query_runner_fn.__name__]}
+        return {
+            **f_map[tap_query_runner_fn.__name__],
+            **f_map[target_query_runner_fn.__name__],
+        }
     return {**f_map[target_query_runner_fn.__name__]}
 
 
-def assert_row_counts_equal(tap_query_runner_fn: callable, target_query_runner_fn: callable) -> None:
+def assert_row_counts_equal(
+    tap_query_runner_fn: callable, target_query_runner_fn: callable
+) -> None:
     """Takes two query runner methods, counts the row numbers in every table in both the
     source and target databases and tests if the row counts are matching.
 
@@ -185,12 +209,20 @@ def assert_row_counts_equal(tap_query_runner_fn: callable, target_query_runner_f
     target_schemas = funcs['target_schemas']
 
     # Generate a dynamic SQLs to count rows in source and target databases
-    source_dynamic_sql_row_count = funcs['source_sql_dynamic_row_count_fn'](source_schemas)
-    target_dynamic_sql_row_count = funcs['target_sql_dynamic_row_count_fn'](target_schemas)
+    source_dynamic_sql_row_count = funcs['source_sql_dynamic_row_count_fn'](
+        source_schemas
+    )
+    target_dynamic_sql_row_count = funcs['target_sql_dynamic_row_count_fn'](
+        target_schemas
+    )
 
     # Count rows
-    source_sql_row_count = _run_sql(tap_query_runner_fn, source_dynamic_sql_row_count)[0][0]
-    target_sql_row_count = _run_sql(target_query_runner_fn, target_dynamic_sql_row_count)[0][0]
+    source_sql_row_count = _run_sql(tap_query_runner_fn, source_dynamic_sql_row_count)[
+        0
+    ][0]
+    target_sql_row_count = _run_sql(
+        target_query_runner_fn, target_dynamic_sql_row_count
+    )[0][0]
 
     # Run the generated SQLs
     row_counts_in_source = _run_sql(tap_query_runner_fn, source_sql_row_count)
@@ -200,11 +232,8 @@ def assert_row_counts_equal(tap_query_runner_fn: callable, target_query_runner_f
     # we fix that by renaming the source tables to names that the target would accept
     if 'target_sql_safe_name_fn' in funcs:
         row_counts_in_source = [
-          (
-            funcs['target_sql_safe_name_fn'](table),
-            row_count
-          )
-          for (table,row_count) in row_counts_in_source
+            (funcs['target_sql_safe_name_fn'](table), row_count)
+            for (table, row_count) in row_counts_in_source
         ]
 
     # Compare the two dataset
@@ -212,10 +241,12 @@ def assert_row_counts_equal(tap_query_runner_fn: callable, target_query_runner_f
 
 
 # pylint: disable=too-many-locals
-def assert_all_columns_exist(tap_query_runner_fn: callable,
-                             target_query_runner_fn: callable,
-                             column_type_mapper_fn: callable = None,
-                             ignore_cols: Union[Set, List] = None) -> None:
+def assert_all_columns_exist(
+    tap_query_runner_fn: callable,
+    target_query_runner_fn: callable,
+    column_type_mapper_fn: callable = None,
+    ignore_cols: Union[Set, List] = None,
+) -> None:
     """Takes two query runner methods, gets the columns list for every table in both the
     source and target database and tests if every column in source exists in the target database.
     Some taps have unsupported column types and these are not part of the schemas published to the target thus
@@ -251,10 +282,7 @@ def _cols_list_to_dict(cols: List) -> dict:
         cols_dict = {}
         for col in cols:
             col_props = col.split(':')
-            cols_dict[col_props[0]] = {
-                'type': col_props[1],
-                'type_extra': col_props[2]
-            }
+            cols_dict[col_props[0]] = {'type': col_props[1], 'type_extra': col_props[2]}
 
         return cols_dict
 
@@ -270,7 +298,11 @@ def _cols_list_to_dict(cols: List) -> dict:
         source_cols = table_cols[1].lower().split(';')
 
         try:
-            target_cols = next(t[1] for t in target_table_cols if t[0].lower() == table_to_check).lower().split(';')
+            target_cols = (
+                next(t[1] for t in target_table_cols if t[0].lower() == table_to_check)
+                .lower()
+                .split(';')
+            )
         except StopIteration as ex:
             ex.args += ('Error', f'{table_to_check} table not found in target')
             raise
@@ -287,25 +319,38 @@ def _cols_list_to_dict(cols: List) -> dict:
             try:
                 assert col_name in target_cols_dict
             except AssertionError as ex:
-                ex.args += ('Error', f'{col_name} column not found in target table {table_to_check}')
+                ex.args += (
+                    'Error',
+                    f'{col_name} column not found in target table {table_to_check}',
+                )
                 raise
 
             # Check if column type is expected in the target table, if mapper function provided
             if column_type_mapper_fn:
                 try:
                     target_col = target_cols_dict[col_name]
-                    exp_col_type = column_type_mapper_fn(col_props['type'], col_props['type_extra']) \
-                        .replace(' NULL', '').lower()
+                    exp_col_type = (
+                        column_type_mapper_fn(
+                            col_props['type'], col_props['type_extra']
+                        )
+                        .replace(' NULL', '')
+                        .lower()
+                    )
                     act_col_type = target_col['type'].lower()
                     assert act_col_type == exp_col_type
                 except AssertionError as ex:
-                    ex.args += ('Error', f'{col_name} column type is not as expected. '
-                                         f'Expected: {exp_col_type} '
-                                         f'Actual: {act_col_type}')
+                    ex.args += (
+                        'Error',
+                        f'{col_name} column type is not as expected. '
+                        f'Expected: {exp_col_type} '
+                        f'Actual: {act_col_type}',
+                    )
                     raise
 
 
-def assert_date_column_naive_in_target(target_query_runner_fn, column_name, full_table_name):
+def assert_date_column_naive_in_target(
+    target_query_runner_fn, column_name, full_table_name
+):
     """
     Checks if all dates in the given column are naive,i.e no timezone
     Args:
@@ -313,19 +358,20 @@ def assert_date_column_naive_in_target(target_query_runner_fn, column_name, full
         column_name: column of timestamp type
         full_table_name: fully qualified table name
     """
-    dates = target_query_runner_fn(
-        f'SELECT {column_name} FROM {full_table_name};')
+    dates = target_query_runner_fn(f'SELECT {column_name} FROM {full_table_name};')
 
     for date in dates:
         if date[0] is not None:
             assert date[0].tzinfo is None
 
 
-def assert_profiling_stats_files_created(stdout: str,
-                                         command: str,
-                                         sync_engines: List = None,
-                                         tap: Union[str, List[str]] = None,
-                                         target: str = None):
+def assert_profiling_stats_files_created(
+    stdout: str,
+    command: str,
+    sync_engines: List = None,
+    tap: Union[str, List[str]] = None,
+    target: str = None,
+):
     """
     Asserts that profiling pstat files were created by checking their existence
     Args:
@@ -339,7 +385,10 @@ def assert_profiling_stats_files_created(stdout: str,
     profiler_dir = tasks.find_profiling_folder(stdout)
 
     # crawl the folder looking for pstat files and strip the folder name from the file name
-    pstat_files = {file[len(f'{profiler_dir}/'):] for file in glob.iglob(f'{profiler_dir}/*.pstat')}
+    pstat_files = {
+        file[len(f'{profiler_dir}/'):]
+        for file in glob.iglob(f'{profiler_dir}/*.pstat')
+    }
 
     assert f'pipelinewise_{command}.pstat' in pstat_files
 
diff --git a/tests/end_to_end/helpers/db.py b/tests/end_to_end/helpers/db.py
index be363efe1..bd7ef8200 100644
--- a/tests/end_to_end/helpers/db.py
+++ b/tests/end_to_end/helpers/db.py
@@ -11,15 +11,14 @@
 
 from pipelinewise.fastsync.commons.target_bigquery import safe_name
 
+
 # pylint: disable=too-many-arguments
 def run_query_postgres(query, host, port, user, password, database):
     """Run and SQL query in a postgres database"""
     result_rows = []
-    with psycopg2.connect(host=host,
-                          port=port,
-                          user=user,
-                          password=password,
-                          database=database) as conn:
+    with psycopg2.connect(
+        host=host, port=port, user=user, password=password, database=database
+    ) as conn:
         conn.set_session(autocommit=True)
         with conn.cursor() as cur:
             cur.execute(query)
@@ -31,13 +30,15 @@ def run_query_postgres(query, host, port, user, password, database):
 def run_query_mysql(query, host, port, user, password, database):
     """Run and SQL query in a mysql database"""
     result_rows = []
-    with pymysql.connect(host=host,
-                         port=port,
-                         user=user,
-                         password=password,
-                         database=database,
-                         charset='utf8mb4',
-                         cursorclass=pymysql.cursors.Cursor) as cur:
+    with pymysql.connect(
+        host=host,
+        port=port,
+        user=user,
+        password=password,
+        database=database,
+        charset='utf8mb4',
+        cursorclass=pymysql.cursors.Cursor,
+    ) as cur:
         cur.execute(query)
         if cur.rowcount > 0:
             result_rows = cur.fetchall()
@@ -47,12 +48,14 @@ def run_query_mysql(query, host, port, user, password, database):
 def run_query_snowflake(query, account, database, warehouse, user, password):
     """Run and SQL query in a snowflake database"""
     result_rows = []
-    with snowflake.connector.connect(account=account,
-                                     database=database,
-                                     warehouse=warehouse,
-                                     user=user,
-                                     password=password,
-                                     autocommit=True) as conn:
+    with snowflake.connector.connect(
+        account=account,
+        database=database,
+        warehouse=warehouse,
+        user=user,
+        password=password,
+        autocommit=True,
+    ) as conn:
         with conn.cursor() as cur:
             cur.execute(query)
             if cur.rowcount > 0:
@@ -70,6 +73,7 @@ def delete_dataset_bigquery(dataset, project):
     client = bigquery.Client(project=project)
     client.delete_dataset(dataset, delete_contents=True, not_found_ok=True)
 
+
 def run_query_bigquery(query, project):
     """Run and SQL query in a BigQuery database"""
     client = bigquery.Client(project=project)
@@ -77,6 +81,7 @@ def run_query_bigquery(query, project):
     query_job.result()
     return [r.values() for r in query_job]
 
+
 def run_query_redshift(query, host, port, user, password, database):
     """Redshift is compatible with postgres"""
     return run_query_postgres(query, host, port, user, password, database)
@@ -152,9 +157,12 @@ def sql_get_columns_snowflake(schemas: list) -> str:
 def sql_get_columns_bigquery(schemas: list) -> str:
     """Generates an SQL command that gives the list of columns of every table
     in a specific schema from a snowflake database"""
-    table_queries = ' UNION ALL '.join(f"""
+    table_queries = ' UNION ALL '.join(
+        f"""
             SELECT table_name, column_name, data_type
-            FROM `{schema}`.INFORMATION_SCHEMA.COLUMNS""" for schema in schemas)
+            FROM `{schema}`.INFORMATION_SCHEMA.COLUMNS"""
+        for schema in schemas
+    )
 
     return f"""
     SELECT table_name, STRING_AGG(CONCAT(column_name, ':', data_type, ':'), ';' ORDER BY column_name)
@@ -254,10 +262,13 @@ def sql_dynamic_row_count_snowflake(schemas: list) -> str:
 def sql_dynamic_row_count_bigquery(schemas: list) -> str:
     """Generates an SQL statement that counts the number of rows in
     every table in a specific schema(s) in a Snowflake database"""
-    table_queries = ' UNION DISTINCT '.join(f"""
+    table_queries = ' UNION DISTINCT '.join(
+        f"""
             SELECT table_schema, table_name
             FROM `{schema}`.INFORMATION_SCHEMA.TABLES
-            WHERE table_type = 'BASE TABLE'""" for schema in schemas)
+            WHERE table_type = 'BASE TABLE'"""
+        for schema in schemas
+    )
 
     return f"""
     WITH table_list AS ({table_queries})
@@ -286,22 +297,26 @@ def sql_dynamic_row_count_redshift(schemas: list) -> str:
              ' UNION ') WITHIN GROUP ( ORDER BY tablename )
            || 'ORDER BY tbl'
       FROM table_list
-    """
+    """  # noqa: E501
 
 
-def get_mongodb_connection(host: str,
-                           port: Union[str, int],
-                           user: str,
-                           password: str,
-                           database: str,
-                           auth_database: str)->Database:
+def get_mongodb_connection(
+    host: str,
+    port: Union[str, int],
+    user: str,
+    password: str,
+    database: str,
+    auth_database: str,
+) -> Database:
     """
     Creates a mongoDB connection to the db to sync from
     Returns: Database instance with established connection
 
     """
-    return pymongo.MongoClient(host=host,
-                               port=int(port),
-                               username=user,
-                               password=password,
-                               authSource=auth_database)[database]
+    return pymongo.MongoClient(
+        host=host,
+        port=int(port),
+        username=user,
+        password=password,
+        authSource=auth_database,
+    )[database]
diff --git a/tests/end_to_end/helpers/env.py b/tests/end_to_end/helpers/env.py
index a4a88e818..9327bff73 100644
--- a/tests/end_to_end/helpers/env.py
+++ b/tests/end_to_end/helpers/env.py
@@ -43,7 +43,9 @@ def _load_env(self):
 
         If optional connector properties are not defined in ../../../dev/project/.env then
         the related test cases will be skipped."""
-        load_dotenv(dotenv_path=os.path.join(DIR, '..', '..', '..', 'dev-project', '.env'))
+        load_dotenv(
+            dotenv_path=os.path.join(DIR, '..', '..', '..', 'dev-project', '.env')
+        )
         self.env = {
             # ------------------------------------------------------------------
             # Tap Postgres is a REQUIRED test connector and test database with test data available
@@ -52,12 +54,27 @@ def _load_env(self):
             'TAP_POSTGRES': {
                 'template_patterns': ['tap_postgres'],
                 'vars': {
-                    'HOST'     : {'value': os.environ.get('TAP_POSTGRES_HOST'), 'required': True},
-                    'PORT'     : {'value': os.environ.get('TAP_POSTGRES_PORT'), 'required': True},
-                    'USER'     : {'value': os.environ.get('TAP_POSTGRES_USER'), 'required': True},
-                    'PASSWORD' : {'value': os.environ.get('TAP_POSTGRES_PASSWORD'), 'required': True},
-                    'DB'       : {'value': os.environ.get('TAP_POSTGRES_DB'), 'required': True}
-                }
+                    'HOST': {
+                        'value': os.environ.get('TAP_POSTGRES_HOST'),
+                        'required': True,
+                    },
+                    'PORT': {
+                        'value': os.environ.get('TAP_POSTGRES_PORT'),
+                        'required': True,
+                    },
+                    'USER': {
+                        'value': os.environ.get('TAP_POSTGRES_USER'),
+                        'required': True,
+                    },
+                    'PASSWORD': {
+                        'value': os.environ.get('TAP_POSTGRES_PASSWORD'),
+                        'required': True,
+                    },
+                    'DB': {
+                        'value': os.environ.get('TAP_POSTGRES_DB'),
+                        'required': True,
+                    },
+                },
             },
             # ------------------------------------------------------------------
             # Tap MySQL is a REQUIRED test connector and test database with test data available
@@ -66,12 +83,12 @@ def _load_env(self):
             'TAP_MYSQL': {
                 'template_patterns': ['tap_mysql'],
                 'vars': {
-                    'HOST'      : {'value': os.environ.get('TAP_MYSQL_HOST')},
-                    'PORT'      : {'value': os.environ.get('TAP_MYSQL_PORT')},
-                    'USER'      : {'value': os.environ.get('TAP_MYSQL_USER')},
-                    'PASSWORD'  : {'value': os.environ.get('TAP_MYSQL_PASSWORD')},
-                    'DB'        : {'value': os.environ.get('TAP_MYSQL_DB')},
-                }
+                    'HOST': {'value': os.environ.get('TAP_MYSQL_HOST')},
+                    'PORT': {'value': os.environ.get('TAP_MYSQL_PORT')},
+                    'USER': {'value': os.environ.get('TAP_MYSQL_USER')},
+                    'PASSWORD': {'value': os.environ.get('TAP_MYSQL_PASSWORD')},
+                    'DB': {'value': os.environ.get('TAP_MYSQL_DB')},
+                },
             },
             # ------------------------------------------------------------------
             # Tap MongoDB is a REQUIRED test connector and test database with test data available
@@ -80,13 +97,25 @@ def _load_env(self):
             'TAP_MONGODB': {
                 'template_patterns': ['tap_postgres'],
                 'vars': {
-                    'HOST': {'value': os.environ.get('TAP_MONGODB_HOST'), 'required': True},
-                    'PORT': {'value': os.environ.get('TAP_MONGODB_PORT'), 'required': True},
-                    'USER': {'value': os.environ.get('TAP_MONGODB_USER'), 'required': True},
-                    'PASSWORD': {'value': os.environ.get('TAP_MONGODB_PASSWORD'), 'required': True},
+                    'HOST': {
+                        'value': os.environ.get('TAP_MONGODB_HOST'),
+                        'required': True,
+                    },
+                    'PORT': {
+                        'value': os.environ.get('TAP_MONGODB_PORT'),
+                        'required': True,
+                    },
+                    'USER': {
+                        'value': os.environ.get('TAP_MONGODB_USER'),
+                        'required': True,
+                    },
+                    'PASSWORD': {
+                        'value': os.environ.get('TAP_MONGODB_PASSWORD'),
+                        'required': True,
+                    },
                     'DB': {'value': os.environ.get('TAP_MONGODB_DB'), 'required': True},
-                    'AUTH_DB': {'value': 'admin', 'required': True}
-                }
+                    'AUTH_DB': {'value': 'admin', 'required': True},
+                },
             },
             # ------------------------------------------------------------------
             # Tap S3 CSV is an OPTIONAL test connector and it requires credentials to a real S3 bucket.
@@ -96,10 +125,12 @@ def _load_env(self):
                 'optional': True,
                 'template_patterns': ['tap_s3_csv'],
                 'vars': {
-                    'AWS_KEY'               : {'value': os.environ.get('TAP_S3_CSV_AWS_KEY')},
-                    'AWS_SECRET_ACCESS_KEY' : {'value': os.environ.get('TAP_S3_CSV_AWS_SECRET_ACCESS_KEY')},
-                    'BUCKET'                : {'value': os.environ.get('TAP_S3_CSV_BUCKET')},
-                }
+                    'AWS_KEY': {'value': os.environ.get('TAP_S3_CSV_AWS_KEY')},
+                    'AWS_SECRET_ACCESS_KEY': {
+                        'value': os.environ.get('TAP_S3_CSV_AWS_SECRET_ACCESS_KEY')
+                    },
+                    'BUCKET': {'value': os.environ.get('TAP_S3_CSV_BUCKET')},
+                },
             },
             # ------------------------------------------------------------------
             # Target Postgres is a REQUIRED test connector and test database available in the docker environment
@@ -107,12 +138,12 @@ def _load_env(self):
             'TARGET_POSTGRES': {
                 'template_patterns': ['target_postgres', 'to_pg'],
                 'vars': {
-                    'HOST'      : {'value': os.environ.get('TARGET_POSTGRES_HOST')},
-                    'PORT'      : {'value': os.environ.get('TARGET_POSTGRES_PORT')},
-                    'USER'      : {'value': os.environ.get('TARGET_POSTGRES_USER')},
-                    'PASSWORD'  : {'value': os.environ.get('TARGET_POSTGRES_PASSWORD')},
-                    'DB'        : {'value': os.environ.get('TARGET_POSTGRES_DB')},
-                }
+                    'HOST': {'value': os.environ.get('TARGET_POSTGRES_HOST')},
+                    'PORT': {'value': os.environ.get('TARGET_POSTGRES_PORT')},
+                    'USER': {'value': os.environ.get('TARGET_POSTGRES_USER')},
+                    'PASSWORD': {'value': os.environ.get('TARGET_POSTGRES_PASSWORD')},
+                    'DB': {'value': os.environ.get('TARGET_POSTGRES_DB')},
+                },
             },
             # ------------------------------------------------------------------
             # Target Snowflake is an OPTIONAL test connector because it's not open sourced and not part of
@@ -123,26 +154,48 @@ def _load_env(self):
                 'optional': True,
                 'template_patterns': ['target_snowflake', 'to_sf'],
                 'vars': {
-                    'ACCOUNT'               : {'value': os.environ.get('TARGET_SNOWFLAKE_ACCOUNT')},
-                    'DBNAME'                : {'value': os.environ.get('TARGET_SNOWFLAKE_DBNAME')},
-                    'USER'                  : {'value': os.environ.get('TARGET_SNOWFLAKE_USER')},
-                    'PASSWORD'              : {'value': os.environ.get('TARGET_SNOWFLAKE_PASSWORD')},
-                    'WAREHOUSE'             : {'value': os.environ.get('TARGET_SNOWFLAKE_WAREHOUSE')},
-                    'AWS_ACCESS_KEY'        : {'value': os.environ.get('TARGET_SNOWFLAKE_AWS_ACCESS_KEY'),
-                                               'optional': True},
-                    'AWS_SECRET_ACCESS_KEY' : {'value': os.environ.get('TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY'),
-                                               'optional': True},
-                    'SESSION_TOKEN'         : {'value': os.environ.get('TARGET_SNOWFLAKE_SESSION_TOKEN'),
-                                               'optional': True},
-                    'S3_BUCKET'             : {'value': os.environ.get('TARGET_SNOWFLAKE_S3_BUCKET')},
-                    'S3_KEY_PREFIX'         : {'value': os.environ.get('TARGET_SNOWFLAKE_S3_KEY_PREFIX')},
-                    'S3_ACL'                : {'value': os.environ.get('TARGET_SNOWFLAKE_S3_ACL'), 'optional': True},
-                    'STAGE'                 : {'value': os.environ.get('TARGET_SNOWFLAKE_STAGE')},
-                    'FILE_FORMAT'           : {'value': os.environ.get('TARGET_SNOWFLAKE_FILE_FORMAT')},
-                    'CLIENT_SIDE_ENCRYPTION_MASTER_KEY':
-                        {'value': os.environ.get('TARGET_SNOWFLAKE_CLIENT_SIDE_ENCRYPTION_MASTER_KEY'),
-                         'optional': True},
-                }
+                    'ACCOUNT': {'value': os.environ.get('TARGET_SNOWFLAKE_ACCOUNT')},
+                    'DBNAME': {'value': os.environ.get('TARGET_SNOWFLAKE_DBNAME')},
+                    'USER': {'value': os.environ.get('TARGET_SNOWFLAKE_USER')},
+                    'PASSWORD': {'value': os.environ.get('TARGET_SNOWFLAKE_PASSWORD')},
+                    'WAREHOUSE': {
+                        'value': os.environ.get('TARGET_SNOWFLAKE_WAREHOUSE')
+                    },
+                    'AWS_ACCESS_KEY': {
+                        'value': os.environ.get('TARGET_SNOWFLAKE_AWS_ACCESS_KEY'),
+                        'optional': True,
+                    },
+                    'AWS_SECRET_ACCESS_KEY': {
+                        'value': os.environ.get(
+                            'TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY'
+                        ),
+                        'optional': True,
+                    },
+                    'SESSION_TOKEN': {
+                        'value': os.environ.get('TARGET_SNOWFLAKE_SESSION_TOKEN'),
+                        'optional': True,
+                    },
+                    'S3_BUCKET': {
+                        'value': os.environ.get('TARGET_SNOWFLAKE_S3_BUCKET')
+                    },
+                    'S3_KEY_PREFIX': {
+                        'value': os.environ.get('TARGET_SNOWFLAKE_S3_KEY_PREFIX')
+                    },
+                    'S3_ACL': {
+                        'value': os.environ.get('TARGET_SNOWFLAKE_S3_ACL'),
+                        'optional': True,
+                    },
+                    'STAGE': {'value': os.environ.get('TARGET_SNOWFLAKE_STAGE')},
+                    'FILE_FORMAT': {
+                        'value': os.environ.get('TARGET_SNOWFLAKE_FILE_FORMAT')
+                    },
+                    'CLIENT_SIDE_ENCRYPTION_MASTER_KEY': {
+                        'value': os.environ.get(
+                            'TARGET_SNOWFLAKE_CLIENT_SIDE_ENCRYPTION_MASTER_KEY'
+                        ),
+                        'optional': True,
+                    },
+                },
             },
             # ------------------------------------------------------------------
             # Target BigQuery is an OPTIONAL test connector because it's not open sourced and not part of
@@ -153,8 +206,8 @@ def _load_env(self):
                 'optional': True,
                 'template_patterns': ['target_bigquery', 'to_bq'],
                 'vars': {
-                    'PROJECT'               : {'value': os.environ.get('TARGET_BIGQUERY_PROJECT')},
-                }
+                    'PROJECT': {'value': os.environ.get('TARGET_BIGQUERY_PROJECT')},
+                },
             },
             # ------------------------------------------------------------------
             # Target Redshift is an OPTIONAL test connector because it's not open sourced and not part of
@@ -165,36 +218,65 @@ def _load_env(self):
                 'optional': True,
                 'template_patterns': ['target_redshift', 'to_rs'],
                 'vars': {
-                    'HOST'                  : {'value': os.environ.get('TARGET_REDSHIFT_HOST')},
-                    'PORT'                  : {'value': os.environ.get('TARGET_REDSHIFT_PORT')},
-                    'USER'                  : {'value': os.environ.get('TARGET_REDSHIFT_USER')},
-                    'PASSWORD'              : {'value': os.environ.get('TARGET_REDSHIFT_PASSWORD')},
-                    'DBNAME'                : {'value': os.environ.get('TARGET_REDSHIFT_DBNAME')},
-                    'AWS_ACCESS_KEY'        : {'value': os.environ.get('TARGET_REDSHIFT_AWS_ACCESS_KEY'),
-                                               'optional': True},
-                    'AWS_SECRET_ACCESS_KEY' : {'value': os.environ.get('TARGET_REDSHIFT_AWS_SECRET_ACCESS_KEY'),
-                                               'optional': True},
-                    'SESSION_TOKEN'         : {'value': os.environ.get('TARGET_REDSHIFT_SESSION_TOKEN'),
-                                               'optional': True},
-                    'COPY_ROLE_ARN'         : {'value': os.environ.get('TARGET_REDSHIFT_COPY_ROLE_ARN'),
-                                               'optional': True},
-                    'S3_BUCKET'             : {'value': os.environ.get('TARGET_REDSHIFT_S3_BUCKET')},
-                    'S3_KEY_PREFIX'         : {'value': os.environ.get('TARGET_REDSHIFT_S3_KEY_PREFIX')},
-                    'S3_ACL'                : {'value': os.environ.get('TARGET_REDSHIFT_S3_ACL'), 'optional': True}
-                }
-            }
+                    'HOST': {'value': os.environ.get('TARGET_REDSHIFT_HOST')},
+                    'PORT': {'value': os.environ.get('TARGET_REDSHIFT_PORT')},
+                    'USER': {'value': os.environ.get('TARGET_REDSHIFT_USER')},
+                    'PASSWORD': {'value': os.environ.get('TARGET_REDSHIFT_PASSWORD')},
+                    'DBNAME': {'value': os.environ.get('TARGET_REDSHIFT_DBNAME')},
+                    'AWS_ACCESS_KEY': {
+                        'value': os.environ.get('TARGET_REDSHIFT_AWS_ACCESS_KEY'),
+                        'optional': True,
+                    },
+                    'AWS_SECRET_ACCESS_KEY': {
+                        'value': os.environ.get(
+                            'TARGET_REDSHIFT_AWS_SECRET_ACCESS_KEY'
+                        ),
+                        'optional': True,
+                    },
+                    'SESSION_TOKEN': {
+                        'value': os.environ.get('TARGET_REDSHIFT_SESSION_TOKEN'),
+                        'optional': True,
+                    },
+                    'COPY_ROLE_ARN': {
+                        'value': os.environ.get('TARGET_REDSHIFT_COPY_ROLE_ARN'),
+                        'optional': True,
+                    },
+                    'S3_BUCKET': {'value': os.environ.get('TARGET_REDSHIFT_S3_BUCKET')},
+                    'S3_KEY_PREFIX': {
+                        'value': os.environ.get('TARGET_REDSHIFT_S3_KEY_PREFIX')
+                    },
+                    'S3_ACL': {
+                        'value': os.environ.get('TARGET_REDSHIFT_S3_ACL'),
+                        'optional': True,
+                    },
+                },
+            },
         }
 
         # Add is_configured keys for every connector
         # Useful to skip certain test cases dynamically when specific tap
         # or target database is not configured
-        self.env['TAP_POSTGRES']['is_configured'] = self._is_env_connector_configured('TAP_POSTGRES')
-        self.env['TAP_MYSQL']['is_configured'] = self._is_env_connector_configured('TAP_MYSQL')
-        self.env['TAP_S3_CSV']['is_configured'] = self._is_env_connector_configured('TAP_S3_CSV')
-        self.env['TARGET_POSTGRES']['is_configured'] = self._is_env_connector_configured('TARGET_POSTGRES')
-        self.env['TARGET_REDSHIFT']['is_configured'] = self._is_env_connector_configured('TARGET_REDSHIFT')
-        self.env['TARGET_SNOWFLAKE']['is_configured'] = self._is_env_connector_configured('TARGET_SNOWFLAKE')
-        self.env['TARGET_BIGQUERY']['is_configured'] = self._is_env_connector_configured('TARGET_BIGQUERY')
+        self.env['TAP_POSTGRES']['is_configured'] = self._is_env_connector_configured(
+            'TAP_POSTGRES'
+        )
+        self.env['TAP_MYSQL']['is_configured'] = self._is_env_connector_configured(
+            'TAP_MYSQL'
+        )
+        self.env['TAP_S3_CSV']['is_configured'] = self._is_env_connector_configured(
+            'TAP_S3_CSV'
+        )
+        self.env['TARGET_POSTGRES'][
+            'is_configured'
+        ] = self._is_env_connector_configured('TARGET_POSTGRES')
+        self.env['TARGET_REDSHIFT'][
+            'is_configured'
+        ] = self._is_env_connector_configured('TARGET_REDSHIFT')
+        self.env['TARGET_SNOWFLAKE'][
+            'is_configured'
+        ] = self._is_env_connector_configured('TARGET_SNOWFLAKE')
+        self.env['TARGET_BIGQUERY'][
+            'is_configured'
+        ] = self._is_env_connector_configured('TARGET_BIGQUERY')
 
     def _get_conn_env_var(self, connector, key):
         """Get the value of a specific variable in the self.env dict"""
@@ -209,14 +291,14 @@ def get_aws_session(self):
         aws_secret_access_key = os.environ.get('TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY')
         if aws_access_key_id is None or aws_secret_access_key is None:
             raise Exception(
-                'Env vars TARGET_SNOWFLAKE_AWS_ACCESS_KEY and TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY are required')
+                'Env vars TARGET_SNOWFLAKE_AWS_ACCESS_KEY and TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY are required'
+            )
 
         return boto3.session.Session(
             aws_access_key_id=aws_access_key_id,
-            aws_secret_access_key=aws_secret_access_key
+            aws_secret_access_key=aws_secret_access_key,
         )
 
-
     def _is_env_connector_configured(self, env_connector):
         """Detect if certain component(s) of env vars group is configured properly"""
         env_conns = []
@@ -235,7 +317,9 @@ def _is_env_connector_configured(self, env_connector):
                     if self.env[env_conn].get('optional'):
                         return False
                     # Value not defined but it's a required property
-                    raise Exception(f'{env_conn}_{key} env var is required but not defined.')
+                    raise Exception(
+                        f'{env_conn}_{key} env var is required but not defined.'
+                    )
         return True
 
     def _find_env_conn_by_template_name(self, template_name):
@@ -290,7 +374,9 @@ def _init_test_project_dir(self, project_dir):
                 if is_configured:
                     template_vars = set(re.findall(r'\$\{(.+?)\}', yaml))
                     for var in template_vars:
-                        yaml = yaml.replace(f'${{{var}}}', self._all_env_vars_to_dict().get(var))
+                        yaml = yaml.replace(
+                            f'${{{var}}}', self._all_env_vars_to_dict().get(var)
+                        )
 
                     # Write the template replaced YAML file
                     with open(yaml_path, 'w+', encoding='utf-8') as f_render:
@@ -314,40 +400,47 @@ def _run_command(args):
 
     def run_query_tap_postgres(self, query):
         """Run and SQL query in tap postgres database"""
-        return db.run_query_postgres(query,
-                                     host=self._get_conn_env_var('TAP_POSTGRES', 'HOST'),
-                                     port=self._get_conn_env_var('TAP_POSTGRES', 'PORT'),
-                                     user=self._get_conn_env_var('TAP_POSTGRES', 'USER'),
-                                     password=self._get_conn_env_var('TAP_POSTGRES', 'PASSWORD'),
-                                     database=self._get_conn_env_var('TAP_POSTGRES', 'DB'))
+        return db.run_query_postgres(
+            query,
+            host=self._get_conn_env_var('TAP_POSTGRES', 'HOST'),
+            port=self._get_conn_env_var('TAP_POSTGRES', 'PORT'),
+            user=self._get_conn_env_var('TAP_POSTGRES', 'USER'),
+            password=self._get_conn_env_var('TAP_POSTGRES', 'PASSWORD'),
+            database=self._get_conn_env_var('TAP_POSTGRES', 'DB'),
+        )
 
     def get_tap_mongodb_connection(self):
         """Create and returns tap mongodb database instance to run queries on"""
-        return db.get_mongodb_connection(host=self._get_conn_env_var('TAP_MONGODB', 'HOST'),
-                                         port=self._get_conn_env_var('TAP_MONGODB', 'PORT'),
-                                         user=self._get_conn_env_var('TAP_MONGODB', 'USER'),
-                                         password=self._get_conn_env_var('TAP_MONGODB', 'PASSWORD'),
-                                         database=self._get_conn_env_var('TAP_MONGODB', 'DB'),
-                                         auth_database=self._get_conn_env_var('TAP_MONGODB', 'AUTH_DB'),
-                                         )
+        return db.get_mongodb_connection(
+            host=self._get_conn_env_var('TAP_MONGODB', 'HOST'),
+            port=self._get_conn_env_var('TAP_MONGODB', 'PORT'),
+            user=self._get_conn_env_var('TAP_MONGODB', 'USER'),
+            password=self._get_conn_env_var('TAP_MONGODB', 'PASSWORD'),
+            database=self._get_conn_env_var('TAP_MONGODB', 'DB'),
+            auth_database=self._get_conn_env_var('TAP_MONGODB', 'AUTH_DB'),
+        )
 
     def run_query_target_postgres(self, query: object) -> object:
         """Run and SQL query in target postgres database"""
-        return db.run_query_postgres(query,
-                                     host=self._get_conn_env_var('TARGET_POSTGRES', 'HOST'),
-                                     port=self._get_conn_env_var('TARGET_POSTGRES', 'PORT'),
-                                     user=self._get_conn_env_var('TARGET_POSTGRES', 'USER'),
-                                     password=self._get_conn_env_var('TARGET_POSTGRES', 'PASSWORD'),
-                                     database=self._get_conn_env_var('TARGET_POSTGRES', 'DB'))
+        return db.run_query_postgres(
+            query,
+            host=self._get_conn_env_var('TARGET_POSTGRES', 'HOST'),
+            port=self._get_conn_env_var('TARGET_POSTGRES', 'PORT'),
+            user=self._get_conn_env_var('TARGET_POSTGRES', 'USER'),
+            password=self._get_conn_env_var('TARGET_POSTGRES', 'PASSWORD'),
+            database=self._get_conn_env_var('TARGET_POSTGRES', 'DB'),
+        )
 
     def run_query_target_redshift(self, query):
         """Run an SQL query in target redshift database"""
-        return db.run_query_redshift(query,
-                                     host=self._get_conn_env_var('TARGET_REDSHIFT', 'HOST'),
-                                     port=self._get_conn_env_var('TARGET_REDSHIFT', 'PORT'),
-                                     user=self._get_conn_env_var('TARGET_REDSHIFT', 'USER'),
-                                     password=self._get_conn_env_var('TARGET_REDSHIFT', 'PASSWORD'),
-                                     database=self._get_conn_env_var('TARGET_REDSHIFT', 'DBNAME'))
+        return db.run_query_redshift(
+            query,
+            host=self._get_conn_env_var('TARGET_REDSHIFT', 'HOST'),
+            port=self._get_conn_env_var('TARGET_REDSHIFT', 'PORT'),
+            user=self._get_conn_env_var('TARGET_REDSHIFT', 'USER'),
+            password=self._get_conn_env_var('TARGET_REDSHIFT', 'PASSWORD'),
+            database=self._get_conn_env_var('TARGET_REDSHIFT', 'DBNAME'),
+        )
 
     # pylint: disable=unnecessary-pass
     def run_query_tap_s3_csv(self, file):
@@ -357,31 +450,37 @@ def run_query_tap_s3_csv(self, file):
 
     def run_query_tap_mysql(self, query):
         """Run and SQL query in tap mysql database"""
-        return db.run_query_mysql(query,
-                                  host=self._get_conn_env_var('TAP_MYSQL', 'HOST'),
-                                  port=int(self._get_conn_env_var('TAP_MYSQL', 'PORT')),
-                                  user=self._get_conn_env_var('TAP_MYSQL', 'USER'),
-                                  password=self._get_conn_env_var('TAP_MYSQL', 'PASSWORD'),
-                                  database=self._get_conn_env_var('TAP_MYSQL', 'DB'))
+        return db.run_query_mysql(
+            query,
+            host=self._get_conn_env_var('TAP_MYSQL', 'HOST'),
+            port=int(self._get_conn_env_var('TAP_MYSQL', 'PORT')),
+            user=self._get_conn_env_var('TAP_MYSQL', 'USER'),
+            password=self._get_conn_env_var('TAP_MYSQL', 'PASSWORD'),
+            database=self._get_conn_env_var('TAP_MYSQL', 'DB'),
+        )
 
     def run_query_target_snowflake(self, query):
         """Run and SQL query in target snowflake database"""
-        return db.run_query_snowflake(query,
-                                      account=self._get_conn_env_var('TARGET_SNOWFLAKE', 'ACCOUNT'),
-                                      database=self._get_conn_env_var('TARGET_SNOWFLAKE', 'DBNAME'),
-                                      warehouse=self._get_conn_env_var('TARGET_SNOWFLAKE', 'WAREHOUSE'),
-                                      user=self._get_conn_env_var('TARGET_SNOWFLAKE', 'USER'),
-                                      password=self._get_conn_env_var('TARGET_SNOWFLAKE', 'PASSWORD'))
+        return db.run_query_snowflake(
+            query,
+            account=self._get_conn_env_var('TARGET_SNOWFLAKE', 'ACCOUNT'),
+            database=self._get_conn_env_var('TARGET_SNOWFLAKE', 'DBNAME'),
+            warehouse=self._get_conn_env_var('TARGET_SNOWFLAKE', 'WAREHOUSE'),
+            user=self._get_conn_env_var('TARGET_SNOWFLAKE', 'USER'),
+            password=self._get_conn_env_var('TARGET_SNOWFLAKE', 'PASSWORD'),
+        )
 
     def delete_dataset_target_bigquery(self, dataset):
         """Run and SQL query in target bigquery database"""
-        return db.delete_dataset_bigquery(dataset,
-                                          project=self._get_conn_env_var('TARGET_BIGQUERY', 'PROJECT'))
+        return db.delete_dataset_bigquery(
+            dataset, project=self._get_conn_env_var('TARGET_BIGQUERY', 'PROJECT')
+        )
 
     def run_query_target_bigquery(self, query):
         """Run and SQL query in target bigquery database"""
-        return db.run_query_bigquery(query,
-                                     project=self._get_conn_env_var('TARGET_BIGQUERY', 'PROJECT'))
+        return db.run_query_bigquery(
+            query, project=self._get_conn_env_var('TARGET_BIGQUERY', 'PROJECT')
+        )
 
     # -------------------------------------------------------------------------
     # Setup methods to initialise source and target databases and to make them
@@ -407,16 +506,23 @@ def setup_tap_mongodb(self):
         db_script = os.path.join(DIR, '..', '..', 'db', 'tap_mongodb.sh')
         self._run_command(db_script)
 
-
     def setup_tap_s3_csv(self):
         """Upload test input files to S3 to be prapared for test run"""
-        mock_data_1 = os.path.join(DIR, '..', 'test-project', 's3_mock_data', 'mock_data_1.csv')
-        mock_data_2 = os.path.join(DIR, '..', 'test-project', 's3_mock_data', 'mock_data_2.csv')
+        mock_data_1 = os.path.join(
+            DIR, '..', 'test-project', 's3_mock_data', 'mock_data_1.csv'
+        )
+        mock_data_2 = os.path.join(
+            DIR, '..', 'test-project', 's3_mock_data', 'mock_data_2.csv'
+        )
 
         bucket = self._get_conn_env_var('TAP_S3_CSV', 'BUCKET')
-        s3 = boto3.client('s3',
-                          aws_access_key_id=self._get_conn_env_var('TAP_S3_CSV', 'AWS_KEY'),
-                          aws_secret_access_key=self._get_conn_env_var('TAP_S3_CSV', 'AWS_SECRET_ACCESS_KEY'))
+        s3 = boto3.client(
+            's3',
+            aws_access_key_id=self._get_conn_env_var('TAP_S3_CSV', 'AWS_KEY'),
+            aws_secret_access_key=self._get_conn_env_var(
+                'TAP_S3_CSV', 'AWS_SECRET_ACCESS_KEY'
+            ),
+        )
 
         s3.upload_file(mock_data_1, bucket, 'ppw_e2e_tap_s3_csv/mock_data_1.csv')
         s3.upload_file(mock_data_2, bucket, 'ppw_e2e_tap_s3_csv/mock_data_2.csv')
@@ -424,28 +530,56 @@ def setup_tap_s3_csv(self):
     def setup_target_postgres(self):
         """Clean postgres target database and prepare for test run"""
         self.run_query_target_postgres('CREATE EXTENSION IF NOT EXISTS pgcrypto')
-        self.run_query_target_postgres('DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres CASCADE')
-        self.run_query_target_postgres('DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_public2 CASCADE')
-        self.run_query_target_postgres('DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical1 CASCADE')
-        self.run_query_target_postgres('DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical2 CASCADE')
-        self.run_query_target_postgres('DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql CASCADE')
-        self.run_query_target_postgres('DROP SCHEMA IF EXISTS ppw_e2e_tap_s3_csv CASCADE')
-        self.run_query_target_postgres('DROP SCHEMA IF EXISTS ppw_e2e_tap_mongodb CASCADE')
+        self.run_query_target_postgres(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres CASCADE'
+        )
+        self.run_query_target_postgres(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_public2 CASCADE'
+        )
+        self.run_query_target_postgres(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical1 CASCADE'
+        )
+        self.run_query_target_postgres(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical2 CASCADE'
+        )
+        self.run_query_target_postgres(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql CASCADE'
+        )
+        self.run_query_target_postgres(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_s3_csv CASCADE'
+        )
+        self.run_query_target_postgres(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_mongodb CASCADE'
+        )
 
         # Clean config directory
         shutil.rmtree(os.path.join(CONFIG_DIR, 'postgres_dwh'), ignore_errors=True)
 
     def setup_target_redshift(self):
         """Clean redshift target database and prepare for test run"""
-        self.run_query_target_redshift('DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres CASCADE')
-        self.run_query_target_redshift('DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_public2 CASCADE')
-        self.run_query_target_redshift('DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical1 CASCADE')
-        self.run_query_target_redshift('DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical2 CASCADE')
-        self.run_query_target_redshift('DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql CASCADE')
-        self.run_query_target_redshift('DROP SCHEMA IF EXISTS ppw_e2e_tap_s3_csv CASCADE')
+        self.run_query_target_redshift(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres CASCADE'
+        )
+        self.run_query_target_redshift(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_public2 CASCADE'
+        )
+        self.run_query_target_redshift(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical1 CASCADE'
+        )
+        self.run_query_target_redshift(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical2 CASCADE'
+        )
+        self.run_query_target_redshift(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql CASCADE'
+        )
+        self.run_query_target_redshift(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_s3_csv CASCADE'
+        )
         self.run_query_target_redshift('DROP SCHEMA IF EXISTS ppw_e2e_helper CASCADE')
         self.run_query_target_redshift('CREATE SCHEMA ppw_e2e_helper')
-        self.run_query_target_redshift('CREATE TABLE ppw_e2e_helper.dual (dummy VARCHAR)')
+        self.run_query_target_redshift(
+            'CREATE TABLE ppw_e2e_helper.dual (dummy VARCHAR)'
+        )
         self.run_query_target_redshift('INSERT INTO ppw_e2e_helper.dual VALUES (\'X\')')
 
         # Clean config directory
@@ -453,13 +587,27 @@ def setup_target_redshift(self):
 
     def setup_target_snowflake(self):
         """Clean snowflake target database and prepare for test run"""
-        self.run_query_target_snowflake('DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres CASCADE')
-        self.run_query_target_snowflake('DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_public2 CASCADE')
-        self.run_query_target_snowflake('DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical1 CASCADE')
-        self.run_query_target_snowflake('DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical2 CASCADE')
-        self.run_query_target_snowflake('DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql CASCADE')
-        self.run_query_target_snowflake('DROP SCHEMA IF EXISTS ppw_e2e_tap_s3_csv CASCADE')
-        self.run_query_target_snowflake('DROP SCHEMA IF EXISTS ppw_e2e_tap_mongodb CASCADE')
+        self.run_query_target_snowflake(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres CASCADE'
+        )
+        self.run_query_target_snowflake(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_public2 CASCADE'
+        )
+        self.run_query_target_snowflake(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical1 CASCADE'
+        )
+        self.run_query_target_snowflake(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical2 CASCADE'
+        )
+        self.run_query_target_snowflake(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql CASCADE'
+        )
+        self.run_query_target_snowflake(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_s3_csv CASCADE'
+        )
+        self.run_query_target_snowflake(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_mongodb CASCADE'
+        )
 
         # Clean config directory
         shutil.rmtree(os.path.join(CONFIG_DIR, 'snowflake'), ignore_errors=True)
diff --git a/tests/end_to_end/helpers/tasks.py b/tests/end_to_end/helpers/tasks.py
index 8717a92dd..9d8c968b3 100644
--- a/tests/end_to_end/helpers/tasks.py
+++ b/tests/end_to_end/helpers/tasks.py
@@ -5,7 +5,9 @@
 
 def run_command(command):
     """Run shell command and return returncode, stdout and stderr"""
-    with subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc:
+    with subprocess.Popen(
+        shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.PIPE
+    ) as proc:
         proc_result = proc.communicate()
         return_code = proc.returncode
         stdout = proc_result[0].decode('utf-8')
@@ -16,12 +18,12 @@ def run_command(command):
 
 def find_run_tap_log_file(stdout, sync_engine=None):
     """Pipelinewise creates log file per running tap instances in a dynamically created directory:
-        ~/.pipelinewise/<TARGET_ID>/<TAP_ID>/log
+    ~/.pipelinewise/<TARGET_ID>/<TAP_ID>/log
 
-        Every log file matches the pattern:
-        <TARGET_ID>-<TAP_ID>-<DATE>_<TIME>.<SYNC_ENGINE>.log.<STATUS>
+    Every log file matches the pattern:
+    <TARGET_ID>-<TAP_ID>-<DATE>_<TIME>.<SYNC_ENGINE>.log.<STATUS>
 
-        The generated full path is logged to STDOUT when tap starting"""
+    The generated full path is logged to STDOUT when tap starting"""
     if sync_engine:
         pattern = re.compile(r'Writing output into (.+\.{}\.log)'.format(sync_engine))
     else:
diff --git a/tests/end_to_end/test_target_bigquery.py b/tests/end_to_end/test_target_bigquery.py
index cd3d462b5..fd1c9a2a9 100644
--- a/tests/end_to_end/test_target_bigquery.py
+++ b/tests/end_to_end/test_target_bigquery.py
@@ -48,7 +48,7 @@ def teardown_method(self):
     @pytest.mark.dependency(name='import_config')
     def test_import_project(self):
         """Import the YAML project with taps and target and do discovery mode
-        to write the JSON files for singer connectors """
+        to write the JSON files for singer connectors"""
 
         # Skip every target_postgres related test if required env vars not provided
         if not self.e2e.env['TARGET_BIGQUERY']['is_configured']:
@@ -63,95 +63,149 @@ def test_import_project(self):
         self.e2e.setup_target_bigquery()
 
         # Import project
-        [return_code, stdout, stderr] = tasks.run_command(f'pipelinewise import_config --dir {self.project_dir}')
+        [return_code, stdout, stderr] = tasks.run_command(
+            f'pipelinewise import_config --dir {self.project_dir}'
+        )
         assertions.assert_command_success(return_code, stdout, stderr)
 
     @pytest.mark.dependency(depends=['import_config'])
     def test_replicate_mariadb_to_bq(self, tap_mariadb_id=TAP_MARIADB_ID):
         """Replicate data from MariaDB to Bigquery"""
         # 1. Run tap first time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(tap_mariadb_id, TARGET_ID, ['fastsync', 'singer'])
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_bigquery)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.e2e.run_query_target_bigquery,
-                                            mysql_to_bigquery.tap_type_to_target_type)
+        assertions.assert_run_tap_success(
+            tap_mariadb_id, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_bigquery
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.e2e.run_query_target_bigquery,
+            mysql_to_bigquery.tap_type_to_target_type,
+        )
 
         # 2. Make changes in MariaDB source database
         #  LOG_BASED
-        self.run_query_tap_mysql('UPDATE weight_unit SET isactive = 0 WHERE weight_unit_id IN (2, 3, 4)')
-        self.run_query_tap_mysql('INSERT INTO edgydata (c_varchar, `group`, `case`, cjson, c_time) VALUES'
-                                 '(\'Lorem ipsum dolor sit amet\', 10, \'A\', \'[]\', \'00:00:00\'),'
-                                 '(\'Thai: แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช\', 20, \'A\', \'{}\', \'12:00:59\'),'
-                                 '(\'Chinese: 和毛泽东 <<重上井冈山>>. 严永欣, 一九八八年.\', null,\'B\', '
-                                 '\'[{"key": "ValueOne", "actions": []}, {"key": "ValueTwo", "actions": []}]\','
-                                 ' \'9:1:00\'),'
-                                 '(\'Special Characters: [\"\\,''!@£$%^&*()]\\\\\', null, \'B\', '
-                                 'null, \'12:00:00\'),'
-                                 '(\'	\', 20, \'B\', null, \'15:36:10\'),'
-                                 '(CONCAT(CHAR(0x0000 using utf16), \'<- null char\'), 20, \'B\', null, \'15:36:10\')')
+        self.run_query_tap_mysql(
+            'UPDATE weight_unit SET isactive = 0 WHERE weight_unit_id IN (2, 3, 4)'
+        )
+        self.run_query_tap_mysql(
+            'INSERT INTO edgydata (c_varchar, `group`, `case`, cjson, c_time) VALUES'
+            '(\'Lorem ipsum dolor sit amet\', 10, \'A\', \'[]\', \'00:00:00\'),'
+            '(\'Thai: แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช\', 20, \'A\', \'{}\', \'12:00:59\'),'
+            '(\'Chinese: 和毛泽东 <<重上井冈山>>. 严永欣, 一九八八年.\', null,\'B\', '
+            '\'[{"key": "ValueOne", "actions": []}, {"key": "ValueTwo", "actions": []}]\','
+            ' \'9:1:00\'),'
+            '(\'Special Characters: [\"\\,'
+            '!@£$%^&*()]\\\\\', null, \'B\', '
+            'null, \'12:00:00\'),'
+            '(\'	\', 20, \'B\', null, \'15:36:10\'),'
+            '(CONCAT(CHAR(0x0000 using utf16), \'<- null char\'), 20, \'B\', null, \'15:36:10\')'
+        )
 
         self.run_query_tap_mysql('UPDATE all_datatypes SET c_point = NULL')
 
         #  INCREMENTAL
-        self.run_query_tap_mysql('INSERT INTO address(isactive, street_number, date_created, date_updated,'
-                                 ' supplier_supplier_id, zip_code_zip_code_id)'
-                                 'VALUES (1, 1234, NOW(), NOW(), 0, 1234)')
-        self.run_query_tap_mysql('UPDATE address SET street_number = 9999, date_updated = NOW()'
-                                 ' WHERE address_id = 1')
+        self.run_query_tap_mysql(
+            'INSERT INTO address(isactive, street_number, date_created, date_updated,'
+            ' supplier_supplier_id, zip_code_zip_code_id)'
+            'VALUES (1, 1234, NOW(), NOW(), 0, 1234)'
+        )
+        self.run_query_tap_mysql(
+            'UPDATE address SET street_number = 9999, date_updated = NOW()'
+            ' WHERE address_id = 1'
+        )
         #  FULL_TABLE
         self.run_query_tap_mysql('DELETE FROM no_pk_table WHERE id > 10')
 
         # 3. Run tap second time - both fastsync and a singer should be triggered, there are some FULL_TABLE
-        assertions.assert_run_tap_success(tap_mariadb_id, TARGET_ID, ['fastsync', 'singer'])
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_bigquery)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.e2e.run_query_target_bigquery,
-                                            mysql_to_bigquery.tap_type_to_target_type, {'blob_col'})
+        assertions.assert_run_tap_success(
+            tap_mariadb_id, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_bigquery
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.e2e.run_query_target_bigquery,
+            mysql_to_bigquery.tap_type_to_target_type,
+            {'blob_col'},
+        )
 
         # Checking if mask-date transformation is working
         result = self.run_query_target_bigquery(
             'SELECT count(1) FROM ppw_e2e_tap_mysql.address '
-            'where EXTRACT(MONTH FROM date_created) != 1 or EXTRACT(DAY FROM date_created) != 1;')[0][0]
+            'where EXTRACT(MONTH FROM date_created) != 1 or EXTRACT(DAY FROM date_created) != 1;'
+        )[0][0]
 
         assert result == 0
 
         # Checking if conditional MASK-NUMBER transformation is working
         result = self.run_query_target_bigquery(
             'SELECT count(1) FROM ppw_e2e_tap_mysql.address '
-            'where zip_code_zip_code_id != 0 and REGEXP_CONTAINS(street_number, \'[801]\');')[0][0]
+            'where zip_code_zip_code_id != 0 and REGEXP_CONTAINS(street_number, \'[801]\');'
+        )[0][0]
 
         assert result == 0
 
         # Checking if conditional SET-NULL transformation is working
         result = self.run_query_target_bigquery(
             'SELECT count(1) FROM ppw_e2e_tap_mysql.edgydata '
-            'where "GROUP" is not null and "CASE" = \'B\';')[0][0]
+            'where "GROUP" is not null and "CASE" = \'B\';'
+        )[0][0]
 
         assert result == 0
 
     @pytest.mark.dependency(depends=['import_config'])
     def test_resync_mariadb_to_bq(self, tap_mariadb_id=TAP_MARIADB_ID):
         """Resync tables from MariaDB to Bigquery"""
-        assertions.assert_resync_tables_success(tap_mariadb_id, TARGET_ID, profiling=True)
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_bigquery)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_bigquery,
-                                            mysql_to_bigquery.tap_type_to_target_type)
+        assertions.assert_resync_tables_success(
+            tap_mariadb_id, TARGET_ID, profiling=True
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_bigquery
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.run_query_target_bigquery,
+            mysql_to_bigquery.tap_type_to_target_type,
+        )
 
     # pylint: disable=invalid-name
     @pytest.mark.dependency(depends=['import_config'])
-    def test_resync_mariadb_to_bq_with_split_large_files(self, tap_mariadb_id=TAP_MARIADB_SPLIT_LARGE_FILES_ID):
+    def test_resync_mariadb_to_bq_with_split_large_files(
+        self, tap_mariadb_id=TAP_MARIADB_SPLIT_LARGE_FILES_ID
+    ):
         """Resync tables from MariaDB to Bigquery using splitting large files option"""
-        assertions.assert_resync_tables_success(tap_mariadb_id, TARGET_ID, profiling=True)
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_bigquery)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_bigquery,
-                                            mysql_to_bigquery.tap_type_to_target_type)
+        assertions.assert_resync_tables_success(
+            tap_mariadb_id, TARGET_ID, profiling=True
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_bigquery
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.run_query_target_bigquery,
+            mysql_to_bigquery.tap_type_to_target_type,
+        )
 
     # pylint: disable=invalid-name
     @pytest.mark.dependency(depends=['import_config'])
-    def test_resync_pg_to_bq_with_split_large_files(self, tap_postgres_id=TAP_POSTGRES_SPLIT_LARGE_FILES_ID):
+    def test_resync_pg_to_bq_with_split_large_files(
+        self, tap_postgres_id=TAP_POSTGRES_SPLIT_LARGE_FILES_ID
+    ):
         """Resync tables from Postgres to Bigquery using splitting large files option"""
-        assertions.assert_resync_tables_success(tap_postgres_id, TARGET_ID, profiling=True)
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_bigquery)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_bigquery,
-                                            mysql_to_bigquery.tap_type_to_target_type)
+        assertions.assert_resync_tables_success(
+            tap_postgres_id, TARGET_ID, profiling=True
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_bigquery
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.run_query_target_bigquery,
+            mysql_to_bigquery.tap_type_to_target_type,
+        )
 
     # pylint: disable=invalid-name
     @pytest.mark.dependency(depends=['import_config'])
@@ -164,64 +218,87 @@ def test_replicate_mariadb_to_bq_with_custom_buffer_size(self):
     def test_replicate_pg_to_bq(self):
         """Replicate data from Postgres to Bigquery"""
         # Run tap first time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer'])
-        assertions.assert_row_counts_equal(self.run_query_tap_postgres, self.run_query_target_bigquery)
-        assertions.assert_all_columns_exist(self.run_query_tap_postgres, self.run_query_target_bigquery)
+        assertions.assert_run_tap_success(
+            TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_postgres, self.run_query_target_bigquery
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_postgres, self.run_query_target_bigquery
+        )
 
         result = self.run_query_target_bigquery(
             'SELECT updated_at FROM '
             'ppw_e2e_tap_postgres.`table_with_space_and_uppercase` '
-            'where cvarchar=\'H\';')[0][0]
+            'where cvarchar=\'H\';'
+        )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 999008, tzinfo=timezone.utc)
 
         result = self.run_query_target_bigquery(
             'SELECT updated_at FROM '
             'ppw_e2e_tap_postgres.`table_with_space_and_uppercase` '
-            'where cvarchar=\'I\';')[0][0]
+            'where cvarchar=\'I\';'
+        )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 999008, tzinfo=timezone.utc)
 
         # 2. Make changes in PG source database
         #  LOG_BASED
-        self.run_query_tap_postgres('insert into public."table_with_space and UPPERCase" (cvarchar, updated_at) values '
-                                    "('X', '2020-01-01 08:53:56.8+10'),"
-                                    "('Y', '2020-12-31 12:59:00.148+00'),"
-                                    "('faaaar future', '15000-05-23 12:40:00.148'),"
-                                    "('BC', '2020-01-23 01:40:00 BC'),"
-                                    "('Z', null),"
-                                    "('W', '2020-03-03 12:30:00');")
+        self.run_query_tap_postgres(
+            'insert into public."table_with_space and UPPERCase" (cvarchar, updated_at) values '
+            "('X', '2020-01-01 08:53:56.8+10'),"
+            "('Y', '2020-12-31 12:59:00.148+00'),"
+            "('faaaar future', '15000-05-23 12:40:00.148'),"
+            "('BC', '2020-01-23 01:40:00 BC'),"
+            "('Z', null),"
+            "('W', '2020-03-03 12:30:00');"
+        )
         #  INCREMENTAL
-        self.run_query_tap_postgres('INSERT INTO public.city (id, name, countrycode, district, population) '
-                                    "VALUES (4080, 'Bath', 'GBR', 'England', 88859)")
-        self.run_query_tap_postgres('UPDATE public.edgydata SET '
-                                    "cjson = json '{\"data\": 1234}', "
-                                    "cjsonb = jsonb '{\"data\": 2345}', "
-                                    "cvarchar = 'Liewe Maatjies UPDATED' WHERE cid = 23")
+        self.run_query_tap_postgres(
+            'INSERT INTO public.city (id, name, countrycode, district, population) '
+            "VALUES (4080, 'Bath', 'GBR', 'England', 88859)"
+        )
+        self.run_query_tap_postgres(
+            'UPDATE public.edgydata SET '
+            "cjson = json '{\"data\": 1234}', "
+            "cjsonb = jsonb '{\"data\": 2345}', "
+            "cvarchar = 'Liewe Maatjies UPDATED' WHERE cid = 23"
+        )
         #  FULL_TABLE
         self.run_query_tap_postgres("DELETE FROM public.country WHERE code = 'UMI'")
 
         # 3. Run tap second time - both fastsync and a singer should be triggered, there are some FULL_TABLE
-        assertions.assert_run_tap_success(TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer'], profiling=True)
-        assertions.assert_row_counts_equal(self.run_query_tap_postgres, self.run_query_target_bigquery)
-        assertions.assert_all_columns_exist(self.run_query_tap_postgres, self.run_query_target_bigquery)
+        assertions.assert_run_tap_success(
+            TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer'], profiling=True
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_postgres, self.run_query_target_bigquery
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_postgres, self.run_query_target_bigquery
+        )
 
         result = self.run_query_target_bigquery(
-            'SELECT updated_at FROM ppw_e2e_tap_postgres.`table_with_space_and_uppercase` where cvarchar=\'X\';')[0][0]
+            'SELECT updated_at FROM ppw_e2e_tap_postgres.`table_with_space_and_uppercase` where cvarchar=\'X\';'
+        )[0][0]
 
         assert result == datetime(2019, 12, 31, 22, 53, 56, 800000, tzinfo=timezone.utc)
 
         result = self.run_query_target_bigquery(
             'SELECT updated_at FROM '
             'ppw_e2e_tap_postgres.`table_with_space_and_uppercase`  '
-            'where cvarchar=\'faaaar future\';')[0][0]
+            'where cvarchar=\'faaaar future\';'
+        )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 999008, tzinfo=timezone.utc)
 
         result = self.run_query_target_bigquery(
             'SELECT updated_at FROM '
             'ppw_e2e_tap_postgres.`table_with_space_and_uppercase`  '
-            'where cvarchar=\'BC\';')[0][0]
+            'where cvarchar=\'BC\';'
+        )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 999008, tzinfo=timezone.utc)
 
@@ -234,18 +311,39 @@ def test_replicate_s3_to_bq(self):
 
         def assert_columns_exist():
             """Helper inner function to test if every table and column exists in target bigquery"""
-            assertions.assert_cols_in_table(self.run_query_target_bigquery, 'ppw_e2e_tap_s3_csv', 'countries',
-                                            ['city', 'country', 'currency', 'id', 'language'])
-            assertions.assert_cols_in_table(self.run_query_target_bigquery, 'ppw_e2e_tap_s3_csv', 'people',
-                                            ['birth_date', 'email', 'first_name', 'gender', 'group', 'id',
-                                             'ip_address', 'is_pensioneer', 'last_name'])
+            assertions.assert_cols_in_table(
+                self.run_query_target_bigquery,
+                'ppw_e2e_tap_s3_csv',
+                'countries',
+                ['city', 'country', 'currency', 'id', 'language'],
+            )
+            assertions.assert_cols_in_table(
+                self.run_query_target_bigquery,
+                'ppw_e2e_tap_s3_csv',
+                'people',
+                [
+                    'birth_date',
+                    'email',
+                    'first_name',
+                    'gender',
+                    'group',
+                    'id',
+                    'ip_address',
+                    'is_pensioneer',
+                    'last_name',
+                ],
+            )
 
         # 1. Run tap first time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer'])
+        assertions.assert_run_tap_success(
+            TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer']
+        )
         assert_columns_exist()
 
         # 2. Run tap second time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer'])
+        assertions.assert_run_tap_success(
+            TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer']
+        )
         assert_columns_exist()
 
     @pytest.mark.dependency(depends=['import_config'])
@@ -254,16 +352,31 @@ def test_replicate_mongodb_to_bq(self):
 
         def assert_columns_exist(table):
             """Helper inner function to test if every table and column exists in the target"""
-            assertions.assert_cols_in_table(self.run_query_target_bigquery, 'ppw_e2e_tap_mongodb', table,
-                                            ['_id', 'document', '_sdc_extracted_at',
-                                             '_sdc_batched_at', '_sdc_deleted_at'])
+            assertions.assert_cols_in_table(
+                self.run_query_target_bigquery,
+                'ppw_e2e_tap_mongodb',
+                table,
+                [
+                    '_id',
+                    'document',
+                    '_sdc_extracted_at',
+                    '_sdc_batched_at',
+                    '_sdc_deleted_at',
+                ],
+            )
 
         def assert_row_counts_equal(target_schema, table, count_in_source):
-            assert count_in_source == \
-                   self.run_query_target_bigquery(f'select count(_id) from {target_schema}.{table}')[0][0]
+            assert (
+                count_in_source
+                == self.run_query_target_bigquery(
+                    f'select count(_id) from {target_schema}.{table}'
+                )[0][0]
+            )
 
         # Run tap first time - fastsync and singer should be triggered
-        assertions.assert_run_tap_success(TAP_MONGODB_ID, TARGET_ID, ['fastsync', 'singer'])
+        assertions.assert_run_tap_success(
+            TAP_MONGODB_ID, TARGET_ID, ['fastsync', 'singer']
+        )
         assert_columns_exist('listings')
         assert_columns_exist('my_collection')
         assert_columns_exist('all_datatypes')
@@ -274,41 +387,59 @@ def assert_row_counts_equal(target_schema, table, count_in_source):
 
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'listings', listing_count)
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'my_collection', my_coll_count)
-        assert_row_counts_equal('ppw_e2e_tap_mongodb', 'all_datatypes', all_datatypes_count)
-
-        result_insert = self.mongodb_con.my_collection.insert_many([
-            {
-                'age': randint(10, 30),
-                'id': 1001,
-                'uuid': uuid.uuid4(),
-                'ts': Timestamp(12030, 500)
-            },
-            {
-                'date': datetime.utcnow(),
-                'id': 1002,
-                'uuid': uuid.uuid4(),
-                'regex': bson.Regex(r'^[A-Z]\\w\\d{2,6}.*$')
-            },
-            {
-                'uuid': uuid.uuid4(),
-                'id': 1003,
-                'decimal': bson.Decimal128(decimal.Decimal('5.64547548425446546546644')),
-                'nested_json': {'a': 1, 'b': 3, 'c': {'key': bson.datetime.datetime(2020, 5, 3, 10, 0, 0)}}
-            }
-        ])
+        assert_row_counts_equal(
+            'ppw_e2e_tap_mongodb', 'all_datatypes', all_datatypes_count
+        )
+
+        result_insert = self.mongodb_con.my_collection.insert_many(
+            [
+                {
+                    'age': randint(10, 30),
+                    'id': 1001,
+                    'uuid': uuid.uuid4(),
+                    'ts': Timestamp(12030, 500),
+                },
+                {
+                    'date': datetime.utcnow(),
+                    'id': 1002,
+                    'uuid': uuid.uuid4(),
+                    'regex': bson.Regex(r'^[A-Z]\\w\\d{2,6}.*$'),
+                },
+                {
+                    'uuid': uuid.uuid4(),
+                    'id': 1003,
+                    'decimal': bson.Decimal128(
+                        decimal.Decimal('5.64547548425446546546644')
+                    ),
+                    'nested_json': {
+                        'a': 1,
+                        'b': 3,
+                        'c': {'key': bson.datetime.datetime(2020, 5, 3, 10, 0, 0)},
+                    },
+                },
+            ]
+        )
         my_coll_count += len(result_insert.inserted_ids)
 
-        result_del = self.mongodb_con.my_collection.delete_one({'_id': result_insert.inserted_ids[0]})
+        result_del = self.mongodb_con.my_collection.delete_one(
+            {'_id': result_insert.inserted_ids[0]}
+        )
         my_coll_count -= result_del.deleted_count
 
-        result_update = self.mongodb_con.my_collection.update_many({}, {'$set': {'id': 0}})
+        result_update = self.mongodb_con.my_collection.update_many(
+            {}, {'$set': {'id': 0}}
+        )
 
         assertions.assert_run_tap_success(TAP_MONGODB_ID, TARGET_ID, ['singer'])
 
-        assert result_update.modified_count == self.run_query_target_bigquery(
-            """select count(_id)
+        assert (
+            result_update.modified_count
+            == self.run_query_target_bigquery(
+                """select count(_id)
                from ppw_e2e_tap_mongodb.my_collection
                where SAFE_CAST(JSON_EXTRACT_SCALAR(document, '$.id') AS INT64) = 0
-               """)[0][0]
+               """
+            )[0][0]
+        )
 
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'my_collection', my_coll_count)
diff --git a/tests/end_to_end/test_target_postgres.py b/tests/end_to_end/test_target_postgres.py
index 6c641329d..182f5edfd 100644
--- a/tests/end_to_end/test_target_postgres.py
+++ b/tests/end_to_end/test_target_postgres.py
@@ -46,7 +46,7 @@ def teardown_method(self):
     @pytest.mark.dependency(name='import_config')
     def test_import_project(self):
         """Import the YAML project with taps and target and do discovery mode
-        to write the JSON files for singer connectors """
+        to write the JSON files for singer connectors"""
 
         # Skip every target_postgres related test if required env vars not provided
         if not self.e2e.env['TARGET_POSTGRES']['is_configured']:
@@ -62,7 +62,8 @@ def test_import_project(self):
 
         # Import project
         [return_code, stdout, stderr] = tasks.run_command(
-            f'pipelinewise import_config --dir {self.project_dir} --profiler')
+            f'pipelinewise import_config --dir {self.project_dir} --profiler'
+        )
 
         assertions.assert_command_success(return_code, stdout, stderr)
         assertions.assert_profiling_stats_files_created(stdout, 'import_project')
@@ -71,70 +72,105 @@ def test_import_project(self):
     def test_replicate_mariadb_to_pg(self, tap_mariadb_id=TAP_MARIADB_ID):
         """Replicate data from MariaDB to Postgres DWH"""
         # 1. Run tap first time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(tap_mariadb_id, TARGET_ID, ['fastsync', 'singer'], profiling=True)
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_postgres)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_postgres,
-                                            mysql_to_postgres.tap_type_to_target_type)
+        assertions.assert_run_tap_success(
+            tap_mariadb_id, TARGET_ID, ['fastsync', 'singer'], profiling=True
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_postgres
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.run_query_target_postgres,
+            mysql_to_postgres.tap_type_to_target_type,
+        )
 
         # 2. Make changes in MariaDB source database
         #  LOG_BASED
-        self.run_query_tap_mysql('UPDATE weight_unit SET isactive = 0 WHERE weight_unit_id IN (2, 3, 4)')
+        self.run_query_tap_mysql(
+            'UPDATE weight_unit SET isactive = 0 WHERE weight_unit_id IN (2, 3, 4)'
+        )
         self.run_query_tap_mysql('ALTER table weight_unit add column bool_col bool;')
-        self.run_query_tap_mysql('INSERT into weight_unit(weight_unit_name, isActive, original_date_created, bool_col) '
-                                 'values (\'Oz\', false, \'2020-07-23 10:00:00\', true);')
+        self.run_query_tap_mysql(
+            'INSERT into weight_unit(weight_unit_name, isActive, original_date_created, bool_col) '
+            'values (\'Oz\', false, \'2020-07-23 10:00:00\', true);'
+        )
         self.run_query_tap_mysql('ALTER table weight_unit add column blob_col blob;')
-        self.run_query_tap_mysql('INSERT into weight_unit(weight_unit_name, isActive, original_date_created, blob_col) '
-                                 'values (\'Oz\', false, \'2020-07-23 10:00:00\', \'blob content\');')
-        self.run_query_tap_mysql('ALTER table weight_unit change column bool_col is_imperial bool;')
-        self.run_query_tap_mysql('UPDATE weight_unit SET is_imperial = false WHERE weight_unit_name like \'%oz%\'')
-
-        self.run_query_tap_mysql('INSERT INTO edgydata (c_varchar, `group`, `case`, cjson, c_time) VALUES'
-                                 '(\'Lorem ipsum dolor sit amet\', 10, \'A\', \'[]\', \'00:00:00\'),'
-                                 '(\'Thai: แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช\', 20, \'A\', \'{}\', \'12:00:59\'),'
-                                 '(\'Chinese: 和毛泽东 <<重上井冈山>>. 严永欣, 一九八八年.\', null,\'B\', '
-                                 '\'[{"key": "ValueOne", "actions": []}, {"key": "ValueTwo", "actions": []}]\','
-                                 ' \'9:1:00\'),'
-                                 '(\'Special Characters: [\"\\,''!@£$%^&*()]\\\\\', null, \'B\', '
-                                 'null, \'12:00:00\'),'
-                                 '(\'	\', 20, \'B\', null, \'15:36:10\'),'
-                                 '(CONCAT(CHAR(0x0000 using utf16), \'<- null char\'), 20, \'B\', null, \'15:36:10\')')
+        self.run_query_tap_mysql(
+            'INSERT into weight_unit(weight_unit_name, isActive, original_date_created, blob_col) '
+            'values (\'Oz\', false, \'2020-07-23 10:00:00\', \'blob content\');'
+        )
+        self.run_query_tap_mysql(
+            'ALTER table weight_unit change column bool_col is_imperial bool;'
+        )
+        self.run_query_tap_mysql(
+            'UPDATE weight_unit SET is_imperial = false WHERE weight_unit_name like \'%oz%\''
+        )
+
+        self.run_query_tap_mysql(
+            'INSERT INTO edgydata (c_varchar, `group`, `case`, cjson, c_time) VALUES'
+            '(\'Lorem ipsum dolor sit amet\', 10, \'A\', \'[]\', \'00:00:00\'),'
+            '(\'Thai: แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช\', 20, \'A\', \'{}\', \'12:00:59\'),'
+            '(\'Chinese: 和毛泽东 <<重上井冈山>>. 严永欣, 一九八八年.\', null,\'B\', '
+            '\'[{"key": "ValueOne", "actions": []}, {"key": "ValueTwo", "actions": []}]\','
+            ' \'9:1:00\'),'
+            '(\'Special Characters: [\"\\,'
+            '!@£$%^&*()]\\\\\', null, \'B\', '
+            'null, \'12:00:00\'),'
+            '(\'	\', 20, \'B\', null, \'15:36:10\'),'
+            '(CONCAT(CHAR(0x0000 using utf16), \'<- null char\'), 20, \'B\', null, \'15:36:10\')'
+        )
 
         self.run_query_tap_mysql('UPDATE all_datatypes SET c_point = NULL')
 
         #  INCREMENTAL
-        self.run_query_tap_mysql('INSERT INTO address(isactive, street_number, date_created, date_updated,'
-                                 ' supplier_supplier_id, zip_code_zip_code_id)'
-                                 'VALUES (1, 1234, NOW(), NOW(), 0, 1234)')
-        self.run_query_tap_mysql('UPDATE address SET street_number = 9999, date_updated = NOW()'
-                                 ' WHERE address_id = 1')
+        self.run_query_tap_mysql(
+            'INSERT INTO address(isactive, street_number, date_created, date_updated,'
+            ' supplier_supplier_id, zip_code_zip_code_id)'
+            'VALUES (1, 1234, NOW(), NOW(), 0, 1234)'
+        )
+        self.run_query_tap_mysql(
+            'UPDATE address SET street_number = 9999, date_updated = NOW()'
+            ' WHERE address_id = 1'
+        )
         #  FULL_TABLE
         self.run_query_tap_mysql('DELETE FROM no_pk_table WHERE id > 10')
 
         # 3. Run tap second time - both fastsync and a singer should be triggered, there are some FULL_TABLE
-        assertions.assert_run_tap_success(tap_mariadb_id, TARGET_ID, ['fastsync', 'singer'], profiling=True)
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_postgres)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_postgres,
-                                            mysql_to_postgres.tap_type_to_target_type, {'blob_col'})
+        assertions.assert_run_tap_success(
+            tap_mariadb_id, TARGET_ID, ['fastsync', 'singer'], profiling=True
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_postgres
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.run_query_target_postgres,
+            mysql_to_postgres.tap_type_to_target_type,
+            {'blob_col'},
+        )
 
         # Checking if mask-date transformation is working
         result = self.run_query_target_postgres(
             'SELECT count(1) FROM ppw_e2e_tap_mysql."address" '
             'where date_part(\'month\',date_created)::int != 1 or '
-            'date_part(\'day\', date_created)::int != 1;')[0][0]
+            'date_part(\'day\', date_created)::int != 1;'
+        )[0][0]
 
         assert result == 0
 
         # Checking if conditional MASK-NUMBER transformation is working
         result = self.run_query_target_postgres(
             'SELECT count(1) FROM ppw_e2e_tap_mysql."address" '
-            'where zip_code_zip_code_id != 0 and street_number ~ \'[801]\';')[0][0]
+            'where zip_code_zip_code_id != 0 and street_number ~ \'[801]\';'
+        )[0][0]
 
         assert result == 0
 
         # Checking if conditional SET-NULL transformation is working
         result = self.run_query_target_postgres(
             'SELECT count(1) FROM ppw_e2e_tap_mysql."edgydata" '
-            'where "group" is not null and "case" = \'B\';')[0][0]
+            'where "group" is not null and "case" = \'B\';'
+        )[0][0]
 
         assert result == 0
 
@@ -143,9 +179,14 @@ def test_resync_mariadb_to_pg(self, tap_mariadb_id=TAP_MARIADB_ID):
         """Resync tables from MariaDB to Postgres DWH"""
         # 1. Run tap first time - both fastsync and a singer should be triggered
         assertions.assert_resync_tables_success(tap_mariadb_id, TARGET_ID)
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_postgres)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_postgres,
-                                            mysql_to_postgres.tap_type_to_target_type)
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_postgres
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.run_query_target_postgres,
+            mysql_to_postgres.tap_type_to_target_type,
+        )
 
     # pylint: disable=invalid-name
     @pytest.mark.dependency(depends=['import_config'])
@@ -158,77 +199,110 @@ def test_replicate_mariadb_to_pg_with_custom_buffer_size(self):
     def test_replicate_pg_to_pg(self):
         """Replicate data from Postgres to Postgres DWH"""
         # 1. Run tap first time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer'])
-        assertions.assert_row_counts_equal(self.run_query_tap_postgres, self.run_query_target_postgres)
-        assertions.assert_all_columns_exist(self.run_query_tap_postgres, self.run_query_target_postgres)
-        assertions.assert_date_column_naive_in_target(self.run_query_target_postgres,
-                                                      'updated_at',
-                                                      'ppw_e2e_tap_postgres."table_with_space and uppercase"')
+        assertions.assert_run_tap_success(
+            TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_postgres, self.run_query_target_postgres
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_postgres, self.run_query_target_postgres
+        )
+        assertions.assert_date_column_naive_in_target(
+            self.run_query_target_postgres,
+            'updated_at',
+            'ppw_e2e_tap_postgres."table_with_space and uppercase"',
+        )
 
         result = self.run_query_target_postgres(
             'SELECT updated_at FROM '
             'ppw_e2e_tap_postgres."table_with_space and uppercase" '
-            'where cvarchar=\'H\';')[0][0]
+            'where cvarchar=\'H\';'
+        )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 999000)
 
         result = self.run_query_target_postgres(
             'SELECT updated_at FROM '
             'ppw_e2e_tap_postgres."table_with_space and uppercase" '
-            'where cvarchar=\'I\';')[0][0]
+            'where cvarchar=\'I\';'
+        )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 999000)
 
         # 2. Make changes in pg source database
         #  LOG_BASED
-        self.run_query_tap_postgres('insert into public."table_with_space and UPPERCase" (cvarchar, updated_at) values '
-                                    "('M', '2020-01-01 08:53:56.8+10'),"
-                                    "('N', '2020-12-31 12:59:00.148+00'),"
-                                    "('Year in the faaaar future', '20000-05-23 12:40:00.148'),"
-                                    "('Year in the BC', '2020-01-23 01:40:00 BC'),"
-                                    "('O', null),"
-                                    "('P', '2020-03-03 12:30:00');")
+        self.run_query_tap_postgres(
+            'insert into public."table_with_space and UPPERCase" (cvarchar, updated_at) values '
+            "('M', '2020-01-01 08:53:56.8+10'),"
+            "('N', '2020-12-31 12:59:00.148+00'),"
+            "('Year in the faaaar future', '20000-05-23 12:40:00.148'),"
+            "('Year in the BC', '2020-01-23 01:40:00 BC'),"
+            "('O', null),"
+            "('P', '2020-03-03 12:30:00');"
+        )
 
         #  INCREMENTAL
-        self.run_query_tap_postgres('INSERT INTO public.city (id, name, countrycode, district, population) '
-                                    "VALUES (4080, 'Bath', 'GBR', 'England', 88859)")
-        self.run_query_tap_postgres('UPDATE public.edgydata SET '
-                                    "cjson = json '{\"data\": 1234}', "
-                                    "cjsonb = jsonb '{\"data\": 2345}', "
-                                    "cvarchar = 'Liewe Maatjies UPDATED' WHERE cid = 23")
+        self.run_query_tap_postgres(
+            'INSERT INTO public.city (id, name, countrycode, district, population) '
+            "VALUES (4080, 'Bath', 'GBR', 'England', 88859)"
+        )
+        self.run_query_tap_postgres(
+            'UPDATE public.edgydata SET '
+            "cjson = json '{\"data\": 1234}', "
+            "cjsonb = jsonb '{\"data\": 2345}', "
+            "cvarchar = 'Liewe Maatjies UPDATED' WHERE cid = 23"
+        )
         #  FULL_TABLE
         self.run_query_tap_postgres("DELETE FROM public.country WHERE code = 'UMI'")
 
         #  LOG_BASED
-        self.run_query_tap_postgres('ALTER TABLE logical1.logical1_table1 ADD COLUMN bool_col bool;')
-        self.run_query_tap_postgres('ALTER TABLE logical1.logical1_table1 RENAME COLUMN cvarchar2 to varchar_col;')
-        self.run_query_tap_postgres('INSERT INTO logical1.logical1_table1 (cvarchar, varchar_col, bool_col) values '
-                                    '(\'insert after alter table\', \'this is renamed column\', true);')
+        self.run_query_tap_postgres(
+            'ALTER TABLE logical1.logical1_table1 ADD COLUMN bool_col bool;'
+        )
+        self.run_query_tap_postgres(
+            'ALTER TABLE logical1.logical1_table1 RENAME COLUMN cvarchar2 to varchar_col;'
+        )
+        self.run_query_tap_postgres(
+            'INSERT INTO logical1.logical1_table1 (cvarchar, varchar_col, bool_col) values '
+            '(\'insert after alter table\', \'this is renamed column\', true);'
+        )
 
         # 3. Run tap second time - both fastsync and a singer should be triggered, there are some FULL_TABLE
-        assertions.assert_run_tap_success(TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer'])
-        assertions.assert_row_counts_equal(self.run_query_tap_postgres, self.run_query_target_postgres)
-        assertions.assert_all_columns_exist(self.run_query_tap_postgres, self.run_query_target_postgres)
-        assertions.assert_date_column_naive_in_target(self.run_query_target_postgres,
-                                                      'updated_at',
-                                                      'ppw_e2e_tap_postgres."table_with_space and uppercase"')
+        assertions.assert_run_tap_success(
+            TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_postgres, self.run_query_target_postgres
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_postgres, self.run_query_target_postgres
+        )
+        assertions.assert_date_column_naive_in_target(
+            self.run_query_target_postgres,
+            'updated_at',
+            'ppw_e2e_tap_postgres."table_with_space and uppercase"',
+        )
 
         result = self.run_query_target_postgres(
-            'SELECT updated_at FROM ppw_e2e_tap_postgres."table_with_space and uppercase" where cvarchar=\'M\';')[0][0]
+            'SELECT updated_at FROM ppw_e2e_tap_postgres."table_with_space and uppercase" where cvarchar=\'M\';'
+        )[0][0]
 
         assert result == datetime(2019, 12, 31, 22, 53, 56, 800000)
 
         result = self.run_query_target_postgres(
             'SELECT updated_at FROM '
             'ppw_e2e_tap_postgres."table_with_space and uppercase" '
-            'where cvarchar=\'Year in the faaaar future\';')[0][0]
+            'where cvarchar=\'Year in the faaaar future\';'
+        )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 999000)
 
         result = self.run_query_target_postgres(
             'SELECT updated_at FROM '
             'ppw_e2e_tap_postgres."table_with_space and uppercase" '
-            'where cvarchar=\'Year in the BC\';')[0][0]
+            'where cvarchar=\'Year in the BC\';'
+        )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 999000)
 
@@ -241,18 +315,39 @@ def test_replicate_s3_to_pg(self):
 
         def assert_columns_exist():
             """Helper inner function to test if every table and column exists in target snowflake"""
-            assertions.assert_cols_in_table(self.run_query_target_postgres, 'ppw_e2e_tap_s3_csv', 'countries',
-                                            ['city', 'country', 'currency', 'id', 'language'])
-            assertions.assert_cols_in_table(self.run_query_target_postgres, 'ppw_e2e_tap_s3_csv', 'people',
-                                            ['birth_date', 'email', 'first_name', 'gender', 'group', 'id',
-                                             'ip_address', 'is_pensioneer', 'last_name'])
+            assertions.assert_cols_in_table(
+                self.run_query_target_postgres,
+                'ppw_e2e_tap_s3_csv',
+                'countries',
+                ['city', 'country', 'currency', 'id', 'language'],
+            )
+            assertions.assert_cols_in_table(
+                self.run_query_target_postgres,
+                'ppw_e2e_tap_s3_csv',
+                'people',
+                [
+                    'birth_date',
+                    'email',
+                    'first_name',
+                    'gender',
+                    'group',
+                    'id',
+                    'ip_address',
+                    'is_pensioneer',
+                    'last_name',
+                ],
+            )
 
         # 1. Run tap first time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer'])
+        assertions.assert_run_tap_success(
+            TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer']
+        )
         assert_columns_exist()
 
         # 2. Run tap second time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer'])
+        assertions.assert_run_tap_success(
+            TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer']
+        )
         assert_columns_exist()
 
     @pytest.mark.dependency(depends=['import_config'])
@@ -261,16 +356,31 @@ def test_replicate_mongodb_to_pg(self):
 
         def assert_columns_exist(table):
             """Helper inner function to test if every table and column exists in the target"""
-            assertions.assert_cols_in_table(self.run_query_target_postgres, 'ppw_e2e_tap_mongodb', table,
-                                            ['_id', 'document', '_sdc_extracted_at',
-                                             '_sdc_batched_at', '_sdc_deleted_at'])
+            assertions.assert_cols_in_table(
+                self.run_query_target_postgres,
+                'ppw_e2e_tap_mongodb',
+                table,
+                [
+                    '_id',
+                    'document',
+                    '_sdc_extracted_at',
+                    '_sdc_batched_at',
+                    '_sdc_deleted_at',
+                ],
+            )
 
         def assert_row_counts_equal(target_schema, table, count_in_source):
-            assert count_in_source == \
-                   self.run_query_target_postgres(f'select count(_id) from {target_schema}.{table}')[0][0]
+            assert (
+                count_in_source
+                == self.run_query_target_postgres(
+                    f'select count(_id) from {target_schema}.{table}'
+                )[0][0]
+            )
 
         # Run tap first time - fastsync and singer should be triggered
-        assertions.assert_run_tap_success(TAP_MONGODB_ID, TARGET_ID, ['fastsync', 'singer'])
+        assertions.assert_run_tap_success(
+            TAP_MONGODB_ID, TARGET_ID, ['fastsync', 'singer']
+        )
         assert_columns_exist('listings')
         assert_columns_exist('my_collection')
         assert_columns_exist('all_datatypes')
@@ -281,38 +391,56 @@ def assert_row_counts_equal(target_schema, table, count_in_source):
 
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'listings', listing_count)
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'my_collection', my_coll_count)
-        assert_row_counts_equal('ppw_e2e_tap_mongodb', 'all_datatypes', all_datatypes_count)
-
-        result_insert = self.mongodb_con.my_collection.insert_many([
-            {
-                'age': randint(10, 30),
-                'id': 1001,
-                'uuid': uuid.uuid4(),
-                'ts': Timestamp(12030, 500)
-            },
-            {
-                'date': datetime.utcnow(),
-                'id': 1002,
-                'uuid': uuid.uuid4(),
-                'regex': bson.Regex(r'^[A-Z]\\w\\d{2,6}.*$')
-            },
-            {
-                'uuid': uuid.uuid4(),
-                'id': 1003,
-                'decimal': bson.Decimal128(decimal.Decimal('5.64547548425446546546644')),
-                'nested_json': {'a': 1, 'b': 3, 'c': {'key': bson.datetime.datetime(2020, 5, 3, 10, 0, 0)}}
-            }
-        ])
+        assert_row_counts_equal(
+            'ppw_e2e_tap_mongodb', 'all_datatypes', all_datatypes_count
+        )
+
+        result_insert = self.mongodb_con.my_collection.insert_many(
+            [
+                {
+                    'age': randint(10, 30),
+                    'id': 1001,
+                    'uuid': uuid.uuid4(),
+                    'ts': Timestamp(12030, 500),
+                },
+                {
+                    'date': datetime.utcnow(),
+                    'id': 1002,
+                    'uuid': uuid.uuid4(),
+                    'regex': bson.Regex(r'^[A-Z]\\w\\d{2,6}.*$'),
+                },
+                {
+                    'uuid': uuid.uuid4(),
+                    'id': 1003,
+                    'decimal': bson.Decimal128(
+                        decimal.Decimal('5.64547548425446546546644')
+                    ),
+                    'nested_json': {
+                        'a': 1,
+                        'b': 3,
+                        'c': {'key': bson.datetime.datetime(2020, 5, 3, 10, 0, 0)},
+                    },
+                },
+            ]
+        )
         my_coll_count += len(result_insert.inserted_ids)
 
-        result_del = self.mongodb_con.my_collection.delete_one({'_id': result_insert.inserted_ids[0]})
+        result_del = self.mongodb_con.my_collection.delete_one(
+            {'_id': result_insert.inserted_ids[0]}
+        )
         my_coll_count -= result_del.deleted_count
 
-        result_update = self.mongodb_con.my_collection.update_many({}, {'$set': {'id': 0}})
+        result_update = self.mongodb_con.my_collection.update_many(
+            {}, {'$set': {'id': 0}}
+        )
 
         assertions.assert_run_tap_success(TAP_MONGODB_ID, TARGET_ID, ['singer'])
 
-        assert result_update.modified_count == self.run_query_target_postgres(
-            'select count(_id) from ppw_e2e_tap_mongodb.my_collection where cast(document->>\'id\' as int) = 0')[0][0]
+        assert (
+            result_update.modified_count
+            == self.run_query_target_postgres(
+                'select count(_id) from ppw_e2e_tap_mongodb.my_collection where cast(document->>\'id\' as int) = 0'
+            )[0][0]
+        )
 
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'my_collection', my_coll_count)
diff --git a/tests/end_to_end/test_target_redshift.py b/tests/end_to_end/test_target_redshift.py
index f70f116ad..76be9131b 100644
--- a/tests/end_to_end/test_target_redshift.py
+++ b/tests/end_to_end/test_target_redshift.py
@@ -38,7 +38,7 @@ def teardown_method(self):
     @pytest.mark.dependency(name='import_config')
     def test_import_project(self):
         """Import the YAML project with taps and target and do discovery mode
-        to write the JSON files for singer connectors """
+        to write the JSON files for singer connectors"""
 
         # Skip every target_postgres related test if env vars not provided
         if not self.e2e.env['TARGET_REDSHIFT']['is_configured']:
@@ -52,45 +52,63 @@ def test_import_project(self):
         self.e2e.setup_target_redshift()
 
         # Import project
-        [return_code, stdout, stderr] = tasks.run_command(f'pipelinewise import_config --dir {self.project_dir}')
+        [return_code, stdout, stderr] = tasks.run_command(
+            f'pipelinewise import_config --dir {self.project_dir}'
+        )
         assertions.assert_command_success(return_code, stdout, stderr)
 
     @pytest.mark.dependency(depends=['import_config'])
     def test_replicate_mariadb_to_rs(self, tap_mariadb_id=TAP_MARIADB_ID):
         """Replicate data from Postgres to Redshift DWH"""
         # 1. Run tap first time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(tap_mariadb_id, TARGET_ID, ['fastsync', 'singer'])
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_redshift)
-        #assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_redshift,
+        assertions.assert_run_tap_success(
+            tap_mariadb_id, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_redshift
+        )
+        # assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_redshift,
         #                                    mysql_to_redshift.tap_type_to_target_type)
 
         # 2. Make changes in MariaDB source database
         #  LOG_BASED
-        self.run_query_tap_mysql('UPDATE weight_unit SET isactive = 0 WHERE weight_unit_id IN (2, 3, 4)')
+        self.run_query_tap_mysql(
+            'UPDATE weight_unit SET isactive = 0 WHERE weight_unit_id IN (2, 3, 4)'
+        )
         self.run_query_tap_mysql('UPDATE all_datatypes SET c_point = NULL')
 
         #  INCREMENTAL
-        self.run_query_tap_mysql('INSERT INTO address(isactive, street_number, date_created, date_updated,'
-                                 ' supplier_supplier_id, zip_code_zip_code_id)'
-                                 'VALUES (1, 1234, NOW(), NOW(), 0, 1234)')
-        self.run_query_tap_mysql('UPDATE address SET street_number = 9999, date_updated = NOW()'
-                                 ' WHERE address_id = 1')
+        self.run_query_tap_mysql(
+            'INSERT INTO address(isactive, street_number, date_created, date_updated,'
+            ' supplier_supplier_id, zip_code_zip_code_id)'
+            'VALUES (1, 1234, NOW(), NOW(), 0, 1234)'
+        )
+        self.run_query_tap_mysql(
+            'UPDATE address SET street_number = 9999, date_updated = NOW()'
+            ' WHERE address_id = 1'
+        )
         #  FULL_TABLE
         self.run_query_tap_mysql('DELETE FROM no_pk_table WHERE id > 10')
 
         # 3. Run tap second time - both fastsync and a singer should be triggered, there are some FULL_TABLE
-        assertions.assert_run_tap_success(tap_mariadb_id, TARGET_ID, ['fastsync', 'singer'])
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_redshift)
-        #assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_redshift,
+        assertions.assert_run_tap_success(
+            tap_mariadb_id, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_redshift
+        )
+        # assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_redshift,
         #                                    mysql_to_redshift.tap_type_to_target_type)
 
     @pytest.mark.dependency(depends=['import_config'])
     def test_resync_mariadb_to_rs(self, tap_mariadb_id=TAP_MARIADB_ID):
         """Resync tables from MariaDB to Redshift DWH"""
         assertions.assert_resync_tables_success(tap_mariadb_id, TARGET_ID)
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_redshift)
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_redshift
+        )
         # assert_all_columns_exist currently not working on Redshift
-        #assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_redshift,
+        # assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_redshift,
         #                                    mysql_to_redshift.tap_type_to_target_type)
 
     # pylint: disable=invalid-name
@@ -104,41 +122,60 @@ def test_replicate_mariadb_to_pg_with_custom_buffer_size(self):
     def test_replicate_pg_to_rs(self):
         """Replicate data from Postgres to Redshift DWH"""
         # 1. Run tap first time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer'])
-        assertions.assert_row_counts_equal(self.run_query_tap_postgres, self.run_query_target_redshift)
+        assertions.assert_run_tap_success(
+            TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_postgres, self.run_query_target_redshift
+        )
         # assert_all_columns_exist currently not working on Redshift
-        #assertions.assert_all_columns_exist(self.run_query_tap_postgres, self.run_query_target_redshift)
-        assertions.assert_date_column_naive_in_target(self.run_query_target_redshift,
-                                                      'updated_at',
-                                                      'ppw_e2e_tap_postgres."table_with_space and uppercase"')
+        # assertions.assert_all_columns_exist(self.run_query_tap_postgres, self.run_query_target_redshift)
+        assertions.assert_date_column_naive_in_target(
+            self.run_query_target_redshift,
+            'updated_at',
+            'ppw_e2e_tap_postgres."table_with_space and uppercase"',
+        )
 
         # 2. Make changes in MariaDB source database
         #  LOG_BASED
-        self.run_query_tap_postgres('insert into public."table_with_space and UPPERCase" (cvarchar, updated_at) values '
-                                    "('M', '2020-01-01 08:53:56.8+10'),"
-                                    "('N', '2020-12-31 12:59:00.148+00'),"
-                                    "('O', null),"
-                                    "('P', '2020-03-03 12:30:00');")
+        self.run_query_tap_postgres(
+            'insert into public."table_with_space and UPPERCase" (cvarchar, updated_at) values '
+            "('M', '2020-01-01 08:53:56.8+10'),"
+            "('N', '2020-12-31 12:59:00.148+00'),"
+            "('O', null),"
+            "('P', '2020-03-03 12:30:00');"
+        )
         #  INCREMENTAL
-        self.run_query_tap_postgres('INSERT INTO public.city (id, name, countrycode, district, population) '
-                                    "VALUES (4080, 'Bath', 'GBR', 'England', 88859)")
-        self.run_query_tap_postgres('UPDATE public.edgydata SET '
-                                    "cjson = json '{\"data\": 1234}', "
-                                    "cjsonb = jsonb '{\"data\": 2345}', "
-                                    "cvarchar = 'Liewe Maatjies UPDATED' WHERE cid = 23")
+        self.run_query_tap_postgres(
+            'INSERT INTO public.city (id, name, countrycode, district, population) '
+            "VALUES (4080, 'Bath', 'GBR', 'England', 88859)"
+        )
+        self.run_query_tap_postgres(
+            'UPDATE public.edgydata SET '
+            "cjson = json '{\"data\": 1234}', "
+            "cjsonb = jsonb '{\"data\": 2345}', "
+            "cvarchar = 'Liewe Maatjies UPDATED' WHERE cid = 23"
+        )
         #  FULL_TABLE
         self.run_query_tap_postgres("DELETE FROM public.country WHERE code = 'UMI'")
 
         # 3. Run tap second time - both fastsync and a singer should be triggered, there are some FULL_TABLE
-        assertions.assert_run_tap_success(TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer'])
-        assertions.assert_row_counts_equal(self.run_query_tap_postgres, self.run_query_target_redshift)
+        assertions.assert_run_tap_success(
+            TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_postgres, self.run_query_target_redshift
+        )
         # assert_all_columns_exist currently not working on Redshift
-        #assertions.assert_all_columns_exist(self.run_query_tap_postgres, self.run_query_target_redshift)
-        assertions.assert_date_column_naive_in_target(self.run_query_target_redshift,
-                                                      'updated_at',
-                                                      'ppw_e2e_tap_postgres."table_with_space and uppercase"')
+        # assertions.assert_all_columns_exist(self.run_query_tap_postgres, self.run_query_target_redshift)
+        assertions.assert_date_column_naive_in_target(
+            self.run_query_target_redshift,
+            'updated_at',
+            'ppw_e2e_tap_postgres."table_with_space and uppercase"',
+        )
         result = self.run_query_target_redshift(
-            'SELECT updated_at FROM ppw_e2e_tap_postgres."table_with_space and uppercase" where cvarchar=\'M\';')[0][0]
+            'SELECT updated_at FROM ppw_e2e_tap_postgres."table_with_space and uppercase" where cvarchar=\'M\';'
+        )[0][0]
 
         assert result == datetime(2019, 12, 31, 22, 53, 56, 800000)
 
@@ -151,15 +188,36 @@ def test_replicate_s3_to_rs(self):
 
         def assert_columns_exist():
             """Helper inner function to test if every table and column exists in target snowflake"""
-            assertions.assert_cols_in_table(self.run_query_target_redshift, 'ppw_e2e_tap_s3_csv', 'countries',
-                                            ['city', 'country', 'currency', 'id', 'language'])
-            assertions.assert_cols_in_table(self.run_query_target_redshift, 'ppw_e2e_tap_s3_csv', 'people',
-                                            ['birth_date', 'email', 'first_name', 'gender', 'group', 'id',
-                                             'ip_address', 'is_pensioneer', 'last_name'])
+            assertions.assert_cols_in_table(
+                self.run_query_target_redshift,
+                'ppw_e2e_tap_s3_csv',
+                'countries',
+                ['city', 'country', 'currency', 'id', 'language'],
+            )
+            assertions.assert_cols_in_table(
+                self.run_query_target_redshift,
+                'ppw_e2e_tap_s3_csv',
+                'people',
+                [
+                    'birth_date',
+                    'email',
+                    'first_name',
+                    'gender',
+                    'group',
+                    'id',
+                    'ip_address',
+                    'is_pensioneer',
+                    'last_name',
+                ],
+            )
 
         # 1. Run tap first time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer'])
+        assertions.assert_run_tap_success(
+            TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer']
+        )
 
         # 2. Run tap second time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer'])
+        assertions.assert_run_tap_success(
+            TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer']
+        )
         assert_columns_exist()
diff --git a/tests/end_to_end/test_target_snowflake.py b/tests/end_to_end/test_target_snowflake.py
index 0beedc288..468f9f0b1 100644
--- a/tests/end_to_end/test_target_snowflake.py
+++ b/tests/end_to_end/test_target_snowflake.py
@@ -51,7 +51,7 @@ def teardown_method(self):
     @pytest.mark.dependency(name='import_config')
     def test_import_project(self):
         """Import the YAML project with taps and target and do discovery mode
-        to write the JSON files for singer connectors """
+        to write the JSON files for singer connectors"""
 
         # Skip every target_postgres related test if required env vars not provided
         if not self.e2e.env['TARGET_SNOWFLAKE']['is_configured']:
@@ -66,95 +66,149 @@ def test_import_project(self):
         self.e2e.setup_target_snowflake()
 
         # Import project
-        [return_code, stdout, stderr] = tasks.run_command(f'pipelinewise import_config --dir {self.project_dir}')
+        [return_code, stdout, stderr] = tasks.run_command(
+            f'pipelinewise import_config --dir {self.project_dir}'
+        )
         assertions.assert_command_success(return_code, stdout, stderr)
 
     @pytest.mark.dependency(depends=['import_config'])
     def test_replicate_mariadb_to_sf(self, tap_mariadb_id=TAP_MARIADB_ID):
         """Replicate data from MariaDB to Snowflake"""
         # 1. Run tap first time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(tap_mariadb_id, TARGET_ID, ['fastsync', 'singer'])
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_snowflake)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.e2e.run_query_target_snowflake,
-                                            mysql_to_snowflake.tap_type_to_target_type)
+        assertions.assert_run_tap_success(
+            tap_mariadb_id, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_snowflake
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.e2e.run_query_target_snowflake,
+            mysql_to_snowflake.tap_type_to_target_type,
+        )
 
         # 2. Make changes in MariaDB source database
         #  LOG_BASED
-        self.run_query_tap_mysql('UPDATE weight_unit SET isactive = 0 WHERE weight_unit_id IN (2, 3, 4)')
-        self.run_query_tap_mysql('INSERT INTO edgydata (c_varchar, `group`, `case`, cjson, c_time) VALUES'
-                                 '(\'Lorem ipsum dolor sit amet\', 10, \'A\', \'[]\', \'00:00:00\'),'
-                                 '(\'Thai: แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช\', 20, \'A\', \'{}\', \'12:00:59\'),'
-                                 '(\'Chinese: 和毛泽东 <<重上井冈山>>. 严永欣, 一九八八年.\', null,\'B\', '
-                                 '\'[{"key": "ValueOne", "actions": []}, {"key": "ValueTwo", "actions": []}]\','
-                                 ' \'9:1:00\'),'
-                                 '(\'Special Characters: [\"\\,''!@£$%^&*()]\\\\\', null, \'B\', '
-                                 'null, \'12:00:00\'),'
-                                 '(\'	\', 20, \'B\', null, \'15:36:10\'),'
-                                 '(CONCAT(CHAR(0x0000 using utf16), \'<- null char\'), 20, \'B\', null, \'15:36:10\')')
+        self.run_query_tap_mysql(
+            'UPDATE weight_unit SET isactive = 0 WHERE weight_unit_id IN (2, 3, 4)'
+        )
+        self.run_query_tap_mysql(
+            'INSERT INTO edgydata (c_varchar, `group`, `case`, cjson, c_time) VALUES'
+            '(\'Lorem ipsum dolor sit amet\', 10, \'A\', \'[]\', \'00:00:00\'),'
+            '(\'Thai: แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช\', 20, \'A\', \'{}\', \'12:00:59\'),'
+            '(\'Chinese: 和毛泽东 <<重上井冈山>>. 严永欣, 一九八八年.\', null,\'B\', '
+            '\'[{"key": "ValueOne", "actions": []}, {"key": "ValueTwo", "actions": []}]\','
+            ' \'9:1:00\'),'
+            '(\'Special Characters: [\"\\,'
+            '!@£$%^&*()]\\\\\', null, \'B\', '
+            'null, \'12:00:00\'),'
+            '(\'	\', 20, \'B\', null, \'15:36:10\'),'
+            '(CONCAT(CHAR(0x0000 using utf16), \'<- null char\'), 20, \'B\', null, \'15:36:10\')'
+        )
 
         self.run_query_tap_mysql('UPDATE all_datatypes SET c_point = NULL')
 
         #  INCREMENTAL
-        self.run_query_tap_mysql('INSERT INTO address(isactive, street_number, date_created, date_updated,'
-                                 ' supplier_supplier_id, zip_code_zip_code_id)'
-                                 'VALUES (1, 1234, NOW(), NOW(), 0, 1234)')
-        self.run_query_tap_mysql('UPDATE address SET street_number = 9999, date_updated = NOW()'
-                                 ' WHERE address_id = 1')
+        self.run_query_tap_mysql(
+            'INSERT INTO address(isactive, street_number, date_created, date_updated,'
+            ' supplier_supplier_id, zip_code_zip_code_id)'
+            'VALUES (1, 1234, NOW(), NOW(), 0, 1234)'
+        )
+        self.run_query_tap_mysql(
+            'UPDATE address SET street_number = 9999, date_updated = NOW()'
+            ' WHERE address_id = 1'
+        )
         #  FULL_TABLE
         self.run_query_tap_mysql('DELETE FROM no_pk_table WHERE id > 10')
 
         # 3. Run tap second time - both fastsync and a singer should be triggered, there are some FULL_TABLE
-        assertions.assert_run_tap_success(tap_mariadb_id, TARGET_ID, ['fastsync', 'singer'])
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_snowflake)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.e2e.run_query_target_snowflake,
-                                            mysql_to_snowflake.tap_type_to_target_type, {'blob_col'})
+        assertions.assert_run_tap_success(
+            tap_mariadb_id, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_snowflake
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.e2e.run_query_target_snowflake,
+            mysql_to_snowflake.tap_type_to_target_type,
+            {'blob_col'},
+        )
 
         # Checking if mask-date transformation is working
         result = self.run_query_target_snowflake(
             'SELECT count(1) FROM ppw_e2e_tap_mysql.address '
-            'where MONTH(date_created) != 1 or DAY(date_created)::int != 1;')[0][0]
+            'where MONTH(date_created) != 1 or DAY(date_created)::int != 1;'
+        )[0][0]
 
         assert result == 0
 
         # Checking if conditional MASK-NUMBER transformation is working
         result = self.run_query_target_snowflake(
             'SELECT count(1) FROM ppw_e2e_tap_mysql.address '
-            'where zip_code_zip_code_id != 0 and street_number REGEXP \'[801]\';')[0][0]
+            'where zip_code_zip_code_id != 0 and street_number REGEXP \'[801]\';'
+        )[0][0]
 
         assert result == 0
 
         # Checking if conditional SET-NULL transformation is working
         result = self.run_query_target_snowflake(
             'SELECT count(1) FROM ppw_e2e_tap_mysql.edgydata '
-            'where "GROUP" is not null and "CASE" = \'B\';')[0][0]
+            'where "GROUP" is not null and "CASE" = \'B\';'
+        )[0][0]
 
         assert result == 0
 
     @pytest.mark.dependency(depends=['import_config'])
     def test_resync_mariadb_to_sf(self, tap_mariadb_id=TAP_MARIADB_ID):
         """Resync tables from MariaDB to Snowflake"""
-        assertions.assert_resync_tables_success(tap_mariadb_id, TARGET_ID, profiling=True)
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_snowflake)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_snowflake,
-                                            mysql_to_snowflake.tap_type_to_target_type)
+        assertions.assert_resync_tables_success(
+            tap_mariadb_id, TARGET_ID, profiling=True
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_snowflake
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.run_query_target_snowflake,
+            mysql_to_snowflake.tap_type_to_target_type,
+        )
 
     # pylint: disable=invalid-name
     @pytest.mark.dependency(depends=['import_config'])
-    def test_resync_mariadb_to_sf_with_split_large_files(self, tap_mariadb_id=TAP_MARIADB_SPLIT_LARGE_FILES_ID):
+    def test_resync_mariadb_to_sf_with_split_large_files(
+        self, tap_mariadb_id=TAP_MARIADB_SPLIT_LARGE_FILES_ID
+    ):
         """Resync tables from MariaDB to Snowflake using splitting large files option"""
-        assertions.assert_resync_tables_success(tap_mariadb_id, TARGET_ID, profiling=True)
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_snowflake)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_snowflake,
-                                            mysql_to_snowflake.tap_type_to_target_type)
+        assertions.assert_resync_tables_success(
+            tap_mariadb_id, TARGET_ID, profiling=True
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_snowflake
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.run_query_target_snowflake,
+            mysql_to_snowflake.tap_type_to_target_type,
+        )
 
     # pylint: disable=invalid-name
     @pytest.mark.dependency(depends=['import_config'])
-    def test_resync_pg_to_sf_with_split_large_files(self, tap_postgres_id=TAP_POSTGRES_SPLIT_LARGE_FILES_ID):
+    def test_resync_pg_to_sf_with_split_large_files(
+        self, tap_postgres_id=TAP_POSTGRES_SPLIT_LARGE_FILES_ID
+    ):
         """Resync tables from Postgres to Snowflake using splitting large files option"""
-        assertions.assert_resync_tables_success(tap_postgres_id, TARGET_ID, profiling=True)
-        assertions.assert_row_counts_equal(self.run_query_tap_mysql, self.run_query_target_snowflake)
-        assertions.assert_all_columns_exist(self.run_query_tap_mysql, self.run_query_target_snowflake,
-                                            mysql_to_snowflake.tap_type_to_target_type)
+        assertions.assert_resync_tables_success(
+            tap_postgres_id, TARGET_ID, profiling=True
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql, self.run_query_target_snowflake
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql,
+            self.run_query_target_snowflake,
+            mysql_to_snowflake.tap_type_to_target_type,
+        )
 
     # pylint: disable=invalid-name
     @pytest.mark.dependency(depends=['import_config'])
@@ -167,71 +221,97 @@ def test_replicate_mariadb_to_sf_with_custom_buffer_size(self):
     def test_replicate_pg_to_sf(self):
         """Replicate data from Postgres to Snowflake"""
         # Run tap first time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer'])
-        assertions.assert_row_counts_equal(self.run_query_tap_postgres, self.run_query_target_snowflake)
-        assertions.assert_all_columns_exist(self.run_query_tap_postgres, self.run_query_target_snowflake)
-        assertions.assert_date_column_naive_in_target(self.run_query_target_snowflake,
-                                                      'updated_at',
-                                                      'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE"')
+        assertions.assert_run_tap_success(
+            TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_postgres, self.run_query_target_snowflake
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_postgres, self.run_query_target_snowflake
+        )
+        assertions.assert_date_column_naive_in_target(
+            self.run_query_target_snowflake,
+            'updated_at',
+            'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE"',
+        )
 
         result = self.run_query_target_snowflake(
             'SELECT updated_at FROM '
             'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE" '
-            'where cvarchar=\'H\';')[0][0]
+            'where cvarchar=\'H\';'
+        )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 998993)
 
         result = self.run_query_target_snowflake(
             'SELECT updated_at FROM '
             'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE" '
-            'where cvarchar=\'I\';')[0][0]
+            'where cvarchar=\'I\';'
+        )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 998993)
 
         # 2. Make changes in PG source database
         #  LOG_BASED
-        self.run_query_tap_postgres('insert into public."table_with_space and UPPERCase" (cvarchar, updated_at) values '
-                                    "('X', '2020-01-01 08:53:56.8+10'),"
-                                    "('Y', '2020-12-31 12:59:00.148+00'),"
-                                    "('faaaar future', '15000-05-23 12:40:00.148'),"
-                                    "('BC', '2020-01-23 01:40:00 BC'),"
-                                    "('Z', null),"
-                                    "('W', '2020-03-03 12:30:00');")
+        self.run_query_tap_postgres(
+            'insert into public."table_with_space and UPPERCase" (cvarchar, updated_at) values '
+            "('X', '2020-01-01 08:53:56.8+10'),"
+            "('Y', '2020-12-31 12:59:00.148+00'),"
+            "('faaaar future', '15000-05-23 12:40:00.148'),"
+            "('BC', '2020-01-23 01:40:00 BC'),"
+            "('Z', null),"
+            "('W', '2020-03-03 12:30:00');"
+        )
         #  INCREMENTAL
-        self.run_query_tap_postgres('INSERT INTO public.city (id, name, countrycode, district, population) '
-                                    "VALUES (4080, 'Bath', 'GBR', 'England', 88859)")
-        self.run_query_tap_postgres('UPDATE public.edgydata SET '
-                                    "cjson = json '{\"data\": 1234}', "
-                                    "cjsonb = jsonb '{\"data\": 2345}', "
-                                    "cvarchar = 'Liewe Maatjies UPDATED' WHERE cid = 23")
+        self.run_query_tap_postgres(
+            'INSERT INTO public.city (id, name, countrycode, district, population) '
+            "VALUES (4080, 'Bath', 'GBR', 'England', 88859)"
+        )
+        self.run_query_tap_postgres(
+            'UPDATE public.edgydata SET '
+            "cjson = json '{\"data\": 1234}', "
+            "cjsonb = jsonb '{\"data\": 2345}', "
+            "cvarchar = 'Liewe Maatjies UPDATED' WHERE cid = 23"
+        )
         #  FULL_TABLE
         self.run_query_tap_postgres("DELETE FROM public.country WHERE code = 'UMI'")
 
         # 3. Run tap second time - both fastsync and a singer should be triggered, there are some FULL_TABLE
-        assertions.assert_run_tap_success(TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer'], profiling=True)
-        assertions.assert_row_counts_equal(self.run_query_tap_postgres, self.run_query_target_snowflake)
-        assertions.assert_all_columns_exist(self.run_query_tap_postgres, self.run_query_target_snowflake)
-        assertions.assert_date_column_naive_in_target(self.run_query_target_snowflake,
-                                                      'updated_at',
-                                                      'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE"')
-
+        assertions.assert_run_tap_success(
+            TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer'], profiling=True
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_postgres, self.run_query_target_snowflake
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_postgres, self.run_query_target_snowflake
+        )
+        assertions.assert_date_column_naive_in_target(
+            self.run_query_target_snowflake,
+            'updated_at',
+            'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE"',
+        )
 
         result = self.run_query_target_snowflake(
-            'SELECT updated_at FROM ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE" where cvarchar=\'X\';')[0][0]
+            'SELECT updated_at FROM ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE" where cvarchar=\'X\';'
+        )[0][0]
 
         assert result == datetime(2019, 12, 31, 22, 53, 56, 800000)
 
         result = self.run_query_target_snowflake(
             'SELECT updated_at FROM '
             'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE" '
-            'where cvarchar=\'faaaar future\';')[0][0]
+            'where cvarchar=\'faaaar future\';'
+        )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 998993)
 
         result = self.run_query_target_snowflake(
             'SELECT updated_at FROM '
             'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE" '
-            'where cvarchar=\'BC\';')[0][0]
+            'where cvarchar=\'BC\';'
+        )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 998993)
 
@@ -250,59 +330,84 @@ def test_replicate_pg_to_sf_with_archive_load_files(self):
         # Delete any dangling files from archive
         files_in_s3_archive = s3_client.list_objects(
             Bucket=s3_bucket,
-            Prefix='{}/postgres_to_sf_archive_load_files/'.format(archive_s3_prefix)).get('Contents', [])
+            Prefix='{}/postgres_to_sf_archive_load_files/'.format(archive_s3_prefix),
+        ).get('Contents', [])
         for file_in_archive in files_in_s3_archive:
             s3_client.delete_object(Bucket=s3_bucket, Key=(file_in_archive['Key']))
 
         # Run tap
-        assertions.assert_run_tap_success(TAP_POSTGRES_ARCHIVE_LOAD_FILES_ID, TARGET_ID, ['fastsync', 'singer'])
+        assertions.assert_run_tap_success(
+            TAP_POSTGRES_ARCHIVE_LOAD_FILES_ID, TARGET_ID, ['fastsync', 'singer']
+        )
 
         # Assert expected files in archive folder
         expected_archive_files_count = {
-            'public.city': 2,       # INCREMENTAL: fastsync and singer
-            'public.country': 1,    # FULL_TABLE : fastsync only
-            'public2.wearehere': 1  # FULL_TABLE : fastsync only
+            'public.city': 2,  # INCREMENTAL: fastsync and singer
+            'public.country': 1,  # FULL_TABLE : fastsync only
+            'public2.wearehere': 1,  # FULL_TABLE : fastsync only
         }
-        for schema_table, expected_archive_files in expected_archive_files_count.items():
+        for (
+            schema_table,
+            expected_archive_files,
+        ) in expected_archive_files_count.items():
             schema, table = schema_table.split('.')
             files_in_s3_archive = s3_client.list_objects(
                 Bucket=s3_bucket,
-                Prefix=('{}/postgres_to_sf_archive_load_files/{}'.format(archive_s3_prefix, table))).get('Contents')
-
-            if files_in_s3_archive is None or len(files_in_s3_archive) != expected_archive_files:
-                raise Exception('files_in_archive for {} is {}. Expected archive files count: {}'.format(
-                    table,
-                    files_in_s3_archive,
-                    expected_archive_files))
+                Prefix=(
+                    '{}/postgres_to_sf_archive_load_files/{}'.format(
+                        archive_s3_prefix, table
+                    )
+                ),
+            ).get('Contents')
+
+            if (
+                files_in_s3_archive is None
+                or len(files_in_s3_archive) != expected_archive_files
+            ):
+                raise Exception(
+                    'files_in_archive for {} is {}. Expected archive files count: {}'.format(
+                        table, files_in_s3_archive, expected_archive_files
+                    )
+                )
 
             # Assert expected metadata
-            archive_metadata = s3_client.head_object(Bucket=s3_bucket, Key=(files_in_s3_archive[0]['Key']))['Metadata']
+            archive_metadata = s3_client.head_object(
+                Bucket=s3_bucket, Key=(files_in_s3_archive[0]['Key'])
+            )['Metadata']
 
             expected_metadata = {
                 'tap': 'postgres_to_sf_archive_load_files',
                 'schema': schema,
                 'table': table,
-                'archived-by': 'pipelinewise_fastsync_postgres_to_snowflake'
+                'archived-by': 'pipelinewise_fastsync_postgres_to_snowflake',
             }
 
             if archive_metadata != expected_metadata:
-                raise Exception('archive_metadata for {} is {}'.format(table, archive_metadata))
+                raise Exception(
+                    'archive_metadata for {} is {}'.format(table, archive_metadata)
+                )
 
             # Assert expected file contents
             with tempfile.NamedTemporaryFile() as tmpfile:
 
                 with open(tmpfile.name, 'wb') as f:
-                    s3_client.download_fileobj(s3_bucket, files_in_s3_archive[0]['Key'], f)
+                    s3_client.download_fileobj(
+                        s3_bucket, files_in_s3_archive[0]['Key'], f
+                    )
 
                 with gzip.open(tmpfile, 'rt') as gzipfile:
                     rows_in_csv = len(gzipfile.readlines())
 
-            rows_in_table = self.run_query_tap_postgres('SELECT COUNT(1) FROM {}'.format(schema_table))[0][0]
+            rows_in_table = self.run_query_tap_postgres(
+                'SELECT COUNT(1) FROM {}'.format(schema_table)
+            )[0][0]
 
             if rows_in_csv != rows_in_table:
-                raise Exception('Rows in csv and db differ: {} vs {}'.format(rows_in_csv, rows_in_table))
-
-
+                raise Exception(
+                    'Rows in csv and db differ: {} vs {}'.format(
+                        rows_in_csv, rows_in_table
+                    )
+                )
 
     @pytest.mark.dependency(depends=['import_config'])
     def test_replicate_s3_to_sf(self):
@@ -313,18 +418,39 @@ def test_replicate_s3_to_sf(self):
 
         def assert_columns_exist():
             """Helper inner function to test if every table and column exists in target snowflake"""
-            assertions.assert_cols_in_table(self.run_query_target_snowflake, 'ppw_e2e_tap_s3_csv', 'countries',
-                                            ['CITY', 'COUNTRY', 'CURRENCY', 'ID', 'LANGUAGE'])
-            assertions.assert_cols_in_table(self.run_query_target_snowflake, 'ppw_e2e_tap_s3_csv', 'people',
-                                            ['BIRTH_DATE', 'EMAIL', 'FIRST_NAME', 'GENDER', 'GROUP', 'ID',
-                                             'IP_ADDRESS', 'IS_PENSIONEER', 'LAST_NAME'])
+            assertions.assert_cols_in_table(
+                self.run_query_target_snowflake,
+                'ppw_e2e_tap_s3_csv',
+                'countries',
+                ['CITY', 'COUNTRY', 'CURRENCY', 'ID', 'LANGUAGE'],
+            )
+            assertions.assert_cols_in_table(
+                self.run_query_target_snowflake,
+                'ppw_e2e_tap_s3_csv',
+                'people',
+                [
+                    'BIRTH_DATE',
+                    'EMAIL',
+                    'FIRST_NAME',
+                    'GENDER',
+                    'GROUP',
+                    'ID',
+                    'IP_ADDRESS',
+                    'IS_PENSIONEER',
+                    'LAST_NAME',
+                ],
+            )
 
         # 1. Run tap first time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer'])
+        assertions.assert_run_tap_success(
+            TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer']
+        )
         assert_columns_exist()
 
         # 2. Run tap second time - both fastsync and a singer should be triggered
-        assertions.assert_run_tap_success(TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer'])
+        assertions.assert_run_tap_success(
+            TAP_S3_CSV_ID, TARGET_ID, ['fastsync', 'singer']
+        )
         assert_columns_exist()
 
     @pytest.mark.dependency(depends=['import_config'])
@@ -333,16 +459,31 @@ def test_replicate_mongodb_to_sf(self):
 
         def assert_columns_exist(table):
             """Helper inner function to test if every table and column exists in the target"""
-            assertions.assert_cols_in_table(self.run_query_target_snowflake, 'ppw_e2e_tap_mongodb', table,
-                                            ['_ID', 'DOCUMENT', '_SDC_EXTRACTED_AT',
-                                             '_SDC_BATCHED_AT', '_SDC_DELETED_AT'])
+            assertions.assert_cols_in_table(
+                self.run_query_target_snowflake,
+                'ppw_e2e_tap_mongodb',
+                table,
+                [
+                    '_ID',
+                    'DOCUMENT',
+                    '_SDC_EXTRACTED_AT',
+                    '_SDC_BATCHED_AT',
+                    '_SDC_DELETED_AT',
+                ],
+            )
 
         def assert_row_counts_equal(target_schema, table, count_in_source):
-            assert count_in_source == \
-                   self.run_query_target_snowflake(f'select count(_id) from {target_schema}.{table}')[0][0]
+            assert (
+                count_in_source
+                == self.run_query_target_snowflake(
+                    f'select count(_id) from {target_schema}.{table}'
+                )[0][0]
+            )
 
         # Run tap first time - fastsync and singer should be triggered
-        assertions.assert_run_tap_success(TAP_MONGODB_ID, TARGET_ID, ['fastsync', 'singer'])
+        assertions.assert_run_tap_success(
+            TAP_MONGODB_ID, TARGET_ID, ['fastsync', 'singer']
+        )
         assert_columns_exist('listings')
         assert_columns_exist('my_collection')
         assert_columns_exist('all_datatypes')
@@ -353,38 +494,56 @@ def assert_row_counts_equal(target_schema, table, count_in_source):
 
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'listings', listing_count)
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'my_collection', my_coll_count)
-        assert_row_counts_equal('ppw_e2e_tap_mongodb', 'all_datatypes', all_datatypes_count)
-
-        result_insert = self.mongodb_con.my_collection.insert_many([
-            {
-                'age': randint(10, 30),
-                'id': 1001,
-                'uuid': uuid.uuid4(),
-                'ts': Timestamp(12030, 500)
-            },
-            {
-                'date': datetime.utcnow(),
-                'id': 1002,
-                'uuid': uuid.uuid4(),
-                'regex': bson.Regex(r'^[A-Z]\\w\\d{2,6}.*$')
-            },
-            {
-                'uuid': uuid.uuid4(),
-                'id': 1003,
-                'decimal': bson.Decimal128(decimal.Decimal('5.64547548425446546546644')),
-                'nested_json': {'a': 1, 'b': 3, 'c': {'key': bson.datetime.datetime(2020, 5, 3, 10, 0, 0)}}
-            }
-        ])
+        assert_row_counts_equal(
+            'ppw_e2e_tap_mongodb', 'all_datatypes', all_datatypes_count
+        )
+
+        result_insert = self.mongodb_con.my_collection.insert_many(
+            [
+                {
+                    'age': randint(10, 30),
+                    'id': 1001,
+                    'uuid': uuid.uuid4(),
+                    'ts': Timestamp(12030, 500),
+                },
+                {
+                    'date': datetime.utcnow(),
+                    'id': 1002,
+                    'uuid': uuid.uuid4(),
+                    'regex': bson.Regex(r'^[A-Z]\\w\\d{2,6}.*$'),
+                },
+                {
+                    'uuid': uuid.uuid4(),
+                    'id': 1003,
+                    'decimal': bson.Decimal128(
+                        decimal.Decimal('5.64547548425446546546644')
+                    ),
+                    'nested_json': {
+                        'a': 1,
+                        'b': 3,
+                        'c': {'key': bson.datetime.datetime(2020, 5, 3, 10, 0, 0)},
+                    },
+                },
+            ]
+        )
         my_coll_count += len(result_insert.inserted_ids)
 
-        result_del = self.mongodb_con.my_collection.delete_one({'_id': result_insert.inserted_ids[0]})
+        result_del = self.mongodb_con.my_collection.delete_one(
+            {'_id': result_insert.inserted_ids[0]}
+        )
         my_coll_count -= result_del.deleted_count
 
-        result_update = self.mongodb_con.my_collection.update_many({}, {'$set': {'id': 0}})
+        result_update = self.mongodb_con.my_collection.update_many(
+            {}, {'$set': {'id': 0}}
+        )
 
         assertions.assert_run_tap_success(TAP_MONGODB_ID, TARGET_ID, ['singer'])
 
-        assert result_update.modified_count == self.run_query_target_snowflake(
-            'select count(_id) from ppw_e2e_tap_mongodb.my_collection where document:id = 0')[0][0]
+        assert (
+            result_update.modified_count
+            == self.run_query_target_snowflake(
+                'select count(_id) from ppw_e2e_tap_mongodb.my_collection where document:id = 0'
+            )[0][0]
+        )
 
         assert_row_counts_equal('ppw_e2e_tap_mongodb', 'my_collection', my_coll_count)
diff --git a/tests/units/cli/cli_args.py b/tests/units/cli/cli_args.py
index 583faa164..4cf1fdcba 100644
--- a/tests/units/cli/cli_args.py
+++ b/tests/units/cli/cli_args.py
@@ -7,19 +7,20 @@
 class CliArgs:
     """Class to simulate argparse command line arguments required by PipelineWise class"""
 
-    def __init__(self,
-                 target='*',
-                 tap='*',
-                 tables=None,
-                 dir='*',
-                 name='*',
-                 secret=None,
-                 string=None,
-                 log='*',
-                 extra_log=False,
-                 debug=False,
-                 profiler=False
-                 ):
+    def __init__(
+        self,
+        target='*',
+        tap='*',
+        tables=None,
+        dir='*',
+        name='*',
+        secret=None,
+        string=None,
+        log='*',
+        extra_log=False,
+        debug=False,
+        profiler=False,
+    ):
         self.target = target
         self.tap = tap
         self.tables = tables
diff --git a/tests/units/cli/test_alert_sender.py b/tests/units/cli/test_alert_sender.py
index 344cdfcd0..feee8a25f 100644
--- a/tests/units/cli/test_alert_sender.py
+++ b/tests/units/cli/test_alert_sender.py
@@ -6,7 +6,9 @@
 from pipelinewise.cli.alert_handlers import errors
 from pipelinewise.cli.alert_sender import AlertHandler, AlertSender
 from pipelinewise.cli.alert_handlers.slack_alert_handler import SlackAlertHandler
-from pipelinewise.cli.alert_handlers.victorops_alert_handler import VictoropsAlertHandler
+from pipelinewise.cli.alert_handlers.victorops_alert_handler import (
+    VictoropsAlertHandler,
+)
 
 
 # pylint: disable=no-self-use,too-few-public-methods
@@ -14,6 +16,7 @@ class TestAlertSender:
     """
     Unit tests for PipelineWise CLI alert sender classes
     """
+
     def test_alert_sender(self):
         """Test function for AlertSender class"""
         # Should raise an exception if alert handlers not initialised by a dictionary
@@ -25,63 +28,93 @@ def test_alert_sender(self):
             AlertSender([1, 2, 3])
 
         # Should get the correct alert handler tuple from a list of alert handlers
-        alert_sender = AlertSender({
-            'handler1': {'unknown-prop1': 'alert-handler-property1'},
-            'handler2': {'unknown-prop2': 'alert-handler-property2'}})
+        alert_sender = AlertSender(
+            {
+                'handler1': {'unknown-prop1': 'alert-handler-property1'},
+                'handler2': {'unknown-prop2': 'alert-handler-property2'},
+            }
+        )
         # pylint: disable=protected-access
-        assert alert_sender._AlertSender__get_alert_handler('handler1') == \
-            AlertHandler(type='handler1', config={'unknown-prop1': 'alert-handler-property1'})
+        assert alert_sender._AlertSender__get_alert_handler('handler1') == AlertHandler(
+            type='handler1', config={'unknown-prop1': 'alert-handler-property1'}
+        )
 
         # Should raise an exception when trying to get a not configured alert handler
         with pytest.raises(errors.NotConfiguredAlertHandlerException):
-            alert_sender = AlertSender({
-                'handler1': {'unknown-prop1': 'alert-handler-property1'},
-                'handler2': {'unknown-prop2': 'alert-handler-property2'}})
+            alert_sender = AlertSender(
+                {
+                    'handler1': {'unknown-prop1': 'alert-handler-property1'},
+                    'handler2': {'unknown-prop2': 'alert-handler-property2'},
+                }
+            )
             # pylint: disable=protected-access
             alert_sender._AlertSender__get_alert_handler('handler3')
 
         # send_to_handler: Should raise an exception if alert handler not configured
         with pytest.raises(errors.NotConfiguredAlertHandlerException):
-            alert_sender = AlertSender({
-                'handler1': {'unknown-prop1': 'alert-handler-property1'},
-                'handler2': {'unknown-prop2': 'alert-handler-property2'}})
+            alert_sender = AlertSender(
+                {
+                    'handler1': {'unknown-prop1': 'alert-handler-property1'},
+                    'handler2': {'unknown-prop2': 'alert-handler-property2'},
+                }
+            )
             alert_sender.send_to_handler('handler3', 'test message to an alert handler')
 
         # send_to_handler: Should raise an exception if alert handler not implemented
         with pytest.raises(errors.NotImplementedAlertHandlerException):
-            alert_sender = AlertSender({
-                'handler1': {'unknown-prop1': 'alert-handler-property1'},
-                'handler2': {'unknown-prop2': 'alert-handler-property2'}})
+            alert_sender = AlertSender(
+                {
+                    'handler1': {'unknown-prop1': 'alert-handler-property1'},
+                    'handler2': {'unknown-prop2': 'alert-handler-property2'},
+                }
+            )
             alert_sender.send_to_handler('handler1', 'test message to an alert handler')
 
         # send_to_all_handlers: Should send an alert if the alert handler configured correctly
         with patch('slack.WebClient.chat_postMessage') as slack_post_message_mock:
             slack_post_message_mock.return_value = []
-            alert_sender = AlertSender({
-                'slack': {'token': 'test-slack-token', 'channel': '#test-channel'},
-                'handler2': {'unknown-prop2': 'alert-handler-property2'}})
-            assert alert_sender.send_to_handler('slack', 'test message to all alert handlers') is True
+            alert_sender = AlertSender(
+                {
+                    'slack': {'token': 'test-slack-token', 'channel': '#test-channel'},
+                    'handler2': {'unknown-prop2': 'alert-handler-property2'},
+                }
+            )
+            assert (
+                alert_sender.send_to_handler(
+                    'slack', 'test message to all alert handlers'
+                )
+                is True
+            )
 
         # send_to_all_handlers: Should raise an exception if alert handler not configured
         with pytest.raises(errors.NotImplementedAlertHandlerException):
-            alert_sender = AlertSender({
-                'handler1': {'unknown-prop1': 'alert-handler-property1'},
-                'handler2': {'unknown-prop2': 'alert-handler-property2'}})
+            alert_sender = AlertSender(
+                {
+                    'handler1': {'unknown-prop1': 'alert-handler-property1'},
+                    'handler2': {'unknown-prop2': 'alert-handler-property2'},
+                }
+            )
             alert_sender.send_to_all_handlers('test message to all alert handlers')
 
         # send_to_all_handlers: Should raise an exception if alert handler not implemented
         with pytest.raises(errors.NotImplementedAlertHandlerException):
-            alert_sender = AlertSender({
-                'handler1': {'unknown-prop1': 'alert-handler-property1'},
-                'handler2': {'unknown-prop2': 'alert-handler-property2'}})
+            alert_sender = AlertSender(
+                {
+                    'handler1': {'unknown-prop1': 'alert-handler-property1'},
+                    'handler2': {'unknown-prop2': 'alert-handler-property2'},
+                }
+            )
             alert_sender.send_to_all_handlers('test message to all alert handlers')
 
         # send_to_all_handlers: Should send an alert if the alert handler configured correctly
         with patch('slack.WebClient.chat_postMessage') as slack_post_message_mock:
             slack_post_message_mock.return_value = []
-            alert_sender = AlertSender({
-                'slack': {'token': 'test-slack-token', 'channel': '#test-channel'}})
-            assert alert_sender.send_to_all_handlers('test message to all alert handlers') == {'sent': 1}
+            alert_sender = AlertSender(
+                {'slack': {'token': 'test-slack-token', 'channel': '#test-channel'}}
+            )
+            assert alert_sender.send_to_all_handlers(
+                'test message to all alert handlers'
+            ) == {'sent': 1}
 
     def test_slack_handler(self):
         """Functions to test slack alert handler"""
@@ -95,13 +128,17 @@ def test_slack_handler(self):
 
         # Should raise an exception if no valid token provided
         with pytest.raises(SlackApiError):
-            slack = SlackAlertHandler({'token': 'invalid-token', 'channel': '#my-channel'})
+            slack = SlackAlertHandler(
+                {'token': 'invalid-token', 'channel': '#my-channel'}
+            )
             slack.send('test message')
 
         # Should send message if valid token and channel provided
         with patch('slack.WebClient.chat_postMessage') as slack_post_message_mock:
             slack_post_message_mock.return_value = []
-            slack = SlackAlertHandler({'token': 'valid-token', 'channel': '#my-channel'})
+            slack = SlackAlertHandler(
+                {'token': 'valid-token', 'channel': '#my-channel'}
+            )
             slack.send('test message')
 
     def test_victorops_handler(self):
@@ -120,7 +157,13 @@ def test_victorops_handler(self):
 
         # Should send alert if valid victorops REST endpoint URL provided
         with patch('requests.post') as victorops_post_message_mock:
-            VictorOpsResponseMock = collections.namedtuple('VictorOpsResponseMock', 'status_code')
-            victorops_post_message_mock.return_value = VictorOpsResponseMock(status_code=200)
-            victorops = VictoropsAlertHandler({'base_url': 'some-url', 'routing_key': 'some-routing-key'})
+            VictorOpsResponseMock = collections.namedtuple(
+                'VictorOpsResponseMock', 'status_code'
+            )
+            victorops_post_message_mock.return_value = VictorOpsResponseMock(
+                status_code=200
+            )
+            victorops = VictoropsAlertHandler(
+                {'base_url': 'some-url', 'routing_key': 'some-routing-key'}
+            )
             victorops.send('test message')
diff --git a/tests/units/cli/test_cli.py b/tests/units/cli/test_cli.py
index db02a4994..d6b769bb3 100644
--- a/tests/units/cli/test_cli.py
+++ b/tests/units/cli/test_cli.py
@@ -36,7 +36,9 @@ class TestCli:
     def setup_method(self):
         """Create CLI arguments"""
         self.args = CliArgs(log='coverage.log')
-        self.pipelinewise = PipelineWise(self.args, CONFIG_DIR, VIRTUALENVS_DIR, PROFILING_DIR)
+        self.pipelinewise = PipelineWise(
+            self.args, CONFIG_DIR, VIRTUALENVS_DIR, PROFILING_DIR
+        )
 
     def teardown_method(self):
         """Delete test directories"""
@@ -48,32 +50,34 @@ def teardown_method(self):
 
     def test_target_dir(self):
         """Singer target connector config path must be relative to the project config dir"""
-        assert \
-            self.pipelinewise.get_target_dir('dummy-target') == \
-            '{}/dummy-target'.format(CONFIG_DIR)
+        assert self.pipelinewise.get_target_dir(
+            'dummy-target'
+        ) == '{}/dummy-target'.format(CONFIG_DIR)
 
     def test_tap_dir(self):
         """Singer tap connector config path must be relative to the target connector config path"""
-        assert \
-            self.pipelinewise.get_tap_dir('dummy-target', 'dummy-tap') == \
-            '{}/dummy-target/dummy-tap'.format(CONFIG_DIR)
+        assert self.pipelinewise.get_tap_dir(
+            'dummy-target', 'dummy-tap'
+        ) == '{}/dummy-target/dummy-tap'.format(CONFIG_DIR)
 
     def test_tap_log_dir(self):
         """Singer tap log path must be relative to the tap connector config path"""
-        assert \
-            self.pipelinewise.get_tap_log_dir('dummy-target', 'dummy-tap') == \
-            '{}/dummy-target/dummy-tap/log'.format(CONFIG_DIR)
+        assert self.pipelinewise.get_tap_log_dir(
+            'dummy-target', 'dummy-tap'
+        ) == '{}/dummy-target/dummy-tap/log'.format(CONFIG_DIR)
 
     def test_connector_bin(self):
         """Singer connector binary must be at a certain location under PIPELINEWISE_HOME .virtualenvs dir"""
-        assert \
-            self.pipelinewise.get_connector_bin('dummy-type') == \
-            '{}/dummy-type/bin/dummy-type'.format(VIRTUALENVS_DIR)
+        assert self.pipelinewise.get_connector_bin(
+            'dummy-type'
+        ) == '{}/dummy-type/bin/dummy-type'.format(VIRTUALENVS_DIR)
 
     def test_not_existing_config_dir(self):
         """Test with not existing config dir"""
         # Create a new pipelinewise object pointing to a not existing config directory
-        pipelinewise_with_no_config = PipelineWise(self.args, 'not-existing-config-dir', VIRTUALENVS_DIR)
+        pipelinewise_with_no_config = PipelineWise(
+            self.args, 'not-existing-config-dir', VIRTUALENVS_DIR
+        )
 
         # It should return and empty config with empty list targets
         # TODO: Make this scenario to fail with error message of "config dir not exists"
@@ -82,19 +86,30 @@ def test_not_existing_config_dir(self):
 
     def test_get_targets(self):
         """Targets should be loaded from JSON as is"""
-        assert self.pipelinewise.get_targets() == cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get(
-            'targets', [])
+        assert self.pipelinewise.get_targets() == cli.utils.load_json(
+            '{}/config.json'.format(CONFIG_DIR)
+        ).get('targets', [])
 
     def test_get_target(self):
         """Selecting target by ID should append connector files"""
         # Get target definitions from JSON file
-        targets = cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get('targets', [])
-        exp_target_one = next((item for item in targets if item['id'] == 'target_one'), False)
-        exp_target_two = next((item for item in targets if item['id'] == 'target_two'), False)
+        targets = cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get(
+            'targets', []
+        )
+        exp_target_one = next(
+            (item for item in targets if item['id'] == 'target_one'), False
+        )
+        exp_target_two = next(
+            (item for item in targets if item['id'] == 'target_two'), False
+        )
 
         # Append the connector file paths to the expected targets
-        exp_target_one['files'] = Config.get_connector_files('{}/target_one'.format(CONFIG_DIR))
-        exp_target_two['files'] = Config.get_connector_files('{}/target_two'.format(CONFIG_DIR))
+        exp_target_one['files'] = Config.get_connector_files(
+            '{}/target_one'.format(CONFIG_DIR)
+        )
+        exp_target_two['files'] = Config.get_connector_files(
+            '{}/target_two'.format(CONFIG_DIR)
+        )
 
         # Getting target by ID should match to original JSON and should contains the connector files list
         assert self.pipelinewise.get_target('target_one') == exp_target_one
@@ -103,19 +118,31 @@ def test_get_target(self):
     def test_get_taps(self):
         """Selecting taps by target ID should append tap statuses"""
         # Get target definitions from JSON file
-        targets = cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get('targets', [])
-        target_one = next((item for item in targets if item['id'] == 'target_one'), False)
-        target_two = next((item for item in targets if item['id'] == 'target_two'), False)
+        targets = cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get(
+            'targets', []
+        )
+        target_one = next(
+            (item for item in targets if item['id'] == 'target_one'), False
+        )
+        target_two = next(
+            (item for item in targets if item['id'] == 'target_two'), False
+        )
 
         # Append the tap statuses to every tap in target_one
         exp_tap_one = target_one['taps'][0]
         exp_tap_two = target_one['taps'][1]
-        exp_tap_one['status'] = self.pipelinewise.detect_tap_status('target_one', exp_tap_one['id'])
-        exp_tap_two['status'] = self.pipelinewise.detect_tap_status('target_one', exp_tap_two['id'])
+        exp_tap_one['status'] = self.pipelinewise.detect_tap_status(
+            'target_one', exp_tap_one['id']
+        )
+        exp_tap_two['status'] = self.pipelinewise.detect_tap_status(
+            'target_one', exp_tap_two['id']
+        )
 
         # Append the tap statuses to every tap in target_one
         exp_tap_three = target_two['taps'][0]
-        exp_tap_three['status'] = self.pipelinewise.detect_tap_status('target_two', exp_tap_three['id'])
+        exp_tap_three['status'] = self.pipelinewise.detect_tap_status(
+            'target_two', exp_tap_three['id']
+        )
 
         # Tap statuses should be appended to every tap
         assert self.pipelinewise.get_taps('target_one') == [exp_tap_one, exp_tap_two]
@@ -124,13 +151,21 @@ def test_get_taps(self):
     def test_get_tap(self):
         """Getting tap by ID should return status, connector and target props as well"""
         # Get target definitions from JSON file
-        targets = cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get('targets', [])
-        target_one = next((item for item in targets if item['id'] == 'target_one'), False)
+        targets = cli.utils.load_json('{}/config.json'.format(CONFIG_DIR)).get(
+            'targets', []
+        )
+        target_one = next(
+            (item for item in targets if item['id'] == 'target_one'), False
+        )
 
         # Append the tap status, files and target keys to the tap
         exp_tap_one = target_one['taps'][0]
-        exp_tap_one['status'] = self.pipelinewise.detect_tap_status('target_one', exp_tap_one['id'])
-        exp_tap_one['files'] = Config.get_connector_files('{}/target_one/tap_one'.format(CONFIG_DIR))
+        exp_tap_one['status'] = self.pipelinewise.detect_tap_status(
+            'target_one', exp_tap_one['id']
+        )
+        exp_tap_one['files'] = Config.get_connector_files(
+            '{}/target_one/tap_one'.format(CONFIG_DIR)
+        )
         exp_tap_one['target'] = self.pipelinewise.get_target('target_one')
 
         # Getting tap by ID should match to original JSON and should contain  status, connector files and target props
@@ -148,7 +183,10 @@ def test_get_taps_from_wrong_target(self):
 
         # Getting not existing from should raise exception
         with pytest.raises(Exception):
-            assert self.pipelinewise.get_tap('not-existing-target', 'not-existing-tap') == {}
+            assert (
+                self.pipelinewise.get_tap('not-existing-target', 'not-existing-tap')
+                == {}
+            )
 
     def test_get_not_existing_tap(self):
         """Test getting not existing tap from existing target"""
@@ -163,23 +201,26 @@ def test_create_filtered_tap_props(self):
             tap_properties_fastsync,
             fastsync_stream_ids,
             tap_properties_singer,
-            singer_stream_ids
+            singer_stream_ids,
         ) = self.pipelinewise.create_filtered_tap_properties(
             target_type=ConnectorType('target-snowflake'),
             tap_type=ConnectorType('tap-mysql'),
-             tap_properties='{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(
-                 os.path.dirname(__file__)),
-             tap_state='{}/resources/sample_json_config/target_one/tap_one/state.json'.format(
-                 os.path.dirname(__file__)),
-             filters={
-                 'selected': True,
-                 'tap_target_pairs': {
-                     ConnectorType.TAP_MYSQL: {ConnectorType.TARGET_SNOWFLAKE},
-                     ConnectorType.TAP_POSTGRES: {ConnectorType.TARGET_SNOWFLAKE}
-                 },
-                 'initial_sync_required': True
-             },
-             create_fallback=True)
+            tap_properties='{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(
+                os.path.dirname(__file__)
+            ),
+            tap_state='{}/resources/sample_json_config/target_one/tap_one/state.json'.format(
+                os.path.dirname(__file__)
+            ),
+            filters={
+                'selected': True,
+                'tap_target_pairs': {
+                    ConnectorType.TAP_MYSQL: {ConnectorType.TARGET_SNOWFLAKE},
+                    ConnectorType.TAP_POSTGRES: {ConnectorType.TARGET_SNOWFLAKE},
+                },
+                'initial_sync_required': True,
+            },
+            create_fallback=True,
+        )
 
         # Fastsync and singer properties should be created
         assert os.path.isfile(tap_properties_fastsync)
@@ -190,8 +231,14 @@ def test_create_filtered_tap_props(self):
         os.remove(tap_properties_singer)
 
         # Fastsync and singer properties should be created
-        assert fastsync_stream_ids == ['db_test_mysql-table_one', 'db_test_mysql-table_two']
-        assert singer_stream_ids == ['db_test_mysql-table_one', 'db_test_mysql-table_two']
+        assert fastsync_stream_ids == [
+            'db_test_mysql-table_one',
+            'db_test_mysql-table_two',
+        ]
+        assert singer_stream_ids == [
+            'db_test_mysql-table_one',
+            'db_test_mysql-table_two',
+        ]
 
     def test_create_filtered_tap_props_no_fastsync(self):
         """Test creating only singer specific properties file"""
@@ -199,23 +246,26 @@ def test_create_filtered_tap_props_no_fastsync(self):
             tap_properties_fastsync,
             fastsync_stream_ids,
             tap_properties_singer,
-            singer_stream_ids
+            singer_stream_ids,
         ) = self.pipelinewise.create_filtered_tap_properties(
             target_type=ConnectorType('target-snowflake'),
             tap_type=ConnectorType('tap-mysql'),
             tap_properties='{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(
-                os.path.dirname(__file__)),
+                os.path.dirname(__file__)
+            ),
             tap_state='{}/resources/sample_json_config/target_one/tap_one/state.json'.format(
-                os.path.dirname(__file__)),
+                os.path.dirname(__file__)
+            ),
             filters={
                 'selected': True,
                 'tap_target_pairs': {
                     ConnectorType.TAP_MYSQL: {ConnectorType.TARGET_REDSHIFT},
-                    ConnectorType.TAP_POSTGRES: {ConnectorType.TARGET_SNOWFLAKE}
+                    ConnectorType.TAP_POSTGRES: {ConnectorType.TARGET_SNOWFLAKE},
                 },
-                'initial_sync_required': True
+                'initial_sync_required': True,
             },
-            create_fallback=True)
+            create_fallback=True,
+        )
 
         # fastsync and singer properties should be created
         assert os.path.isfile(tap_properties_fastsync)
@@ -227,7 +277,10 @@ def test_create_filtered_tap_props_no_fastsync(self):
 
         # only singer properties should be created
         assert fastsync_stream_ids == []
-        assert singer_stream_ids == ['db_test_mysql-table_one', 'db_test_mysql-table_two']
+        assert singer_stream_ids == [
+            'db_test_mysql-table_one',
+            'db_test_mysql-table_two',
+        ]
 
     def test_merge_empty_catalog(self):
         """Merging two empty singer schemas should be another empty"""
@@ -237,56 +290,110 @@ def test_merge_empty_catalog(self):
     def test_merge_empty_stream_catalog(self):
         """Merging empty schemas should be empty"""
         # TODO: Check if pipelinewise.merge_schemas is required at all or not
-        assert self.pipelinewise.merge_schemas({'streams': []}, {'streams': []}) == {'streams': []}
+        assert self.pipelinewise.merge_schemas({'streams': []}, {'streams': []}) == {
+            'streams': []
+        }
 
     def test_merge_same_catalog(self):
         """Test merging not empty schemas"""
         # TODO: Check if pipelinewise.merge_schemas is required at all or not
         tap_one_catalog = cli.utils.load_json(
-            '{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(os.path.dirname(__file__)))
+            '{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(
+                os.path.dirname(__file__)
+            )
+        )
 
-        assert self.pipelinewise.merge_schemas(tap_one_catalog, tap_one_catalog) == tap_one_catalog
+        assert (
+            self.pipelinewise.merge_schemas(tap_one_catalog, tap_one_catalog)
+            == tap_one_catalog
+        )
 
     def test_merge_updated_catalog(self):
         """Test merging not empty schemas"""
         # TODO: Check if pipelinewise.merge_schemas is required at all or not
         tap_one_catalog = cli.utils.load_json(
-            '{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(os.path.dirname(__file__)))
+            '{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(
+                os.path.dirname(__file__)
+            )
+        )
         tap_one_updated_catalog = cli.utils.load_json(
             '{}/resources/sample_json_config/target_one/tap_one/properties_updated.json'.format(
-                os.path.dirname(__file__)))
+                os.path.dirname(__file__)
+            )
+        )
 
-        assert self.pipelinewise.merge_schemas(tap_one_catalog, tap_one_updated_catalog) == tap_one_catalog
+        assert (
+            self.pipelinewise.merge_schemas(tap_one_catalog, tap_one_updated_catalog)
+            == tap_one_catalog
+        )
 
     def test_make_default_selection(self):
         """Test if streams selected correctly in catalog JSON"""
         tap_one_catalog = cli.utils.load_json(
-            '{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(os.path.dirname(__file__)))
-        tap_one_selection_file = '{}/resources/sample_json_config/target_one/tap_one/selection.json'.format(
-            os.path.dirname(__file__))
+            '{}/resources/sample_json_config/target_one/tap_one/properties.json'.format(
+                os.path.dirname(__file__)
+            )
+        )
+        tap_one_selection_file = (
+            '{}/resources/sample_json_config/target_one/tap_one/selection.json'.format(
+                os.path.dirname(__file__)
+            )
+        )
 
         # Update catalog selection
-        tap_one_with_selection = self.pipelinewise.make_default_selection(tap_one_catalog, tap_one_selection_file)
+        tap_one_with_selection = self.pipelinewise.make_default_selection(
+            tap_one_catalog, tap_one_selection_file
+        )
 
         # Table one has to be selected with LOG_BASED replication method
-        assert tap_one_with_selection['streams'][0]['metadata'][0]['metadata']['selected'] is True
-        assert tap_one_with_selection['streams'][0]['metadata'][0]['metadata']['replication-method'] == 'LOG_BASED'
+        assert (
+            tap_one_with_selection['streams'][0]['metadata'][0]['metadata']['selected']
+            is True
+        )
+        assert (
+            tap_one_with_selection['streams'][0]['metadata'][0]['metadata'][
+                'replication-method'
+            ]
+            == 'LOG_BASED'
+        )
 
         # Table two has to be selected with INCREMENTAL replication method
-        assert tap_one_with_selection['streams'][1]['metadata'][0]['metadata']['selected'] is True
-        assert tap_one_with_selection['streams'][1]['metadata'][0]['metadata']['replication-method'] == 'INCREMENTAL'
-        assert tap_one_with_selection['streams'][1]['metadata'][0]['metadata']['replication-key'] == 'id'
+        assert (
+            tap_one_with_selection['streams'][1]['metadata'][0]['metadata']['selected']
+            is True
+        )
+        assert (
+            tap_one_with_selection['streams'][1]['metadata'][0]['metadata'][
+                'replication-method'
+            ]
+            == 'INCREMENTAL'
+        )
+        assert (
+            tap_one_with_selection['streams'][1]['metadata'][0]['metadata'][
+                'replication-key'
+            ]
+            == 'id'
+        )
 
         # Table three should not be selected
-        assert tap_one_with_selection['streams'][2]['metadata'][0]['metadata']['selected'] is False
+        assert (
+            tap_one_with_selection['streams'][2]['metadata'][0]['metadata']['selected']
+            is False
+        )
 
     def test_target_config(self):
         """Test merging target config.json and inheritable_config.json"""
-        target_config = '{}/resources/target-config.json'.format(os.path.dirname(__file__))
-        tap_inheritable_config = '{}/resources/tap-inheritable-config.json'.format(os.path.dirname(__file__))
+        target_config = '{}/resources/target-config.json'.format(
+            os.path.dirname(__file__)
+        )
+        tap_inheritable_config = '{}/resources/tap-inheritable-config.json'.format(
+            os.path.dirname(__file__)
+        )
 
         # The merged JSON written into a temp file
-        temp_file = self.pipelinewise.create_consumable_target_config(target_config, tap_inheritable_config)
+        temp_file = self.pipelinewise.create_consumable_target_config(
+            target_config, tap_inheritable_config
+        )
         cons_targ_config = cli.utils.load_json(temp_file)
 
         # The merged object needs
@@ -312,9 +419,9 @@ def test_target_config(self):
             'schema_mapping': {
                 'jira': {
                     'target_schema': 'jira_clear',
-                    'target_schema_select_permissions': ['grp_power']
+                    'target_schema_select_permissions': ['grp_power'],
                 }
-            }
+            },
         }
 
         # Remove temp file with merged JSON
@@ -327,11 +434,15 @@ def test_invalid_target_config(self):
 
         # Merging invalid or not existing JSONs should raise exception
         with pytest.raises(Exception):
-            self.pipelinewise.create_consumable_target_config(target_config, tap_inheritable_config)
+            self.pipelinewise.create_consumable_target_config(
+                target_config, tap_inheritable_config
+            )
 
     def test_send_alert(self):
         """Test if alert"""
-        with patch('pipelinewise.cli.alert_sender.AlertSender.send_to_all_handlers') as aler_sender_mock:
+        with patch(
+            'pipelinewise.cli.alert_sender.AlertSender.send_to_all_handlers'
+        ) as aler_sender_mock:
             aler_sender_mock.return_value = {'sent': 1}
             # Should send alert and should return stats if alerting enabled on the tap
             self.pipelinewise.tap = self.pipelinewise.get_tap('target_one', 'tap_one')
@@ -363,7 +474,9 @@ def test_command_init(self):
         pipelinewise.init()
 
         # The test project should contain every sample YAML file
-        for sample_yaml in os.listdir('{}/../../../pipelinewise/cli/samples'.format(os.path.dirname(__file__))):
+        for sample_yaml in os.listdir(
+            '{}/../../../pipelinewise/cli/samples'.format(os.path.dirname(__file__))
+        ):
             assert os.path.isfile(os.path.join(TEST_PROJECT_DIR, sample_yaml))
 
         # Re-creating project should reaise exception of directory not empty
@@ -381,13 +494,16 @@ def test_command_status(self, capsys):
 
         # Exact output match
         # pylint: disable=line-too-long
-        assert stdout == """Tap ID     Tap Type      Target ID    Target Type       Enabled    Status          Last Sync    Last Sync Result
+        assert (
+            stdout
+            == """Tap ID     Tap Type      Target ID    Target Type       Enabled    Status          Last Sync    Last Sync Result
 ---------  ------------  -----------  ----------------  ---------  --------------  -----------  ------------------
 tap_one    tap-mysql     target_one   target-snowflake  True       ready                        unknown
 tap_two    tap-postgres  target_one   target-snowflake  True       ready                        unknown
 tap_three  tap-mysql     target_two   target-s3-csv     True       not-configured               unknown
 3 pipeline(s)
 """
+        )
 
     def test_command_discover_tap(self):
         """Test discover tap command"""
@@ -401,7 +517,6 @@ def test_command_discover_tap(self):
         exp_err_pattern = '/tap-mysql/bin/tap-mysql: No such file or directory'
         assert exp_err_pattern in result
 
-
     def _test_command_run_tap(self):
         """Test run tap command"""
         args = CliArgs(target='target_one', tap='tap_one')
@@ -430,8 +545,11 @@ def test_command_stop_tap(self):
         # 1. Start the pipelinewise mock executable that's running
         #    linux piped dummy tap and target connectors
         with pidfile.PIDFile(pipelinewise.tap['files']['pidfile']):
-            os.spawnl(os.P_NOWAIT, f'{RESOURCES_DIR}/test_stop_tap/scheduler-mock.sh',
-                      'test_stop_tap/scheduler-mock.sh')
+            os.spawnl(
+                os.P_NOWAIT,
+                f'{RESOURCES_DIR}/test_stop_tap/scheduler-mock.sh',
+                'test_stop_tap/scheduler-mock.sh',
+            )
             # Wait 5 seconds making sure the dummy tap is running
             time.sleep(5)
 
@@ -481,7 +599,7 @@ def test_exit_gracefully(self):
 
     def test_validate_command_1(self):
         """Test validate command should fail because of missing replication key for incremental"""
-        test_validate_command_dir = \
+        test_validate_command_dir =\
             f'{os.path.dirname(__file__)}/resources/test_validate_command/missing_replication_key_incremental'
 
         args = CliArgs(dir=test_validate_command_dir)
@@ -492,7 +610,7 @@ def test_validate_command_1(self):
 
     def test_validate_command_2(self):
         """Test validate command should succeed"""
-        test_validate_command_dir = \
+        test_validate_command_dir =\
             f'{os.path.dirname(__file__)}/resources/test_validate_command/missing_replication_key'
 
         args = CliArgs(dir=test_validate_command_dir)
@@ -512,7 +630,7 @@ def test_validate_command_3(self):
 
     def test_validate_command_4(self):
         """Test validate command should fail because of duplicate targets"""
-        test_validate_command_dir = \
+        test_validate_command_dir =\
             f'{os.path.dirname(__file__)}/resources/test_validate_command/test_yaml_config_two_targets'
 
         args = CliArgs(dir=test_validate_command_dir)
@@ -526,65 +644,164 @@ def test_post_import_checks(self):
         """Test post import checks"""
         args = CliArgs()
         pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
-        test_files_dir = '{}/resources/test_post_import_checks'.format(os.path.dirname(__file__))
-
-        tap_pk_required = cli.utils.load_json('{}/tap_config_pk_required.json'.format(test_files_dir))
-        tap_pk_not_required = cli.utils.load_json('{}/tap_config_pk_not_required.json'.format(test_files_dir))
-        tap_pk_not_defined = cli.utils.load_json('{}/tap_config_pk_not_defined.json'.format(test_files_dir))
-        tap_with_pk = cli.utils.load_json('{}//tap_properties_with_pk.json'.format(test_files_dir))
+        test_files_dir = '{}/resources/test_post_import_checks'.format(
+            os.path.dirname(__file__)
+        )
+
+        tap_pk_required = cli.utils.load_json(
+            '{}/tap_config_pk_required.json'.format(test_files_dir)
+        )
+        tap_pk_not_required = cli.utils.load_json(
+            '{}/tap_config_pk_not_required.json'.format(test_files_dir)
+        )
+        tap_pk_not_defined = cli.utils.load_json(
+            '{}/tap_config_pk_not_defined.json'.format(test_files_dir)
+        )
+        tap_with_pk = cli.utils.load_json(
+            '{}//tap_properties_with_pk.json'.format(test_files_dir)
+        )
         tap_with_no_pk_full_table = cli.utils.load_json(
-            '{}//tap_properties_with_no_pk_full_table.json'.format(test_files_dir))
+            '{}//tap_properties_with_no_pk_full_table.json'.format(test_files_dir)
+        )
         tap_with_no_pk_incremental = cli.utils.load_json(
-            '{}//tap_properties_with_no_pk_incremental.json'.format(test_files_dir))
+            '{}//tap_properties_with_no_pk_incremental.json'.format(test_files_dir)
+        )
         tap_with_no_pk_log_based = cli.utils.load_json(
-            '{}//tap_properties_with_no_pk_log_based.json'.format(test_files_dir))
+            '{}//tap_properties_with_no_pk_log_based.json'.format(test_files_dir)
+        )
         tap_with_no_pk_not_selected = cli.utils.load_json(
-            '{}//tap_properties_with_no_pk_not_selected.json'.format(test_files_dir))
+            '{}//tap_properties_with_no_pk_not_selected.json'.format(test_files_dir)
+        )
 
-        with patch('pipelinewise.cli.pipelinewise.commands.run_command') as run_command_mock:
+        with patch(
+            'pipelinewise.cli.pipelinewise.commands.run_command'
+        ) as run_command_mock:
             # Test scenarios when post import checks should pass
-            assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_pk, 'snowflake') == []
-            assert pipelinewise._run_post_import_tap_checks(tap_pk_not_required, tap_with_pk, 'snowflake') == []
-            assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_full_table, 'snowflake') \
-                   == []
-            assert pipelinewise._run_post_import_tap_checks(tap_pk_not_required, tap_with_no_pk_incremental,
-                                                            'snowflake') == []
-            assert pipelinewise._run_post_import_tap_checks(tap_pk_not_required, tap_with_no_pk_log_based,
-                                                            'snowflake') == []
-            assert pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_not_selected,
-                                                            'snowflake') == []
-            assert pipelinewise._run_post_import_tap_checks(tap_pk_not_defined, tap_with_no_pk_full_table,
-                                                            'snowflake') == []
+            assert (
+                pipelinewise._run_post_import_tap_checks(
+                    tap_pk_required, tap_with_pk, 'snowflake'
+                )
+                == []
+            )
+            assert (
+                pipelinewise._run_post_import_tap_checks(
+                    tap_pk_not_required, tap_with_pk, 'snowflake'
+                )
+                == []
+            )
+            assert (
+                pipelinewise._run_post_import_tap_checks(
+                    tap_pk_required, tap_with_no_pk_full_table, 'snowflake'
+                )
+                == []
+            )
+            assert (
+                pipelinewise._run_post_import_tap_checks(
+                    tap_pk_not_required, tap_with_no_pk_incremental, 'snowflake'
+                )
+                == []
+            )
+            assert (
+                pipelinewise._run_post_import_tap_checks(
+                    tap_pk_not_required, tap_with_no_pk_log_based, 'snowflake'
+                )
+                == []
+            )
+            assert (
+                pipelinewise._run_post_import_tap_checks(
+                    tap_pk_required, tap_with_no_pk_not_selected, 'snowflake'
+                )
+                == []
+            )
+            assert (
+                pipelinewise._run_post_import_tap_checks(
+                    tap_pk_not_defined, tap_with_no_pk_full_table, 'snowflake'
+                )
+                == []
+            )
 
             # Test scenarios when post import checks should fail due to primary keys not exists
-            assert len(pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_incremental,
-                                                                'snowflake')) == 1
-            assert len(pipelinewise._run_post_import_tap_checks(tap_pk_required, tap_with_no_pk_log_based,
-                                                                'snowflake')) == 1
-            assert len(pipelinewise._run_post_import_tap_checks(tap_pk_not_defined, tap_with_no_pk_incremental,
-                                                                'snowflake')) == 1
-            assert len(pipelinewise._run_post_import_tap_checks(tap_pk_not_defined, tap_with_no_pk_log_based,
-                                                                'snowflake')) == 1
+            assert (
+                len(
+                    pipelinewise._run_post_import_tap_checks(
+                        tap_pk_required, tap_with_no_pk_incremental, 'snowflake'
+                    )
+                )
+                == 1
+            )
+            assert (
+                len(
+                    pipelinewise._run_post_import_tap_checks(
+                        tap_pk_required, tap_with_no_pk_log_based, 'snowflake'
+                    )
+                )
+                == 1
+            )
+            assert (
+                len(
+                    pipelinewise._run_post_import_tap_checks(
+                        tap_pk_not_defined, tap_with_no_pk_incremental, 'snowflake'
+                    )
+                )
+                == 1
+            )
+            assert (
+                len(
+                    pipelinewise._run_post_import_tap_checks(
+                        tap_pk_not_defined, tap_with_no_pk_log_based, 'snowflake'
+                    )
+                )
+                == 1
+            )
 
             # Test scenarios when post import checks should fail due to transformations validation command fails
-            tap_with_trans = cli.utils.\
-                load_json('{}/tap_config_with_transformations.json'.format(test_files_dir))
-
-            run_command_mock.return_value = (1, None, 'transformation `HASH` cannot be applied')
-
-            assert len(pipelinewise._run_post_import_tap_checks(tap_with_trans, tap_with_no_pk_not_selected,
-                                                                'snowflake')) == 1
-
-            assert len(pipelinewise._run_post_import_tap_checks(tap_with_trans, tap_with_no_pk_incremental,
-                                                                'snowflake')) == 2
+            tap_with_trans = cli.utils.load_json(
+                '{}/tap_config_with_transformations.json'.format(test_files_dir)
+            )
+
+            run_command_mock.return_value = (
+                1,
+                None,
+                'transformation `HASH` cannot be applied',
+            )
+
+            assert (
+                len(
+                    pipelinewise._run_post_import_tap_checks(
+                        tap_with_trans, tap_with_no_pk_not_selected, 'snowflake'
+                    )
+                )
+                == 1
+            )
+
+            assert (
+                len(
+                    pipelinewise._run_post_import_tap_checks(
+                        tap_with_trans, tap_with_no_pk_incremental, 'snowflake'
+                    )
+                )
+                == 2
+            )
 
             # mock successful transformation validation command
             run_command_mock.return_value = (0, None, None)
 
-            assert len(pipelinewise._run_post_import_tap_checks(tap_with_trans, tap_with_no_pk_not_selected,
-                                                                'snowflake')) == 0
-
-            assert len(pipelinewise._run_post_import_tap_checks(tap_with_trans, tap_with_no_pk_incremental,
-                                                                'snowflake')) == 1
+            assert (
+                len(
+                    pipelinewise._run_post_import_tap_checks(
+                        tap_with_trans, tap_with_no_pk_not_selected, 'snowflake'
+                    )
+                )
+                == 0
+            )
+
+            assert (
+                len(
+                    pipelinewise._run_post_import_tap_checks(
+                        tap_with_trans, tap_with_no_pk_incremental, 'snowflake'
+                    )
+                )
+                == 1
+            )
 
             assert run_command_mock.call_count == 4
diff --git a/tests/units/cli/test_cli_utils.py b/tests/units/cli/test_cli_utils.py
index 20039c619..069bfae58 100644
--- a/tests/units/cli/test_cli_utils.py
+++ b/tests/units/cli/test_cli_utils.py
@@ -270,9 +270,8 @@ def test_delete_keys(self):
         # Delete multiple keys from list of nested dictionaries
         assert cli.utils.delete_keys_from_dict(
             [{'foo': 'bar', 'foo2': 'bar2'},
-             {'foo3': {'nested_foo': 'nested_bar', 'nested_foo2': 'nested_bar2'}}], ['foo2', 'nested_foo']) == \
-               [{'foo': 'bar'},
-                {'foo3': {'nested_foo2': 'nested_bar2'}}]
+             {'foo3': {'nested_foo': 'nested_bar', 'nested_foo2': 'nested_bar2'}}],
+            ['foo2', 'nested_foo']) == [{'foo': 'bar'}, {'foo3': {'nested_foo2': 'nested_bar2'}}]
 
     def test_silentremove(self):
         """Test removing functions"""
diff --git a/tests/units/cli/test_cli_utils_tap_github.py b/tests/units/cli/test_cli_utils_tap_github.py
index 5f6db4d5a..54dcb4f22 100644
--- a/tests/units/cli/test_cli_utils_tap_github.py
+++ b/tests/units/cli/test_cli_utils_tap_github.py
@@ -31,7 +31,9 @@ def test_should_pass_with_valid_json_schema(self):
         actual_yaml = cli.utils.load_yaml(TAP_GITHUB_YAML)
         self.assertIsNone(cli.utils.validate(actual_yaml, schema))
 
-    def test_should_pass_if_organization_and_repos_include_missing_but_repository_exists(self):
+    def test_should_pass_if_organization_and_repos_include_missing_but_repository_exists(
+        self,
+    ):
         """
         Test should pass if organization and repos include missing but repository exists
         """
@@ -43,7 +45,9 @@ def test_should_pass_if_organization_and_repos_include_missing_but_repository_ex
 
         self.assertIsNone(cli.utils.validate(actual_yaml, schema))
 
-    def test_should_pass_if_organization_and_repository_missing_but_repos_include_exists(self):
+    def test_should_pass_if_organization_and_repository_missing_but_repos_include_exists(
+        self,
+    ):
         """
         Test should pass if organization and repository missing but repos_include exists
         """
diff --git a/tests/units/cli/test_commands.py b/tests/units/cli/test_commands.py
index 7d70669ca..50046f5eb 100644
--- a/tests/units/cli/test_commands.py
+++ b/tests/units/cli/test_commands.py
@@ -28,153 +28,188 @@ def test_exists_and_executable(self):
     def test_build_tap_command(self):
         """Tests the function that generates tap executable command"""
         # State file should not be included if state file path not passed
-        tap = commands.TapParams(id='my_tap_mysql',
-                                 type='tap_mysql',
-                                 bin='/bin/tap_mysql.py',
-                                 python_bin='/bin/python3',
-                                 config='.ppw/config.json',
-                                 properties='.ppw/properties.json',
-                                 state=None
-                                 )
+        tap = commands.TapParams(
+            id='my_tap_mysql',
+            type='tap_mysql',
+            bin='/bin/tap_mysql.py',
+            python_bin='/bin/python3',
+            config='.ppw/config.json',
+            properties='.ppw/properties.json',
+            state=None,
+        )
 
         # profiling is not enabled
-        command = commands.build_tap_command(tap=tap,
-                                             profiling_mode=False,
-                                             profiling_dir=None)
+        command = commands.build_tap_command(
+            tap=tap, profiling_mode=False, profiling_dir=None
+        )
 
-        assert command == '/bin/tap_mysql.py --config .ppw/config.json --catalog .ppw/properties.json '
+        assert (
+            command
+            == '/bin/tap_mysql.py --config .ppw/config.json --catalog .ppw/properties.json '
+        )
 
         # profiling is enabled
-        command = commands.build_tap_command(tap=tap,
-                                             profiling_mode=True,
-                                             profiling_dir='./profiling')
+        command = commands.build_tap_command(
+            tap=tap, profiling_mode=True, profiling_dir='./profiling'
+        )
 
-        assert command == '/bin/python3 -m cProfile -o ./profiling/tap_my_tap_mysql.pstat /bin/tap_mysql.py ' \
-                          '--config .ppw/config.json --catalog .ppw/properties.json '
+        assert (
+            command
+            == '/bin/python3 -m cProfile -o ./profiling/tap_my_tap_mysql.pstat /bin/tap_mysql.py '
+            '--config .ppw/config.json --catalog .ppw/properties.json '
+        )
 
         # State file should not be included if state file passed but file not exists
-        tap = commands.TapParams(id='my_tap_mysql',
-                                 type='tap_mysql',
-                                 bin='/bin/tap_mysql.py',
-                                 python_bin='/bin/python3',
-                                 config='.ppw/config.json',
-                                 properties='.ppw/properties.json',
-                                 state='.pipelinewise/state.json'
-                                 )
+        tap = commands.TapParams(
+            id='my_tap_mysql',
+            type='tap_mysql',
+            bin='/bin/tap_mysql.py',
+            python_bin='/bin/python3',
+            config='.ppw/config.json',
+            properties='.ppw/properties.json',
+            state='.pipelinewise/state.json',
+        )
 
         # profiling is not enabled
-        command = commands.build_tap_command(tap=tap,
-                                             profiling_dir=None,
-                                             profiling_mode=False
-                                             )
+        command = commands.build_tap_command(
+            tap=tap, profiling_dir=None, profiling_mode=False
+        )
 
-        assert command == '/bin/tap_mysql.py --config .ppw/config.json --catalog .ppw/properties.json '
+        assert (
+            command
+            == '/bin/tap_mysql.py --config .ppw/config.json --catalog .ppw/properties.json '
+        )
 
         # profiling is enabled
-        command = commands.build_tap_command(tap=tap,
-                                             profiling_mode=True,
-                                             profiling_dir='./profiling'
-                                             )
+        command = commands.build_tap_command(
+            tap=tap, profiling_mode=True, profiling_dir='./profiling'
+        )
 
-        assert command == '/bin/python3 -m cProfile -o ./profiling/tap_my_tap_mysql.pstat /bin/tap_mysql.py ' \
-                          '--config .ppw/config.json --catalog .ppw/properties.json '
+        assert (
+            command
+            == '/bin/python3 -m cProfile -o ./profiling/tap_my_tap_mysql.pstat /bin/tap_mysql.py '
+            '--config .ppw/config.json --catalog .ppw/properties.json '
+        )
 
         # State file should be included if state file passed and file exists
         state_mock = __file__
 
-        tap = commands.TapParams(id='my_tap_mysql',
-                                 type='tap_mysql',
-                                 bin='/bin/tap_mysql.py',
-                                 python_bin='/bin/python3',
-                                 config='.ppw/config.json',
-                                 properties='.ppw/properties.json',
-                                 state=state_mock
-                                 )
+        tap = commands.TapParams(
+            id='my_tap_mysql',
+            type='tap_mysql',
+            bin='/bin/tap_mysql.py',
+            python_bin='/bin/python3',
+            config='.ppw/config.json',
+            properties='.ppw/properties.json',
+            state=state_mock,
+        )
         # profiling is not enabled
-        command = commands.build_tap_command(tap=tap,
-                                             profiling_mode=False,
-                                             profiling_dir=None)
+        command = commands.build_tap_command(
+            tap=tap, profiling_mode=False, profiling_dir=None
+        )
 
-        assert command == f'/bin/tap_mysql.py --config .ppw/config.json --catalog .ppw/properties.json ' \
-                          f'--state {state_mock}'
+        assert (
+            command
+            == f'/bin/tap_mysql.py --config .ppw/config.json --catalog .ppw/properties.json '
+            f'--state {state_mock}'
+        )
 
         # profiling is enabled
-        command = commands.build_tap_command(tap=tap,
-                                             profiling_mode=True,
-                                             profiling_dir='./profiling')
+        command = commands.build_tap_command(
+            tap=tap, profiling_mode=True, profiling_dir='./profiling'
+        )
 
-        assert command == f'/bin/python3 -m cProfile -o ./profiling/tap_my_tap_mysql.pstat /bin/tap_mysql.py ' \
-                          f'--config .ppw/config.json --catalog .ppw/properties.json --state {state_mock}'
+        assert (
+            command
+            == f'/bin/python3 -m cProfile -o ./profiling/tap_my_tap_mysql.pstat /bin/tap_mysql.py '
+            f'--config .ppw/config.json --catalog .ppw/properties.json --state {state_mock}'
+        )
 
     def test_build_target_command(self):
         """Tests the function that generates target executable command"""
         # Should return a input piped command with an executable target command
 
-        target = commands.TargetParams(id='my_target',
-                                       type='target-snowflake',
-                                       bin='/bin/target_postgres.py',
-                                       python_bin='/bin/python',
-                                       config='.ppw/config.json')
+        target = commands.TargetParams(
+            id='my_target',
+            type='target-snowflake',
+            bin='/bin/target_postgres.py',
+            python_bin='/bin/python',
+            config='.ppw/config.json',
+        )
 
         # profiling is not enabled
-        command = commands.build_target_command(target=target,
-                                                profiling_mode=False,
-                                                profiling_dir=None)
+        command = commands.build_target_command(
+            target=target, profiling_mode=False, profiling_dir=None
+        )
 
         assert command == '/bin/target_postgres.py --config .ppw/config.json'
 
         # profiling is enabled
-        command = commands.build_target_command(target=target,
-                                                profiling_mode=True,
-                                                profiling_dir='./profiling')
-        assert command == '/bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py' \
-                          ' --config .ppw/config.json'
+        command = commands.build_target_command(
+            target=target, profiling_mode=True, profiling_dir='./profiling'
+        )
+        assert (
+            command
+            == '/bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py'
+            ' --config .ppw/config.json'
+        )
 
     def test_build_transform_command(self):
         """Tests the function that generates transform executable command"""
         # Should return empty string if config file exists but no transformation
-        transform_config = '{}/resources/transform-config-empty.json'.format(os.path.dirname(__file__))
+        transform_config = '{}/resources/transform-config-empty.json'.format(
+            os.path.dirname(__file__)
+        )
         transform_bin = '/bin/transform_field.py'
 
-        transform = commands.TransformParams(config=transform_config,
-                                             bin=transform_bin,
-                                             python_bin='/bin/python',
-                                             tap_id='my_tap',
-                                             target_id='my_target')
+        transform = commands.TransformParams(
+            config=transform_config,
+            bin=transform_bin,
+            python_bin='/bin/python',
+            tap_id='my_tap',
+            target_id='my_target',
+        )
         # profiling disabled
-        command = commands.build_transformation_command(transform=transform,
-                                                        profiling_mode=False,
-                                                        profiling_dir=None)
+        command = commands.build_transformation_command(
+            transform=transform, profiling_mode=False, profiling_dir=None
+        )
         assert command is None
 
         # profiling enabled
-        command = commands.build_transformation_command(transform=transform,
-                                                        profiling_mode=True,
-                                                        profiling_dir='./profiling')
+        command = commands.build_transformation_command(
+            transform=transform, profiling_mode=True, profiling_dir='./profiling'
+        )
         assert command is None
 
         # Should return a input piped command with an executable transform command
-        transform_config = '{}/resources/transform-config.json'.format(os.path.dirname(__file__))
-
-        transform = commands.TransformParams(config=transform_config,
-                                             bin=transform_bin,
-                                             python_bin='/bin/python',
-                                             tap_id='my_tap',
-                                             target_id='my_target')
+        transform_config = '{}/resources/transform-config.json'.format(
+            os.path.dirname(__file__)
+        )
+
+        transform = commands.TransformParams(
+            config=transform_config,
+            bin=transform_bin,
+            python_bin='/bin/python',
+            tap_id='my_tap',
+            target_id='my_target',
+        )
         # profiling disabled
-        command = commands.build_transformation_command(transform=transform,
-                                                        profiling_mode=False,
-                                                        profiling_dir=None)
+        command = commands.build_transformation_command(
+            transform=transform, profiling_mode=False, profiling_dir=None
+        )
 
         assert command == f'/bin/transform_field.py --config {transform_config}'
 
         # profiling enabled
-        command = commands.build_transformation_command(transform=transform,
-                                                        profiling_mode=True,
-                                                        profiling_dir='./profiling')
+        command = commands.build_transformation_command(
+            transform=transform, profiling_mode=True, profiling_dir='./profiling'
+        )
 
-        assert command == f'/bin/python -m cProfile -o ./profiling/transformation_my_tap_my_target.pstat ' \
-                          f'/bin/transform_field.py --config {transform_config}'
+        assert (
+            command
+            == f'/bin/python -m cProfile -o ./profiling/transformation_my_tap_my_target.pstat '
+            f'/bin/transform_field.py --config {transform_config}'
+        )
 
     # pylint: disable=invalid-name
     def test_build_stream_buffer_command(self):
@@ -186,315 +221,432 @@ def test_build_stream_buffer_command(self):
         assert commands.build_stream_buffer_command(buffer_size=-10) is None
 
         # Should use the minimum buffer size if enabled but less than minimal buffer size
-        assert commands.build_stream_buffer_command(buffer_size=1) == f'mbuffer -m {commands.MIN_STREAM_BUFFER_SIZE}M'
+        assert (
+            commands.build_stream_buffer_command(buffer_size=1)
+            == f'mbuffer -m {commands.MIN_STREAM_BUFFER_SIZE}M'
+        )
 
         # Should raise StreamBufferTooLargeException if buffer_size is greater than the max allowed
         with pytest.raises(StreamBufferTooLargeException):
-            commands.build_stream_buffer_command(buffer_size=commands.MAX_STREAM_BUFFER_SIZE + 1000)
+            commands.build_stream_buffer_command(
+                buffer_size=commands.MAX_STREAM_BUFFER_SIZE + 1000
+            )
 
         # Should use custom buffer size if between max and min buffer size
-        assert commands.build_stream_buffer_command(buffer_size=100) == 'mbuffer -m 100M'
+        assert (
+            commands.build_stream_buffer_command(buffer_size=100) == 'mbuffer -m 100M'
+        )
 
         # Should use custom buffer binary executable if bin parameter provided
-        assert commands.build_stream_buffer_command(buffer_size=100, stream_buffer_bin='dummy_buffer') == \
-               'dummy_buffer -m 100M'
+        assert (
+            commands.build_stream_buffer_command(
+                buffer_size=100, stream_buffer_bin='dummy_buffer'
+            )
+            == 'dummy_buffer -m 100M'
+        )
 
         # Should log mbuffer status to log file with .running extension
-        assert commands.build_stream_buffer_command(buffer_size=100, log_file='stream_buffer.log') == \
-               'mbuffer -m 100M -q -l stream_buffer.log.running'
+        assert (
+            commands.build_stream_buffer_command(
+                buffer_size=100, log_file='stream_buffer.log'
+            )
+            == 'mbuffer -m 100M -q -l stream_buffer.log.running'
+        )
 
     def test_build_singer_command(self):
         """Tests the function that generates the full singer singer command
         that connects the required components with linux pipes"""
-        transform_config = '{}/resources/transform-config.json'.format(os.path.dirname(__file__))
-        transform_config_empty = '{}/resources/transform-config-empty.json'.format(os.path.dirname(__file__))
+        transform_config = '{}/resources/transform-config.json'.format(
+            os.path.dirname(__file__)
+        )
+        transform_config_empty = '{}/resources/transform-config-empty.json'.format(
+            os.path.dirname(__file__)
+        )
         state_mock = __file__
 
         # Should generate a command with tap state and transformation
-        tap_params = commands.TapParams(id='my_tap',
-                                        type='tap-mysql',
-                                        bin='/bin/tap_mysql.py',
-                                        python_bin='/bin/python',
-                                        config='.ppw/config.json',
-                                        properties='.ppw/properties.json',
-                                        state=state_mock)
-
-        target_params = commands.TargetParams(id='my_target',
-                                              type='target-postgres',
-                                              bin='/bin/target_postgres.py',
-                                              python_bin='/bin/python',
-                                              config='.ppw/config.json')
-
-        transform_params = commands.TransformParams(bin='/bin/transform_field.py',
-                                                    python_bin='/bin/python',
-                                                    config=transform_config,
-                                                    tap_id='my_tap',
-                                                    target_id='my_target')
+        tap_params = commands.TapParams(
+            id='my_tap',
+            type='tap-mysql',
+            bin='/bin/tap_mysql.py',
+            python_bin='/bin/python',
+            config='.ppw/config.json',
+            properties='.ppw/properties.json',
+            state=state_mock,
+        )
+
+        target_params = commands.TargetParams(
+            id='my_target',
+            type='target-postgres',
+            bin='/bin/target_postgres.py',
+            python_bin='/bin/python',
+            config='.ppw/config.json',
+        )
+
+        transform_params = commands.TransformParams(
+            bin='/bin/transform_field.py',
+            python_bin='/bin/python',
+            config=transform_config,
+            tap_id='my_tap',
+            target_id='my_target',
+        )
         # profiling disabled
-        command = commands.build_singer_command(tap_params,
-                                                target_params,
-                                                transform_params)
-
-        assert command == f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json ' \
-                          f'--state {state_mock}' \
-                          f' | /bin/transform_field.py --config {transform_config}' \
-                          ' | /bin/target_postgres.py --config .ppw/config.json'
+        command = commands.build_singer_command(
+            tap_params, target_params, transform_params
+        )
+
+        assert (
+            command
+            == f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
+            f'--state {state_mock}'
+            f' | /bin/transform_field.py --config {transform_config}'
+            ' | /bin/target_postgres.py --config .ppw/config.json'
+        )
 
         # profiling enabled
-        command = commands.build_singer_command(tap_params,
-                                                target_params,
-                                                transform_params,
-                                                profiling_mode=True,
-                                                profiling_dir='./profiling')
-
-        assert command == f'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat ' \
-                          f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json ' \
-                          f'--state {state_mock}' \
-                          f' | /bin/python -m cProfile -o ./profiling/transformation_my_tap_my_target.pstat ' \
-                          f'/bin/transform_field.py --config {transform_config}' \
-                          ' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py ' \
-                          '--config .ppw/config.json'
+        command = commands.build_singer_command(
+            tap_params,
+            target_params,
+            transform_params,
+            profiling_mode=True,
+            profiling_dir='./profiling',
+        )
+
+        assert (
+            command == f'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat '
+            f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
+            f'--state {state_mock}'
+            f' | /bin/python -m cProfile -o ./profiling/transformation_my_tap_my_target.pstat '
+            f'/bin/transform_field.py --config {transform_config}'
+            ' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py '
+            '--config .ppw/config.json'
+        )
 
         # Should generate a command with tap state and transformation and stream buffer
 
         # profiling disabled
-        command = commands.build_singer_command(tap_params, target_params, transform_params, stream_buffer_size=10)
-
-        assert command == f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json ' \
-                          f'--state {state_mock}' \
-                          f' | /bin/transform_field.py --config {transform_config}' \
-                          ' | mbuffer -m 10M' \
-                          ' | /bin/target_postgres.py --config .ppw/config.json'
+        command = commands.build_singer_command(
+            tap_params, target_params, transform_params, stream_buffer_size=10
+        )
+
+        assert (
+            command
+            == f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
+            f'--state {state_mock}'
+            f' | /bin/transform_field.py --config {transform_config}'
+            ' | mbuffer -m 10M'
+            ' | /bin/target_postgres.py --config .ppw/config.json'
+        )
 
         # profiling enabled
-        command = commands.build_singer_command(tap_params,
-                                                target_params,
-                                                transform_params,
-                                                profiling_mode=True,
-                                                profiling_dir='./profiling',
-                                                stream_buffer_size=10)
-
-        assert command == f'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat /bin/tap_mysql.py ' \
-                          f'--config .ppw/config.json --properties .ppw/properties.json --state {state_mock}' \
-                          f' | /bin/python -m cProfile -o ./profiling/transformation_my_tap_my_target.pstat ' \
-                          f'/bin/transform_field.py --config {transform_config}' \
-                          ' | mbuffer -m 10M' \
-                          ' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py' \
-                          ' --config .ppw/config.json'
+        command = commands.build_singer_command(
+            tap_params,
+            target_params,
+            transform_params,
+            profiling_mode=True,
+            profiling_dir='./profiling',
+            stream_buffer_size=10,
+        )
+
+        assert (
+            command
+            == f'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat /bin/tap_mysql.py '
+            f'--config .ppw/config.json --properties .ppw/properties.json --state {state_mock}'
+            f' | /bin/python -m cProfile -o ./profiling/transformation_my_tap_my_target.pstat '
+            f'/bin/transform_field.py --config {transform_config}'
+            ' | mbuffer -m 10M'
+            ' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py'
+            ' --config .ppw/config.json'
+        )
 
         # Should generate a command without state and with transformation
-        tap_params = commands.TapParams(id='my_tap',
-                                        type='tap-mysql',
-                                        bin='/bin/tap_mysql.py',
-                                        python_bin='/bin/python',
-                                        config='.ppw/config.json',
-                                        properties='.ppw/properties.json',
-                                        state=None)
+        tap_params = commands.TapParams(
+            id='my_tap',
+            type='tap-mysql',
+            bin='/bin/tap_mysql.py',
+            python_bin='/bin/python',
+            config='.ppw/config.json',
+            properties='.ppw/properties.json',
+            state=None,
+        )
 
         # profiling disabled
-        command = commands.build_singer_command(tap_params, target_params, transform_params)
+        command = commands.build_singer_command(
+            tap_params, target_params, transform_params
+        )
 
-        assert command == f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json ' \
-                          f' | /bin/transform_field.py --config {transform_config}' \
-                          ' | /bin/target_postgres.py --config .ppw/config.json'
+        assert (
+            command
+            == f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
+            f' | /bin/transform_field.py --config {transform_config}'
+            ' | /bin/target_postgres.py --config .ppw/config.json'
+        )
 
         # profiling enabled
-        command = commands.build_singer_command(tap_params,
-                                                target_params,
-                                                transform_params,
-                                                profiling_mode=True,
-                                                profiling_dir='./profiling')
-
-        assert command == f'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat ' \
-                          f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json ' \
-                          f' | /bin/python -m cProfile -o ./profiling/transformation_my_tap_my_target.pstat ' \
-                          f'/bin/transform_field.py --config {transform_config}' \
-                          ' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat ' \
-                          '/bin/target_postgres.py --config .ppw/config.json'
+        command = commands.build_singer_command(
+            tap_params,
+            target_params,
+            transform_params,
+            profiling_mode=True,
+            profiling_dir='./profiling',
+        )
+
+        assert (
+            command == f'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat '
+            f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
+            f' | /bin/python -m cProfile -o ./profiling/transformation_my_tap_my_target.pstat '
+            f'/bin/transform_field.py --config {transform_config}'
+            ' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat '
+            '/bin/target_postgres.py --config .ppw/config.json'
+        )
 
         # Should generate a command with state and without transformation
-        tap_params = commands.TapParams(id='my_tap',
-                                        type='tap-mysql',
-                                        bin='/bin/tap_mysql.py',
-                                        python_bin='/bin/python',
-                                        config='.ppw/config.json',
-                                        properties='.ppw/properties.json',
-                                        state=state_mock)
-
-        transform_params = commands.TransformParams(bin='/bin/transform_field.py',
-                                                    python_bin='/bin/python',
-                                                    config=transform_config_empty,
-                                                    tap_id='my_tap',
-                                                    target_id='my_target')
+        tap_params = commands.TapParams(
+            id='my_tap',
+            type='tap-mysql',
+            bin='/bin/tap_mysql.py',
+            python_bin='/bin/python',
+            config='.ppw/config.json',
+            properties='.ppw/properties.json',
+            state=state_mock,
+        )
+
+        transform_params = commands.TransformParams(
+            bin='/bin/transform_field.py',
+            python_bin='/bin/python',
+            config=transform_config_empty,
+            tap_id='my_tap',
+            target_id='my_target',
+        )
         # profiling disabled
-        command = commands.build_singer_command(tap_params, target_params, transform_params)
+        command = commands.build_singer_command(
+            tap_params, target_params, transform_params
+        )
 
-        assert command == f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json ' \
-                          f'--state {state_mock}' \
-                          ' | /bin/target_postgres.py --config .ppw/config.json'
+        assert (
+            command
+            == f'/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
+            f'--state {state_mock}'
+            ' | /bin/target_postgres.py --config .ppw/config.json'
+        )
 
         # profiling enabled
-        command = commands.build_singer_command(tap_params,
-                                                target_params,
-                                                transform_params,
-                                                profiling_mode=True,
-                                                profiling_dir='./profiling')
-
-        assert command == f'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat /bin/tap_mysql.py ' \
-                          f'--config .ppw/config.json --properties .ppw/properties.json ' \
-                          f'--state {state_mock}' \
-                          ' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py ' \
-                          '--config .ppw/config.json'
+        command = commands.build_singer_command(
+            tap_params,
+            target_params,
+            transform_params,
+            profiling_mode=True,
+            profiling_dir='./profiling',
+        )
+
+        assert (
+            command
+            == f'/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat /bin/tap_mysql.py '
+            f'--config .ppw/config.json --properties .ppw/properties.json '
+            f'--state {state_mock}'
+            ' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py '
+            '--config .ppw/config.json'
+        )
 
         # Should generate a command without state and transformation
-        tap_params = commands.TapParams(id='my_tap',
-                                        type='tap-mysql',
-                                        bin='/bin/tap_mysql.py',
-                                        python_bin='/bin/python',
-                                        config='.ppw/config.json',
-                                        properties='.ppw/properties.json',
-                                        state='.ppw/state.json')
+        tap_params = commands.TapParams(
+            id='my_tap',
+            type='tap-mysql',
+            bin='/bin/tap_mysql.py',
+            python_bin='/bin/python',
+            config='.ppw/config.json',
+            properties='.ppw/properties.json',
+            state='.ppw/state.json',
+        )
         # profiling disabled
-        command = commands.build_singer_command(tap_params, target_params, transform_params)
+        command = commands.build_singer_command(
+            tap_params, target_params, transform_params
+        )
 
-        assert command == '/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json ' \
-                          ' | /bin/target_postgres.py --config .ppw/config.json'
+        assert (
+            command
+            == '/bin/tap_mysql.py --config .ppw/config.json --properties .ppw/properties.json '
+            ' | /bin/target_postgres.py --config .ppw/config.json'
+        )
 
         # profiling enabled
-        command = commands.build_singer_command(tap_params,
-                                                target_params,
-                                                transform_params,
-                                                profiling_mode=True,
-                                                profiling_dir='./profiling')
-
-        assert command == '/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat /bin/tap_mysql.py ' \
-                          '--config .ppw/config.json --properties .ppw/properties.json ' \
-                          ' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py ' \
-                          '--config .ppw/config.json'
+        command = commands.build_singer_command(
+            tap_params,
+            target_params,
+            transform_params,
+            profiling_mode=True,
+            profiling_dir='./profiling',
+        )
+
+        assert (
+            command
+            == '/bin/python -m cProfile -o ./profiling/tap_my_tap.pstat /bin/tap_mysql.py '
+            '--config .ppw/config.json --properties .ppw/properties.json '
+            ' | /bin/python -m cProfile -o ./profiling/target_my_target.pstat /bin/target_postgres.py '
+            '--config .ppw/config.json'
+        )
 
     def test_build_fastsync_command(self):
         """Tests the function that generates the fastsync command"""
-        transform_config = '{}/resources/transform-config.json'.format(os.path.dirname(__file__))
+        transform_config = '{}/resources/transform-config.json'.format(
+            os.path.dirname(__file__)
+        )
         state_mock = __file__
         venv_dir = '.dummy_venv_dir'
         temp_dir = 'dummy_temp_dir'
 
         # Should generate a fastsync command with transformation
 
-        tap_params = commands.TapParams(id='my_tap',
-                                        type='tap-mysql',
-                                        bin='/bin/tap_mysql.py',
-                                        python_bin='/tap-mysql/bin/python',
-                                        config='.ppw/tap_config.json',
-                                        properties='.ppw/properties.json',
-                                        state=state_mock)
-
-        target_params = commands.TargetParams(id='my_target',
-                                              type='target-postgres',
-                                              bin='/bin/target_postgres.py',
-                                              python_bin='/target-postgres/bin/python',
-                                              config='.ppw/target_config.json')
-
-        transform_params = commands.TransformParams(bin='/bin/transform_field.py',
-                                                    python_bin='transform/bin/python',
-                                                    config=None,
-                                                    tap_id='my_tap',
-                                                    target_id='my_target')
+        tap_params = commands.TapParams(
+            id='my_tap',
+            type='tap-mysql',
+            bin='/bin/tap_mysql.py',
+            python_bin='/tap-mysql/bin/python',
+            config='.ppw/tap_config.json',
+            properties='.ppw/properties.json',
+            state=state_mock,
+        )
+
+        target_params = commands.TargetParams(
+            id='my_target',
+            type='target-postgres',
+            bin='/bin/target_postgres.py',
+            python_bin='/target-postgres/bin/python',
+            config='.ppw/target_config.json',
+        )
+
+        transform_params = commands.TransformParams(
+            bin='/bin/transform_field.py',
+            python_bin='transform/bin/python',
+            config=None,
+            tap_id='my_tap',
+            target_id='my_target',
+        )
 
         # profiling disabled
-        command = commands.build_fastsync_command(tap_params, target_params, transform_params, venv_dir, temp_dir)
-        assert command == '.dummy_venv_dir/pipelinewise/bin/mysql-to-postgres' \
-                          ' --tap .ppw/tap_config.json' \
-                          ' --properties .ppw/properties.json' \
-                          f' --state {state_mock}' \
-                          ' --target .ppw/target_config.json' \
-                          ' --temp_dir dummy_temp_dir'
+        command = commands.build_fastsync_command(
+            tap_params, target_params, transform_params, venv_dir, temp_dir
+        )
+        assert (
+            command == '.dummy_venv_dir/pipelinewise/bin/mysql-to-postgres'
+            ' --tap .ppw/tap_config.json'
+            ' --properties .ppw/properties.json'
+            f' --state {state_mock}'
+            ' --target .ppw/target_config.json'
+            ' --temp_dir dummy_temp_dir'
+        )
 
         # profiling enabled
-        command = commands.build_fastsync_command(tap_params,
-                                                  target_params,
-                                                  transform_params,
-                                                  venv_dir,
-                                                  temp_dir,
-                                                  profiling_mode=True,
-                                                  profiling_dir='./profiling')
-
-        assert command == '.dummy_venv_dir/pipelinewise/bin/python -m cProfile ' \
-                          '-o ./profiling/fastsync_my_tap_my_target.pstat' \
-                          ' .dummy_venv_dir/pipelinewise/bin/mysql-to-postgres' \
-                          ' --tap .ppw/tap_config.json' \
-                          ' --properties .ppw/properties.json' \
-                          f' --state {state_mock}' \
-                          ' --target .ppw/target_config.json' \
-                          ' --temp_dir dummy_temp_dir'
+        command = commands.build_fastsync_command(
+            tap_params,
+            target_params,
+            transform_params,
+            venv_dir,
+            temp_dir,
+            profiling_mode=True,
+            profiling_dir='./profiling',
+        )
+
+        assert (
+            command == '.dummy_venv_dir/pipelinewise/bin/python -m cProfile '
+            '-o ./profiling/fastsync_my_tap_my_target.pstat'
+            ' .dummy_venv_dir/pipelinewise/bin/mysql-to-postgres'
+            ' --tap .ppw/tap_config.json'
+            ' --properties .ppw/properties.json'
+            f' --state {state_mock}'
+            ' --target .ppw/target_config.json'
+            ' --temp_dir dummy_temp_dir'
+        )
 
         # Should generate a fastsync command with transformation
-        transform_params = commands.TransformParams(bin='/bin/transform_field.py',
-                                                    python_bin='transform/bin/python',
-                                                    config=transform_config,
-                                                    tap_id='my_tap',
-                                                    target_id='my_target')
+        transform_params = commands.TransformParams(
+            bin='/bin/transform_field.py',
+            python_bin='transform/bin/python',
+            config=transform_config,
+            tap_id='my_tap',
+            target_id='my_target',
+        )
 
         # profiling disabled
-        command = commands.build_fastsync_command(tap_params, target_params, transform_params, venv_dir, temp_dir)
-        assert command == '.dummy_venv_dir/pipelinewise/bin/mysql-to-postgres' \
-                          ' --tap .ppw/tap_config.json' \
-                          ' --properties .ppw/properties.json' \
-                          f' --state {state_mock}' \
-                          ' --target .ppw/target_config.json' \
-                          ' --temp_dir dummy_temp_dir' \
-                          f' --transform {transform_config}'
+        command = commands.build_fastsync_command(
+            tap_params, target_params, transform_params, venv_dir, temp_dir
+        )
+        assert (
+            command == '.dummy_venv_dir/pipelinewise/bin/mysql-to-postgres'
+            ' --tap .ppw/tap_config.json'
+            ' --properties .ppw/properties.json'
+            f' --state {state_mock}'
+            ' --target .ppw/target_config.json'
+            ' --temp_dir dummy_temp_dir'
+            f' --transform {transform_config}'
+        )
 
         # profiling enabled
-        command = commands.build_fastsync_command(tap_params,
-                                                  target_params,
-                                                  transform_params,
-                                                  venv_dir,
-                                                  temp_dir,
-                                                  profiling_mode=True,
-                                                  profiling_dir='./profiling')
-
-        assert command == '.dummy_venv_dir/pipelinewise/bin/python -m cProfile' \
-                          ' -o ./profiling/fastsync_my_tap_my_target.pstat' \
-                          ' .dummy_venv_dir/pipelinewise/bin/mysql-to-postgres' \
-                          ' --tap .ppw/tap_config.json' \
-                          ' --properties .ppw/properties.json' \
-                          f' --state {state_mock}' \
-                          ' --target .ppw/target_config.json' \
-                          ' --temp_dir dummy_temp_dir' \
-                          f' --transform {transform_config}'
+        command = commands.build_fastsync_command(
+            tap_params,
+            target_params,
+            transform_params,
+            venv_dir,
+            temp_dir,
+            profiling_mode=True,
+            profiling_dir='./profiling',
+        )
+
+        assert (
+            command == '.dummy_venv_dir/pipelinewise/bin/python -m cProfile'
+            ' -o ./profiling/fastsync_my_tap_my_target.pstat'
+            ' .dummy_venv_dir/pipelinewise/bin/mysql-to-postgres'
+            ' --tap .ppw/tap_config.json'
+            ' --properties .ppw/properties.json'
+            f' --state {state_mock}'
+            ' --target .ppw/target_config.json'
+            ' --temp_dir dummy_temp_dir'
+            f' --transform {transform_config}'
+        )
 
         # Should generate a fastsync command with specific list of tables
-        command = commands.build_fastsync_command(tap_params, target_params, transform_params, venv_dir, temp_dir,
-                                                  tables='public.table_one,public.table_two')
-        assert command == '.dummy_venv_dir/pipelinewise/bin/mysql-to-postgres' \
-                          ' --tap .ppw/tap_config.json' \
-                          ' --properties .ppw/properties.json' \
-                          f' --state {state_mock}' \
-                          ' --target .ppw/target_config.json' \
-                          ' --temp_dir dummy_temp_dir' \
-                          f' --transform {transform_config}' \
-                          ' --tables public.table_one,public.table_two'
+        command = commands.build_fastsync_command(
+            tap_params,
+            target_params,
+            transform_params,
+            venv_dir,
+            temp_dir,
+            tables='public.table_one,public.table_two',
+        )
+        assert (
+            command == '.dummy_venv_dir/pipelinewise/bin/mysql-to-postgres'
+            ' --tap .ppw/tap_config.json'
+            ' --properties .ppw/properties.json'
+            f' --state {state_mock}'
+            ' --target .ppw/target_config.json'
+            ' --temp_dir dummy_temp_dir'
+            f' --transform {transform_config}'
+            ' --tables public.table_one,public.table_two'
+        )
 
     def test_run_command(self):
         """Test run command functions
 
-            Run command runs everything enclosed by /bin/bash -o pipefail -c '{}'
-            This means arguments should pass as plain string after the command
+        Run command runs everything enclosed by /bin/bash -o pipefail -c '{}'
+        This means arguments should pass as plain string after the command
 
-            Return value is an array of: [return_code, stdout, stderr]
+        Return value is an array of: [return_code, stdout, stderr]
         """
         # Printing something to stdout should return 0
         [returncode, stdout, stderr] = commands.run_command('echo this is a test line')
         assert [returncode, stdout, stderr] == [0, 'this is a test line\n', '']
 
         # Running an invalid command should return 127 and some error message to stdout
-        [returncode, stdout, stderr] = commands.run_command('invalid-command this is an invalid command')
+        [returncode, stdout, stderr] = commands.run_command(
+            'invalid-command this is an invalid command'
+        )
         assert [returncode, stdout] == [127, '']
         assert stderr != ''
 
         # If loggin enabled then a success command should create log file with success status
-        [returncode, stdout, stderr] = commands.run_command('echo this is a test line', log_file='./test.log')
+        [returncode, stdout, stderr] = commands.run_command(
+            'echo this is a test line', log_file='./test.log'
+        )
         assert [returncode, stdout, stderr] == [0, 'this is a test line\n', None]
         assert os.path.isfile('test.log.success')
         os.remove('test.log.success')
@@ -505,6 +657,8 @@ def test_run_command(self):
         # TODO: Sync failed command execution behaviour with logging and no-logging option
         #       Both should return [rc, stdout, stderr] list or both should raise exception
         with pytest.raises(Exception):
-            commands.run_command('invalid-command this is an invalid command', log_file='./test.log')
+            commands.run_command(
+                'invalid-command this is an invalid command', log_file='./test.log'
+            )
         assert os.path.isfile('test.log.failed')
         os.remove('test.log.failed')
diff --git a/tests/units/cli/test_config.py b/tests/units/cli/test_config.py
index 54a3ec98a..a68b1064c 100644
--- a/tests/units/cli/test_config.py
+++ b/tests/units/cli/test_config.py
@@ -7,6 +7,7 @@
 
 PIPELINEWISE_TEST_HOME = '/tmp/.pipelinewise'
 
+
 # Todo: Inherit from unittest.TestCase
 # pylint: disable=no-self-use,fixme
 class TestConfig:
@@ -25,37 +26,39 @@ def test_constructor(self):
 
     def test_connector_files(self):
         """Every singer connector must have a list of JSON files at certain locations"""
-        assert Config.get_connector_files('/var/singer-connector') == \
-            {
-                'config': '/var/singer-connector/config.json',
-                'inheritable_config': '/var/singer-connector/inheritable_config.json',
-                'properties': '/var/singer-connector/properties.json',
-                'state': '/var/singer-connector/state.json',
-                'transformation': '/var/singer-connector/transformation.json',
-                'selection': '/var/singer-connector/selection.json',
-                'pidfile': '/var/singer-connector/pipelinewise.pid'
-            }
+        assert Config.get_connector_files('/var/singer-connector') == {
+            'config': '/var/singer-connector/config.json',
+            'inheritable_config': '/var/singer-connector/inheritable_config.json',
+            'properties': '/var/singer-connector/properties.json',
+            'state': '/var/singer-connector/state.json',
+            'transformation': '/var/singer-connector/transformation.json',
+            'selection': '/var/singer-connector/selection.json',
+            'pidfile': '/var/singer-connector/pipelinewise.pid',
+        }
 
     def test_from_yamls(self):
         """Test creating Config object using YAML configuration directory as the input"""
 
         # Create Config object by parsing target and tap YAMLs in a directory
-        yaml_config_dir = '{}/resources/test_yaml_config'.format(os.path.dirname(__file__))
-        vault_secret = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__))
+        yaml_config_dir = f'{os.path.dirname(__file__)}/resources/test_yaml_config'
+
+        vault_secret = f'{os.path.dirname(__file__)}/resources/vault-secret.txt'
 
         # Parse YAML files and create the config object
-        config = Config.from_yamls(PIPELINEWISE_TEST_HOME, yaml_config_dir, vault_secret)
+        config = Config.from_yamls(
+            PIPELINEWISE_TEST_HOME, yaml_config_dir, vault_secret
+        )
 
         # config dir and path should be generated automatically
         assert config.config_dir == PIPELINEWISE_TEST_HOME
-        assert config.config_path == '{}/config.json'.format(PIPELINEWISE_TEST_HOME)
+        assert config.config_path == f'{PIPELINEWISE_TEST_HOME}/config.json'
 
         # Vault encrypted alert handlers should be loaded into global config
         assert config.global_config == {
             'alert_handlers': {
                 'slack': {
                     'token': 'Vault Encrypted Secret Fruit',
-                    'channel': '#slack-channel'
+                    'channel': '#slack-channel',
                 }
             }
         }
@@ -78,67 +81,68 @@ def test_from_yamls(self):
                     's3_key_prefix': 's3_prefix/',
                     'stage': 'foo_stage',
                     'user': 'user',
-                    'warehouse': 'MY_WAREHOUSE'
+                    'warehouse': 'MY_WAREHOUSE',
                 },
                 'files': {
-                    'config': '{}/test_snowflake_target/config.json'.format(PIPELINEWISE_TEST_HOME),
-                    'inheritable_config':
-                        '{}/test_snowflake_target/inheritable_config.json'.format(PIPELINEWISE_TEST_HOME),
-                    'properties': '{}/test_snowflake_target/properties.json'.format(PIPELINEWISE_TEST_HOME),
-                    'selection': '{}/test_snowflake_target/selection.json'.format(PIPELINEWISE_TEST_HOME),
-                    'state': '{}/test_snowflake_target/state.json'.format(PIPELINEWISE_TEST_HOME),
-                    'transformation': '{}/test_snowflake_target/transformation.json'.format(PIPELINEWISE_TEST_HOME),
-                    'pidfile': '{}/test_snowflake_target/pipelinewise.pid'.format(PIPELINEWISE_TEST_HOME),
+                    'config': f'{PIPELINEWISE_TEST_HOME}/test_snowflake_target/config.json',
+                    'inheritable_config': f'{ PIPELINEWISE_TEST_HOME}/test_snowflake_target/inheritable_config.json',
+                    'properties': f'{PIPELINEWISE_TEST_HOME}/test_snowflake_target/properties.json',
+                    'selection': f'{PIPELINEWISE_TEST_HOME}/test_snowflake_target/selection.json',
+                    'state': f'{PIPELINEWISE_TEST_HOME}/test_snowflake_target/state.json',
+                    'transformation': f'{PIPELINEWISE_TEST_HOME}/test_snowflake_target/transformation.json',
+                    'pidfile': f'{PIPELINEWISE_TEST_HOME}/test_snowflake_target/pipelinewise.pid',
                 },
-                'taps': [{
-                    'id': 'mysql_sample',
-                    'name': 'Sample MySQL Database',
-                    'type': 'tap-mysql',
-                    'owner': 'somebody@foo.com',
-                    'target': 'test_snowflake_target',
-                    'batch_size_rows': 20000,
-                    'batch_wait_limit_seconds': 3600,
-                    'split_large_files': True,
-                    'split_file_chunk_size_mb': 500,
-                    'split_file_max_chunks': 25,
-                    'db_conn': {
-                        'dbname': '<DB_NAME>',
-                        'host': '<HOST>',
-                        'password': '<PASSWORD>',
-                        'port': 3306,
-                        'user': '<USER>'
-                    },
-                    'files': {
-                        'config':
-                            '{}/test_snowflake_target/mysql_sample/config.json'.format(PIPELINEWISE_TEST_HOME),
-                        'inheritable_config':
-                            '{}/test_snowflake_target/mysql_sample/inheritable_config.json'.format(
-                                PIPELINEWISE_TEST_HOME),
-                        'properties':
-                            '{}/test_snowflake_target/mysql_sample/properties.json'.format(PIPELINEWISE_TEST_HOME),
-                        'selection':
-                            '{}/test_snowflake_target/mysql_sample/selection.json'.format(PIPELINEWISE_TEST_HOME),
-                        'state':
-                            '{}/test_snowflake_target/mysql_sample/state.json'.format(PIPELINEWISE_TEST_HOME),
-                        'transformation':
-                            '{}/test_snowflake_target/mysql_sample/transformation.json'.format(PIPELINEWISE_TEST_HOME),
-                        'pidfile': '{}/test_snowflake_target/mysql_sample/pipelinewise.pid'.format(
-                            PIPELINEWISE_TEST_HOME)
-                    },
-                    'schemas': [{
-                        'source_schema': 'my_db',
-                        'target_schema': 'repl_my_db',
-                        'target_schema_select_permissions': ['grp_stats'],
-                        'tables': [{
-                            'table_name': 'table_one',
-                            'replication_method': 'INCREMENTAL',
-                            'replication_key': 'last_update'
-                        }, {
-                            'table_name': 'table_two',
-                            'replication_method': 'LOG_BASED'
-                        }]
-                    }]
-                }]
+                'taps': [
+                    {
+                        'id': 'mysql_sample',
+                        'name': 'Sample MySQL Database',
+                        'type': 'tap-mysql',
+                        'owner': 'somebody@foo.com',
+                        'target': 'test_snowflake_target',
+                        'batch_size_rows': 20000,
+                        'batch_wait_limit_seconds': 3600,
+                        'split_large_files': True,
+                        'split_file_chunk_size_mb': 500,
+                        'split_file_max_chunks': 25,
+                        'db_conn': {
+                            'dbname': '<DB_NAME>',
+                            'host': '<HOST>',
+                            'password': '<PASSWORD>',
+                            'port': 3306,
+                            'user': '<USER>',
+                        },
+                        'files': {
+                            'config': f'{PIPELINEWISE_TEST_HOME}/test_snowflake_target/mysql_sample/config.json',
+                            'inheritable_config': f'{PIPELINEWISE_TEST_HOME}'
+                                                  f'/test_snowflake_target/mysql_sample/inheritable_config.json',
+                            'properties': f'{PIPELINEWISE_TEST_HOME}/'
+                                          f'test_snowflake_target/mysql_sample/properties.json',
+                            'selection': f'{PIPELINEWISE_TEST_HOME}/test_snowflake_target/mysql_sample/selection.json',
+                            'state': f'{PIPELINEWISE_TEST_HOME}/test_snowflake_target/mysql_sample/state.json',
+                            'transformation': f'{PIPELINEWISE_TEST_HOME}'
+                                              f'/test_snowflake_target/mysql_sample/transformation.json',
+                            'pidfile': f'{PIPELINEWISE_TEST_HOME}/test_snowflake_target/mysql_sample/pipelinewise.pid',
+                        },
+                        'schemas': [
+                            {
+                                'source_schema': 'my_db',
+                                'target_schema': 'repl_my_db',
+                                'target_schema_select_permissions': ['grp_stats'],
+                                'tables': [
+                                    {
+                                        'table_name': 'table_one',
+                                        'replication_method': 'INCREMENTAL',
+                                        'replication_key': 'last_update',
+                                    },
+                                    {
+                                        'table_name': 'table_two',
+                                        'replication_method': 'LOG_BASED',
+                                    },
+                                ],
+                            }
+                        ],
+                    }
+                ],
             }
         }
 
@@ -146,7 +150,9 @@ def test_from_invalid_mongodb_yamls(self):
         """Test creating Config object using invalid YAML configuration directory"""
 
         # Initialising config object with a tap that's referencing an unknown target should exit
-        yaml_config_dir = '{}/resources/test_invalid_tap_mongo_yaml_config'.format(os.path.dirname(__file__))
+        yaml_config_dir = '{}/resources/test_invalid_tap_mongo_yaml_config'.format(
+            os.path.dirname(__file__)
+        )
         vault_secret = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__))
         print(yaml_config_dir)
         with pytest.raises(SystemExit) as pytest_wrapped_e:
@@ -163,10 +169,14 @@ def test_from_invalid_yamls(self):
 
         # Initialising Config object with a not existing directory should raise an exception
         with pytest.raises(Exception):
-            Config.from_yamls(PIPELINEWISE_TEST_HOME, 'not-existing-yaml-config-directory')
+            Config.from_yamls(
+                PIPELINEWISE_TEST_HOME, 'not-existing-yaml-config-directory'
+            )
 
         # Initialising config object with a tap that's referencing an unknown target should exit
-        yaml_config_dir = '{}/resources/test_invalid_yaml_config'.format(os.path.dirname(__file__))
+        yaml_config_dir = '{}/resources/test_invalid_yaml_config'.format(
+            os.path.dirname(__file__)
+        )
         vault_secret = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__))
 
         with pytest.raises(SystemExit) as pytest_wrapped_e:
@@ -185,7 +195,9 @@ def test_from_invalid_yamls_fails(self):
 
         # Initialising Config object with a not existing directory should raise an exception
         with pytest.raises(Exception):
-            Config.from_yamls(PIPELINEWISE_TEST_HOME, 'not-existing-yaml-config-directory')
+            Config.from_yamls(
+                PIPELINEWISE_TEST_HOME, 'not-existing-yaml-config-directory'
+            )
 
         # Initialising config object with a tap that's referencing an unknown target should exit
         yaml_config_dir = f'{os.path.dirname(__file__)}/resources/test_invalid_yaml_config_with_duplicate_targets'
@@ -202,27 +214,30 @@ def test_getters(self):
 
         # Target and tap directory should be g
         assert config.get_temp_dir() == '{}/tmp'.format(PIPELINEWISE_TEST_HOME)
-        assert config.get_target_dir('test-target-id') == '{}/test-target-id'.format(PIPELINEWISE_TEST_HOME)
-        assert config.get_tap_dir('test-target-id',
-                                  'test-tap-id') == '{}/test-target-id/test-tap-id'.format(PIPELINEWISE_TEST_HOME)
-
-        assert \
-            config.get_connector_files('/var/singer-connector') == \
-            {
-                'config': '/var/singer-connector/config.json',
-                'inheritable_config': '/var/singer-connector/inheritable_config.json',
-                'properties': '/var/singer-connector/properties.json',
-                'state': '/var/singer-connector/state.json',
-                'transformation': '/var/singer-connector/transformation.json',
-                'selection': '/var/singer-connector/selection.json',
-                'pidfile': '/var/singer-connector/pipelinewise.pid'
-            }
+        assert config.get_target_dir('test-target-id') == '{}/test-target-id'.format(
+            PIPELINEWISE_TEST_HOME
+        )
+        assert config.get_tap_dir(
+            'test-target-id', 'test-tap-id'
+        ) == '{}/test-target-id/test-tap-id'.format(PIPELINEWISE_TEST_HOME)
+
+        assert config.get_connector_files('/var/singer-connector') == {
+            'config': '/var/singer-connector/config.json',
+            'inheritable_config': '/var/singer-connector/inheritable_config.json',
+            'properties': '/var/singer-connector/properties.json',
+            'state': '/var/singer-connector/state.json',
+            'transformation': '/var/singer-connector/transformation.json',
+            'selection': '/var/singer-connector/selection.json',
+            'pidfile': '/var/singer-connector/pipelinewise.pid',
+        }
 
     def test_save_config(self):
         """Test config target and tap JSON save functionalities"""
 
         # Load a full configuration set from YAML files
-        yaml_config_dir = '{}/resources/test_yaml_config'.format(os.path.dirname(__file__))
+        yaml_config_dir = '{}/resources/test_yaml_config'.format(
+            os.path.dirname(__file__)
+        )
         vault_secret = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__))
 
         json_config_dir = './pipelinewise-test-config'
@@ -233,23 +248,38 @@ def test_save_config(self):
 
         # Check if every required JSON file created, both for target and tap
         main_config_json = '{}/config.json'.format(json_config_dir)
-        target_config_json = '{}/test_snowflake_target/config.json'.format(json_config_dir)
-        tap_config_json = '{}/test_snowflake_target/mysql_sample/config.json'.format(json_config_dir)
-        tap_inheritable_config_json = '{}/test_snowflake_target/mysql_sample/inheritable_config.json'.format(
-            json_config_dir)
-        tap_selection_json = '{}/test_snowflake_target/mysql_sample/selection.json'.format(json_config_dir)
-        tap_transformation_json = '{}/test_snowflake_target/mysql_sample/transformation.json'.format(json_config_dir)
+        target_config_json = '{}/test_snowflake_target/config.json'.format(
+            json_config_dir
+        )
+        tap_config_json = '{}/test_snowflake_target/mysql_sample/config.json'.format(
+            json_config_dir
+        )
+        tap_inheritable_config_json = (
+            '{}/test_snowflake_target/mysql_sample/inheritable_config.json'.format(
+                json_config_dir
+            )
+        )
+        tap_selection_json = (
+            '{}/test_snowflake_target/mysql_sample/selection.json'.format(
+                json_config_dir
+            )
+        )
+        tap_transformation_json = (
+            '{}/test_snowflake_target/mysql_sample/transformation.json'.format(
+                json_config_dir
+            )
+        )
 
         # Check content of the generated JSON files
         assert cli.utils.load_json(main_config_json) == {
             'alert_handlers': {
                 'slack': {
                     'token': 'Vault Encrypted Secret Fruit',
-                    'channel': '#slack-channel'
+                    'channel': '#slack-channel',
                 }
             },
-            'targets':
-                [{
+            'targets': [
+                {
                     'id': 'test_snowflake_target',
                     'type': 'target-snowflake',
                     'name': 'Test Target Connector',
@@ -264,8 +294,9 @@ def test_save_config(self):
                             'send_alert': True,
                             'enabled': True,
                         }
-                    ]
-                }]
+                    ],
+                }
+            ],
         }
         assert cli.utils.load_json(target_config_json) == {
             'account': 'account',
@@ -279,7 +310,7 @@ def test_save_config(self):
             's3_key_prefix': 's3_prefix/',
             'stage': 'foo_stage',
             'user': 'user',
-            'warehouse': 'MY_WAREHOUSE'
+            'warehouse': 'MY_WAREHOUSE',
         }
         assert cli.utils.load_json(tap_config_json) == {
             'dbname': '<DB_NAME>',
@@ -287,24 +318,19 @@ def test_save_config(self):
             'port': 3306,
             'user': '<USER>',
             'password': '<PASSWORD>',
-            'server_id': cli.utils.load_json(tap_config_json)['server_id']
+            'server_id': cli.utils.load_json(tap_config_json)['server_id'],
         }
         assert cli.utils.load_json(tap_selection_json) == {
             'selection': [
                 {
                     'replication_key': 'last_update',
                     'replication_method': 'INCREMENTAL',
-                    'tap_stream_id': 'my_db-table_one'
+                    'tap_stream_id': 'my_db-table_one',
                 },
-                {
-                    'replication_method': 'LOG_BASED',
-                    'tap_stream_id': 'my_db-table_two'
-                }
+                {'replication_method': 'LOG_BASED', 'tap_stream_id': 'my_db-table_two'},
             ]
         }
-        assert cli.utils.load_json(tap_transformation_json) == {
-            'transformations': []
-        }
+        assert cli.utils.load_json(tap_transformation_json) == {'transformations': []}
         assert cli.utils.load_json(tap_inheritable_config_json) == {
             'batch_size_rows': 20000,
             'batch_wait_limit_seconds': 3600,
@@ -317,20 +343,19 @@ def test_save_config(self):
             'schema_mapping': {
                 'my_db': {
                     'target_schema': 'repl_my_db',
-                    'target_schema_select_permissions': ['grp_stats']
+                    'target_schema_select_permissions': ['grp_stats'],
                 }
             },
             'temp_dir': './pipelinewise-test-config/tmp',
             'tap_id': 'mysql_sample',
-            'query_tag':
-                '{"ppw_component": "tap-mysql", "tap_id": "mysql_sample", '
-                '"database": "{{database}}", "schema": "{{schema}}", "table": "{{table}}"}',
+            'query_tag': '{"ppw_component": "tap-mysql", "tap_id": "mysql_sample", '
+            '"database": "{{database}}", "schema": "{{schema}}", "table": "{{table}}"}',
             'validate_records': False,
             'add_metadata_columns': False,
             'split_large_files': True,
             'split_file_chunk_size_mb': 500,
             'split_file_max_chunks': 25,
-            'archive_load_files': False
+            'archive_load_files': False,
         }
 
         # Delete the generated JSON config directory
diff --git a/tests/units/fastsync/assertions.py b/tests/units/fastsync/assertions.py
index 98ce7a39c..127620cb3 100644
--- a/tests/units/fastsync/assertions.py
+++ b/tests/units/fastsync/assertions.py
@@ -8,47 +8,55 @@
 
 FASTSYNC_NS = Namespace(
     **{
-        'tap': {
-            'bucket': 'testBucket'
-        },
+        'tap': {'bucket': 'testBucket'},
         'properties': {},
         'target': {},
         'transform': {},
         'temp_dir': '',
         'state': '',
-    })
+    }
+)
 
 
-def _create_object_names_to_mock(package_nm: str,
-                                 tap_class_nm: str,
-                                 target_class_nm: str):
+def _create_object_names_to_mock(
+    package_nm: str, tap_class_nm: str, target_class_nm: str
+):
     """Function to generate dynamic object names"""
-    ObjectNames = collections.namedtuple('ObjectNames', ['full_tap_class_nm',
-                                                         'full_target_class_nm',
-                                                         'sync_table_fn_nm',
-                                                         'utils_module_nm',
-                                                         'multiproc_module_nm',
-                                                         'os_module_nm'])
-    return ObjectNames(full_tap_class_nm=f'{package_nm}.{tap_class_nm}',
-                       full_target_class_nm=f'{package_nm}.{target_class_nm}',
-                       sync_table_fn_nm=f'{package_nm}.sync_table',
-                       utils_module_nm=f'{package_nm}.utils',
-                       multiproc_module_nm=f'{package_nm}.multiprocessing',
-                       os_module_nm=f'{package_nm}.os')
+    ObjectNames = collections.namedtuple(
+        'ObjectNames',
+        [
+            'full_tap_class_nm',
+            'full_target_class_nm',
+            'sync_table_fn_nm',
+            'utils_module_nm',
+            'multiproc_module_nm',
+            'os_module_nm',
+        ],
+    )
+    return ObjectNames(
+        full_tap_class_nm=f'{package_nm}.{tap_class_nm}',
+        full_target_class_nm=f'{package_nm}.{target_class_nm}',
+        sync_table_fn_nm=f'{package_nm}.sync_table',
+        utils_module_nm=f'{package_nm}.utils',
+        multiproc_module_nm=f'{package_nm}.multiprocessing',
+        os_module_nm=f'{package_nm}.os',
+    )
 
 
 # pylint: disable=missing-function-docstring,unused-variable
-def assert_sync_table_returns_true_on_success(sync_table: callable,
-                                              package_nm: str,
-                                              tap_class_nm: str,
-                                              target_class_nm: str) -> None:
+def assert_sync_table_returns_true_on_success(
+    sync_table: callable, package_nm: str, tap_class_nm: str, target_class_nm: str
+) -> None:
     """Tests if fastsync sync table function returns true on success"""
-    objects_to_mock = _create_object_names_to_mock(package_nm, tap_class_nm, target_class_nm)
+    objects_to_mock = _create_object_names_to_mock(
+        package_nm, tap_class_nm, target_class_nm
+    )
 
     class LockMock:
         """
         Lock Mock
         """
+
         @staticmethod
         def acquire():
             print('Acquired lock')
@@ -61,19 +69,29 @@ def release():
         with patch(objects_to_mock.full_target_class_nm) as target_mock:
             with patch(objects_to_mock.utils_module_nm) as utils_mock:
                 with patch(objects_to_mock.multiproc_module_nm) as multiproc_mock:
-                    with patch(objects_to_mock.os_module_nm) as os_mock:
+                    with patch(objects_to_mock.os_module_nm):
                         utils_mock.get_target_schema.return_value = 'my-target-schema'
                         tap_mock.return_value.map_column_types_to_target.return_value = {
-                            'columns': ['id INTEGER', 'is_test SMALLINT', 'age INTEGER', 'name VARCHAR'],
-                            'primary_key': 'id,name'
+                            'columns': [
+                                'id INTEGER',
+                                'is_test SMALLINT',
+                                'age INTEGER',
+                                'name VARCHAR',
+                            ],
+                            'primary_key': 'id,name',
                         }
 
                         target_mock.return_value.upload_to_s3.return_value = 's3_key'
                         utils_mock.return_value.get_bookmark_for_table.return_value = {
                             'modified_since': '2019-11-18'
                         }
-                        utils_mock.return_value.get_grantees.return_value = ['role_1', 'role_2']
-                        utils_mock.return_value.get_bookmark_for_table.return_value = None
+                        utils_mock.return_value.get_grantees.return_value = [
+                            'role_1',
+                            'role_2',
+                        ]
+                        utils_mock.return_value.get_bookmark_for_table.return_value = (
+                            None
+                        )
 
                         multiproc_mock.lock.return_value = LockMock()
 
@@ -84,16 +102,17 @@ def release():
 
 
 # pylint: disable=missing-function-docstring,unused-variable,invalid-name
-def assert_sync_table_exception_on_failed_copy(sync_table: callable,
-                                               package_nm: str,
-                                               tap_class_nm: str,
-                                               target_class_nm: str) -> None:
-    objects_to_mock = _create_object_names_to_mock(package_nm, tap_class_nm, target_class_nm)
+def assert_sync_table_exception_on_failed_copy(
+    sync_table: callable, package_nm: str, tap_class_nm: str, target_class_nm: str
+) -> None:
+    objects_to_mock = _create_object_names_to_mock(
+        package_nm, tap_class_nm, target_class_nm
+    )
 
     with patch(objects_to_mock.full_tap_class_nm) as tap_mock:
-        with patch(objects_to_mock.full_target_class_nm) as target_mock:
+        with patch(objects_to_mock.full_target_class_nm):
             with patch(objects_to_mock.utils_module_nm) as utils_mock:
-                with patch(objects_to_mock.multiproc_module_nm) as multiproc_mock:
+                with patch(objects_to_mock.multiproc_module_nm):
                     utils_mock.get_target_schema.return_value = 'my-target-schema'
                     utils_mock.gen_export_filename.return_value = 'my-export-file'
                     tap_mock.return_value.copy_table.side_effect = Exception('Boooom')
@@ -105,32 +124,40 @@ def assert_sync_table_exception_on_failed_copy(sync_table: callable,
 
 
 # pylint: disable=missing-function-docstring,unused-variable,invalid-name
-def assert_main_impl_exit_normally_on_success(main_impl: callable,
-                                              package_nm: str,
-                                              tap_class_nm: str,
-                                              target_class_nm: str) -> None:
-    objects_to_mock = _create_object_names_to_mock(package_nm, tap_class_nm, target_class_nm)
+def assert_main_impl_exit_normally_on_success(
+    main_impl: callable, package_nm: str, tap_class_nm: str, target_class_nm: str
+) -> None:
+    objects_to_mock = _create_object_names_to_mock(
+        package_nm, tap_class_nm, target_class_nm
+    )
 
     with patch(objects_to_mock.utils_module_nm) as utils_mock:
-        with patch(objects_to_mock.full_target_class_nm) as target_mock:
-            with patch(objects_to_mock.sync_table_fn_nm) as sync_table_mock:
+        with patch(objects_to_mock.full_target_class_nm):
+            with patch(objects_to_mock.sync_table_fn_nm):
                 with patch(objects_to_mock.multiproc_module_nm) as multiproc_mock:
                     with patch(objects_to_mock.full_tap_class_nm) as tap_mock:
                         tap_mock.return_value.drop_slot.side_effect = None
 
-                        ns = Namespace(**{
-                            'tables': ['table_1', 'table_2', 'table_3', 'table_4'],
-                            'target': 'sf',
-                            'transform': None,
-                            'drop_pg_slot': False,
-                            'tap': {}
-                        })
+                        ns = Namespace(
+                            **{
+                                'tables': ['table_1', 'table_2', 'table_3', 'table_4'],
+                                'target': 'sf',
+                                'transform': None,
+                                'drop_pg_slot': False,
+                                'tap': {},
+                            }
+                        )
 
                         utils_mock.parse_args.return_value = ns
                         utils_mock.get_pool_size.return_value = 10
 
                         mock_enter = Mock()
-                        mock_enter.return_value.map.return_value = [True, True, True, True]
+                        mock_enter.return_value.map.return_value = [
+                            True,
+                            True,
+                            True,
+                            True,
+                        ]
 
                         pool_mock = Mock(spec_set=multiprocessing.Pool).return_value
 
@@ -151,32 +178,42 @@ def assert_main_impl_exit_normally_on_success(main_impl: callable,
 
 
 # pylint: disable=missing-function-docstring,unused-variable,invalid-name
-def assert_main_impl_should_exit_with_error_on_failure(main_impl: callable,
-                                                       package_nm: str,
-                                                       tap_class_nm: str,
-                                                       target_class_nm: str) -> None:
-    objects_to_mock = _create_object_names_to_mock(package_nm, tap_class_nm, target_class_nm)
+def assert_main_impl_should_exit_with_error_on_failure(
+    main_impl: callable, package_nm: str, tap_class_nm: str, target_class_nm: str
+) -> None:
+    objects_to_mock = _create_object_names_to_mock(
+        package_nm, tap_class_nm, target_class_nm
+    )
 
     with patch(objects_to_mock.utils_module_nm) as utils_mock:
-        with patch(objects_to_mock.full_target_class_nm) as target_mock:
-            with patch(objects_to_mock.sync_table_fn_nm) as sync_table_mock:
+        with patch(objects_to_mock.full_target_class_nm):
+            with patch(objects_to_mock.sync_table_fn_nm):
                 with patch(objects_to_mock.multiproc_module_nm) as multiproc_mock:
                     with patch(objects_to_mock.full_tap_class_nm) as tap_mock:
                         tap_mock.return_value.drop_slot.side_effect = None
 
-                        ns = Namespace(**{
-                            'tables': ['table_1', 'table_2', 'table_3', 'table_4'],
-                            'target': 'sf',
-                            'transform': None,
-                            'drop_pg_slot': True,
-                            'tap': {'fastsync_parallelism': 4,},
-                        })
+                        ns = Namespace(
+                            **{
+                                'tables': ['table_1', 'table_2', 'table_3', 'table_4'],
+                                'target': 'sf',
+                                'transform': None,
+                                'drop_pg_slot': True,
+                                'tap': {
+                                    'fastsync_parallelism': 4,
+                                },
+                            }
+                        )
 
                         utils_mock.parse_args.return_value = ns
                         utils_mock.get_pool_size.return_value = 10
 
                         mock_enter = Mock()
-                        mock_enter.return_value.map.return_value = [True, True, 'Critical: random error', True]
+                        mock_enter.return_value.map.return_value = [
+                            True,
+                            True,
+                            'Critical: random error',
+                            True,
+                        ]
 
                         pool_mock = Mock(spec_set=multiprocessing.Pool).return_value
 
@@ -192,7 +229,9 @@ def assert_main_impl_should_exit_with_error_on_failure(main_impl: callable,
                             assert utils_mock.parse_args.call_count == 1
                             assert mock_enter.return_value.map.call_count == 1
                             assert tap_mock.return_value.drop_slot.call_count == 1
-                            utils_mock.get_pool_size.assert_called_once_with({
-                                'fastsync_parallelism': 4,
-                            })
+                            utils_mock.get_pool_size.assert_called_once_with(
+                                {
+                                    'fastsync_parallelism': 4,
+                                }
+                            )
                             multiproc_mock.Pool.assert_called_once_with(10)
diff --git a/tests/units/fastsync/commons/test_fastsync_tap_mongodb.py b/tests/units/fastsync/commons/test_fastsync_tap_mongodb.py
index 8b5afaf30..081506c73 100644
--- a/tests/units/fastsync/commons/test_fastsync_tap_mongodb.py
+++ b/tests/units/fastsync/commons/test_fastsync_tap_mongodb.py
@@ -46,7 +46,7 @@ def generate_all_datatypes_doc():
         'string_field': random_string_generator(100),
         'object_field': {
             'obj_field_1_key': 'obj_field_1_val',
-            'obj_field_2_key': 'obj_field_2_val'
+            'obj_field_2_key': 'obj_field_2_val',
         },
         'array_field': [
             None,
@@ -62,13 +62,19 @@ def generate_all_datatypes_doc():
         'null_field': None,
         'regex_field': regex,
         '32_bit_integer_field': 32,
-        'timestamp_field': bson.timestamp.Timestamp(int(time.time()), random.randint(0, 100)),
+        'timestamp_field': bson.timestamp.Timestamp(
+            int(time.time()), random.randint(0, 100)
+        ),
         '64_bit_integer_field': 34359738368,
-        'decimal_field': bson.Decimal128(decimal.Decimal(f'{random.randrange(-100, 100) / 33}')),
+        'decimal_field': bson.Decimal128(
+            decimal.Decimal(f'{random.randrange(-100, 100) / 33}')
+        ),
         'javaScript_field': bson.code.Code('var x, y, z;'),
-        'javaScript_with_scope_field': bson.code.Code('function incrementX() { x++; }', scope={'x': 1}),
+        'javaScript_with_scope_field': bson.code.Code(
+            'function incrementX() { x++; }', scope={'x': 1}
+        ),
         'min_key_field': bson.min_key.MinKey(),
-        'max_key_field': bson.max_key.MaxKey()
+        'max_key_field': bson.max_key.MaxKey(),
     }
 
 
@@ -80,19 +86,21 @@ class TestFastSyncTapMongoDB(TestCase):
 
     def setUp(self) -> None:
         """Initialise test FastSyncTapPostgres object"""
-        self.connection_config = {'host': 'foo.com',
-                                  'port': 3306,
-                                  'user': 'my_user',
-                                  'password': 'secret',
-                                  'auth_database': 'admin',
-                                  'database': 'my_db',
-                                  'ssl': 'true'
-                                  }
-        self.mongo = FastSyncTapMongoDB(self.connection_config,
-                                        lambda x: {
-                                            'string': 'text',
-                                            'date': 'time with timezone'
-                                        }.get(x, 'default'))
+        self.connection_config = {
+            'host': 'foo.com',
+            'port': 3306,
+            'user': 'my_user',
+            'password': 'secret',
+            'auth_database': 'admin',
+            'database': 'my_db',
+            'ssl': 'true',
+        }
+        self.mongo = FastSyncTapMongoDB(
+            self.connection_config,
+            lambda x: {'string': 'text', 'date': 'time with timezone'}.get(
+                x, 'default'
+            ),
+        )
 
     def tearDown(self):
         try:
@@ -102,13 +110,13 @@ def tearDown(self):
 
     def test_open_connections(self):
         """
-            Test open_connection method
-            it should create a Database Mock
+        Test open_connection method
+        it should create a Database Mock
         """
-        with patch('pipelinewise.fastsync.commons.tap_mongodb.MongoClient') as mongo_client_mock:
-            mongo_client_mock.return_value = {
-                'my_db': Mock(spec_set=Database)
-            }
+        with patch(
+            'pipelinewise.fastsync.commons.tap_mongodb.MongoClient'
+        ) as mongo_client_mock:
+            mongo_client_mock.return_value = {'my_db': Mock(spec_set=Database)}
             self.mongo.open_connection()
 
         self.assertIsInstance(self.mongo.database, Mock)
@@ -134,7 +142,11 @@ def test_copy_table_with_collection_not_found_expect_exception(self):
         error
         """
         self.mongo.database = Mock(spec_set=Database).return_value
-        self.mongo.database.list_collection_names.return_value = ['col1', 'col2', 'col3']
+        self.mongo.database.list_collection_names.return_value = [
+            'col1',
+            'col2',
+            'col3',
+        ]
 
         with self.assertRaises(TableNotFoundError):
             self.mongo.copy_table('my_col', TEST_EXPORT_FILE, 'tmp')
@@ -147,23 +159,35 @@ def test_copy_table_with_collection_found_but_export_failed_expect_exception(sel
         error
         """
         self.mongo.database = Mock(spec_set=Database).return_value
-        self.mongo.database.list_collection_names.return_value = ['col1', 'col2', 'col3', 'my_col']
-
-        with patch('pipelinewise.fastsync.commons.tap_mongodb.subprocess.call') as call_mock:
+        self.mongo.database.list_collection_names.return_value = [
+            'col1',
+            'col2',
+            'col3',
+            'my_col',
+        ]
+
+        with patch(
+            'pipelinewise.fastsync.commons.tap_mongodb.subprocess.call'
+        ) as call_mock:
             call_mock.return_value = 1
 
             with self.assertRaises(ExportError):
                 self.mongo.copy_table('my_col', TEST_EXPORT_FILE, 'tmp')
 
-            call_mock.assert_called_once_with([
-                'mongodump',
-                '--uri', '"mongodb://my_user:secret@foo.com:3306/my_db'
-                         '?authSource=admin&readPreference=secondaryPreferred&ssl=true"',
-                '--forceTableScan',
-                '--gzip',
-                '-c', 'my_col',
-                '-o', 'tmp'
-            ])
+            call_mock.assert_called_once_with(
+                [
+                    'mongodump',
+                    '--uri',
+                    '"mongodb://my_user:secret@foo.com:3306/my_db'
+                    '?authSource=admin&readPreference=secondaryPreferred&ssl=true"',
+                    '--forceTableScan',
+                    '--gzip',
+                    '-c',
+                    'my_col',
+                    '-o',
+                    'tmp',
+                ]
+            )
 
         self.assertEqual(self.mongo.database.list_collection_names.call_count, 1)
 
@@ -172,22 +196,36 @@ def test_copy_table_with_collection_found_success(self):
         Test copy_table method with a collection name that's in the db, the copy should continue successfully
         """
         self.mongo.database = Mock(spec_set=Database).return_value
-        self.mongo.database.list_collection_names.return_value = ['col1', 'col2', 'col3', 'my_col']
-
-        with patch('pipelinewise.fastsync.commons.tap_mongodb.subprocess.call') as call_mock:
+        self.mongo.database.list_collection_names.return_value = [
+            'col1',
+            'col2',
+            'col3',
+            'my_col',
+        ]
+
+        with patch(
+            'pipelinewise.fastsync.commons.tap_mongodb.subprocess.call'
+        ) as call_mock:
             call_mock.return_value = 0
 
-            with patch('pipelinewise.fastsync.commons.tap_mongodb.os.remove') as os_remove_mock:
+            with patch(
+                'pipelinewise.fastsync.commons.tap_mongodb.os.remove'
+            ) as os_remove_mock:
                 os_remove_mock.return_value = True
 
-                with patch('pipelinewise.fastsync.commons.tap_mongodb.gzip') as gzip_mock:
+                with patch(
+                    'pipelinewise.fastsync.commons.tap_mongodb.gzip'
+                ) as gzip_mock:
                     mock_enter = Mock()
 
-                    with patch('pipelinewise.fastsync.commons.tap_mongodb.bson.decode_file_iter') as \
-                            bson_decode_iter_mock:
+                    with patch(
+                        'pipelinewise.fastsync.commons.tap_mongodb.bson.decode_file_iter'
+                    ) as bson_decode_iter_mock:
 
                         # generate 10 documents
-                        bson_decode_iter_mock.return_value = (generate_all_datatypes_doc() for _ in range(10))
+                        bson_decode_iter_mock.return_value = (
+                            generate_all_datatypes_doc() for _ in range(10)
+                        )
 
                         mock_enter.return_value.open.return_value = Mock()
 
@@ -196,18 +234,27 @@ def test_copy_table_with_collection_found_success(self):
 
                         self.mongo.copy_table('my_col', TEST_EXPORT_FILE, 'tmp')
 
-                        call_mock.assert_called_once_with([
-                            'mongodump',
-                            '--uri', '"mongodb://my_user:secret@foo.com:3306/my_db'
-                                     '?authSource=admin&readPreference=secondaryPreferred&ssl=true"',
-                            '--forceTableScan',
-                            '--gzip',
-                            '-c', 'my_col',
-                            '-o', 'tmp'
-                        ])
+                        call_mock.assert_called_once_with(
+                            [
+                                'mongodump',
+                                '--uri',
+                                '"mongodb://my_user:secret@foo.com:3306/my_db'
+                                '?authSource=admin&readPreference=secondaryPreferred&ssl=true"',
+                                '--forceTableScan',
+                                '--gzip',
+                                '-c',
+                                'my_col',
+                                '-o',
+                                'tmp',
+                            ]
+                        )
 
                         os_remove_mock.assert_has_calls(
-                            [call('tmp/my_db/my_col.metadata.json.gz'), call('tmp/my_db/my_col.bson.gz')])
+                            [
+                                call('tmp/my_db/my_col.metadata.json.gz'),
+                                call('tmp/my_db/my_col.bson.gz'),
+                            ]
+                        )
                         self.assertEqual(os_remove_mock.call_count, 2)
                         self.assertEqual(bson_decode_iter_mock.call_count, 1)
 
@@ -217,21 +264,16 @@ def test_fetch_current_log_pos_return_first_token(self):
         """
         cursor_mock = Mock(spec_set=DatabaseChangeStream).return_value
         type(cursor_mock).alive = PropertyMock(return_value=True)
-        type(cursor_mock).resume_token = PropertyMock(side_effect=[
-            {
-                '_data': 'token1',
-                '_typeBits': b'\x81\x80'
-            },
-            {
-                '_data': 'token2',
-            },
-
-            {
-                '_data': 'token3'
-            },
-            {
-                '_data': 'token4'
-            }])
+        type(cursor_mock).resume_token = PropertyMock(
+            side_effect=[
+                {'_data': 'token1', '_typeBits': b'\x81\x80'},
+                {
+                    '_data': 'token2',
+                },
+                {'_data': 'token3'},
+                {'_data': 'token4'},
+            ]
+        )
         cursor_mock.try_next.side_effect = [{}, {}, {}]
 
         mock_enter = Mock()
@@ -244,11 +286,9 @@ def test_fetch_current_log_pos_return_first_token(self):
         self.mongo.database = Mock(spec_set=Database).return_value
         self.mongo.database.watch.return_value = mock_watch
 
-        self.assertDictEqual({
-            'token': {
-                '_data': 'token1'
-            }
-        }, self.mongo.fetch_current_log_pos())
+        self.assertDictEqual(
+            {'token': {'_data': 'token1'}}, self.mongo.fetch_current_log_pos()
+        )
 
     def test_fetch_current_incremental_key_pos(self):
         """
@@ -262,8 +302,16 @@ def test_map_column_types_to_target(self):
         test map_column_types_to_target method, it shoudl retuns columns mapping using the mocked callable in the setup
 
         """
-        self.assertDictEqual({
-            'columns': ['_ID text', 'DOCUMENT default', '_SDC_EXTRACTED_AT default',
-                        '_SDC_BATCHED_AT default', '_SDC_DELETED_AT text'],
-            'primary_key': ['_ID']
-        }, self.mongo.map_column_types_to_target())
+        self.assertDictEqual(
+            {
+                'columns': [
+                    '_ID text',
+                    'DOCUMENT default',
+                    '_SDC_EXTRACTED_AT default',
+                    '_SDC_BATCHED_AT default',
+                    '_SDC_DELETED_AT text',
+                ],
+                'primary_key': ['_ID'],
+            },
+            self.mongo.map_column_types_to_target(),
+        )
diff --git a/tests/units/fastsync/commons/test_fastsync_tap_mysql.py b/tests/units/fastsync/commons/test_fastsync_tap_mysql.py
index eafe4f616..d1b67af9b 100644
--- a/tests/units/fastsync/commons/test_fastsync_tap_mysql.py
+++ b/tests/units/fastsync/commons/test_fastsync_tap_mysql.py
@@ -10,6 +10,7 @@ class FastSyncTapMySqlMock(FastSyncTapMySql):
     """
     Mocked FastSyncTapMySql class
     """
+
     def __init__(self, connection_config, tap_type_to_target_type=None):
         super().__init__(connection_config, tap_type_to_target_type)
 
@@ -34,13 +35,16 @@ class TestFastSyncTapMySql(TestCase):
     """
     Unit tests for fastsync tap mysql
     """
+
     def setUp(self) -> None:
         """Initialise test FastSyncTapPostgres object"""
-        self.connection_config = {'host': 'foo.com',
-                                  'port': 3306,
-                                  'user': 'my_user',
-                                  'password': 'secret',
-                                  'dbname': 'my_db'}
+        self.connection_config = {
+            'host': 'foo.com',
+            'port': 3306,
+            'user': 'my_user',
+            'password': 'secret',
+            'dbname': 'my_db',
+        }
         self.mysql = None
 
     def test_open_connections_with_default_session_sqls(self):
@@ -58,10 +62,14 @@ def test_open_connections_with_session_sqls(self):
         """Custom session parameters should be applied if defined"""
         session_sqls = [
             'SET SESSION max_statement_time=0',
-            'SET SESSION wait_timeout=28800'
+            'SET SESSION wait_timeout=28800',
         ]
-        self.mysql = FastSyncTapMySqlMock(connection_config={**self.connection_config,
-                                                             **{'session_sqls': session_sqls}})
+        self.mysql = FastSyncTapMySqlMock(
+            connection_config={
+                **self.connection_config,
+                **{'session_sqls': session_sqls},
+            }
+        )
         with patch('pymysql.connect') as mysql_connect_mock:
             mysql_connect_mock.return_value = []
             self.mysql.open_connections()
@@ -75,15 +83,21 @@ def test_open_connections_with_invalid_session_sqls(self):
         session_sqls = [
             'SET SESSION max_statement_time=0',
             'INVALID-SQL-SHOULD-BE-SILENTLY-IGNORED',
-            'SET SESSION wait_timeout=28800'
+            'SET SESSION wait_timeout=28800',
         ]
-        self.mysql = FastSyncTapMySqlMock(connection_config={**self.connection_config,
-                                                             **{'session_sqls': session_sqls}})
+        self.mysql = FastSyncTapMySqlMock(
+            connection_config={
+                **self.connection_config,
+                **{'session_sqls': session_sqls},
+            }
+        )
         with patch('pymysql.connect') as mysql_connect_mock:
             mysql_connect_mock.return_value = []
             self.mysql.open_connections()
 
         # Test if session variables applied on both connections
-        assert self.mysql.executed_queries == ['SET SESSION max_statement_time=0',
-                                               'SET SESSION wait_timeout=28800']
+        assert self.mysql.executed_queries == [
+            'SET SESSION max_statement_time=0',
+            'SET SESSION wait_timeout=28800',
+        ]
         assert self.mysql.executed_queries_unbuffered == self.mysql.executed_queries
diff --git a/tests/units/fastsync/commons/test_fastsync_tap_postgres.py b/tests/units/fastsync/commons/test_fastsync_tap_postgres.py
index 16b6045ec..269e4e6a0 100644
--- a/tests/units/fastsync/commons/test_fastsync_tap_postgres.py
+++ b/tests/units/fastsync/commons/test_fastsync_tap_postgres.py
@@ -11,9 +11,10 @@ class TestFastSyncTapPostgres(TestCase):
 
     def setUp(self) -> None:
         """Initialise test FastSyncTapPostgres object"""
-        self.postgres = FastSyncTapPostgres(connection_config={'dbname': 'test_database',
-                                                               'tap_id': 'test_tap'},
-                                            tap_type_to_target_type={})
+        self.postgres = FastSyncTapPostgres(
+            connection_config={'dbname': 'test_database', 'tap_id': 'test_tap'},
+            tap_type_to_target_type={},
+        )
         self.postgres.executed_queries_primary_host = []
         self.postgres.executed_queries = []
 
@@ -25,27 +26,41 @@ def primary_host_query_mock(query, _=None):
     def test_generate_repl_slot_name(self):
         """Validate if the replication slot name generated correctly"""
         # Provide only database name
-        assert self.postgres.generate_replication_slot_name('some_db') == 'pipelinewise_some_db'
+        assert (
+            self.postgres.generate_replication_slot_name('some_db')
+            == 'pipelinewise_some_db'
+        )
 
         # Provide database name and tap_id
-        assert self.postgres.generate_replication_slot_name('some_db',
-                                                            'some_tap') == 'pipelinewise_some_db_some_tap'
+        assert (
+            self.postgres.generate_replication_slot_name('some_db', 'some_tap')
+            == 'pipelinewise_some_db_some_tap'
+        )
 
         # Provide database name, tap_id and prefix
-        assert self.postgres.generate_replication_slot_name('some_db',
-                                                            'some_tap',
-                                                            prefix='custom_prefix') == 'custom_prefix_some_db_some_tap'
+        assert (
+            self.postgres.generate_replication_slot_name(
+                'some_db', 'some_tap', prefix='custom_prefix'
+            )
+            == 'custom_prefix_some_db_some_tap'
+        )
 
         # Replication slot name should be lowercase
-        assert self.postgres.generate_replication_slot_name('SoMe_DB',
-                                                            'SoMe_TaP') == 'pipelinewise_some_db_some_tap'
+        assert (
+            self.postgres.generate_replication_slot_name('SoMe_DB', 'SoMe_TaP')
+            == 'pipelinewise_some_db_some_tap'
+        )
 
         # Invalid characters should be replaced by underscores
-        assert self.postgres.generate_replication_slot_name('some-db',
-                                                            'some-tap') == 'pipelinewise_some_db_some_tap'
+        assert (
+            self.postgres.generate_replication_slot_name('some-db', 'some-tap')
+            == 'pipelinewise_some_db_some_tap'
+        )
 
-        assert self.postgres.generate_replication_slot_name('some.db',
-                                                            'some.tap') == 'pipelinewise_some_db_some_tap'
+        assert (
+            self.postgres.generate_replication_slot_name('some.db', 'some.tap')
+            == 'pipelinewise_some_db_some_tap'
+        )
 
     def test_create_replication_slot_1(self):
         """
@@ -70,7 +85,7 @@ def execute_mock(query):
         self.postgres.create_replication_slot()
         assert self.postgres.executed_queries_primary_host == [
             "SELECT * FROM pg_replication_slots WHERE slot_name = 'pipelinewise_test_database';",
-            "SELECT * FROM pg_create_logical_replication_slot('pipelinewise_test_database_test_tap', 'wal2json')"
+            "SELECT * FROM pg_create_logical_replication_slot('pipelinewise_test_database_test_tap', 'wal2json')",
         ]
 
     def test_create_replication_slot_2(self):
@@ -96,7 +111,7 @@ def execute_mock(query):
         self.postgres.create_replication_slot()
         assert self.postgres.executed_queries_primary_host == [
             "SELECT * FROM pg_replication_slots WHERE slot_name = 'pipelinewise_test_database';",
-            "SELECT * FROM pg_create_logical_replication_slot('pipelinewise_test_database', 'wal2json')"
+            "SELECT * FROM pg_create_logical_replication_slot('pipelinewise_test_database', 'wal2json')",
         ]
 
     @patch('pipelinewise.fastsync.commons.tap_postgres.psycopg2.connect')
@@ -112,12 +127,15 @@ def test_get_connection_to_primary(self, connect_mock):
             'port': 'my_primary_port',
         }
 
-        self.assertEqual(FastSyncTapPostgres.get_connection(creds, prioritize_primary=True),
-                         connect_mock.return_value)
+        self.assertEqual(
+            FastSyncTapPostgres.get_connection(creds, prioritize_primary=True),
+            connect_mock.return_value,
+        )
 
         connect_mock.assert_called_once_with(
             f"host='{creds['host']}' port='{creds['port']}' user='{creds['user']}' password='{creds['password']}' "
-            f"dbname='{creds['dbname']}'")
+            f"dbname='{creds['dbname']}'"
+        )
 
         self.assertTrue(connect_mock.autocommit)
 
@@ -138,13 +156,16 @@ def test_get_connection_to_sec(self, connect_mock):
             'replica_port': 'my_replica_port',
         }
 
-        self.assertEqual(FastSyncTapPostgres.get_connection(creds, prioritize_primary=False),
-                         connect_mock.return_value)
+        self.assertEqual(
+            FastSyncTapPostgres.get_connection(creds, prioritize_primary=False),
+            connect_mock.return_value,
+        )
 
         connect_mock.assert_called_once_with(
             f"host='{creds['replica_host']}' port='{creds['replica_port']}' user='{creds['replica_user']}' password"
             f"='{creds['replica_password']}' "
-            f"dbname='{creds['dbname']}'")
+            f"dbname='{creds['dbname']}'"
+        )
 
         self.assertTrue(connect_mock.autocommit)
 
@@ -162,12 +183,15 @@ def test_get_connection_fallback(self, connect_mock):
             'port': 'my_primary_port',
         }
 
-        self.assertEqual(FastSyncTapPostgres.get_connection(creds, prioritize_primary=False),
-                         connect_mock.return_value)
+        self.assertEqual(
+            FastSyncTapPostgres.get_connection(creds, prioritize_primary=False),
+            connect_mock.return_value,
+        )
 
         connect_mock.assert_called_once_with(
             f"host='{creds['replica_host']}' port='{creds['port']}' user='{creds['user']}' password"
-            f"='{creds['password']}' dbname='{creds['dbname']}'")
+            f"='{creds['password']}' dbname='{creds['dbname']}'"
+        )
 
         self.assertTrue(connect_mock.autocommit)
 
@@ -182,15 +206,18 @@ def test_get_connection_ssl(self, connect_mock):
             'password': 'my_primary_user',
             'dbname': 'my_db',
             'port': 'my_primary_port',
-            'ssl': 'true'
+            'ssl': 'true',
         }
 
-        self.assertEqual(FastSyncTapPostgres.get_connection(creds, prioritize_primary=False),
-                         connect_mock.return_value)
+        self.assertEqual(
+            FastSyncTapPostgres.get_connection(creds, prioritize_primary=False),
+            connect_mock.return_value,
+        )
 
         connect_mock.assert_called_once_with(
             f"host='{creds['host']}' port='{creds['port']}' user='{creds['user']}' password"
-            f"='{creds['password']}' dbname='{creds['dbname']}' sslmode='require'")
+            f"='{creds['password']}' dbname='{creds['dbname']}' sslmode='require'"
+        )
 
         self.assertTrue(connect_mock.autocommit)
 
@@ -199,6 +226,7 @@ def test_drop_slot_v15(self, connect_mock):
         """
         Check that dropping slots works fine for v15 slots
         """
+
         def execute_mock(query):
             print('Mocked execute called')
             self.postgres.executed_queries_primary_host.append(query)
@@ -210,13 +238,15 @@ def execute_mock(query):
             'dbname': 'my_db',
             'port': 'my_primary_port',
             'ssl': 'true',
-            'tap_id': 'tap_test'
+            'tap_id': 'tap_test',
         }
 
         # mock cursor with execute method
         cursor_mock = MagicMock().return_value
         cursor_mock.__enter__.return_value.execute.side_effect = execute_mock
-        type(cursor_mock.__enter__.return_value).rowcount = PropertyMock(side_effect=[1, 2])
+        type(cursor_mock.__enter__.return_value).rowcount = PropertyMock(
+            side_effect=[1, 2]
+        )
 
         # mock PG connection instance with ability to open cursor
         pg_con = Mock()
@@ -237,6 +267,7 @@ def test_drop_slot_v16(self, connect_mock):
         """
         Check that dropping slots works fine for v16 slots
         """
+
         def execute_mock(query):
             print('Mocked execute called')
             self.postgres.executed_queries_primary_host.append(query)
@@ -248,13 +279,15 @@ def execute_mock(query):
             'dbname': 'my_db',
             'port': 'my_primary_port',
             'ssl': 'true',
-            'tap_id': 'tap_test'
+            'tap_id': 'tap_test',
         }
 
         # mock cursor with execute method
         cursor_mock = MagicMock().return_value
         cursor_mock.__enter__.return_value.execute.side_effect = execute_mock
-        type(cursor_mock.__enter__.return_value).rowcount = PropertyMock(side_effect=[0, 1])
+        type(cursor_mock.__enter__.return_value).rowcount = PropertyMock(
+            side_effect=[0, 1]
+        )
 
         # mock PG connection instance with ability to open cursor
         pg_con = Mock()
diff --git a/tests/units/fastsync/commons/test_fastsync_tap_s3_csv.py b/tests/units/fastsync/commons/test_fastsync_tap_s3_csv.py
index 012d12656..d334ef22b 100644
--- a/tests/units/fastsync/commons/test_fastsync_tap_s3_csv.py
+++ b/tests/units/fastsync/commons/test_fastsync_tap_s3_csv.py
@@ -19,20 +19,11 @@ def setUp(self) -> None:
             'aws_endpoint_url': 'https://aws.com/random-url',
             'start_date': '2000-01-01',
             'tables': [
-                {
-                    'table_name': 'table 1',
-                    'key_properties': None
-                }, {
-                    'table_name': 'table 2',
-                    'key_properties': []
-                }, {
-                    'table_name': 'table 3',
-                    'key_properties': ['key_1']
-                }, {
-                    'table_name': 'table 4',
-                    'key_properties': ['key_2', 'key_3']
-                }
-            ]
+                {'table_name': 'table 1', 'key_properties': None},
+                {'table_name': 'table 2', 'key_properties': []},
+                {'table_name': 'table 3', 'key_properties': ['key_1']},
+                {'table_name': 'table 4', 'key_properties': ['key_2', 'key_3']},
+            ],
         }
 
         def tap_type_to_target_type(tap_type):
@@ -40,10 +31,12 @@ def tap_type_to_target_type(tap_type):
                 'boolean': 'boolean',
                 'integer': 'number',
                 'number': 'number',
-                'string': 'varchar'
+                'string': 'varchar',
             }.get(tap_type, 'varchar')
 
-        with patch('pipelinewise.fastsync.commons.tap_s3_csv.S3Helper') as s3_helper_mock:
+        with patch(
+            'pipelinewise.fastsync.commons.tap_s3_csv.S3Helper'
+        ) as s3_helper_mock:
             s3_helper_mock.return_value.list_files_in_bucket.return_value = []
             self.fs_tap_s3_csv = FastSyncTapS3Csv(con_config, tap_type_to_target_type)
 
@@ -52,21 +45,32 @@ def test_copy_table_given_an_invalid_file_path_throws_exception(self):
             self.fs_tap_s3_csv.copy_table('table_1', 'invalid_file_path.csv')
 
     def test_copy_table_given_a_valid_file_path(self):
-        with patch('pipelinewise.fastsync.commons.tap_s3_csv.S3Helper') as s3_helper_mock:
+        with patch(
+            'pipelinewise.fastsync.commons.tap_s3_csv.S3Helper'
+        ) as s3_helper_mock:
             s3_helper_mock.get_input_files_for_table.return_value = [
                 {
                     'key': 'file_1.csv',
-                    'last_modified': datetime.datetime.strptime('2001-07-13', '%Y-%m-%d')
-                }, {
+                    'last_modified': datetime.datetime.strptime(
+                        '2001-07-13', '%Y-%m-%d'
+                    ),
+                },
+                {
                     'key': 'file_2.csv',
-                    'last_modified': datetime.datetime.strptime('2001-10-05', '%Y-%m-%d')
-                }
+                    'last_modified': datetime.datetime.strptime(
+                        '2001-10-05', '%Y-%m-%d'
+                    ),
+                },
             ]
 
-            with patch.object(self.fs_tap_s3_csv, '_get_file_records') as get_file_rec_mock:
+            with patch.object(
+                self.fs_tap_s3_csv, '_get_file_records'
+            ) as get_file_rec_mock:
                 get_file_rec_mock.return_value = 'test'
 
-                with patch('pipelinewise.fastsync.commons.tap_s3_csv.gzip') as gzip_mock:
+                with patch(
+                    'pipelinewise.fastsync.commons.tap_s3_csv.gzip'
+                ) as gzip_mock:
                     mock_enter = Mock()
                     mock_enter.return_value.open.return_value = ''
 
@@ -77,93 +81,117 @@ def test_copy_table_given_a_valid_file_path(self):
 
                     self.assertEqual(2, get_file_rec_mock.call_count)
                     self.assertIn('table 2', self.fs_tap_s3_csv.tables_last_modified)
-                    self.assertEqual('2001-10-05',
-                                     self.fs_tap_s3_csv.tables_last_modified['table 2'].strftime('%Y-%m-%d'))
+                    self.assertEqual(
+                        '2001-10-05',
+                        self.fs_tap_s3_csv.tables_last_modified['table 2'].strftime(
+                            '%Y-%m-%d'
+                        ),
+                    )
 
     def test_get_file_records(self):
         with patch.object(S3Helper, 'get_file_handle') as get_file_handle_mock:
             handle = Mock().return_value
-            handle.configure_mock(**{
-                '_raw_stream.return_value': 'file handle'
-            })
+            handle.configure_mock(**{'_raw_stream.return_value': 'file handle'})
 
             get_file_handle_mock.return_value = handle.return_value
 
-            with patch('pipelinewise.fastsync.commons.tap_s3_csv.singer_encodings_csv') as singer_encodings_csv_mock:
+            with patch(
+                'pipelinewise.fastsync.commons.tap_s3_csv.singer_encodings_csv'
+            ) as singer_encodings_csv_mock:
                 singer_encodings_csv_mock.get_row_iterator.return_value = [
                     {
                         'id': 1,
                         'group': 'A',
-                    }, {
-                        'id': 2,
-                        'group': 'A',
-                        'test': True
-                    }, {
+                    },
+                    {'id': 2, 'group': 'A', 'test': True},
+                    {
                         'id': 3,
                         'group': 'B',
-                    }
+                    },
                 ]
 
-                with patch('pipelinewise.fastsync.commons.tap_s3_csv.datetime') as datetime_mock:
-                    datetime_mock.utcnow.return_value.strftime.return_value = '2019-11-21'
+                with patch(
+                    'pipelinewise.fastsync.commons.tap_s3_csv.datetime'
+                ) as datetime_mock:
+                    datetime_mock.utcnow.return_value.strftime.return_value = (
+                        '2019-11-21'
+                    )
 
                     records = []
                     headers = set()
 
-                    self.fs_tap_s3_csv._get_file_records('s3 path 1',
-                                                         {},
-                                                         records,
-                                                         headers)
-
-                    self.assertListEqual([
-                        {
-                            S3Helper.SDC_SOURCE_BUCKET_COLUMN: 'testBucket',
-                            S3Helper.SDC_SOURCE_FILE_COLUMN: 's3 path 1',
-                            S3Helper.SDC_SOURCE_LINENO_COLUMN: 1,
-                            '_SDC_EXTRACTED_AT': '2019-11-21',
-                            '_SDC_BATCHED_AT': '2019-11-21',
-                            '_SDC_DELETED_AT': None,
-                            '"ID"': 1,
-                            '"GROUP"': 'A',
-                        }, {
-                            S3Helper.SDC_SOURCE_BUCKET_COLUMN: 'testBucket',
-                            S3Helper.SDC_SOURCE_FILE_COLUMN: 's3 path 1',
-                            S3Helper.SDC_SOURCE_LINENO_COLUMN: 2,
-                            '_SDC_EXTRACTED_AT': '2019-11-21',
-                            '_SDC_BATCHED_AT': '2019-11-21',
-                            '_SDC_DELETED_AT': None,
-                            '"ID"': 2,
-                            '"GROUP"': 'A',
-                            '"TEST"': True,
-                        }, {
-                            S3Helper.SDC_SOURCE_BUCKET_COLUMN: 'testBucket',
-                            S3Helper.SDC_SOURCE_FILE_COLUMN: 's3 path 1',
-                            S3Helper.SDC_SOURCE_LINENO_COLUMN: 3,
-                            '_SDC_EXTRACTED_AT': '2019-11-21',
-                            '_SDC_BATCHED_AT': '2019-11-21',
-                            '_SDC_DELETED_AT': None,
-                            '"ID"': 3,
-                            '"GROUP"': 'B',
-                        }
-                    ], records)
+                    self.fs_tap_s3_csv._get_file_records(
+                        's3 path 1', {}, records, headers
+                    )
+
+                    self.assertListEqual(
+                        [
+                            {
+                                S3Helper.SDC_SOURCE_BUCKET_COLUMN: 'testBucket',
+                                S3Helper.SDC_SOURCE_FILE_COLUMN: 's3 path 1',
+                                S3Helper.SDC_SOURCE_LINENO_COLUMN: 1,
+                                '_SDC_EXTRACTED_AT': '2019-11-21',
+                                '_SDC_BATCHED_AT': '2019-11-21',
+                                '_SDC_DELETED_AT': None,
+                                '"ID"': 1,
+                                '"GROUP"': 'A',
+                            },
+                            {
+                                S3Helper.SDC_SOURCE_BUCKET_COLUMN: 'testBucket',
+                                S3Helper.SDC_SOURCE_FILE_COLUMN: 's3 path 1',
+                                S3Helper.SDC_SOURCE_LINENO_COLUMN: 2,
+                                '_SDC_EXTRACTED_AT': '2019-11-21',
+                                '_SDC_BATCHED_AT': '2019-11-21',
+                                '_SDC_DELETED_AT': None,
+                                '"ID"': 2,
+                                '"GROUP"': 'A',
+                                '"TEST"': True,
+                            },
+                            {
+                                S3Helper.SDC_SOURCE_BUCKET_COLUMN: 'testBucket',
+                                S3Helper.SDC_SOURCE_FILE_COLUMN: 's3 path 1',
+                                S3Helper.SDC_SOURCE_LINENO_COLUMN: 3,
+                                '_SDC_EXTRACTED_AT': '2019-11-21',
+                                '_SDC_BATCHED_AT': '2019-11-21',
+                                '_SDC_DELETED_AT': None,
+                                '"ID"': 3,
+                                '"GROUP"': 'B',
+                            },
+                        ],
+                        records,
+                    )
 
                     self.assertSetEqual(
                         {
-                            '"ID"', '"GROUP"', '"TEST"',
-                            S3Helper.SDC_SOURCE_LINENO_COLUMN, S3Helper.SDC_SOURCE_FILE_COLUMN,
-                            S3Helper.SDC_SOURCE_BUCKET_COLUMN, '_SDC_EXTRACTED_AT',
-                            '_SDC_BATCHED_AT', '_SDC_DELETED_AT'
+                            '"ID"',
+                            '"GROUP"',
+                            '"TEST"',
+                            S3Helper.SDC_SOURCE_LINENO_COLUMN,
+                            S3Helper.SDC_SOURCE_FILE_COLUMN,
+                            S3Helper.SDC_SOURCE_BUCKET_COLUMN,
+                            '_SDC_EXTRACTED_AT',
+                            '_SDC_BATCHED_AT',
+                            '_SDC_DELETED_AT',
                         },
-                        headers)
-
-    def test_fetch_current_incremental_key_pos_with_no_tables_in_dictionary_returns_empty_dict(self):
-        self.assertFalse(self.fs_tap_s3_csv.fetch_current_incremental_key_pos('table-x'))
-
-    def test_fetch_current_incremental_key_pos_with_tables_in_dictionary_returns_empty_dict(self):
+                        headers,
+                    )
+
+    def test_fetch_current_incremental_key_pos_with_no_tables_in_dictionary_returns_empty_dict(
+        self,
+    ):
+        self.assertFalse(
+            self.fs_tap_s3_csv.fetch_current_incremental_key_pos('table-x')
+        )
+
+    def test_fetch_current_incremental_key_pos_with_tables_in_dictionary_returns_empty_dict(
+        self,
+    ):
         dt = datetime.datetime.strptime('2019-11-21', '%Y-%m-%d')
         self.fs_tap_s3_csv.tables_last_modified['table-x'] = dt
-        self.assertEqual({'modified_since': dt.isoformat()},
-                         self.fs_tap_s3_csv.fetch_current_incremental_key_pos('table-x', 'key'))
+        self.assertEqual(
+            {'modified_since': dt.isoformat()},
+            self.fs_tap_s3_csv.fetch_current_incremental_key_pos('table-x', 'key'),
+        )
 
     def test_get_primary_keys_with_table_that_has_no_keys_returns_none(self):
         self.assertIsNone(self.fs_tap_s3_csv._get_primary_keys({}))
@@ -172,62 +200,82 @@ def test_get_primary_keys_with_table_that_has_empty_keys_list_returns_none(self)
         self.assertIsNone(self.fs_tap_s3_csv._get_primary_keys({'key_properties': []}))
 
     def test_get_primary_keys_with_table_that_has_1_key_returns_one_safe_key(self):
-        self.assertEqual(['"KEY_1"'], self.fs_tap_s3_csv._get_primary_keys({'key_properties': ['key_1']}))
-
-    def test_get_primary_keys_with_table_that_has_2_keys_returns_concatenated_keys(self):
-        self.assertIn(self.fs_tap_s3_csv._get_primary_keys({'key_properties': ['key_2', 'key_3']}),
-                      [['"KEY_2"', '"KEY_3"'], ['"KEY_3"', '"KEY_2"']])
+        self.assertEqual(
+            ['"KEY_1"'],
+            self.fs_tap_s3_csv._get_primary_keys({'key_properties': ['key_1']}),
+        )
+
+    def test_get_primary_keys_with_table_that_has_2_keys_returns_concatenated_keys(
+        self,
+    ):
+        self.assertIn(
+            self.fs_tap_s3_csv._get_primary_keys(
+                {'key_properties': ['key_2', 'key_3']}
+            ),
+            [['"KEY_2"', '"KEY_3"'], ['"KEY_3"', '"KEY_2"']],
+        )
 
     def test_get_table_columns(self):
         output = list(
-            self.fs_tap_s3_csv._get_table_columns(f'{os.path.dirname(__file__)}/resources/dummy_data.csv.gz'))
-
-        self.assertListEqual([
-            ('Region', 'string'),
-            ('Country', 'string'),
-            ('Item Type', 'string'),
-            ('Sales Channel', 'string'),
-            ('Order Priority', 'string'),
-            ('Order Date', 'string'),
-            ('Order ID', 'integer'),
-            ('Ship Date', 'string'),
-            ('Units Sold', 'integer'),
-            ('Unit Price', 'number'),
-            ('Unit Cost', 'number'),
-            ('Total Revenue', 'number'),
-            ('Total Cost', 'number'),
-            ('Total Profit', 'number'),
-        ], output)
+            self.fs_tap_s3_csv._get_table_columns(
+                f'{os.path.dirname(__file__)}/resources/dummy_data.csv.gz'
+            )
+        )
+
+        self.assertListEqual(
+            [
+                ('Region', 'string'),
+                ('Country', 'string'),
+                ('Item Type', 'string'),
+                ('Sales Channel', 'string'),
+                ('Order Priority', 'string'),
+                ('Order Date', 'string'),
+                ('Order ID', 'integer'),
+                ('Ship Date', 'string'),
+                ('Units Sold', 'integer'),
+                ('Unit Price', 'number'),
+                ('Unit Cost', 'number'),
+                ('Total Revenue', 'number'),
+                ('Total Cost', 'number'),
+                ('Total Profit', 'number'),
+            ],
+            output,
+        )
 
     def test_map_column_types_to_target(self):
         output = self.fs_tap_s3_csv.map_column_types_to_target(
-            f'{os.path.dirname(__file__)}/resources/dummy_data.csv.gz', 'table 1')
-
-        self.assertDictEqual({
-            'columns': [
-                'Region varchar',
-                'Country varchar',
-                'Item Type varchar',
-                'Sales Channel varchar',
-                'Order Priority varchar',
-                'Order Date varchar',
-                'Order ID number',
-                'Ship Date varchar',
-                'Units Sold number',
-                'Unit Price number',
-                'Unit Cost number',
-                'Total Revenue number',
-                'Total Cost number',
-                'Total Profit number',
-            ],
-            'primary_key': None
-        }, output)
+            f'{os.path.dirname(__file__)}/resources/dummy_data.csv.gz', 'table 1'
+        )
+
+        self.assertDictEqual(
+            {
+                'columns': [
+                    'Region varchar',
+                    'Country varchar',
+                    'Item Type varchar',
+                    'Sales Channel varchar',
+                    'Order Priority varchar',
+                    'Order Date varchar',
+                    'Order ID number',
+                    'Ship Date varchar',
+                    'Units Sold number',
+                    'Unit Price number',
+                    'Unit Cost number',
+                    'Total Revenue number',
+                    'Total Cost number',
+                    'Total Profit number',
+                ],
+                'primary_key': None,
+            },
+            output,
+        )
 
 
 class TestS3Helper(TestCase):
     """
     Unit tests for S3Helper in tap s3 csv
     """
+
     # pylint: disable=R0201
     def test_setup_aws_client_with_creds(self):
         """
@@ -236,7 +284,7 @@ def test_setup_aws_client_with_creds(self):
         config = {
             'aws_access_key_id': 'some key id',
             'aws_secret_access_key': 'some secret key',
-            'aws_session_token': 'token'
+            'aws_session_token': 'token',
         }
 
         with patch('pipelinewise.fastsync.commons.tap_s3_csv.boto3') as boto3_mock:
@@ -245,7 +293,7 @@ def test_setup_aws_client_with_creds(self):
             boto3_mock.setup_default_session.assert_called_once_with(
                 aws_access_key_id=config['aws_access_key_id'],
                 aws_secret_access_key=config['aws_secret_access_key'],
-                aws_session_token=config['aws_session_token']
+                aws_session_token=config['aws_session_token'],
             )
 
     # pylint: disable=R0201
@@ -257,7 +305,7 @@ def test_setup_aws_client_with_env_creds(self):
         config = {
             'aws_access_key_id': 'some key id',
             'aws_secret_access_key': 'some secret key',
-            'aws_session_token': 'token'
+            'aws_session_token': 'token',
         }
 
         os.environ['AWS_ACCESS_KEY_ID'] = config['aws_access_key_id']
@@ -270,7 +318,7 @@ def test_setup_aws_client_with_env_creds(self):
             boto3_mock.setup_default_session.assert_called_once_with(
                 aws_access_key_id=config['aws_access_key_id'],
                 aws_secret_access_key=config['aws_secret_access_key'],
-                aws_session_token=config['aws_session_token']
+                aws_session_token=config['aws_session_token'],
             )
 
         os.environ.pop('AWS_ACCESS_KEY_ID')
@@ -325,8 +373,6 @@ def test_setup_aws_client_with_no_profile_nor_creds(self):
         with patch('pipelinewise.fastsync.commons.tap_s3_csv.boto3') as boto3_mock:
             S3Helper.setup_aws_client({})
 
-            boto3_mock.setup_default_session.assert_called_once_with(
-                profile_name=None
-            )
+            boto3_mock.setup_default_session.assert_called_once_with(profile_name=None)
 
     # TODO: add unit tests for the rest of class methods.
diff --git a/tests/units/fastsync/commons/test_fastsync_target_bigquery.py b/tests/units/fastsync/commons/test_fastsync_target_bigquery.py
index f75b051ef..d82bbe2c9 100644
--- a/tests/units/fastsync/commons/test_fastsync_target_bigquery.py
+++ b/tests/units/fastsync/commons/test_fastsync_target_bigquery.py
@@ -3,6 +3,7 @@
 from google.cloud import bigquery
 from pipelinewise.fastsync.commons.target_bigquery import FastSyncTargetBigquery
 
+
 @pytest.fixture(name='query_result')
 def fixture_query_result():
     """
@@ -12,6 +13,7 @@ def fixture_query_result():
     mocked_qr.return_value = []
     return mocked_qr
 
+
 # pylint: disable=W0613
 @pytest.fixture(name='bigquery_job')
 def fixture_bigquery_job(query_result):
@@ -24,6 +26,7 @@ def fixture_bigquery_job(query_result):
     mocked_qj.result().total_rows = 0
     return mocked_qj
 
+
 @pytest.fixture(name='bigquery_job_config')
 def fixture_bigquery_job_config():
     """
@@ -32,10 +35,12 @@ def fixture_bigquery_job_config():
     mocked_qc = Mock()
     return mocked_qc
 
+
 class FastSyncTargetBigqueryMock(FastSyncTargetBigquery):
     """
     Mocked FastSyncTargetBigquery class
     """
+
     def __init__(self, connection_config, transformation_config=None):
         super().__init__(connection_config, transformation_config)
         self.executed_queries = []
@@ -46,10 +51,13 @@ class TestFastSyncTargetBigquery:
     """
     Unit tests for fastsync target bigquery
     """
+
     def setup_method(self):
         """Initialise test FastSyncTargetPostgres object"""
-        self.bigquery = FastSyncTargetBigqueryMock(connection_config={'project_id': 'dummy-project', 'location': 'EU'},
-                                                   transformation_config={})
+        self.bigquery = FastSyncTargetBigqueryMock(
+            connection_config={'project_id': 'dummy-project', 'location': 'EU'},
+            transformation_config={},
+        )
 
     @patch('pipelinewise.fastsync.commons.target_bigquery.bigquery.Client')
     def test_open_connection(self, client):
@@ -68,99 +76,119 @@ def test_drop_table(self, client):
         """Validate if drop table queries generated correctly"""
         self.bigquery.drop_table('test_schema', 'test_table')
         client().query.assert_called_with(
-            'DROP TABLE IF EXISTS test_schema.`test_table`', job_config=ANY)
+            'DROP TABLE IF EXISTS test_schema.`test_table`', job_config=ANY
+        )
 
         self.bigquery.drop_table('test_schema', 'test_table', is_temporary=True)
         client().query.assert_called_with(
-            'DROP TABLE IF EXISTS test_schema.`test_table_temp`', job_config=ANY)
+            'DROP TABLE IF EXISTS test_schema.`test_table_temp`', job_config=ANY
+        )
 
         self.bigquery.drop_table('test_schema', 'UPPERCASE_TABLE')
         client().query.assert_called_with(
-            'DROP TABLE IF EXISTS test_schema.`uppercase_table`', job_config=ANY)
+            'DROP TABLE IF EXISTS test_schema.`uppercase_table`', job_config=ANY
+        )
 
         self.bigquery.drop_table('test_schema', 'UPPERCASE_TABLE', is_temporary=True)
         client().query.assert_called_with(
-            'DROP TABLE IF EXISTS test_schema.`uppercase_table_temp`', job_config=ANY)
+            'DROP TABLE IF EXISTS test_schema.`uppercase_table_temp`', job_config=ANY
+        )
 
         self.bigquery.drop_table('test_schema', 'test_table_with_space')
         client().query.assert_called_with(
-            'DROP TABLE IF EXISTS test_schema.`test_table_with_space`', job_config=ANY)
+            'DROP TABLE IF EXISTS test_schema.`test_table_with_space`', job_config=ANY
+        )
 
-        self.bigquery.drop_table('test_schema', 'test table with space', is_temporary=True)
+        self.bigquery.drop_table(
+            'test_schema', 'test table with space', is_temporary=True
+        )
         client().query.assert_called_with(
-            'DROP TABLE IF EXISTS test_schema.`test_table_with_space_temp`', job_config=ANY)
+            'DROP TABLE IF EXISTS test_schema.`test_table_with_space_temp`',
+            job_config=ANY,
+        )
 
     @patch('pipelinewise.fastsync.commons.target_bigquery.bigquery.Client')
     def test_create_table(self, client):
         """Validate if create table queries generated correctly"""
         # Create table with standard table and column names
-        self.bigquery.create_table(target_schema='test_schema',
-                                    table_name='test_table',
-                                    columns=['`id` INTEGER',
-                                             '`txt` STRING'])
+        self.bigquery.create_table(
+            target_schema='test_schema',
+            table_name='test_table',
+            columns=['`id` INTEGER', '`txt` STRING'],
+        )
         client().query.assert_called_with(
             'CREATE OR REPLACE TABLE test_schema.`test_table` ('
             '`id` integer,`txt` string,'
             '_sdc_extracted_at timestamp,'
             '_sdc_batched_at timestamp,'
             '_sdc_deleted_at timestamp)',
-            job_config=ANY)
+            job_config=ANY,
+        )
 
         # Create table with reserved words in table and column names
-        self.bigquery.create_table(target_schema='test_schema',
-                                    table_name='order',
-                                    columns=['`id` INTEGER',
-                                             '`txt` STRING',
-                                             '`select` STRING'])
+        self.bigquery.create_table(
+            target_schema='test_schema',
+            table_name='order',
+            columns=['`id` INTEGER', '`txt` STRING', '`select` STRING'],
+        )
         client().query.assert_called_with(
             'CREATE OR REPLACE TABLE test_schema.`order` ('
             '`id` integer,`txt` string,`select` string,'
             '_sdc_extracted_at timestamp,'
             '_sdc_batched_at timestamp,'
             '_sdc_deleted_at timestamp)',
-            job_config=ANY)
+            job_config=ANY,
+        )
 
         # Create table with mixed lower and uppercase and space characters
-        self.bigquery.create_table(target_schema='test_schema',
-                                    table_name='TABLE with SPACE',
-                                    columns=['`ID` INTEGER',
-                                             '`COLUMN WITH SPACE` STRING'])
+        self.bigquery.create_table(
+            target_schema='test_schema',
+            table_name='TABLE with SPACE',
+            columns=['`ID` INTEGER', '`COLUMN WITH SPACE` STRING'],
+        )
         client().query.assert_called_with(
             'CREATE OR REPLACE TABLE test_schema.`table_with_space` ('
             '`id` integer,`column with space` string,'
             '_sdc_extracted_at timestamp,'
             '_sdc_batched_at timestamp,'
             '_sdc_deleted_at timestamp)',
-            job_config=ANY)
+            job_config=ANY,
+        )
 
         # Create table with no primary key
-        self.bigquery.create_table(target_schema='test_schema',
-                                    table_name='test_table_no_pk',
-                                    columns=['`ID` INTEGER',
-                                             '`TXT` STRING'])
+        self.bigquery.create_table(
+            target_schema='test_schema',
+            table_name='test_table_no_pk',
+            columns=['`ID` INTEGER', '`TXT` STRING'],
+        )
         client().query.assert_called_with(
             'CREATE OR REPLACE TABLE test_schema.`test_table_no_pk` ('
             '`id` integer,`txt` string,'
             '_sdc_extracted_at timestamp,'
             '_sdc_batched_at timestamp,'
             '_sdc_deleted_at timestamp)',
-            job_config=ANY)
+            job_config=ANY,
+        )
 
     @patch('pipelinewise.fastsync.commons.target_bigquery.bigquery.LoadJobConfig')
     @patch('pipelinewise.fastsync.commons.target_bigquery.bigquery.Client')
-    def test_copy_to_table(self, client, load_job_config, bigquery_job_config, bigquery_job):
+    def test_copy_to_table(
+        self, client, load_job_config, bigquery_job_config, bigquery_job
+    ):
         """Validate if COPY command generated correctly"""
         # COPY table with standard table and column names
         client().load_table_from_file.return_value = bigquery_job
         load_job_config.return_value = bigquery_job_config
         mocked_open = mock_open()
         with patch('pipelinewise.fastsync.commons.target_bigquery.open', mocked_open):
-            self.bigquery.copy_to_table(filepath='/path/to/dummy-file.csv.gz',
-                                         target_schema='test_schema',
-                                         table_name='test_table',
-                                         size_bytes=1000,
-                                         is_temporary=False,
-                                         skip_csv_header=False)
+            self.bigquery.copy_to_table(
+                filepath='/path/to/dummy-file.csv.gz',
+                target_schema='test_schema',
+                table_name='test_table',
+                size_bytes=1000,
+                is_temporary=False,
+                skip_csv_header=False,
+            )
         mocked_open.assert_called_with('/path/to/dummy-file.csv.gz', 'rb')
         assert bigquery_job_config.source_format == bigquery.SourceFormat.CSV
         assert bigquery_job_config.write_disposition == 'WRITE_TRUNCATE'
@@ -172,12 +200,14 @@ def test_copy_to_table(self, client, load_job_config, bigquery_job_config, bigqu
 
         # COPY table with reserved word in table and column names in temp table
         with patch('pipelinewise.fastsync.commons.target_bigquery.open', mocked_open):
-            self.bigquery.copy_to_table(filepath='/path/to/full-file.csv.gz',
-                                         target_schema='test_schema',
-                                         table_name='full',
-                                         size_bytes=1000,
-                                         is_temporary=True,
-                                         skip_csv_header=False)
+            self.bigquery.copy_to_table(
+                filepath='/path/to/full-file.csv.gz',
+                target_schema='test_schema',
+                table_name='full',
+                size_bytes=1000,
+                is_temporary=True,
+                skip_csv_header=False,
+            )
         mocked_open.assert_called_with('/path/to/full-file.csv.gz', 'rb')
         assert bigquery_job_config.source_format == bigquery.SourceFormat.CSV
         assert bigquery_job_config.write_disposition == 'WRITE_TRUNCATE'
@@ -189,19 +219,23 @@ def test_copy_to_table(self, client, load_job_config, bigquery_job_config, bigqu
 
         # COPY table with space and uppercase in table name and s3 key
         with patch('pipelinewise.fastsync.commons.target_bigquery.open', mocked_open):
-            self.bigquery.copy_to_table(filepath='/path/to/file with space.csv.gz',
-                                         target_schema='test_schema',
-                                         table_name='table with SPACE and UPPERCASE',
-                                         size_bytes=1000,
-                                         is_temporary=True,
-                                         skip_csv_header=False)
+            self.bigquery.copy_to_table(
+                filepath='/path/to/file with space.csv.gz',
+                target_schema='test_schema',
+                table_name='table with SPACE and UPPERCASE',
+                size_bytes=1000,
+                is_temporary=True,
+                skip_csv_header=False,
+            )
         mocked_open.assert_called_with('/path/to/file with space.csv.gz', 'rb')
         assert bigquery_job_config.source_format == bigquery.SourceFormat.CSV
         assert bigquery_job_config.write_disposition == 'WRITE_TRUNCATE'
         assert bigquery_job_config.allow_quoted_newlines is True
         assert bigquery_job_config.skip_leading_rows == 0
         client().dataset.assert_called_with('test_schema')
-        client().dataset().table.assert_called_with('table_with_space_and_uppercase_temp')
+        client().dataset().table.assert_called_with(
+            'table_with_space_and_uppercase_temp'
+        )
         assert client().load_table_from_file.call_count == 3
 
     @patch('pipelinewise.fastsync.commons.target_bigquery.bigquery.Client')
@@ -209,77 +243,100 @@ def test_grant_select_on_table(self, client, bigquery_job):
         """Validate if GRANT command generated correctly"""
         # GRANT table with standard table and column names
         client().query.return_value = bigquery_job
-        self.bigquery.grant_select_on_table(target_schema='test_schema',
-                                             table_name='test_table',
-                                             role='test_role',
-                                             is_temporary=False)
+        self.bigquery.grant_select_on_table(
+            target_schema='test_schema',
+            table_name='test_table',
+            role='test_role',
+            is_temporary=False,
+        )
         client().query.assert_called_with(
-            'GRANT SELECT ON test_schema.`test_table` TO ROLE test_role', job_config=ANY)
+            'GRANT SELECT ON test_schema.`test_table` TO ROLE test_role', job_config=ANY
+        )
 
         # GRANT table with reserved word in table and column names in temp table
-        self.bigquery.grant_select_on_table(target_schema='test_schema',
-                                             table_name='full',
-                                             role='test_role',
-                                             is_temporary=False)
+        self.bigquery.grant_select_on_table(
+            target_schema='test_schema',
+            table_name='full',
+            role='test_role',
+            is_temporary=False,
+        )
         client().query.assert_called_with(
-            'GRANT SELECT ON test_schema.`full` TO ROLE test_role', job_config=ANY)
+            'GRANT SELECT ON test_schema.`full` TO ROLE test_role', job_config=ANY
+        )
 
         # GRANT table with with space and uppercase in table name and s3 key
-        self.bigquery.grant_select_on_table(target_schema='test_schema',
-                                             table_name='table with SPACE and UPPERCASE',
-                                             role='test_role',
-                                             is_temporary=False)
+        self.bigquery.grant_select_on_table(
+            target_schema='test_schema',
+            table_name='table with SPACE and UPPERCASE',
+            role='test_role',
+            is_temporary=False,
+        )
         client().query.assert_called_with(
-            'GRANT SELECT ON test_schema.`table_with_space_and_uppercase` TO ROLE test_role', job_config=ANY)
+            'GRANT SELECT ON test_schema.`table_with_space_and_uppercase` TO ROLE test_role',
+            job_config=ANY,
+        )
 
     @patch('pipelinewise.fastsync.commons.target_bigquery.bigquery.Client')
     def test_grant_usage_on_schema(self, client):
         """Validate if GRANT command generated correctly"""
-        self.bigquery.grant_usage_on_schema(target_schema='test_schema',
-                                             role='test_role')
+        self.bigquery.grant_usage_on_schema(
+            target_schema='test_schema', role='test_role'
+        )
         client().query.assert_called_with(
-            'GRANT USAGE ON SCHEMA test_schema TO ROLE test_role', job_config=ANY)
+            'GRANT USAGE ON SCHEMA test_schema TO ROLE test_role', job_config=ANY
+        )
 
     @patch('pipelinewise.fastsync.commons.target_bigquery.bigquery.Client')
     def test_grant_select_on_schema(self, client):
         """Validate if GRANT command generated correctly"""
-        self.bigquery.grant_select_on_schema(target_schema='test_schema',
-                                              role='test_role')
+        self.bigquery.grant_select_on_schema(
+            target_schema='test_schema', role='test_role'
+        )
         client().query.assert_called_with(
-            'GRANT SELECT ON ALL TABLES IN SCHEMA test_schema TO ROLE test_role', job_config=ANY)
+            'GRANT SELECT ON ALL TABLES IN SCHEMA test_schema TO ROLE test_role',
+            job_config=ANY,
+        )
 
     @patch('pipelinewise.fastsync.commons.target_bigquery.bigquery.CopyJobConfig')
     @patch('pipelinewise.fastsync.commons.target_bigquery.bigquery.Client')
-    def test_swap_tables(self, client, copy_job_config, bigquery_job_config, bigquery_job):
+    def test_swap_tables(
+        self, client, copy_job_config, bigquery_job_config, bigquery_job
+    ):
         """Validate if swap table commands generated correctly"""
         # Swap tables with standard table and column names
         client().copy_table.return_value = bigquery_job
         copy_job_config.return_value = bigquery_job_config
-        self.bigquery.swap_tables(schema='test_schema',
-                                   table_name='test_table')
+        self.bigquery.swap_tables(schema='test_schema', table_name='test_table')
         assert bigquery_job_config.write_disposition == 'WRITE_TRUNCATE'
         client().copy_table.assert_called_with(
             'dummy-project.test_schema.test_table_temp',
             'dummy-project.test_schema.test_table',
-            job_config=ANY)
-        client().delete_table.assert_called_with('dummy-project.test_schema.test_table_temp')
+            job_config=ANY,
+        )
+        client().delete_table.assert_called_with(
+            'dummy-project.test_schema.test_table_temp'
+        )
 
         # Swap tables with reserved word in table and column names in temp table
-        self.bigquery.swap_tables(schema='test_schema',
-                                   table_name='full')
+        self.bigquery.swap_tables(schema='test_schema', table_name='full')
         assert bigquery_job_config.write_disposition == 'WRITE_TRUNCATE'
         client().copy_table.assert_called_with(
             'dummy-project.test_schema.full_temp',
             'dummy-project.test_schema.full',
-            job_config=ANY)
+            job_config=ANY,
+        )
         client().delete_table.assert_called_with('dummy-project.test_schema.full_temp')
 
         # Swap tables with with space and uppercase in table name and s3 key
-        self.bigquery.swap_tables(schema='test_schema',
-                                   table_name='table with SPACE and UPPERCASE')
+        self.bigquery.swap_tables(
+            schema='test_schema', table_name='table with SPACE and UPPERCASE'
+        )
         assert bigquery_job_config.write_disposition == 'WRITE_TRUNCATE'
         client().copy_table.assert_called_with(
             'dummy-project.test_schema.table_with_space_and_uppercase_temp',
             'dummy-project.test_schema.table_with_space_and_uppercase',
-            job_config=ANY)
-        client().delete_table.assert_called_with('dummy-project.test_schema.table_with_space_and_uppercase_temp')
+            job_config=ANY,
+        )
+        client().delete_table.assert_called_with(
+            'dummy-project.test_schema.table_with_space_and_uppercase_temp'
+        )
diff --git a/tests/units/fastsync/commons/test_fastsync_target_postgres.py b/tests/units/fastsync/commons/test_fastsync_target_postgres.py
index 4e07154d2..65647cd5a 100644
--- a/tests/units/fastsync/commons/test_fastsync_target_postgres.py
+++ b/tests/units/fastsync/commons/test_fastsync_target_postgres.py
@@ -24,12 +24,16 @@ class TestFastSyncTargetPostgres(TestCase):
 
     def setUp(self) -> None:
         """Initialise test FastSyncTargetPostgres object"""
-        self.postgres = FastSyncTargetPostgresMock(connection_config={}, transformation_config={})
+        self.postgres = FastSyncTargetPostgresMock(
+            connection_config={}, transformation_config={}
+        )
 
     def test_create_schema(self):
         """Validate if create schema queries generated correctly"""
         self.postgres.create_schema('new_schema')
-        assert self.postgres.executed_queries == ['CREATE SCHEMA IF NOT EXISTS new_schema']
+        assert self.postgres.executed_queries == [
+            'CREATE SCHEMA IF NOT EXISTS new_schema'
+        ]
 
     def test_drop_table(self):
         """Validate if drop table queries generated correctly"""
@@ -38,164 +42,198 @@ def test_drop_table(self):
         self.postgres.drop_table('test_schema', 'UPPERCASE_TABLE')
         self.postgres.drop_table('test_schema', 'UPPERCASE_TABLE', is_temporary=True)
         self.postgres.drop_table('test_schema', 'test table with space')
-        self.postgres.drop_table('test_schema', 'test table with space', is_temporary=True)
+        self.postgres.drop_table(
+            'test_schema', 'test table with space', is_temporary=True
+        )
         assert self.postgres.executed_queries == [
             'DROP TABLE IF EXISTS test_schema."test_table"',
             'DROP TABLE IF EXISTS test_schema."test_table_temp"',
             'DROP TABLE IF EXISTS test_schema."uppercase_table"',
             'DROP TABLE IF EXISTS test_schema."uppercase_table_temp"',
             'DROP TABLE IF EXISTS test_schema."test table with space"',
-            'DROP TABLE IF EXISTS test_schema."test table with space_temp"']
+            'DROP TABLE IF EXISTS test_schema."test table with space_temp"',
+        ]
 
     def test_create_table(self):
         """Validate if create table queries generated correctly"""
         # Create table with standard table and column names
         self.postgres.executed_queries = []
-        self.postgres.create_table(target_schema='test_schema',
-                                   table_name='test_table',
-                                   columns=['"id" INTEGER',
-                                            '"txt" CHARACTER VARYING'],
-                                   primary_key=['"id"'])
+        self.postgres.create_table(
+            target_schema='test_schema',
+            table_name='test_table',
+            columns=['"id" INTEGER', '"txt" CHARACTER VARYING'],
+            primary_key=['"id"'],
+        )
         assert self.postgres.executed_queries == [
             'CREATE TABLE IF NOT EXISTS test_schema."test_table" ('
             '"id" integer,"txt" character varying,'
             '_sdc_extracted_at timestamp without time zone,'
             '_sdc_batched_at timestamp without time zone,'
             '_sdc_deleted_at character varying'
-            ', PRIMARY KEY ("id"))']
+            ', PRIMARY KEY ("id"))'
+        ]
 
         # Create table with reserved words in table and column names
         self.postgres.executed_queries = []
-        self.postgres.create_table(target_schema='test_schema',
-                                   table_name='ORDER',
-                                   columns=['"id" INTEGER',
-                                            '"txt" CHARACTER VARYING',
-                                            '"SELECT" CHARACTER VARYING'],
-                                   primary_key=['"id"'])
+        self.postgres.create_table(
+            target_schema='test_schema',
+            table_name='ORDER',
+            columns=[
+                '"id" INTEGER',
+                '"txt" CHARACTER VARYING',
+                '"SELECT" CHARACTER VARYING',
+            ],
+            primary_key=['"id"'],
+        )
         assert self.postgres.executed_queries == [
             'CREATE TABLE IF NOT EXISTS test_schema."order" ('
             '"id" integer,"txt" character varying,"select" character varying,'
             '_sdc_extracted_at timestamp without time zone,'
             '_sdc_batched_at timestamp without time zone,'
             '_sdc_deleted_at character varying'
-            ', PRIMARY KEY ("id"))']
+            ', PRIMARY KEY ("id"))'
+        ]
 
         # Create table with mixed lower and uppercase and space characters
         self.postgres.executed_queries = []
-        self.postgres.create_table(target_schema='test_schema',
-                                   table_name='TABLE with SPACE',
-                                   columns=['"id" INTEGER',
-                                            '"column_with space" CHARACTER VARYING'],
-                                   primary_key=['"id"'])
+        self.postgres.create_table(
+            target_schema='test_schema',
+            table_name='TABLE with SPACE',
+            columns=['"id" INTEGER', '"column_with space" CHARACTER VARYING'],
+            primary_key=['"id"'],
+        )
         assert self.postgres.executed_queries == [
             'CREATE TABLE IF NOT EXISTS test_schema."table with space" ('
             '"id" integer,"column_with space" character varying,'
             '_sdc_extracted_at timestamp without time zone,'
             '_sdc_batched_at timestamp without time zone,'
             '_sdc_deleted_at character varying'
-            ', PRIMARY KEY ("id"))']
+            ', PRIMARY KEY ("id"))'
+        ]
 
         # Create table with composite primary key
         self.postgres.executed_queries = []
-        self.postgres.create_table(target_schema='test_schema',
-                                   table_name='TABLE with SPACE',
-                                   columns=['"id" INTEGER',
-                                            '"num" INTEGER',
-                                            '"column_with space" CHARACTER VARYING'],
-                                   primary_key=['"id"', '"num"'])
+        self.postgres.create_table(
+            target_schema='test_schema',
+            table_name='TABLE with SPACE',
+            columns=[
+                '"id" INTEGER',
+                '"num" INTEGER',
+                '"column_with space" CHARACTER VARYING',
+            ],
+            primary_key=['"id"', '"num"'],
+        )
         assert self.postgres.executed_queries == [
             'CREATE TABLE IF NOT EXISTS test_schema."table with space" ('
             '"id" integer,"num" integer,"column_with space" character varying,'
             '_sdc_extracted_at timestamp without time zone,'
             '_sdc_batched_at timestamp without time zone,'
             '_sdc_deleted_at character varying'
-            ', PRIMARY KEY ("id","num"))']
+            ', PRIMARY KEY ("id","num"))'
+        ]
 
         # Create table with no primary key
         self.postgres.executed_queries = []
-        self.postgres.create_table(target_schema='test_schema',
-                                   table_name='test_table_no_pk',
-                                   columns=['"id" INTEGER',
-                                            '"txt" CHARACTER VARYING'],
-                                   primary_key=None)
+        self.postgres.create_table(
+            target_schema='test_schema',
+            table_name='test_table_no_pk',
+            columns=['"id" INTEGER', '"txt" CHARACTER VARYING'],
+            primary_key=None,
+        )
         assert self.postgres.executed_queries == [
             'CREATE TABLE IF NOT EXISTS test_schema."test_table_no_pk" ('
             '"id" integer,"txt" character varying,'
             '_sdc_extracted_at timestamp without time zone,'
             '_sdc_batched_at timestamp without time zone,'
-            '_sdc_deleted_at character varying)']
+            '_sdc_deleted_at character varying)'
+        ]
 
     def test_grant_select_on_table(self):
         """Validate if GRANT command generated correctly"""
         # GRANT table with standard table and column names
         self.postgres.executed_queries = []
-        self.postgres.grant_select_on_table(target_schema='test_schema',
-                                            table_name='test_table',
-                                            role='test_role',
-                                            is_temporary=False)
+        self.postgres.grant_select_on_table(
+            target_schema='test_schema',
+            table_name='test_table',
+            role='test_role',
+            is_temporary=False,
+        )
         assert self.postgres.executed_queries == [
-            'GRANT SELECT ON test_schema."test_table" TO GROUP test_role']
+            'GRANT SELECT ON test_schema."test_table" TO GROUP test_role'
+        ]
 
         # GRANT table with reserved word in table and column names in temp table
         self.postgres.executed_queries = []
-        self.postgres.grant_select_on_table(target_schema='test_schema',
-                                            table_name='full',
-                                            role='test_role',
-                                            is_temporary=False)
+        self.postgres.grant_select_on_table(
+            target_schema='test_schema',
+            table_name='full',
+            role='test_role',
+            is_temporary=False,
+        )
         assert self.postgres.executed_queries == [
-            'GRANT SELECT ON test_schema."full" TO GROUP test_role']
+            'GRANT SELECT ON test_schema."full" TO GROUP test_role'
+        ]
 
         # GRANT table with with space and uppercase in table name and s3 key
         self.postgres.executed_queries = []
-        self.postgres.grant_select_on_table(target_schema='test_schema',
-                                            table_name='table with SPACE and UPPERCASE',
-                                            role='test_role',
-                                            is_temporary=False)
+        self.postgres.grant_select_on_table(
+            target_schema='test_schema',
+            table_name='table with SPACE and UPPERCASE',
+            role='test_role',
+            is_temporary=False,
+        )
         assert self.postgres.executed_queries == [
-            'GRANT SELECT ON test_schema."table with space and uppercase" TO GROUP test_role']
+            'GRANT SELECT ON test_schema."table with space and uppercase" TO GROUP test_role'
+        ]
 
     def test_grant_usage_on_schema(self):
         """Validate if GRANT command generated correctly"""
         self.postgres.executed_queries = []
-        self.postgres.grant_usage_on_schema(target_schema='test_schema',
-                                            role='test_role')
+        self.postgres.grant_usage_on_schema(
+            target_schema='test_schema', role='test_role'
+        )
         assert self.postgres.executed_queries == [
-            'GRANT USAGE ON SCHEMA test_schema TO GROUP test_role']
+            'GRANT USAGE ON SCHEMA test_schema TO GROUP test_role'
+        ]
 
     def test_grant_select_on_schema(self):
         """Validate if GRANT command generated correctly"""
         self.postgres.executed_queries = []
-        self.postgres.grant_select_on_schema(target_schema='test_schema',
-                                             role='test_role')
+        self.postgres.grant_select_on_schema(
+            target_schema='test_schema', role='test_role'
+        )
         assert self.postgres.executed_queries == [
-            'GRANT SELECT ON ALL TABLES IN SCHEMA test_schema TO GROUP test_role']
+            'GRANT SELECT ON ALL TABLES IN SCHEMA test_schema TO GROUP test_role'
+        ]
 
     def test_swap_tables(self):
         """Validate if swap table commands generated correctly"""
         # Swap tables with standard table and column names
         self.postgres.executed_queries = []
-        self.postgres.swap_tables(schema='test_schema',
-                                  table_name='test_table')
+        self.postgres.swap_tables(schema='test_schema', table_name='test_table')
         assert self.postgres.executed_queries == [
             'DROP TABLE IF EXISTS test_schema."test_table"',
-            'ALTER TABLE test_schema."test_table_temp" RENAME TO "test_table"']
+            'ALTER TABLE test_schema."test_table_temp" RENAME TO "test_table"',
+        ]
 
         # Swap tables with reserved word in table and column names in temp table
         self.postgres.executed_queries = []
-        self.postgres.swap_tables(schema='test_schema',
-                                  table_name='full')
+        self.postgres.swap_tables(schema='test_schema', table_name='full')
         assert self.postgres.executed_queries == [
             'DROP TABLE IF EXISTS test_schema."full"',
-            'ALTER TABLE test_schema."full_temp" RENAME TO "full"']
+            'ALTER TABLE test_schema."full_temp" RENAME TO "full"',
+        ]
 
         # Swap tables with with space and uppercase in table name
         self.postgres.executed_queries = []
-        self.postgres.swap_tables(schema='test_schema',
-                                  table_name='table with SPACE and UPPERCASE')
+        self.postgres.swap_tables(
+            schema='test_schema', table_name='table with SPACE and UPPERCASE'
+        )
         assert self.postgres.executed_queries == [
             'DROP TABLE IF EXISTS test_schema."table with space and uppercase"',
             'ALTER TABLE test_schema."table with space and uppercase_temp" '
-            'RENAME TO "table with space and uppercase"']
+            'RENAME TO "table with space and uppercase"',
+        ]
 
     def test_obfuscate_columns_case1(self):
         """
@@ -223,7 +261,7 @@ def test_obfuscate_columns_case2(self):
                 {
                     'field_id': 'col_7',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'RANDOM'
+                    'type': 'RANDOM',
                 }
             ]
         }
@@ -246,34 +284,34 @@ def test_obfuscate_columns_case3(self):
                 {
                     'field_id': 'col_1',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'SET-NULL'
+                    'type': 'SET-NULL',
                 },
                 {
                     'field_id': 'col_2',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'MASK-HIDDEN'
+                    'type': 'MASK-HIDDEN',
                 },
                 {
                     'field_id': 'col_3',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'MASK-DATE'
+                    'type': 'MASK-DATE',
                 },
                 {
                     'field_id': 'col_4',
                     'tap_stream_name': 'public-my_table',
                     'safe_field_id': '"col_4"',
-                    'type': 'MASK-NUMBER'
+                    'type': 'MASK-NUMBER',
                 },
                 {
                     'field_id': 'col_5',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'HASH'
+                    'type': 'HASH',
                 },
                 {
                     'field_id': 'col_6',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'HASH-SKIP-FIRST-5'
-                }
+                    'type': 'HASH-SKIP-FIRST-5',
+                },
             ]
         }
 
@@ -291,7 +329,8 @@ def test_obfuscate_columns_case3(self):
                 '"col_5" = ENCODE(DIGEST("col_5", \'sha256\'), \'hex\'), '
                 '"col_6" = CONCAT(SUBSTRING("col_6", 1, 5), '
                 'ENCODE(DIGEST(SUBSTRING("col_6", 5 + 1), \'sha256\'), \'hex\'));'
-            ])
+            ],
+        )
 
     def test_obfuscate_columns_case4(self):
         """
@@ -306,59 +345,44 @@ def test_obfuscate_columns_case4(self):
                 {
                     'field_id': 'col_1',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'SET-NULL'
+                    'type': 'SET-NULL',
                 },
                 {
                     'field_id': 'col_2',
                     'tap_stream_name': 'public-my_table',
                     'type': 'MASK-HIDDEN',
                     'when': [
-                        {
-                            'column': 'col_4',
-                            'safe_column': '"col_4"',
-                            'equals': None
-                        },
+                        {'column': 'col_4', 'safe_column': '"col_4"', 'equals': None},
                         {
                             'column': 'col_1',
-                        }
-                    ]
+                        },
+                    ],
                 },
                 {
                     'field_id': 'col_3',
                     'tap_stream_name': 'public-my_table',
                     'type': 'MASK-DATE',
-                    'when': [
-                        {
-                            'column': 'col_5',
-                            'equals': 'some_value'
-                        }
-                    ]
+                    'when': [{'column': 'col_5', 'equals': 'some_value'}],
                 },
                 {
                     'field_id': 'col_4',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'MASK-NUMBER'
+                    'type': 'MASK-NUMBER',
                 },
                 {
                     'field_id': 'col_5',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'HASH'
+                    'type': 'HASH',
                 },
                 {
                     'field_id': 'col_6',
                     'tap_stream_name': 'public-my_table',
                     'type': 'HASH-SKIP-FIRST-5',
                     'when': [
-                        {
-                            'column': 'col_1',
-                            'equals': 30
-                        },
-                        {
-                            'column': 'col_2',
-                            'regex_match': r'[0-9]{3}\.[0-9]{3}'
-                        }
-                    ]
-                }
+                        {'column': 'col_1', 'equals': 30},
+                        {'column': 'col_2', 'regex_match': r'[0-9]{3}\.[0-9]{3}'},
+                    ],
+                },
             ]
         }
 
@@ -375,9 +399,8 @@ def test_obfuscate_columns_case4(self):
                 'UPDATE "my_schema"."my_table_temp" SET '
                 '"col_6" = CONCAT(SUBSTRING("col_6", 1, 5), '
                 'ENCODE(DIGEST(SUBSTRING("col_6", 5 + 1), \'sha256\'), \'hex\')) WHERE ("col_1" = 30) AND '
-                '("col_2" ~ \'[0-9]{3}\.[0-9]{3}\');', # pylint: disable=W1401
-
+                '("col_2" ~ \'[0-9]{3}\.[0-9]{3}\');',  # pylint: disable=W1401  # noqa: W605
                 'UPDATE "my_schema"."my_table_temp" SET "col_1" = NULL, '
-                '"col_4" = 0, "col_5" = ENCODE(DIGEST("col_5", \'sha256\'), \'hex\');'
-            ]
+                '"col_4" = 0, "col_5" = ENCODE(DIGEST("col_5", \'sha256\'), \'hex\');',
+            ],
         )
diff --git a/tests/units/fastsync/commons/test_fastsync_target_snowflake.py b/tests/units/fastsync/commons/test_fastsync_target_snowflake.py
index cf465f375..c66a527ac 100644
--- a/tests/units/fastsync/commons/test_fastsync_target_snowflake.py
+++ b/tests/units/fastsync/commons/test_fastsync_target_snowflake.py
@@ -50,14 +50,17 @@ class TestFastSyncTargetSnowflake(TestCase):
 
     def setUp(self) -> None:
         """Initialise test FastSyncTargetPostgres object"""
-        self.snowflake = FastSyncTargetSnowflakeMock(connection_config={'s3_bucket': 'dummy_bucket',
-                                                                        'stage': 'dummy_stage'},
-                                                     transformation_config={})
+        self.snowflake = FastSyncTargetSnowflakeMock(
+            connection_config={'s3_bucket': 'dummy_bucket', 'stage': 'dummy_stage'},
+            transformation_config={},
+        )
 
     def test_create_schema(self):
         """Validate if create schema queries generated correctly"""
         self.snowflake.create_schema('new_schema')
-        self.assertListEqual(self.snowflake.executed_queries, ['CREATE SCHEMA IF NOT EXISTS new_schema'])
+        self.assertListEqual(
+            self.snowflake.executed_queries, ['CREATE SCHEMA IF NOT EXISTS new_schema']
+        )
 
     def test_drop_table(self):
         """Validate if drop table queries generated correctly"""
@@ -66,208 +69,247 @@ def test_drop_table(self):
         self.snowflake.drop_table('test_schema', 'UPPERCASE_TABLE')
         self.snowflake.drop_table('test_schema', 'UPPERCASE_TABLE', is_temporary=True)
         self.snowflake.drop_table('test_schema', 'test table with space')
-        self.snowflake.drop_table('test_schema', 'test table with space', is_temporary=True)
+        self.snowflake.drop_table(
+            'test_schema', 'test table with space', is_temporary=True
+        )
         assert self.snowflake.executed_queries == [
             'DROP TABLE IF EXISTS test_schema."TEST_TABLE"',
             'DROP TABLE IF EXISTS test_schema."TEST_TABLE_TEMP"',
             'DROP TABLE IF EXISTS test_schema."UPPERCASE_TABLE"',
             'DROP TABLE IF EXISTS test_schema."UPPERCASE_TABLE_TEMP"',
             'DROP TABLE IF EXISTS test_schema."TEST TABLE WITH SPACE"',
-            'DROP TABLE IF EXISTS test_schema."TEST TABLE WITH SPACE_TEMP"']
+            'DROP TABLE IF EXISTS test_schema."TEST TABLE WITH SPACE_TEMP"',
+        ]
 
     def test_create_table(self):
         """Validate if create table queries generated correctly"""
         # Create table with standard table and column names
         self.snowflake.executed_queries = []
-        self.snowflake.create_table(target_schema='test_schema',
-                                    table_name='test_table',
-                                    columns=['"ID" INTEGER',
-                                             '"TXT" VARCHAR'],
-                                    primary_key=['"ID"'])
+        self.snowflake.create_table(
+            target_schema='test_schema',
+            table_name='test_table',
+            columns=['"ID" INTEGER', '"TXT" VARCHAR'],
+            primary_key=['"ID"'],
+        )
         assert self.snowflake.executed_queries == [
             'CREATE OR REPLACE TABLE test_schema."TEST_TABLE" ('
             '"ID" INTEGER,"TXT" VARCHAR,'
             '_SDC_EXTRACTED_AT TIMESTAMP_NTZ,'
             '_SDC_BATCHED_AT TIMESTAMP_NTZ,'
             '_SDC_DELETED_AT VARCHAR'
-            ', PRIMARY KEY ("ID"))']
+            ', PRIMARY KEY ("ID"))'
+        ]
 
         # Create table with reserved words in table and column names
         self.snowflake.executed_queries = []
-        self.snowflake.create_table(target_schema='test_schema',
-                                    table_name='order',
-                                    columns=['"ID" INTEGER',
-                                             '"TXT" VARCHAR',
-                                             '"SELECT" VARCHAR'],
-                                    primary_key=['"ID"'])
+        self.snowflake.create_table(
+            target_schema='test_schema',
+            table_name='order',
+            columns=['"ID" INTEGER', '"TXT" VARCHAR', '"SELECT" VARCHAR'],
+            primary_key=['"ID"'],
+        )
         assert self.snowflake.executed_queries == [
             'CREATE OR REPLACE TABLE test_schema."ORDER" ('
             '"ID" INTEGER,"TXT" VARCHAR,"SELECT" VARCHAR,'
             '_SDC_EXTRACTED_AT TIMESTAMP_NTZ,'
             '_SDC_BATCHED_AT TIMESTAMP_NTZ,'
             '_SDC_DELETED_AT VARCHAR'
-            ', PRIMARY KEY ("ID"))']
+            ', PRIMARY KEY ("ID"))'
+        ]
 
         # Create table with mixed lower and uppercase and space characters
         self.snowflake.executed_queries = []
-        self.snowflake.create_table(target_schema='test_schema',
-                                    table_name='TABLE with SPACE',
-                                    columns=['"ID" INTEGER',
-                                             '"COLUMN WITH SPACE" CHARACTER VARYING'],
-                                    primary_key=['"ID"'])
+        self.snowflake.create_table(
+            target_schema='test_schema',
+            table_name='TABLE with SPACE',
+            columns=['"ID" INTEGER', '"COLUMN WITH SPACE" CHARACTER VARYING'],
+            primary_key=['"ID"'],
+        )
         assert self.snowflake.executed_queries == [
             'CREATE OR REPLACE TABLE test_schema."TABLE WITH SPACE" ('
             '"ID" INTEGER,"COLUMN WITH SPACE" CHARACTER VARYING,'
             '_SDC_EXTRACTED_AT TIMESTAMP_NTZ,'
             '_SDC_BATCHED_AT TIMESTAMP_NTZ,'
             '_SDC_DELETED_AT VARCHAR'
-            ', PRIMARY KEY ("ID"))']
+            ', PRIMARY KEY ("ID"))'
+        ]
 
         # Create table with composite primary key
         self.snowflake.executed_queries = []
-        self.snowflake.create_table(target_schema='test_schema',
-                                    table_name='TABLE with SPACE',
-                                    columns=['"ID" INTEGER',
-                                             '"NUM" INTEGER',
-                                             '"COLUMN WITH SPACE" CHARACTER VARYING'],
-                                    primary_key=['"ID", "NUM"'])
+        self.snowflake.create_table(
+            target_schema='test_schema',
+            table_name='TABLE with SPACE',
+            columns=[
+                '"ID" INTEGER',
+                '"NUM" INTEGER',
+                '"COLUMN WITH SPACE" CHARACTER VARYING',
+            ],
+            primary_key=['"ID", "NUM"'],
+        )
         assert self.snowflake.executed_queries == [
             'CREATE OR REPLACE TABLE test_schema."TABLE WITH SPACE" ('
             '"ID" INTEGER,"NUM" INTEGER,"COLUMN WITH SPACE" CHARACTER VARYING,'
             '_SDC_EXTRACTED_AT TIMESTAMP_NTZ,'
             '_SDC_BATCHED_AT TIMESTAMP_NTZ,'
             '_SDC_DELETED_AT VARCHAR'
-            ', PRIMARY KEY ("ID", "NUM"))']
+            ', PRIMARY KEY ("ID", "NUM"))'
+        ]
 
         # Create table with no primary key
         self.snowflake.executed_queries = []
-        self.snowflake.create_table(target_schema='test_schema',
-                                    table_name='test_table_no_pk',
-                                    columns=['"ID" INTEGER',
-                                             '"TXT" CHARACTER VARYING'],
-                                    primary_key=None)
+        self.snowflake.create_table(
+            target_schema='test_schema',
+            table_name='test_table_no_pk',
+            columns=['"ID" INTEGER', '"TXT" CHARACTER VARYING'],
+            primary_key=None,
+        )
         assert self.snowflake.executed_queries == [
             'CREATE OR REPLACE TABLE test_schema."TEST_TABLE_NO_PK" ('
             '"ID" INTEGER,"TXT" CHARACTER VARYING,'
             '_SDC_EXTRACTED_AT TIMESTAMP_NTZ,'
             '_SDC_BATCHED_AT TIMESTAMP_NTZ,'
-            '_SDC_DELETED_AT VARCHAR)']
+            '_SDC_DELETED_AT VARCHAR)'
+        ]
 
     def test_copy_to_table(self):
         """Validate if COPY command generated correctly"""
         # COPY table with standard table and column names
         self.snowflake.executed_queries = []
-        self.snowflake.copy_to_table(s3_key='s3_key',
-                                     target_schema='test_schema',
-                                     table_name='test_table',
-                                     size_bytes=1000,
-                                     is_temporary=False,
-                                     skip_csv_header=False)
+        self.snowflake.copy_to_table(
+            s3_key='s3_key',
+            target_schema='test_schema',
+            table_name='test_table',
+            size_bytes=1000,
+            is_temporary=False,
+            skip_csv_header=False,
+        )
         assert self.snowflake.executed_queries == [
             'COPY INTO test_schema."TEST_TABLE" FROM \'@dummy_stage/s3_key\''
             ' FILE_FORMAT = (type=CSV escape=\'\\x1e\' escape_unenclosed_field=\'\\x1e\''
             ' field_optionally_enclosed_by=\'\"\' skip_header=0'
-            ' compression=GZIP binary_format=HEX)']
+            ' compression=GZIP binary_format=HEX)'
+        ]
 
         # COPY table with reserved word in table and column names in temp table
         self.snowflake.executed_queries = []
-        self.snowflake.copy_to_table(s3_key='s3_key',
-                                     target_schema='test_schema',
-                                     table_name='full',
-                                     size_bytes=1000,
-                                     is_temporary=True,
-                                     skip_csv_header=False)
+        self.snowflake.copy_to_table(
+            s3_key='s3_key',
+            target_schema='test_schema',
+            table_name='full',
+            size_bytes=1000,
+            is_temporary=True,
+            skip_csv_header=False,
+        )
         assert self.snowflake.executed_queries == [
             'COPY INTO test_schema."FULL_TEMP" FROM \'@dummy_stage/s3_key\''
             ' FILE_FORMAT = (type=CSV escape=\'\\x1e\' escape_unenclosed_field=\'\\x1e\''
             ' field_optionally_enclosed_by=\'\"\' skip_header=0'
-            ' compression=GZIP binary_format=HEX)']
+            ' compression=GZIP binary_format=HEX)'
+        ]
 
         # COPY table with space and uppercase in table name and s3 key
         self.snowflake.executed_queries = []
-        self.snowflake.copy_to_table(s3_key='s3 key with space',
-                                     target_schema='test_schema',
-                                     table_name='table with SPACE and UPPERCASE',
-                                     size_bytes=1000,
-                                     is_temporary=True,
-                                     skip_csv_header=False)
+        self.snowflake.copy_to_table(
+            s3_key='s3 key with space',
+            target_schema='test_schema',
+            table_name='table with SPACE and UPPERCASE',
+            size_bytes=1000,
+            is_temporary=True,
+            skip_csv_header=False,
+        )
         assert self.snowflake.executed_queries == [
             'COPY INTO test_schema."TABLE WITH SPACE AND UPPERCASE_TEMP" FROM \'@dummy_stage/s3 key with space\''
             ' FILE_FORMAT = (type=CSV escape=\'\\x1e\' escape_unenclosed_field=\'\\x1e\''
             ' field_optionally_enclosed_by=\'\"\' skip_header=0'
-            ' compression=GZIP binary_format=HEX)']
+            ' compression=GZIP binary_format=HEX)'
+        ]
 
     def test_grant_select_on_table(self):
         """Validate if GRANT command generated correctly"""
         # GRANT table with standard table and column names
         self.snowflake.executed_queries = []
-        self.snowflake.grant_select_on_table(target_schema='test_schema',
-                                             table_name='test_table',
-                                             role='test_role',
-                                             is_temporary=False)
+        self.snowflake.grant_select_on_table(
+            target_schema='test_schema',
+            table_name='test_table',
+            role='test_role',
+            is_temporary=False,
+        )
         assert self.snowflake.executed_queries == [
-            'GRANT SELECT ON test_schema."TEST_TABLE" TO ROLE test_role']
+            'GRANT SELECT ON test_schema."TEST_TABLE" TO ROLE test_role'
+        ]
 
         # GRANT table with reserved word in table and column names in temp table
         self.snowflake.executed_queries = []
-        self.snowflake.grant_select_on_table(target_schema='test_schema',
-                                             table_name='full',
-                                             role='test_role',
-                                             is_temporary=False)
+        self.snowflake.grant_select_on_table(
+            target_schema='test_schema',
+            table_name='full',
+            role='test_role',
+            is_temporary=False,
+        )
         assert self.snowflake.executed_queries == [
-            'GRANT SELECT ON test_schema."FULL" TO ROLE test_role']
+            'GRANT SELECT ON test_schema."FULL" TO ROLE test_role'
+        ]
 
         # GRANT table with with space and uppercase in table name and s3 key
         self.snowflake.executed_queries = []
-        self.snowflake.grant_select_on_table(target_schema='test_schema',
-                                             table_name='table with SPACE and UPPERCASE',
-                                             role='test_role',
-                                             is_temporary=False)
+        self.snowflake.grant_select_on_table(
+            target_schema='test_schema',
+            table_name='table with SPACE and UPPERCASE',
+            role='test_role',
+            is_temporary=False,
+        )
         assert self.snowflake.executed_queries == [
-            'GRANT SELECT ON test_schema."TABLE WITH SPACE AND UPPERCASE" TO ROLE test_role']
+            'GRANT SELECT ON test_schema."TABLE WITH SPACE AND UPPERCASE" TO ROLE test_role'
+        ]
 
     def test_grant_usage_on_schema(self):
         """Validate if GRANT command generated correctly"""
         self.snowflake.executed_queries = []
-        self.snowflake.grant_usage_on_schema(target_schema='test_schema',
-                                             role='test_role')
+        self.snowflake.grant_usage_on_schema(
+            target_schema='test_schema', role='test_role'
+        )
         assert self.snowflake.executed_queries == [
-            'GRANT USAGE ON SCHEMA test_schema TO ROLE test_role']
+            'GRANT USAGE ON SCHEMA test_schema TO ROLE test_role'
+        ]
 
     def test_grant_select_on_schema(self):
         """Validate if GRANT command generated correctly"""
         self.snowflake.executed_queries = []
-        self.snowflake.grant_select_on_schema(target_schema='test_schema',
-                                              role='test_role')
+        self.snowflake.grant_select_on_schema(
+            target_schema='test_schema', role='test_role'
+        )
         assert self.snowflake.executed_queries == [
-            'GRANT SELECT ON ALL TABLES IN SCHEMA test_schema TO ROLE test_role']
+            'GRANT SELECT ON ALL TABLES IN SCHEMA test_schema TO ROLE test_role'
+        ]
 
     def test_swap_tables(self):
         """Validate if swap table commands generated correctly"""
         # Swap tables with standard table and column names
         self.snowflake.executed_queries = []
-        self.snowflake.swap_tables(schema='test_schema',
-                                   table_name='test_table')
+        self.snowflake.swap_tables(schema='test_schema', table_name='test_table')
         assert self.snowflake.executed_queries == [
             'ALTER TABLE test_schema."TEST_TABLE_TEMP" SWAP WITH test_schema."TEST_TABLE"',
-            'DROP TABLE IF EXISTS test_schema."TEST_TABLE_TEMP"']
+            'DROP TABLE IF EXISTS test_schema."TEST_TABLE_TEMP"',
+        ]
 
         # Swap tables with reserved word in table and column names in temp table
         self.snowflake.executed_queries = []
-        self.snowflake.swap_tables(schema='test_schema',
-                                   table_name='full')
+        self.snowflake.swap_tables(schema='test_schema', table_name='full')
         assert self.snowflake.executed_queries == [
             'ALTER TABLE test_schema."FULL_TEMP" SWAP WITH test_schema."FULL"',
-            'DROP TABLE IF EXISTS test_schema."FULL_TEMP"']
+            'DROP TABLE IF EXISTS test_schema."FULL_TEMP"',
+        ]
 
         # Swap tables with with space and uppercase in table name and s3 key
         self.snowflake.executed_queries = []
-        self.snowflake.swap_tables(schema='test_schema',
-                                   table_name='table with SPACE and UPPERCASE')
+        self.snowflake.swap_tables(
+            schema='test_schema', table_name='table with SPACE and UPPERCASE'
+        )
         assert self.snowflake.executed_queries == [
             'ALTER TABLE test_schema."TABLE WITH SPACE AND UPPERCASE_TEMP" '
             'SWAP WITH test_schema."TABLE WITH SPACE AND UPPERCASE"',
-            'DROP TABLE IF EXISTS test_schema."TABLE WITH SPACE AND UPPERCASE_TEMP"']
+            'DROP TABLE IF EXISTS test_schema."TABLE WITH SPACE AND UPPERCASE_TEMP"',
+        ]
 
     def test_create_query_tag(self):
         """Validate if query tag generated correctly"""
@@ -279,7 +321,7 @@ def test_create_query_tag(self):
             'tap_id': None,
             'database': 'fake_db',
             'schema': None,
-            'table': None
+            'table': None,
         }
 
         # passing invalid query_tag_props (string)
@@ -288,7 +330,7 @@ def test_create_query_tag(self):
             'tap_id': None,
             'database': 'fake_db',
             'schema': None,
-            'table': None
+            'table': None,
         }
 
         # passing invalid query_tag_props (number)
@@ -298,7 +340,7 @@ def test_create_query_tag(self):
             'tap_id': 'fake_tap',
             'database': 'fake_db',
             'schema': None,
-            'table': None
+            'table': None,
         }
 
         # passing invalid query_tag_props (array)
@@ -308,7 +350,7 @@ def test_create_query_tag(self):
             'tap_id': 'fake_tap',
             'database': 'fake_db',
             'schema': None,
-            'table': None
+            'table': None,
         }
 
         # passing invalid query_tag_props
@@ -318,29 +360,33 @@ def test_create_query_tag(self):
             'tap_id': 'fake_tap',
             'database': 'fake_db',
             'schema': None,
-            'table': None
+            'table': None,
         }
 
         # passing valid query_props
         self.snowflake.connection_config['tap_id'] = 'fake_tap'
-        assert json.loads(self.snowflake.create_query_tag({'schema': 'fake_schema',
-                                                           'table': 'fake_table'})) == \
-               {
-                   'ppw_component': 'fastsync',
-                   'tap_id': 'fake_tap',
-                   'database': 'fake_db',
-                   'schema': 'fake_schema',
-                   'table': 'fake_table'
-               }
+        assert json.loads(
+            self.snowflake.create_query_tag(
+                {'schema': 'fake_schema', 'table': 'fake_table'}
+            )
+        ) == {
+            'ppw_component': 'fastsync',
+            'tap_id': 'fake_tap',
+            'database': 'fake_db',
+            'schema': 'fake_schema',
+            'table': 'fake_table',
+        }
 
         # passing partial query_props
         self.snowflake.connection_config['tap_id'] = 'fake_tap'
-        assert json.loads(self.snowflake.create_query_tag({'schema': 'fake_schema'})) == {
+        assert json.loads(
+            self.snowflake.create_query_tag({'schema': 'fake_schema'})
+        ) == {
             'ppw_component': 'fastsync',
             'tap_id': 'fake_tap',
             'database': 'fake_db',
             'schema': 'fake_schema',
-            'table': None
+            'table': None,
         }
 
         # passing partial query_props
@@ -350,17 +396,19 @@ def test_create_query_tag(self):
             'tap_id': 'fake_tap',
             'database': 'fake_db',
             'schema': None,
-            'table': 'fake_table'
+            'table': 'fake_table',
         }
 
         # passing not supported query_props
         self.snowflake.connection_config['tap_id'] = 'fake_tap'
-        assert json.loads(self.snowflake.create_query_tag({'fake_prop': 'fake_value'})) == {
+        assert json.loads(
+            self.snowflake.create_query_tag({'fake_prop': 'fake_value'})
+        ) == {
             'ppw_component': 'fastsync',
             'tap_id': 'fake_tap',
             'database': 'fake_db',
             'schema': None,
-            'table': None
+            'table': None,
         }
 
     def test_obfuscate_columns_case1(self):
@@ -389,7 +437,7 @@ def test_obfuscate_columns_case2(self):
                 {
                     'field_id': 'col_7',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'RANDOM'
+                    'type': 'RANDOM',
                 }
             ]
         }
@@ -412,34 +460,34 @@ def test_obfuscate_columns_case3(self):
                 {
                     'field_id': 'col_1',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'SET-NULL'
+                    'type': 'SET-NULL',
                 },
                 {
                     'field_id': 'col_2',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'MASK-HIDDEN'
+                    'type': 'MASK-HIDDEN',
                 },
                 {
                     'field_id': 'col_3',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'MASK-DATE'
+                    'type': 'MASK-DATE',
                 },
                 {
                     'field_id': 'col_4',
                     'tap_stream_name': 'public-my_table',
                     'safe_field_id': '"COL_4"',
-                    'type': 'MASK-NUMBER'
+                    'type': 'MASK-NUMBER',
                 },
                 {
                     'field_id': 'col_5',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'HASH'
+                    'type': 'HASH',
                 },
                 {
                     'field_id': 'col_6',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'HASH-SKIP-FIRST-5'
-                }
+                    'type': 'HASH-SKIP-FIRST-5',
+                },
             ]
         }
 
@@ -455,7 +503,8 @@ def test_obfuscate_columns_case3(self):
                 '"COL_4" = 0, '
                 '"COL_5" = SHA2("COL_5", 256), '
                 '"COL_6" = CONCAT(SUBSTRING("COL_6", 1, 5), SHA2(SUBSTRING("COL_6", 5 + 1), 256));'
-            ])
+            ],
+        )
 
     def test_obfuscate_columns_case4(self):
         """
@@ -470,59 +519,44 @@ def test_obfuscate_columns_case4(self):
                 {
                     'field_id': 'col_1',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'SET-NULL'
+                    'type': 'SET-NULL',
                 },
                 {
                     'field_id': 'col_2',
                     'tap_stream_name': 'public-my_table',
                     'type': 'MASK-HIDDEN',
                     'when': [
-                        {
-                            'column': 'col_4',
-                            'safe_column': '"COL_4"',
-                            'equals': None
-                        },
+                        {'column': 'col_4', 'safe_column': '"COL_4"', 'equals': None},
                         {
                             'column': 'col_1',
-                        }
-                    ]
+                        },
+                    ],
                 },
                 {
                     'field_id': 'col_3',
                     'tap_stream_name': 'public-my_table',
                     'type': 'MASK-DATE',
-                    'when': [
-                        {
-                            'column': 'col_5',
-                            'equals': 'some_value'
-                        }
-                    ]
+                    'when': [{'column': 'col_5', 'equals': 'some_value'}],
                 },
                 {
                     'field_id': 'col_4',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'MASK-NUMBER'
+                    'type': 'MASK-NUMBER',
                 },
                 {
                     'field_id': 'col_5',
                     'tap_stream_name': 'public-my_table',
-                    'type': 'HASH'
+                    'type': 'HASH',
                 },
                 {
                     'field_id': 'col_6',
                     'tap_stream_name': 'public-my_table',
                     'type': 'HASH-SKIP-FIRST-5',
                     'when': [
-                        {
-                            'column': 'col_1',
-                            'equals': 30
-                        },
-                        {
-                            'column': 'col_2',
-                            'regex_match': r'[0-9]{3}\.[0-9]{3}'
-                        }
-                    ]
-                }
+                        {'column': 'col_1', 'equals': 30},
+                        {'column': 'col_2', 'regex_match': r'[0-9]{3}\.[0-9]{3}'},
+                    ],
+                },
             ]
         }
 
@@ -537,9 +571,9 @@ def test_obfuscate_columns_case4(self):
                 'UPDATE "MY_SCHEMA"."MY_TABLE_TEMP" SET '
                 '"COL_6" = CONCAT(SUBSTRING("COL_6", 1, 5), SHA2(SUBSTRING("COL_6", 5 + 1), 256)) '
                 'WHERE ("COL_1" = 30) AND ("COL_2" '
-                'REGEXP \'[0-9]{3}\.[0-9]{3}\');',  # pylint: disable=W1401
-                'UPDATE "MY_SCHEMA"."MY_TABLE_TEMP" SET "COL_1" = NULL, "COL_4" = 0, "COL_5" = SHA2("COL_5", 256);'
-            ]
+                'REGEXP \'[0-9]{3}\.[0-9]{3}\');',  # pylint: disable=W1401  # noqa: W605
+                'UPDATE "MY_SCHEMA"."MY_TABLE_TEMP" SET "COL_1" = NULL, "COL_4" = 0, "COL_5" = SHA2("COL_5", 256);',
+            ],
         )
 
     # pylint: disable=invalid-name
@@ -551,7 +585,10 @@ def test_default_archive_destination(self):
         self.snowflake.s3.copy_object = mock_copy_object
         self.snowflake.connection_config['s3_bucket'] = 'some_bucket'
         self.snowflake.copy_to_archive(
-            'snowflake-import/ppw_20210615115603_fastsync.csv.gz', 'some-tap', 'some_schema.some_table')
+            'snowflake-import/ppw_20210615115603_fastsync.csv.gz',
+            'some-tap',
+            'some_schema.some_table',
+        )
 
         mock_copy_object.assert_called_with(
             Bucket='some_bucket',
@@ -561,9 +598,10 @@ def test_default_archive_destination(self):
                 'tap': 'some-tap',
                 'schema': 'some_schema',
                 'table': 'some_table',
-                'archived-by': 'pipelinewise_fastsync_postgres_to_snowflake'
+                'archived-by': 'pipelinewise_fastsync_postgres_to_snowflake',
             },
-            MetadataDirective='REPLACE')
+            MetadataDirective='REPLACE',
+        )
 
     # pylint: disable=invalid-name
     def test_custom_archive_destination(self):
@@ -573,10 +611,17 @@ def test_custom_archive_destination(self):
         mock_copy_object = MagicMock()
         self.snowflake.s3.copy_object = mock_copy_object
         self.snowflake.connection_config['s3_bucket'] = 'some_bucket'
-        self.snowflake.connection_config['archive_load_files_s3_bucket'] = 'archive_bucket'
-        self.snowflake.connection_config['archive_load_files_s3_prefix'] = 'archive_folder'
+        self.snowflake.connection_config[
+            'archive_load_files_s3_bucket'
+        ] = 'archive_bucket'
+        self.snowflake.connection_config[
+            'archive_load_files_s3_prefix'
+        ] = 'archive_folder'
         self.snowflake.copy_to_archive(
-            'snowflake-import/ppw_20210615115603_fastsync.csv.gz', 'some-tap', 'some_schema.some_table')
+            'snowflake-import/ppw_20210615115603_fastsync.csv.gz',
+            'some-tap',
+            'some_schema.some_table',
+        )
 
         mock_copy_object.assert_called_with(
             Bucket='archive_bucket',
@@ -586,9 +631,10 @@ def test_custom_archive_destination(self):
                 'tap': 'some-tap',
                 'schema': 'some_schema',
                 'table': 'some_table',
-                'archived-by': 'pipelinewise_fastsync_postgres_to_snowflake'
+                'archived-by': 'pipelinewise_fastsync_postgres_to_snowflake',
             },
-            MetadataDirective='REPLACE')
+            MetadataDirective='REPLACE',
+        )
 
     # pylint: disable=invalid-name
     def test_copied_archive_metadata(self):
@@ -596,13 +642,18 @@ def test_copied_archive_metadata(self):
         Validate parameters passed to s3 copy_object method when custom s3 bucket and folder are not defined
         """
         mock_head_object = MagicMock()
-        mock_head_object.return_value = {'Metadata': {'copied-old-key': 'copied-old-value'}}
+        mock_head_object.return_value = {
+            'Metadata': {'copied-old-key': 'copied-old-value'}
+        }
         mock_copy_object = MagicMock()
         self.snowflake.s3.copy_object = mock_copy_object
         self.snowflake.s3.head_object = mock_head_object
         self.snowflake.connection_config['s3_bucket'] = 'some_bucket'
         self.snowflake.copy_to_archive(
-            'snowflake-import/ppw_20210615115603_fastsync.csv.gz', 'some-tap', 'some_schema.some_table')
+            'snowflake-import/ppw_20210615115603_fastsync.csv.gz',
+            'some-tap',
+            'some_schema.some_table',
+        )
 
         mock_copy_object.assert_called_with(
             Bucket='some_bucket',
@@ -613,6 +664,7 @@ def test_copied_archive_metadata(self):
                 'tap': 'some-tap',
                 'schema': 'some_schema',
                 'table': 'some_table',
-                'archived-by': 'pipelinewise_fastsync_postgres_to_snowflake'
+                'archived-by': 'pipelinewise_fastsync_postgres_to_snowflake',
             },
-            MetadataDirective='REPLACE')
+            MetadataDirective='REPLACE',
+        )
diff --git a/tests/units/fastsync/commons/test_fastsync_utils.py b/tests/units/fastsync/commons/test_fastsync_utils.py
index a81289c91..d138fdb60 100644
--- a/tests/units/fastsync/commons/test_fastsync_utils.py
+++ b/tests/units/fastsync/commons/test_fastsync_utils.py
@@ -19,18 +19,14 @@ class MySqlMock:
     """
 
     def fetch_current_log_pos(self):
-        return {
-            'log_file': 'mysqld-bin.000001',
-            'log_pos': '123456',
-            'version': 1
-        }
+        return {'log_file': 'mysqld-bin.000001', 'log_pos': '123456', 'version': 1}
 
     # pylint: disable=unused-argument
     def fetch_current_incremental_key_pos(self, table, replication_key):
         return {
             'replication_key': replication_key,
             'replication_key_value': 123456,
-            'version': 1
+            'version': 1,
         }
 
 
@@ -40,17 +36,14 @@ class PostgresMock:
     """
 
     def fetch_current_log_pos(self):
-        return {
-            'lsn': '16/B374D848',
-            'version': 1
-        }
+        return {'lsn': '16/B374D848', 'version': 1}
 
     # pylint: disable=unused-argument
     def fetch_current_incremental_key_pos(self, table, replication_key):
         return {
             'replication_key': replication_key,
             'replication_key_value': 123456,
-            'version': 1
+            'version': 1,
         }
 
 
@@ -61,9 +54,7 @@ class S3CsvMock:
 
     # pylint: disable=unused-argument
     def fetch_current_incremental_key_pos(self, table, replication_key):
-        return {
-            'modified_since': '2019-11-15T07:39:44.171098'
-        }
+        return {'modified_since': '2019-11-15T07:39:44.171098'}
 
 
 class TestFastSyncUtils(TestCase):
@@ -75,56 +66,57 @@ def test_tablename_to_dict(self):
         """Test identifying schema and table names from fully qualified table names"""
 
         # Format: <CATALOG>.<SCHEMA>.<TABLE>
-        assert utils.tablename_to_dict('my_catalog.my_schema.my_table') == \
-               {
-                   'catalog_name': 'my_catalog',
-                   'schema_name': 'my_schema',
-                   'table_name': 'my_table',
-                   'temp_table_name': 'my_table_temp'
-               }
+        assert utils.tablename_to_dict('my_catalog.my_schema.my_table') == {
+            'catalog_name': 'my_catalog',
+            'schema_name': 'my_schema',
+            'table_name': 'my_table',
+            'temp_table_name': 'my_table_temp',
+        }
 
         # Format: <SCHEMA>.<TABLE>
-        assert utils.tablename_to_dict('my_schema.my_table') == \
-               {
-                   'catalog_name': None,
-                   'schema_name': 'my_schema',
-                   'table_name': 'my_table',
-                   'temp_table_name': 'my_table_temp'
-               }
+        assert utils.tablename_to_dict('my_schema.my_table') == {
+            'catalog_name': None,
+            'schema_name': 'my_schema',
+            'table_name': 'my_table',
+            'temp_table_name': 'my_table_temp',
+        }
 
         # Format: <TABLE>
-        assert utils.tablename_to_dict('my_table') == \
-               {
-                   'catalog_name': None,
-                   'schema_name': None,
-                   'table_name': 'my_table',
-                   'temp_table_name': 'my_table_temp'
-               }
+        assert utils.tablename_to_dict('my_table') == {
+            'catalog_name': None,
+            'schema_name': None,
+            'table_name': 'my_table',
+            'temp_table_name': 'my_table_temp',
+        }
 
         # Format: <CATALOG>.<SCHEMA>.<TABLE>.<SOMETHING>
-        assert utils.tablename_to_dict('my_catalog.my_schema.my_table.foo') == \
-               {
-                   'catalog_name': 'my_catalog',
-                   'schema_name': 'my_schema',
-                   'table_name': 'my_table_foo',
-                   'temp_table_name': 'my_table_foo_temp'
-               }
+        assert utils.tablename_to_dict('my_catalog.my_schema.my_table.foo') == {
+            'catalog_name': 'my_catalog',
+            'schema_name': 'my_schema',
+            'table_name': 'my_table_foo',
+            'temp_table_name': 'my_table_foo_temp',
+        }
 
         # Format: <CATALOG>.<SCHEMA>.<TABLE>.<SOMETHING>
         # Custom separator
-        assert utils.tablename_to_dict('my_catalog-my_schema-my_table-foo', separator='-') == \
-               {
-                   'catalog_name': 'my_catalog',
-                   'schema_name': 'my_schema',
-                   'table_name': 'my_table_foo',
-                   'temp_table_name': 'my_table_foo_temp'
-               }
+        assert utils.tablename_to_dict(
+            'my_catalog-my_schema-my_table-foo', separator='-'
+        ) == {
+            'catalog_name': 'my_catalog',
+            'schema_name': 'my_schema',
+            'table_name': 'my_table_foo',
+            'temp_table_name': 'my_table_foo_temp',
+        }
 
     def test_get_tables_from_properties(self):
         """Test getting selected tables from tap properties JSON"""
         # Load MySQL and Postgres properties JSON
-        mysql_properties = utils.load_json('{}/properties_mysql.json'.format(RESOURCES_DIR))
-        postgres_properties = utils.load_json('{}/properties_postgres.json'.format(RESOURCES_DIR))
+        mysql_properties = utils.load_json(
+            '{}/properties_mysql.json'.format(RESOURCES_DIR)
+        )
+        postgres_properties = utils.load_json(
+            '{}/properties_postgres.json'.format(RESOURCES_DIR)
+        )
 
         # Get list of selected tables
         # MySQL and Postgres schemas defined at different keys. get_tables_from_properties function
@@ -133,18 +125,13 @@ def test_get_tables_from_properties(self):
         postgres_tables = utils.get_tables_from_properties(postgres_properties)
 
         # MySQL schema
-        assert mysql_tables == \
-               {
-                   'mysql_source_db.address',
-                   'mysql_source_db.order',
-                   'mysql_source_db.weight_unit'
-               }
-
-        assert postgres_tables == \
-               {
-                   'public.city',
-                   'public.country'
-               }
+        assert mysql_tables == {
+            'mysql_source_db.address',
+            'mysql_source_db.order',
+            'mysql_source_db.weight_unit',
+        }
+
+        assert postgres_tables == {'public.city', 'public.country'}
 
     def test_get_tables_from_properties_for_s3_csv(self):
         properties = utils.load_json('{}/properties_s3_csv.json'.format(RESOURCES_DIR))
@@ -152,61 +139,71 @@ def test_get_tables_from_properties_for_s3_csv(self):
         s3_csv_tables = utils.get_tables_from_properties(properties)
 
         # MySQL schema
-        assert s3_csv_tables == \
-               {
-                   'applications',
-                   'candidate_survey_questions',
-                   'interviews',
-               }
+        assert s3_csv_tables == {
+            'applications',
+            'candidate_survey_questions',
+            'interviews',
+        }
 
     def test_get_bookmark_for_table_mysql(self):
         """Test bookmark extractors for MySQL taps"""
         # Load MySQL and Postgres properties JSON
-        mysql_properties = utils.load_json('{}/properties_mysql.json'.format(RESOURCES_DIR))
+        mysql_properties = utils.load_json(
+            '{}/properties_mysql.json'.format(RESOURCES_DIR)
+        )
 
         # MySQL: mysql_source_db.order is LOG_BASED
-        assert utils.get_bookmark_for_table('mysql_source_db.order', mysql_properties, MySqlMock()) == {
-            'log_file': 'mysqld-bin.000001',
-            'log_pos': '123456',
-            'version': 1
-        }
+        assert utils.get_bookmark_for_table(
+            'mysql_source_db.order', mysql_properties, MySqlMock()
+        ) == {'log_file': 'mysqld-bin.000001', 'log_pos': '123456', 'version': 1}
 
         # MySQL: mysql_source_db.address is INCREMENTAL
-        assert utils.get_bookmark_for_table('mysql_source_db.address', mysql_properties, MySqlMock()) == {
+        assert utils.get_bookmark_for_table(
+            'mysql_source_db.address', mysql_properties, MySqlMock()
+        ) == {
             'replication_key': 'date_updated',
             'replication_key_value': 123456,
-            'version': 1
+            'version': 1,
         }
 
         # MySQL mysql_source_db.foo not exists
-        assert utils.get_bookmark_for_table('mysql_source_db.foo', mysql_properties, MySqlMock()) == {}
+        assert (
+            utils.get_bookmark_for_table(
+                'mysql_source_db.foo', mysql_properties, MySqlMock()
+            )
+            == {}
+        )
 
     def test_get_bookmark_for_table_postgresl(self):
         """Test bookmark extractors for Postgres taps"""
         # Load Postgres properties JSON
-        postgres_properties = utils.load_json('{}/properties_postgres.json'.format(RESOURCES_DIR))
+        postgres_properties = utils.load_json(
+            '{}/properties_postgres.json'.format(RESOURCES_DIR)
+        )
 
         # Postgres: public.countrylanguage is LOG_BASED
-        assert utils.get_bookmark_for_table('public.countrylanguage', postgres_properties, PostgresMock()) == {
-            'lsn': '16/B374D848',
-            'version': 1
-        }
+        assert utils.get_bookmark_for_table(
+            'public.countrylanguage', postgres_properties, PostgresMock()
+        ) == {'lsn': '16/B374D848', 'version': 1}
 
         # Postgres: postgres_source_db.public.city is INCREMENTAL
-        assert utils.get_bookmark_for_table('public.city',
-                                            postgres_properties,
-                                            PostgresMock(),
-                                            dbname='postgres_source_db') == {
-                                                'replication_key': 'id',
-                                                'replication_key_value': 123456,
-                                                'version': 1
-                                            }
+        assert utils.get_bookmark_for_table(
+            'public.city',
+            postgres_properties,
+            PostgresMock(),
+            dbname='postgres_source_db',
+        ) == {'replication_key': 'id', 'replication_key_value': 123456, 'version': 1}
 
         # Postgres: postgres_source_db.public.foo not exists
-        assert utils.get_bookmark_for_table('public.foo',
-                                            postgres_properties,
-                                            PostgresMock(),
-                                            dbname='postgres_source_db') == {}
+        assert (
+            utils.get_bookmark_for_table(
+                'public.foo',
+                postgres_properties,
+                PostgresMock(),
+                dbname='postgres_source_db',
+            )
+            == {}
+        )
 
     def test_get_bookmark_for_table_tap_s3_csv(self):
         """Test bookmark extractors for S3 CSV taps"""
@@ -214,12 +211,19 @@ def test_get_bookmark_for_table_tap_s3_csv(self):
         properties = utils.load_json('{}/properties_s3_csv.json'.format(RESOURCES_DIR))
 
         # applications is INCREMENTAL
-        assert utils.get_bookmark_for_table('applications', properties, S3CsvMock()) == {
+        assert utils.get_bookmark_for_table(
+            'applications', properties, S3CsvMock()
+        ) == {
             'modified_since': '2019-11-15T07:39:44.171098',
         }
 
         # candidate_survey_questions is Full table
-        assert utils.get_bookmark_for_table('candidate_survey_questions', properties, S3CsvMock()) == {}
+        assert (
+            utils.get_bookmark_for_table(
+                'candidate_survey_questions', properties, S3CsvMock()
+            )
+            == {}
+        )
 
         # foo not exists
         assert utils.get_bookmark_for_table('foo', properties, S3CsvMock()) == {}
@@ -238,7 +242,10 @@ def test_get_target_schema(self):
 
         # Default_target_schema should define the target_schema
         target_config_with_default = {'default_target_schema': 'target_schema'}
-        assert utils.get_target_schema(target_config_with_default, 'foo.foo') == 'target_schema'
+        assert (
+            utils.get_target_schema(target_config_with_default, 'foo.foo')
+            == 'target_schema'
+        )
 
         # Empty schema_mapping should raise exception
         with pytest.raises(Exception):
@@ -247,28 +254,33 @@ def test_get_target_schema(self):
 
         # Missing schema in schema_mapping should raise exception
         with pytest.raises(Exception):
-            target_config_with_missing_schema_mapping = {'schema_mapping': {'foo2': {'target_schema': 'foo2'}}}
-            utils.get_target_schema(target_config_with_missing_schema_mapping, 'foo.foo')
+            target_config_with_missing_schema_mapping = {
+                'schema_mapping': {'foo2': {'target_schema': 'foo2'}}
+            }
+            utils.get_target_schema(
+                target_config_with_missing_schema_mapping, 'foo.foo'
+            )
 
         # Target schema should be extracted from schema_mapping
-        target_config_with_schema_mapping = {'schema_mapping': {'foo': {'target_schema': 'foo'}}}
-        assert utils.get_target_schema(target_config_with_schema_mapping, 'foo.foo') == 'foo'
+        target_config_with_schema_mapping = {
+            'schema_mapping': {'foo': {'target_schema': 'foo'}}
+        }
+        assert (
+            utils.get_target_schema(target_config_with_schema_mapping, 'foo.foo')
+            == 'foo'
+        )
 
         # If target schema exist in schema_mapping then should not use the default_target_schema
         target_config = {
             'default_target_schema': 'target_schema',
-            'schema_mapping': {'foo': {'target_schema': 'foo'}}
+            'schema_mapping': {'foo': {'target_schema': 'foo'}},
         }
         assert utils.get_target_schema(target_config, 'foo.foo') == 'foo'
 
         # If target schema not exist in schema_mapping then should return the default_target_schema
         target_config = {
             'default_target_schema': 'target_schema',
-            'schema_mapping': {
-                'foo2': {
-                    'target_schema': 'foo2'
-                }
-            }
+            'schema_mapping': {'foo2': {'target_schema': 'foo2'}},
         }
         assert utils.get_target_schema(target_config, 'foo.foo') == 'target_schema'
 
@@ -279,63 +291,75 @@ def test_get_grantees(self):
         assert utils.get_grantees(target_config_with_empty_grantees, 'foo.foo') == []
 
         # Empty default_target_schema_select_permissions should return empty list
-        target_config_with_default_empty = {'default_target_schema_select_permissions': ''}
+        target_config_with_default_empty = {
+            'default_target_schema_select_permissions': ''
+        }
         assert utils.get_grantees(target_config_with_default_empty, 'foo.foo') == []
 
         # default_target_schema_select_permissions as string should return list
-        target_config_with_default_as_string = {'default_target_schema_select_permissions': 'grantee'}
-        assert utils.get_grantees(target_config_with_default_as_string, 'foo.foo') == ['grantee']
+        target_config_with_default_as_string = {
+            'default_target_schema_select_permissions': 'grantee'
+        }
+        assert utils.get_grantees(target_config_with_default_as_string, 'foo.foo') == [
+            'grantee'
+        ]
 
         # default_target_schema_select_permissions as list should return list
-        target_config_with_default_as_list = {'default_target_schema_select_permissions': ['grantee1']}
-        assert utils.get_grantees(target_config_with_default_as_list, 'foo.foo') == ['grantee1']
+        target_config_with_default_as_list = {
+            'default_target_schema_select_permissions': ['grantee1']
+        }
+        assert utils.get_grantees(target_config_with_default_as_list, 'foo.foo') == [
+            'grantee1'
+        ]
 
         # default_target_schema_select_permissions as list should return list
-        target_config_with_default_as_list = {'default_target_schema_select_permissions': ['grantee1', 'grantee2']}
-        assert utils.get_grantees(target_config_with_default_as_list, 'foo.foo') == ['grantee1', 'grantee2']
+        target_config_with_default_as_list = {
+            'default_target_schema_select_permissions': ['grantee1', 'grantee2']
+        }
+        assert utils.get_grantees(target_config_with_default_as_list, 'foo.foo') == [
+            'grantee1',
+            'grantee2',
+        ]
 
         # Empty schema_mapping should return empty list
         target_config_with_empty_schema_mapping = {'schema_mapping': {}}
-        assert utils.get_grantees(target_config_with_empty_schema_mapping, 'foo.foo') == []
+        assert (
+            utils.get_grantees(target_config_with_empty_schema_mapping, 'foo.foo') == []
+        )
 
         # Missing schema in schema_mapping should return empty list
         target_config_with_missing_schema_mapping = {
-            'schema_mapping': {
-                'foo2': {
-                    'target_schema_select_permissions': 'grantee'
-                }
-            }
+            'schema_mapping': {'foo2': {'target_schema_select_permissions': 'grantee'}}
         }
-        assert utils.get_grantees(target_config_with_missing_schema_mapping, 'foo.foo') == []
+        assert (
+            utils.get_grantees(target_config_with_missing_schema_mapping, 'foo.foo')
+            == []
+        )
 
         # Grantees as string should be extracted from schema_mapping
         target_config_with_missing_schema_mapping = {
-            'schema_mapping': {
-                'foo': {
-                    'target_schema_select_permissions': 'grantee'
-                }
-            }
+            'schema_mapping': {'foo': {'target_schema_select_permissions': 'grantee'}}
         }
-        assert utils.get_grantees(target_config_with_missing_schema_mapping, 'foo.foo') == ['grantee']
+        assert utils.get_grantees(
+            target_config_with_missing_schema_mapping, 'foo.foo'
+        ) == ['grantee']
 
         # Grantees as list should be extracted from schema_mapping
         target_config_with_missing_schema_mapping = {
             'schema_mapping': {
-                'foo': {
-                    'target_schema_select_permissions': ['grantee1', 'grantee2']
-                }
+                'foo': {'target_schema_select_permissions': ['grantee1', 'grantee2']}
             }
         }
-        assert utils.get_grantees(target_config_with_missing_schema_mapping, 'foo.foo') == ['grantee1', 'grantee2']
+        assert utils.get_grantees(
+            target_config_with_missing_schema_mapping, 'foo.foo'
+        ) == ['grantee1', 'grantee2']
 
         # If grantees exist in schema_mapping then should not use the default_target_schema_select_permissions
         target_config = {
             'default_target_schema_select_permissions': ['grantee1', 'grantee2'],
             'schema_mapping': {
-                'foo': {
-                    'target_schema_select_permissions': ['grantee3', 'grantee4']
-                }
-            }
+                'foo': {'target_schema_select_permissions': ['grantee3', 'grantee4']}
+            },
         }
         assert utils.get_grantees(target_config, 'foo.foo') == ['grantee3', 'grantee4']
 
@@ -343,10 +367,8 @@ def test_get_grantees(self):
         target_config = {
             'default_target_schema_select_permissions': ['grantee1', 'grantee2'],
             'schema_mapping': {
-                'foo2': {
-                    'target_schema_select_permissions': ['grantee3', 'grantee4']
-                }
-            }
+                'foo2': {'target_schema_select_permissions': ['grantee3', 'grantee4']}
+            },
         }
         assert utils.get_grantees(target_config, 'foo.foo') == ['grantee1', 'grantee2']
 
@@ -354,27 +376,29 @@ def test_get_grantees(self):
         target_config_with_default_as_dict = {
             'default_target_schema_select_permissions': {
                 'users': 'grantee_user1',
-                'groups': 'grantee_group1'
+                'groups': 'grantee_group1',
             }
         }
         assert utils.get_grantees(target_config_with_default_as_dict, 'foo.foo') == {
             'users': ['grantee_user1'],
-            'groups': ['grantee_group1']
+            'groups': ['grantee_group1'],
         }
 
         # default_target_schema_select_permissions as dict with list should return dict
         target_config_with_default_as_dict = {
             'default_target_schema_select_permissions': {
                 'users': ['grantee_user1', 'grantee_user2'],
-                'groups': ['grantee_group1', 'grantee_group2']
+                'groups': ['grantee_group1', 'grantee_group2'],
             }
         }
         assert utils.get_grantees(target_config_with_default_as_dict, 'foo.foo') == {
             'users': ['grantee_user1', 'grantee_user2'],
-            'groups': ['grantee_group1', 'grantee_group2']
+            'groups': ['grantee_group1', 'grantee_group2'],
         }
 
-    @patch('pipelinewise.fastsync.commons.utils.multiprocessing.cpu_count', return_value=10)
+    @patch(
+        'pipelinewise.fastsync.commons.utils.multiprocessing.cpu_count', return_value=10
+    )
     def test_get_cpu_cores_should_succeed(self, _):
         assert utils.get_cpu_cores() == 10
 
@@ -384,29 +408,36 @@ def test_check_config_with_all_required_keys_present_should_succeed(self):
 
         utils.check_config(config, required_keys)
 
-    def test_check_config_with_some_required_keys_not_present_should_raise_exception(self):
+    def test_check_config_with_some_required_keys_not_present_should_raise_exception(
+        self,
+    ):
         config = {'key1': 1, 'key2': 2, 'key3': 3}
         required_keys = {'key1', 'key4'}
 
         with pytest.raises(Exception):
             utils.check_config(config, required_keys)
 
-    @patch('pipelinewise.fastsync.commons.utils.multiprocessing.cpu_count', return_value=10)
+    @patch(
+        'pipelinewise.fastsync.commons.utils.multiprocessing.cpu_count', return_value=10
+    )
     def test_get_pool_size_without_custom_size(self, _):
         """
         Calling get_pool_size without providing fastsync_parallelism return cpu core count
         """
         assert utils.get_pool_size({}) == 10
 
-    @patch('pipelinewise.fastsync.commons.utils.multiprocessing.cpu_count', return_value=10)
+    @patch(
+        'pipelinewise.fastsync.commons.utils.multiprocessing.cpu_count', return_value=10
+    )
     def test_get_pool_size_with_custom_size_small(self, _):
         """
         Calling get_pool_size with fastsync_parallelism smaller than cpu core count return the fastsync_parallelism
         """
         assert utils.get_pool_size({'fastsync_parallelism': 2}) == 2
 
-
-    @patch('pipelinewise.fastsync.commons.utils.multiprocessing.cpu_count', return_value=10)
+    @patch(
+        'pipelinewise.fastsync.commons.utils.multiprocessing.cpu_count', return_value=10
+    )
     def test_get_pool_size_with_custom_size_big(self, _):
         """
         Calling get_pool_size with fastsync_parallelism greater than cpu core count return the cpu core count
@@ -417,19 +448,23 @@ def test_get_pool_size_with_custom_size_big(self, _):
     @mock.patch('pipelinewise.fastsync.commons.utils.check_config')
     @mock.patch('pipelinewise.fastsync.commons.utils.load_json')
     @mock.patch('argparse.ArgumentParser.parse_args')
-    def test_parse_args_without_tables(self, mock_args, load_json_mock, check_config_mock, get_tables_prop_mock):
+    def test_parse_args_without_tables(
+        self, mock_args, load_json_mock, check_config_mock, get_tables_prop_mock
+    ):
         """
         test args parsing:
             not tables are specified, this should return a tables equal to the list of selected tables
         """
-        mock_args.return_value = argparse.Namespace(**{
-            'tap': './tap.yml',
-            'properties': './prop.json',
-            'transform': None,
-            'target': './target.yml',
-            'tables': None,
-            'temp_dir': './'
-        })
+        mock_args.return_value = argparse.Namespace(
+            **{
+                'tap': './tap.yml',
+                'properties': './prop.json',
+                'transform': None,
+                'target': './target.yml',
+                'tables': None,
+                'temp_dir': './',
+            }
+        )
 
         load_json_mock.return_value = {}
         check_config_mock.return_value = None
@@ -442,34 +477,39 @@ def test_parse_args_without_tables(self, mock_args, load_json_mock, check_config
         self.assertEqual(check_config_mock.call_count, 2)
 
         self.assertDictEqual(
-            vars(args), {
+            vars(args),
+            {
                 'tables': {'schema.table_1', 'schema.table_2'},
                 'tap': {},
                 'target': {},
                 'transform': {},
                 'properties': {},
-                'temp_dir': './'
-            }
+                'temp_dir': './',
+            },
         )
 
     @mock.patch('pipelinewise.fastsync.commons.utils.get_tables_from_properties')
     @mock.patch('pipelinewise.fastsync.commons.utils.check_config')
     @mock.patch('pipelinewise.fastsync.commons.utils.load_json')
     @mock.patch('argparse.ArgumentParser.parse_args')
-    def test_parse_args_with_all_tables(self, mock_args, load_json_mock, check_config_mock, get_tables_prop_mock):
+    def test_parse_args_with_all_tables(
+        self, mock_args, load_json_mock, check_config_mock, get_tables_prop_mock
+    ):
         """
         test args parsing:
             all selected tables are specified
         """
-        mock_args.return_value = argparse.Namespace(**{
-            'tap': './tap.yml',
-            'properties': './prop.json',
-            'transform': None,
-            'drop_pg_slot': True,
-            'target': './target.yml',
-            'tables': 'schema.table_1,schema.table_2',
-            'temp_dir': './'
-        })
+        mock_args.return_value = argparse.Namespace(
+            **{
+                'tap': './tap.yml',
+                'properties': './prop.json',
+                'transform': None,
+                'drop_pg_slot': True,
+                'target': './target.yml',
+                'tables': 'schema.table_1,schema.table_2',
+                'temp_dir': './',
+            }
+        )
 
         load_json_mock.return_value = {}
         check_config_mock.return_value = None
@@ -482,34 +522,39 @@ def test_parse_args_with_all_tables(self, mock_args, load_json_mock, check_confi
         self.assertEqual(check_config_mock.call_count, 2)
 
         self.assertDictEqual(
-            vars(args), {
+            vars(args),
+            {
                 'tables': {'schema.table_1', 'schema.table_2'},
                 'drop_pg_slot': True,
                 'tap': {},
                 'target': {},
                 'transform': {},
                 'properties': {},
-                'temp_dir': './'
-            }
+                'temp_dir': './',
+            },
         )
 
     @mock.patch('pipelinewise.fastsync.commons.utils.get_tables_from_properties')
     @mock.patch('pipelinewise.fastsync.commons.utils.check_config')
     @mock.patch('pipelinewise.fastsync.commons.utils.load_json')
     @mock.patch('argparse.ArgumentParser.parse_args')
-    def test_parse_args_with_table_found(self, mock_args, load_json_mock, check_config_mock, get_tables_prop_mock):
+    def test_parse_args_with_table_found(
+        self, mock_args, load_json_mock, check_config_mock, get_tables_prop_mock
+    ):
         """
         test args parsing:
             one table is specified out of 2, this should return a drop_pg_slot = False
         """
-        mock_args.return_value = argparse.Namespace(**{
-            'tap': './tap.yml',
-            'properties': './prop.json',
-            'transform': None,
-            'target': './target.yml',
-            'tables': 'schema.table_2',
-            'temp_dir': './'
-        })
+        mock_args.return_value = argparse.Namespace(
+            **{
+                'tap': './tap.yml',
+                'properties': './prop.json',
+                'transform': None,
+                'target': './target.yml',
+                'tables': 'schema.table_2',
+                'temp_dir': './',
+            }
+        )
 
         load_json_mock.return_value = {}
         check_config_mock.return_value = None
@@ -522,34 +567,38 @@ def test_parse_args_with_table_found(self, mock_args, load_json_mock, check_conf
         self.assertEqual(check_config_mock.call_count, 2)
 
         self.assertDictEqual(
-            vars(args), {
+            vars(args),
+            {
                 'tables': {'schema.table_2'},
                 'tap': {},
                 'target': {},
                 'transform': {},
                 'properties': {},
-                'temp_dir': './'
-            }
+                'temp_dir': './',
+            },
         )
 
     @mock.patch('pipelinewise.fastsync.commons.utils.get_tables_from_properties')
     @mock.patch('pipelinewise.fastsync.commons.utils.check_config')
     @mock.patch('pipelinewise.fastsync.commons.utils.load_json')
     @mock.patch('argparse.ArgumentParser.parse_args')
-    def test_parse_args_with_table_not_selected(self, mock_args, load_json_mock, check_config_mock,
-                                                get_tables_prop_mock):
+    def test_parse_args_with_table_not_selected(
+        self, mock_args, load_json_mock, check_config_mock, get_tables_prop_mock
+    ):
         """
         test args parsing:
             one table not found in selected tables, this should throw a  NotSelectedTableException exception
         """
-        mock_args.return_value = argparse.Namespace(**{
-            'tap': './tap.yml',
-            'properties': './prop.json',
-            'transform': None,
-            'target': './target.yml',
-            'tables': 'schema.table_not_selected',
-            'temp_dir': './'
-        })
+        mock_args.return_value = argparse.Namespace(
+            **{
+                'tap': './tap.yml',
+                'properties': './prop.json',
+                'transform': None,
+                'target': './target.yml',
+                'tables': 'schema.table_not_selected',
+                'temp_dir': './',
+            }
+        )
 
         load_json_mock.return_value = {}
         check_config_mock.return_value = None
@@ -570,9 +619,15 @@ def test_gen_export_filename(self):
         # including timestamps with milliseconds and random generated string
         #
         # Example: pipelinewise_tap_table_20210316-111338-878470_fastsync_L5C6VG9W.csv.gz
-        self.assertRegex(utils.gen_export_filename('tap', 'table'),
-                         r'pipelinewise_tap_table_(\d{8})-(\d{6})-(\d{6})_fastsync_(.{8}).csv.gz')
+        self.assertRegex(
+            utils.gen_export_filename('tap', 'table'),
+            r'pipelinewise_tap_table_(\d{8})-(\d{6})-(\d{6})_fastsync_(.{8}).csv.gz',
+        )
 
         # Generate filename with custom suffic, postfix and extension
-        self.assertEqual(utils.gen_export_filename('tap', 'table', suffix='suffix', postfix='postfix', ext='ext'),
-                         'pipelinewise_tap_table_suffix_fastsync_postfix.ext')
+        self.assertEqual(
+            utils.gen_export_filename(
+                'tap', 'table', suffix='suffix', postfix='postfix', ext='ext'
+            ),
+            'pipelinewise_tap_table_suffix_fastsync_postfix.ext',
+        )
diff --git a/tests/units/fastsync/commons/test_split_gzip.py b/tests/units/fastsync/commons/test_split_gzip.py
index bb5c1f672..2766e6163 100644
--- a/tests/units/fastsync/commons/test_split_gzip.py
+++ b/tests/units/fastsync/commons/test_split_gzip.py
@@ -32,6 +32,7 @@ class TestSplitGzipFile(TestCase):
     """
     Unit tests for SplitGzipFile
     """
+
     filename = '{}_{}_tmp'.format('@test', os.getpid())
 
     def setUp(self):
@@ -55,8 +56,12 @@ def test_bytes_to_megabytes(self):
 
         # Using SI kilo unit
         self.assertEqual(round(gzip_splitter._bytes_to_megabytes(1000), 5), 0.00095)
-        self.assertEqual(round(gzip_splitter._bytes_to_megabytes(1000 ** 2 * 10), 5), 9.53674)
-        self.assertEqual(round(gzip_splitter._bytes_to_megabytes(1000 ** 3), 5), 953.67432)
+        self.assertEqual(
+            round(gzip_splitter._bytes_to_megabytes(1000 ** 2 * 10), 5), 9.53674
+        )
+        self.assertEqual(
+            round(gzip_splitter._bytes_to_megabytes(1000 ** 3), 5), 953.67432
+        )
 
     def test_parameter_validation(self):
         """
@@ -75,10 +80,14 @@ def test_gen_export_chunk_filename(self):
         Test generating chunked filenames
         """
         # split_large_files should be disabled when max_chunks is zero
-        gzip_splitter = split_gzip.SplitGzipFile('basefile', chunk_size_mb=1000, max_chunks=0)
+        gzip_splitter = split_gzip.SplitGzipFile(
+            'basefile', chunk_size_mb=1000, max_chunks=0
+        )
         self.assertEqual(gzip_splitter._gen_chunk_filename(), 'basefile')
         # first chunk should be part nr 1
-        gzip_splitter = split_gzip.SplitGzipFile('basefile', chunk_size_mb=1000, max_chunks=20)
+        gzip_splitter = split_gzip.SplitGzipFile(
+            'basefile', chunk_size_mb=1000, max_chunks=20
+        )
         self.assertEqual(gzip_splitter._gen_chunk_filename(), 'basefile.part00001')
         # generated file part should be in sync with chunk_seq
         gzip_splitter.chunk_seq = 5
@@ -116,7 +125,9 @@ def test_write_with_single_chunk(self):
         Write all data into one chunk
         """
         # test data fits into one chunk
-        with split_gzip.SplitGzipFile(self.filename, 'wb', chunk_size_mb=1000, max_chunks=20) as f_write:
+        with split_gzip.SplitGzipFile(
+            self.filename, 'wb', chunk_size_mb=1000, max_chunks=20
+        ) as f_write:
             f_write.write(DATA_WITH_100_BYTES * 50)
 
         with gzip.open(f'{self.filename}.part00001', 'rb') as f_read:
@@ -129,10 +140,13 @@ def test_write_with_multiple_chunks(self):
         Write data into multiple gzip files
         """
         # test data fits into one chunk
-        with split_gzip.SplitGzipFile(self.filename, 'wb',
-                                      chunk_size_mb=split_gzip.SplitGzipFile._bytes_to_megabytes(200),
-                                      max_chunks=20,
-                                      est_compr_rate=1) as f_write:
+        with split_gzip.SplitGzipFile(
+            self.filename,
+            'wb',
+            chunk_size_mb=split_gzip.SplitGzipFile._bytes_to_megabytes(200),
+            max_chunks=20,
+            est_compr_rate=1,
+        ) as f_write:
             # Write 1100 bytes of test data
             for _ in itertools.repeat(None, 11):
                 f_write.write(DATA_WITH_100_BYTES)
diff --git a/tests/units/fastsync/commons/test_transform_utils.py b/tests/units/fastsync/commons/test_transform_utils.py
index f6aad82fe..bdc99f764 100644
--- a/tests/units/fastsync/commons/test_transform_utils.py
+++ b/tests/units/fastsync/commons/test_transform_utils.py
@@ -1,6 +1,9 @@
 import unittest
 
-from pipelinewise.fastsync.commons.transform_utils import TransformationHelper, SQLFlavor
+from pipelinewise.fastsync.commons.transform_utils import (
+    TransformationHelper,
+    SQLFlavor,
+)
 
 
 class TestTransformHelper(unittest.TestCase):
@@ -17,9 +20,9 @@ def test_get_trans_in_sql_case1(self):
 
         transformations = []
 
-        trans = TransformationHelper.get_trans_in_sql_flavor(table_name,
-                                                             transformations,
-                                                             SQLFlavor('snowflake'))
+        trans = TransformationHelper.get_trans_in_sql_flavor(
+            table_name, transformations, SQLFlavor('snowflake')
+        )
         self.assertFalse(trans)
 
     def test_get_trans_in_sql_case2(self):
@@ -31,9 +34,9 @@ def test_get_trans_in_sql_case2(self):
 
         transformations = []
 
-        trans = TransformationHelper.get_trans_in_sql_flavor(table_name,
-                                                             transformations,
-                                                             SQLFlavor('postgres'))
+        trans = TransformationHelper.get_trans_in_sql_flavor(
+            table_name, transformations, SQLFlavor('postgres')
+        )
         self.assertFalse(trans)
 
     def test_get_trans_in_sql_case3(self):
@@ -47,19 +50,19 @@ def test_get_trans_in_sql_case3(self):
             {
                 'field_id': 'col_7',
                 'tap_stream_name': 'public-my_table',
-                'type': 'RANDOM'
+                'type': 'RANDOM',
             },
             {
                 'field_id': 'col_1',
                 'tap_stream_name': 'public-my_other_table',
-                'type': 'HASH'
-            }
+                'type': 'HASH',
+            },
         ]
 
         with self.assertRaises(ValueError):
-            TransformationHelper.get_trans_in_sql_flavor(table_name,
-                                                         transformations,
-                                                         SQLFlavor('snowflake'))
+            TransformationHelper.get_trans_in_sql_flavor(
+                table_name, transformations, SQLFlavor('snowflake')
+            )
 
     def test_get_trans_in_sql_case4(self):
         """
@@ -72,75 +75,59 @@ def test_get_trans_in_sql_case4(self):
             {
                 'field_id': 'col_1',
                 'tap_stream_name': 'public-my_table',
-                'type': 'SET-NULL'
+                'type': 'SET-NULL',
             },
             {
                 'field_id': 'col_2',
                 'tap_stream_name': 'public-my_table',
-                'type': 'MASK-HIDDEN'
+                'type': 'MASK-HIDDEN',
             },
             {
                 'field_id': 'col_3',
                 'tap_stream_name': 'public-my_table',
-                'type': 'MASK-DATE'
+                'type': 'MASK-DATE',
             },
             {
                 'field_id': 'col_4',
                 'tap_stream_name': 'public-my_table',
                 'safe_field_id': '"COL_4"',
-                'type': 'MASK-NUMBER'
-            },
-            {
-                'field_id': 'col_5',
-                'tap_stream_name': 'public-my_table',
-                'type': 'HASH'
+                'type': 'MASK-NUMBER',
             },
+            {'field_id': 'col_5', 'tap_stream_name': 'public-my_table', 'type': 'HASH'},
             {
                 'field_id': 'col_1',
                 'tap_stream_name': 'public-my_other_table',
-                'type': 'HASH'
+                'type': 'HASH',
             },
             {
                 'field_id': 'col_6',
                 'tap_stream_name': 'public-my_table',
-                'type': 'HASH-SKIP-FIRST-5'
-            }
+                'type': 'HASH-SKIP-FIRST-5',
+            },
         ]
 
         trans = TransformationHelper.get_trans_in_sql_flavor(
-            table_name,
-            transformations,
-            SQLFlavor('snowflake'))
+            table_name, transformations, SQLFlavor('snowflake')
+        )
 
         self.assertListEqual(
             trans,
             [
-                {
-                    'trans': '"COL_1" = NULL',
-                    'conditions': None
-                },
-                {
-                    'trans': '"COL_2" = \'hidden\'',
-                    'conditions': None
-                },
+                {'trans': '"COL_1" = NULL', 'conditions': None},
+                {'trans': '"COL_2" = \'hidden\'', 'conditions': None},
                 {
                     'trans': '"COL_3" = TIMESTAMP_NTZ_FROM_PARTS(DATE_FROM_PARTS(YEAR("COL_3"), 1, 1),'
-                             'TO_TIME("COL_3"))',
-                    'conditions': None
-                },
-                {
-                    'trans': '"COL_4" = 0',
-                    'conditions': None
-                },
-                {
-                    'trans': '"COL_5" = SHA2("COL_5", 256)',
-                    'conditions': None
+                    'TO_TIME("COL_3"))',
+                    'conditions': None,
                 },
+                {'trans': '"COL_4" = 0', 'conditions': None},
+                {'trans': '"COL_5" = SHA2("COL_5", 256)', 'conditions': None},
                 {
                     'trans': '"COL_6" = CONCAT(SUBSTRING("COL_6", 1, 5), SHA2(SUBSTRING("COL_6", 5 + 1), 256))',
-                    'conditions': None
-                }
-            ])
+                    'conditions': None,
+                },
+            ],
+        )
 
     def test_get_trans_in_sql_case5(self):
         """
@@ -153,82 +140,69 @@ def test_get_trans_in_sql_case5(self):
             {
                 'field_id': 'col_1',
                 'tap_stream_name': 'public-my_table',
-                'type': 'SET-NULL'
+                'type': 'SET-NULL',
             },
             {
                 'field_id': 'col_2',
                 'tap_stream_name': 'public-my_table',
-                'type': 'MASK-HIDDEN'
+                'type': 'MASK-HIDDEN',
             },
             {
                 'field_id': 'col_3',
                 'tap_stream_name': 'public-my_table',
-                'type': 'MASK-DATE'
+                'type': 'MASK-DATE',
             },
             {
                 'field_id': 'col_4',
                 'tap_stream_name': 'public-my_table',
                 'safe_field_id': '"COL_4"',
-                'type': 'MASK-NUMBER'
-            },
-            {
-                'field_id': 'col_5',
-                'tap_stream_name': 'public-my_table',
-                'type': 'HASH'
+                'type': 'MASK-NUMBER',
             },
+            {'field_id': 'col_5', 'tap_stream_name': 'public-my_table', 'type': 'HASH'},
             {
                 'field_id': 'col_1',
                 'tap_stream_name': 'public-my_other_table',
-                'type': 'HASH'
+                'type': 'HASH',
             },
             {
                 'field_id': 'col_6',
                 'tap_stream_name': 'public-my_table',
-                'type': 'HASH-SKIP-FIRST-5'
-            }
+                'type': 'HASH-SKIP-FIRST-5',
+            },
         ]
 
         trans = TransformationHelper.get_trans_in_sql_flavor(
-            table_name,
-            transformations,
-            SQLFlavor('postgres'))
+            table_name, transformations, SQLFlavor('postgres')
+        )
 
         self.assertListEqual(
             trans,
             [
-                {
-                    'trans': '"col_1" = NULL',
-                    'conditions': None
-                },
-                {
-                    'trans': '"col_2" = \'hidden\'',
-                    'conditions': None
-                },
+                {'trans': '"col_1" = NULL', 'conditions': None},
+                {'trans': '"col_2" = \'hidden\'', 'conditions': None},
                 {
                     'trans': '"col_3" = MAKE_TIMESTAMP('
-                             'DATE_PART(\'year\', "col_3")::int, '
-                             '1, '
-                             '1, '
-                             'DATE_PART(\'hour\', "col_3")::int, '
-                             'DATE_PART(\'minute\', "col_3")::int, '
-                             'DATE_PART(\'second\', "col_3")::double precision'
-                             ')',
-                    'conditions': None
-                },
-                {
-                    'trans': '"col_4" = 0',
-                    'conditions': None
+                    'DATE_PART(\'year\', "col_3")::int, '
+                    '1, '
+                    '1, '
+                    'DATE_PART(\'hour\', "col_3")::int, '
+                    'DATE_PART(\'minute\', "col_3")::int, '
+                    'DATE_PART(\'second\', "col_3")::double precision'
+                    ')',
+                    'conditions': None,
                 },
+                {'trans': '"col_4" = 0', 'conditions': None},
                 {
                     'trans': '"col_5" = ENCODE(DIGEST("col_5", \'sha256\'), \'hex\')',
-                    'conditions': None
+                    'conditions': None,
                 },
                 {
                     'trans': '"col_6" = CONCAT(SUBSTRING("col_6", 1, 5), '
-                             'ENCODE(DIGEST(SUBSTRING("col_6", 5 + 1), \'sha256\'), \'hex\'))',
-                    'conditions': None
-                }
-            ])
+                    'ENCODE(DIGEST(SUBSTRING("col_6", 5 + 1), \'sha256\'), \'hex\'))',
+                    'conditions': None,
+                },
+            ],
+        )
 
     def test_get_trans_in_sql_case6(self):
         """
@@ -241,70 +215,50 @@ def test_get_trans_in_sql_case6(self):
             {
                 'field_id': 'col_1',
                 'tap_stream_name': 'public-my_table',
-                'type': 'SET-NULL'
+                'type': 'SET-NULL',
             },
             {
                 'field_id': 'col_2',
                 'tap_stream_name': 'public-my_table',
                 'type': 'MASK-HIDDEN',
                 'when': [
-                    {
-                        'column': 'col_4',
-                        'safe_column': '"COL_4"',
-                        'equals': None
-                    },
+                    {'column': 'col_4', 'safe_column': '"COL_4"', 'equals': None},
                     {
                         'column': 'col_1',
-                    }
-                ]
+                    },
+                ],
             },
             {
                 'field_id': 'col_3',
                 'tap_stream_name': 'public-my_table',
                 'type': 'MASK-DATE',
-                'when': [
-                    {
-                        'column': 'col_5',
-                        'equals': 'some_value'
-                    }
-                ]
+                'when': [{'column': 'col_5', 'equals': 'some_value'}],
             },
             {
                 'field_id': 'col_4',
                 'tap_stream_name': 'public-my_table',
-                'type': 'MASK-NUMBER'
-            },
-            {
-                'field_id': 'col_5',
-                'tap_stream_name': 'public-my_table',
-                'type': 'HASH'
+                'type': 'MASK-NUMBER',
             },
+            {'field_id': 'col_5', 'tap_stream_name': 'public-my_table', 'type': 'HASH'},
             {
                 'field_id': 'col_10',
                 'tap_stream_name': 'public-my_other_table',
-                'type': 'HASH'
+                'type': 'HASH',
             },
             {
                 'field_id': 'col_6',
                 'tap_stream_name': 'public-my_table',
                 'type': 'HASH-SKIP-FIRST-5',
                 'when': [
-                    {
-                        'column': 'col_1',
-                        'equals': 30
-                    },
-                    {
-                        'column': 'col_2',
-                        'regex_match': r'[0-9]{3}\.[0-9]{3}'
-                    }
-                ]
-            }
+                    {'column': 'col_1', 'equals': 30},
+                    {'column': 'col_2', 'regex_match': r'[0-9]{3}\.[0-9]{3}'},
+                ],
+            },
         ]
 
         trans = TransformationHelper.get_trans_in_sql_flavor(
-            table_name,
-            transformations,
-            SQLFlavor('snowflake'))
+            table_name, transformations, SQLFlavor('snowflake')
+        )
 
         self.assertListEqual(
             trans,
@@ -319,7 +273,7 @@ def test_get_trans_in_sql_case6(self):
                 },
                 {
                     'trans': '"COL_3" = TIMESTAMP_NTZ_FROM_PARTS(DATE_FROM_PARTS(YEAR("COL_3"), 1, 1),'
-                             'TO_TIME("COL_3"))',
+                    'TO_TIME("COL_3"))',
                     'conditions': '("COL_5" = \'some_value\')',
                 },
                 {
@@ -333,9 +287,9 @@ def test_get_trans_in_sql_case6(self):
                 {
                     'trans': '"COL_6" = CONCAT(SUBSTRING("COL_6", 1, 5), SHA2(SUBSTRING("COL_6", 5 + 1), 256))',
                     'conditions': '("COL_1" = 30) AND ("COL_2" '
-                                  'REGEXP \'[0-9]{3}\.[0-9]{3}\')',  # pylint: disable=W1401
+                    'REGEXP \'[0-9]{3}\.[0-9]{3}\')',  # pylint: disable=W1401  # noqa: W605
                 },
-            ]
+            ],
         )
 
     def test_get_trans_in_sql_case7(self):
@@ -349,70 +303,50 @@ def test_get_trans_in_sql_case7(self):
             {
                 'field_id': 'col_1',
                 'tap_stream_name': 'public-my_table',
-                'type': 'SET-NULL'
+                'type': 'SET-NULL',
             },
             {
                 'field_id': 'col_2',
                 'tap_stream_name': 'public-my_table',
                 'type': 'MASK-HIDDEN',
                 'when': [
-                    {
-                        'column': 'col_4',
-                        'safe_column': '"COL_4"',
-                        'equals': None
-                    },
+                    {'column': 'col_4', 'safe_column': '"COL_4"', 'equals': None},
                     {
                         'column': 'col_1',
-                    }
-                ]
+                    },
+                ],
             },
             {
                 'field_id': 'col_3',
                 'tap_stream_name': 'public-my_table',
                 'type': 'MASK-DATE',
-                'when': [
-                    {
-                        'column': 'col_5',
-                        'equals': 'some_value'
-                    }
-                ]
+                'when': [{'column': 'col_5', 'equals': 'some_value'}],
             },
             {
                 'field_id': 'col_4',
                 'tap_stream_name': 'public-my_table',
-                'type': 'MASK-NUMBER'
-            },
-            {
-                'field_id': 'col_5',
-                'tap_stream_name': 'public-my_table',
-                'type': 'HASH'
+                'type': 'MASK-NUMBER',
             },
+            {'field_id': 'col_5', 'tap_stream_name': 'public-my_table', 'type': 'HASH'},
             {
                 'field_id': 'col_10',
                 'tap_stream_name': 'public-my_other_table',
-                'type': 'HASH'
+                'type': 'HASH',
             },
             {
                 'field_id': 'col_6',
                 'tap_stream_name': 'public-my_table',
                 'type': 'HASH-SKIP-FIRST-5',
                 'when': [
-                    {
-                        'column': 'col_1',
-                        'equals': 30
-                    },
-                    {
-                        'column': 'col_2',
-                        'regex_match': r'[0-9]{3}\.[0-9]{3}'
-                    }
-                ]
-            }
+                    {'column': 'col_1', 'equals': 30},
+                    {'column': 'col_2', 'regex_match': r'[0-9]{3}\.[0-9]{3}'},
+                ],
+            },
         ]
 
         trans = TransformationHelper.get_trans_in_sql_flavor(
-            table_name,
-            transformations,
-            SQLFlavor('postgres'))
+            table_name, transformations, SQLFlavor('postgres')
+        )
 
         self.assertListEqual(
             trans,
@@ -427,13 +361,13 @@ def test_get_trans_in_sql_case7(self):
                 },
                 {
                     'trans': '"col_3" = MAKE_TIMESTAMP('
-                             'DATE_PART(\'year\', "col_3")::int, '
-                             '1, '
-                             '1, '
-                             'DATE_PART(\'hour\', "col_3")::int, '
-                             'DATE_PART(\'minute\', "col_3")::int, '
-                             'DATE_PART(\'second\', "col_3")::double precision'
-                             ')',
+                    'DATE_PART(\'year\', "col_3")::int, '
+                    '1, '
+                    '1, '
+                    'DATE_PART(\'hour\', "col_3")::int, '
+                    'DATE_PART(\'minute\', "col_3")::int, '
+                    'DATE_PART(\'second\', "col_3")::double precision'
+                    ')',
                     'conditions': '("col_5" = \'some_value\')',
                 },
                 {
@@ -446,8 +380,8 @@ def test_get_trans_in_sql_case7(self):
                 },
                 {
                     'trans': '"col_6" = CONCAT(SUBSTRING("col_6", 1, 5), ENCODE(DIGEST(SUBSTRING("col_6", 5 + 1), '
-                             '\'sha256\'), \'hex\'))',
-                    'conditions': '("col_1" = 30) AND ("col_2" ~ \'[0-9]{3}\.[0-9]{3}\')',  # pylint: disable=W1401
+                    '\'sha256\'), \'hex\'))',
+                    'conditions': '("col_1" = 30) AND ("col_2" ~ \'[0-9]{3}\.[0-9]{3}\')',  # pylint: disable=W1401  # noqa: W605, E501
                 },
-            ]
+            ],
         )
diff --git a/tests/units/fastsync/test_mongodb_to_bigquery.py b/tests/units/fastsync/test_mongodb_to_bigquery.py
index 88ed4a620..cd3b630ab 100644
--- a/tests/units/fastsync/test_mongodb_to_bigquery.py
+++ b/tests/units/fastsync/test_mongodb_to_bigquery.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.mongodb_to_bigquery import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.mongodb_to_bigquery import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.mongodb_to_bigquery'
 TAP = 'FastSyncTapMongoDB'
@@ -13,7 +17,10 @@ class MongoDBToBigquery(unittest.TestCase):
     """
     Unit tests for fastsync MongoDB to postgres
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('STRING', tap_type_to_target_type('object'))
 
     def test_tap_type_to_target_type_with_undefined_tap_type_returns_STRING(self):
@@ -21,19 +28,27 @@ def test_tap_type_to_target_type_with_undefined_tap_type_returns_STRING(self):
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_mongodb_to_postgres.py b/tests/units/fastsync/test_mongodb_to_postgres.py
index 8a3a82c23..f8f12b459 100644
--- a/tests/units/fastsync/test_mongodb_to_postgres.py
+++ b/tests/units/fastsync/test_mongodb_to_postgres.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.mongodb_to_postgres import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.mongodb_to_postgres import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.mongodb_to_postgres'
 TAP = 'FastSyncTapMongoDB'
@@ -13,27 +17,40 @@ class MongoDBToPostgres(unittest.TestCase):
     """
     Unit tests for fastsync MongoDB to postgres
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('JSONB', tap_type_to_target_type('object'))
 
-    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(self):
+    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(
+        self,
+    ):
         self.assertEqual('CHARACTER VARYING', tap_type_to_target_type('random-type'))
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_mongodb_to_snowflake.py b/tests/units/fastsync/test_mongodb_to_snowflake.py
index 084038470..a3f46191a 100644
--- a/tests/units/fastsync/test_mongodb_to_snowflake.py
+++ b/tests/units/fastsync/test_mongodb_to_snowflake.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.mongodb_to_snowflake import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.mongodb_to_snowflake import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.mongodb_to_snowflake'
 TAP = 'FastSyncTapMongoDB'
@@ -13,27 +17,40 @@ class MongoDBToSnowflake(unittest.TestCase):
     """
     Unit tests for fastsync MongoDB to postgres
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('VARIANT', tap_type_to_target_type('object'))
 
-    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(self):
+    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(
+        self,
+    ):
         self.assertEqual('VARCHAR', tap_type_to_target_type('random-type'))
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_mysql_to_bigquery.py b/tests/units/fastsync/test_mysql_to_bigquery.py
index b41a5e99b..1f6f22cca 100644
--- a/tests/units/fastsync/test_mysql_to_bigquery.py
+++ b/tests/units/fastsync/test_mysql_to_bigquery.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.mysql_to_bigquery import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.mysql_to_bigquery import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.mysql_to_bigquery'
 TAP = 'FastSyncTapMySql'
@@ -13,7 +17,10 @@ class MySQLToBigQuery(unittest.TestCase):
     """
     Unit tests for fastsync mysql to bigquery
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('STRING', tap_type_to_target_type('binary', None))
         self.assertEqual('STRING', tap_type_to_target_type('geometry', None))
         self.assertEqual('STRING', tap_type_to_target_type('point', None))
@@ -25,23 +32,33 @@ def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target
         self.assertEqual('STRING', tap_type_to_target_type('geometrycollection', None))
 
     def test_tap_type_to_target_type_with_undefined_tap_type_returns_STRING(self):
-        self.assertEqual('STRING', tap_type_to_target_type('random-type', 'random-type'))
+        self.assertEqual(
+            'STRING', tap_type_to_target_type('random-type', 'random-type')
+        )
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_mysql_to_postgres.py b/tests/units/fastsync/test_mysql_to_postgres.py
index bee9982aa..bca128ed6 100644
--- a/tests/units/fastsync/test_mysql_to_postgres.py
+++ b/tests/units/fastsync/test_mysql_to_postgres.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.mysql_to_postgres import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.mysql_to_postgres import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.mysql_to_postgres'
 TAP = 'FastSyncTapMySql'
@@ -13,7 +17,10 @@ class S3CsvToPostgres(unittest.TestCase):
     """
     Unit tests for fastsync mysql to postgres
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('CHARACTER VARYING', tap_type_to_target_type('binary', None))
         self.assertEqual('JSONB', tap_type_to_target_type('geometry', None))
         self.assertEqual('JSONB', tap_type_to_target_type('point', None))
@@ -24,24 +31,36 @@ def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target
         self.assertEqual('JSONB', tap_type_to_target_type('multipolygon', None))
         self.assertEqual('JSONB', tap_type_to_target_type('geometrycollection', None))
 
-    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(self):
-        self.assertEqual('CHARACTER VARYING', tap_type_to_target_type('random-type', 'random-type'))
+    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(
+        self,
+    ):
+        self.assertEqual(
+            'CHARACTER VARYING', tap_type_to_target_type('random-type', 'random-type')
+        )
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_mysql_to_redshift.py b/tests/units/fastsync/test_mysql_to_redshift.py
index 56d7c979b..4b1d50977 100644
--- a/tests/units/fastsync/test_mysql_to_redshift.py
+++ b/tests/units/fastsync/test_mysql_to_redshift.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.mysql_to_redshift import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.mysql_to_redshift import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.mysql_to_redshift'
 TAP = 'FastSyncTapMySql'
@@ -13,35 +17,70 @@ class S3CsvToPostgres(unittest.TestCase):
     """
     Unit tests for fastsync mysql to redshift
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
-        self.assertEqual('CHARACTER VARYING(65535)', tap_type_to_target_type('binary', None))
-        self.assertEqual('CHARACTER VARYING(10000)', tap_type_to_target_type('geometry', None))
-        self.assertEqual('CHARACTER VARYING(10000)', tap_type_to_target_type('point', None))
-        self.assertEqual('CHARACTER VARYING(10000)', tap_type_to_target_type('linestring', None))
-        self.assertEqual('CHARACTER VARYING(10000)', tap_type_to_target_type('polygon', None))
-        self.assertEqual('CHARACTER VARYING(10000)', tap_type_to_target_type('multipoint', None))
-        self.assertEqual('CHARACTER VARYING(10000)', tap_type_to_target_type('multilinestring', None))
-        self.assertEqual('CHARACTER VARYING(10000)', tap_type_to_target_type('multipolygon', None))
-        self.assertEqual('CHARACTER VARYING(10000)', tap_type_to_target_type('geometrycollection', None))
-
-    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(self):
-        self.assertEqual('CHARACTER VARYING(10000)', tap_type_to_target_type('random-type', 'random-type'))
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
+        self.assertEqual(
+            'CHARACTER VARYING(65535)', tap_type_to_target_type('binary', None)
+        )
+        self.assertEqual(
+            'CHARACTER VARYING(10000)', tap_type_to_target_type('geometry', None)
+        )
+        self.assertEqual(
+            'CHARACTER VARYING(10000)', tap_type_to_target_type('point', None)
+        )
+        self.assertEqual(
+            'CHARACTER VARYING(10000)', tap_type_to_target_type('linestring', None)
+        )
+        self.assertEqual(
+            'CHARACTER VARYING(10000)', tap_type_to_target_type('polygon', None)
+        )
+        self.assertEqual(
+            'CHARACTER VARYING(10000)', tap_type_to_target_type('multipoint', None)
+        )
+        self.assertEqual(
+            'CHARACTER VARYING(10000)', tap_type_to_target_type('multilinestring', None)
+        )
+        self.assertEqual(
+            'CHARACTER VARYING(10000)', tap_type_to_target_type('multipolygon', None)
+        )
+        self.assertEqual(
+            'CHARACTER VARYING(10000)',
+            tap_type_to_target_type('geometrycollection', None),
+        )
+
+    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(
+        self,
+    ):
+        self.assertEqual(
+            'CHARACTER VARYING(10000)',
+            tap_type_to_target_type('random-type', 'random-type'),
+        )
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_mysql_to_snowflake.py b/tests/units/fastsync/test_mysql_to_snowflake.py
index 21bc07108..73831fe70 100644
--- a/tests/units/fastsync/test_mysql_to_snowflake.py
+++ b/tests/units/fastsync/test_mysql_to_snowflake.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.mysql_to_snowflake import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.mysql_to_snowflake import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.mysql_to_snowflake'
 TAP = 'FastSyncTapMySql'
@@ -13,7 +17,10 @@ class S3CsvToPostgres(unittest.TestCase):
     """
     Unit tests for fastsync mysql to snowflake
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('BINARY', tap_type_to_target_type('binary', None))
         self.assertEqual('VARIANT', tap_type_to_target_type('geometry', None))
         self.assertEqual('VARIANT', tap_type_to_target_type('point', None))
@@ -24,24 +31,36 @@ def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target
         self.assertEqual('VARIANT', tap_type_to_target_type('multipolygon', None))
         self.assertEqual('VARIANT', tap_type_to_target_type('geometrycollection', None))
 
-    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(self):
-        self.assertEqual('VARCHAR', tap_type_to_target_type('random-type', 'random-type'))
+    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(
+        self,
+    ):
+        self.assertEqual(
+            'VARCHAR', tap_type_to_target_type('random-type', 'random-type')
+        )
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_postgres_to_bigquery.py b/tests/units/fastsync/test_postgres_to_bigquery.py
index a1b680414..069b7fd4a 100644
--- a/tests/units/fastsync/test_postgres_to_bigquery.py
+++ b/tests/units/fastsync/test_postgres_to_bigquery.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.postgres_to_bigquery import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.postgres_to_bigquery import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.postgres_to_bigquery'
 TAP = 'FastSyncTapPostgres'
@@ -13,7 +17,10 @@ class PostgresToBigquery(unittest.TestCase):
     """
     Unit tests for fastsync postgres to bigquery
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('INT64', tap_type_to_target_type('serial'))
 
     def test_tap_type_to_target_type_with_undefined_tap_type_returns_STRING(self):
@@ -21,19 +28,27 @@ def test_tap_type_to_target_type_with_undefined_tap_type_returns_STRING(self):
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_postgres_to_postgres.py b/tests/units/fastsync/test_postgres_to_postgres.py
index ddc89ad7f..e821ee22a 100644
--- a/tests/units/fastsync/test_postgres_to_postgres.py
+++ b/tests/units/fastsync/test_postgres_to_postgres.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.postgres_to_postgres import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.postgres_to_postgres import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.postgres_to_postgres'
 TAP = 'FastSyncTapPostgres'
@@ -13,27 +17,40 @@ class PostgresToPostgres(unittest.TestCase):
     """
     Unit tests for postgres postgres to postgres
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('DOUBLE PRECISION', tap_type_to_target_type('serial'))
 
-    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(self):
+    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(
+        self,
+    ):
         self.assertEqual('CHARACTER VARYING', tap_type_to_target_type('random-type'))
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_postgres_to_redshift.py b/tests/units/fastsync/test_postgres_to_redshift.py
index e92c23828..b1a9b8930 100644
--- a/tests/units/fastsync/test_postgres_to_redshift.py
+++ b/tests/units/fastsync/test_postgres_to_redshift.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.postgres_to_redshift import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.postgres_to_redshift import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.postgres_to_redshift'
 TAP = 'FastSyncTapPostgres'
@@ -13,27 +17,42 @@ class PostgresToRedshift(unittest.TestCase):
     """
     Unit tests for postgres postgres to redshift
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('NUMERIC NULL', tap_type_to_target_type('serial'))
 
-    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(self):
-        self.assertEqual('CHARACTER VARYING(10000)', tap_type_to_target_type('random-type'))
+    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(
+        self,
+    ):
+        self.assertEqual(
+            'CHARACTER VARYING(10000)', tap_type_to_target_type('random-type')
+        )
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_postgres_to_snowflake.py b/tests/units/fastsync/test_postgres_to_snowflake.py
index 7e0afc6e1..7db82d791 100644
--- a/tests/units/fastsync/test_postgres_to_snowflake.py
+++ b/tests/units/fastsync/test_postgres_to_snowflake.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.postgres_to_snowflake import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.postgres_to_snowflake import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.postgres_to_snowflake'
 TAP = 'FastSyncTapPostgres'
@@ -13,27 +17,40 @@ class PostgresToSnowflake(unittest.TestCase):
     """
     Unit tests for fastsync postgres to snowflake
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('NUMBER', tap_type_to_target_type('serial'))
 
-    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(self):
+    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(
+        self,
+    ):
         self.assertEqual('VARCHAR', tap_type_to_target_type('random-type'))
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_s3_csv_to_bigquery.py b/tests/units/fastsync/test_s3_csv_to_bigquery.py
index 28a94d470..fd20a9d9a 100644
--- a/tests/units/fastsync/test_s3_csv_to_bigquery.py
+++ b/tests/units/fastsync/test_s3_csv_to_bigquery.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.s3_csv_to_bigquery import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.s3_csv_to_bigquery import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.s3_csv_to_bigquery'
 TAP = 'FastSyncTapS3Csv'
@@ -13,7 +17,10 @@ class S3CsvToPostgres(unittest.TestCase):
     """
     Unit tests for fastsync s3 csv to bigquery
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('INT64', tap_type_to_target_type('integer'))
 
     def test_tap_type_to_target_type_with_undefined_tap_type_returns_STRING(self):
@@ -21,19 +28,27 @@ def test_tap_type_to_target_type_with_undefined_tap_type_returns_STRING(self):
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_s3_csv_to_postgres.py b/tests/units/fastsync/test_s3_csv_to_postgres.py
index 1b2850f7a..22ea86047 100644
--- a/tests/units/fastsync/test_s3_csv_to_postgres.py
+++ b/tests/units/fastsync/test_s3_csv_to_postgres.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.s3_csv_to_postgres import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.s3_csv_to_postgres import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.s3_csv_to_postgres'
 TAP = 'FastSyncTapS3Csv'
@@ -13,27 +17,40 @@ class S3CsvToPostgres(unittest.TestCase):
     """
     Unit tests for fastsync s3 csv to postgres
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('DOUBLE PRECISION', tap_type_to_target_type('number'))
 
-    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(self):
+    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(
+        self,
+    ):
         self.assertEqual('CHARACTER VARYING', tap_type_to_target_type('random-type'))
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_s3_csv_to_redshift.py b/tests/units/fastsync/test_s3_csv_to_redshift.py
index af40e4fa5..4bf7bdb09 100644
--- a/tests/units/fastsync/test_s3_csv_to_redshift.py
+++ b/tests/units/fastsync/test_s3_csv_to_redshift.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.s3_csv_to_redshift import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.s3_csv_to_redshift import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.s3_csv_to_redshift'
 TAP = 'FastSyncTapS3Csv'
@@ -13,27 +17,40 @@ class S3CsvToRedshift(unittest.TestCase):
     """
     Unit tests for fastsync s3 csv to redshift
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('NUMERIC', tap_type_to_target_type('integer'))
 
-    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(self):
+    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(
+        self,
+    ):
         self.assertEqual('CHARACTER VARYING', tap_type_to_target_type('random-type'))
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/fastsync/test_s3_csv_to_snowflake.py b/tests/units/fastsync/test_s3_csv_to_snowflake.py
index c319d0896..eeba7ba57 100644
--- a/tests/units/fastsync/test_s3_csv_to_snowflake.py
+++ b/tests/units/fastsync/test_s3_csv_to_snowflake.py
@@ -1,7 +1,11 @@
 import unittest
 from . import assertions
 
-from pipelinewise.fastsync.s3_csv_to_snowflake import tap_type_to_target_type, sync_table, main_impl
+from pipelinewise.fastsync.s3_csv_to_snowflake import (
+    tap_type_to_target_type,
+    sync_table,
+    main_impl,
+)
 
 PACKAGE_IN_SCOPE = 'pipelinewise.fastsync.s3_csv_to_snowflake'
 TAP = 'FastSyncTapS3Csv'
@@ -13,27 +17,40 @@ class S3CsvToPostgres(unittest.TestCase):
     """
     Unit tests for fastsync s3 csv to snowflake
     """
-    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(self):
+
+    def test_tap_type_to_target_type_with_defined_tap_type_returns_equivalent_target_type(
+        self,
+    ):
         self.assertEqual('INTEGER', tap_type_to_target_type('integer'))
 
-    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(self):
+    def test_tap_type_to_target_type_with_undefined_tap_type_returns_CHARACTER_VARYING(
+        self,
+    ):
         self.assertEqual('VARCHAR', tap_type_to_target_type('random-type'))
 
     @staticmethod
     def test_sync_table_runs_successfully_returns_true():
-        assertions.assert_sync_table_returns_true_on_success(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_returns_true_on_success(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_sync_table_exception_on_copy_table_returns_failed_table_name_and_exception():
-        assertions.assert_sync_table_exception_on_failed_copy(sync_table, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_sync_table_exception_on_failed_copy(
+            sync_table, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_all_tables_synced_successfully_should_exit_normally():
-        assertions.assert_main_impl_exit_normally_on_success(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_exit_normally_on_success(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
     @staticmethod
     def test_main_impl_with_one_table_fails_to_sync_should_exit_with_error():
-        assertions.assert_main_impl_should_exit_with_error_on_failure(main_impl, PACKAGE_IN_SCOPE, TAP, TARGET)
+        assertions.assert_main_impl_should_exit_with_error_on_failure(
+            main_impl, PACKAGE_IN_SCOPE, TAP, TARGET
+        )
 
 
 if __name__ == '__main__':
diff --git a/tests/units/test_logger.py b/tests/units/test_logger.py
index be3068be3..c54e9eb55 100644
--- a/tests/units/test_logger.py
+++ b/tests/units/test_logger.py
@@ -5,6 +5,7 @@
 
 from tests.units.cli.cli_args import CliArgs
 
+
 # pylint: disable=no-self-use
 class TestLogging:
     """
@@ -20,14 +21,18 @@ def test_logging_default(self):
         """Debug option should be disabled by default, LOGGING_CONF_FILE"""
         args = CliArgs()
         Logger(debug=args.debug)
-        path = os.path.join(Path(__file__).parent, '..', '..', 'pipelinewise', 'logging.conf')
+        path = os.path.join(
+            Path(__file__).parent, '..', '..', 'pipelinewise', 'logging.conf'
+        )
         assert os.environ['LOGGING_CONF_FILE'] == os.path.abspath(path)
 
     def test_logging_debug(self):
         """Providing debug option should set LOGGING_CONF_FILE env var"""
         args = CliArgs(debug=True)
         Logger(debug=args.debug)
-        path = os.path.join(Path(__file__).parent, '..', '..', 'pipelinewise', 'logging_debug.conf')
+        path = os.path.join(
+            Path(__file__).parent, '..', '..', 'pipelinewise', 'logging_debug.conf'
+        )
         assert os.environ['LOGGING_CONF_FILE'] == os.path.abspath(path)
 
     def test_custom_logging_conf(self):
diff --git a/tests/units/test_utils.py b/tests/units/test_utils.py
index 3d947b0d5..5bea460fa 100644
--- a/tests/units/test_utils.py
+++ b/tests/units/test_utils.py
@@ -7,6 +7,7 @@ class TestUtils(unittest.TestCase):
     """
     Unit Tests for PipelineWise common utils
     """
+
     def test_safe_column_name_case_1(self):
         """
         Given an all lower case word would be wrapped in double quotes and capitalized
@@ -47,5 +48,6 @@ def test_safe_column_name_is_null(self):
 
         self.assertIsNone(utils.safe_column_name(input_name))
 
+
 if __name__ == '__main__':
     unittest.main()

From d7ab63461f5a07ab71b1d1e863f88a6112a626e5 Mon Sep 17 00:00:00 2001
From: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
Date: Wed, 22 Sep 2021 14:02:07 +0300
Subject: [PATCH 19/78] [gh-issue-800] Add all connectors to enum (#802)

---
 docs/project/contribution.rst | 24 ++++++++++++++----------
 pipelinewise/cli/constants.py | 20 ++++++++++++++++++--
 2 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/docs/project/contribution.rst b/docs/project/contribution.rst
index 2340f5d2d..f8f7f1fc8 100644
--- a/docs/project/contribution.rst
+++ b/docs/project/contribution.rst
@@ -13,7 +13,7 @@ To add new item to the supported :ref:`taps` or :ref:`targets` please follow the
 Adding new tap
 ''''''''''''''
 
-1. Make sure your tap is following the Singer specification by following the following criterias:
+1. Make sure your tap is following the Singer specification by following the following criteria:
 
    * Configuration JSON is mandatory and defined with the ``--config`` CLI argument
 
@@ -54,38 +54,42 @@ Adding new tap
   | **tap_config_extras**                        | Anything else that's required in the tap ``config.json`` to run. This can be static or dynamically generated values at runtime.        |
   +----------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------+  
 
-3. Add your tap id to the allowed values in `tap.json <https://github.com/transferwise/pipelinewise/blob/master/pipelinewise/cli/schemas/tap.json#L149>`_
+3. Add your tap type to the allowed values in `tap.json <https://github.com/transferwise/pipelinewise/blob/master/pipelinewise/cli/schemas/tap.json#L149>`_
+4. Add your tap type to `ConnectorType Enum <https://github.com/transferwise/pipelinewise/blob/acde3c18f4dd116113f47f3dc50d31fdcf59a1d6/pipelinewise/cli/constants.py#L4>`_
 
-4. Add your tap to the `singer-connectors <https://github.com/transferwise/pipelinewise/tree/master/singer-connectors>`_ directory.
+5. Add your tap to the `singer-connectors <https://github.com/transferwise/pipelinewise/tree/master/singer-connectors>`_ directory.
    The new directory should have only one ``requirements.txt`` file with a reference and version of the tap in PyPI.
    Some taps are forks of the community versions and customised to PipelineWise. If new fork or project required in PyPI please
    mention this the PR. The Wise team will create the PyPi package.
 
-5. Document your tap in the :ref:`taps_list` section with some YAML examples.
+6. Document your tap in the :ref:`taps_list` section with some YAML examples.
    The editable documentation is at `GitHub Taps in RST format <https://github.com/transferwise/pipelinewise/tree/master/docs/connectors/taps>`_.
 
-6. Send a Pull Request to the `PipelineWise Github Repository <https://github.com/transferwise/pipelinewise>`_.
+7. Send a Pull Request to the `PipelineWise Github Repository <https://github.com/transferwise/pipelinewise>`_.
 
 
 Adding new target
 '''''''''''''''''
 
-1. Make sure your target is following the Singer specification and meets the following criterias:
+1. Make sure your target is following the Singer specification and meets the following criteria:
 
    * Configuration JSON is mandatory and defined with the ``--config`` CLI argument
 
    * State messages printed to standard output. PipelineWise compatible target connectors should send
      `Singer State Messages <https://github.com/singer-io/getting-started/blob/master/docs/SPEC.md#state-message>`_
      in the original format to ``STDOUT``. Other components of PipelineWise will process it at a later stage making it
-     compatible across every target connetor.
+     compatible across every target connector.
 
 2. Add your target to `singer-connectors <https://github.com/transferwise/pipelinewise/tree/master/singer-connectors>`_ directory.
    The new directory should have only one ``requirements.txt`` file with a reference and version of the tap in PyPI.
    Some taps are forks of the community versions and customised to PipelineWise. If new fork or project required in PyPI please
    mention this the PR. The Wise team will create the PyPi package.
 
-3. Document your target in the :ref:`targets_list` section with some YAML examples.
-   The editable documentation is at `GitHub Targets in RST format <https://github.com/transferwise/pipelinewise/tree/master/docs/connectors/targets>`_.
+3. Add your target type to the allowed values in `target.json <https://github.com/transferwise/pipelinewise/blob/master/pipelinewise/cli/schemas/target.json#L204>`_
 
-4. Send a Pull Request to the `PipelineWise Github Repository <https://github.com/transferwise/pipelinewise>`_.
+4. Add your target type to `ConnectorType Enum <https://github.com/transferwise/pipelinewise/blob/acde3c18f4dd116113f47f3dc50d31fdcf59a1d6/pipelinewise/cli/constants.py#L4>`_
 
+5. Document your target in the :ref:`targets_list` section with some YAML examples.
+   The editable documentation is at `GitHub Targets in RST format <https://github.com/transferwise/pipelinewise/tree/master/docs/connectors/targets>`_.
+
+6. Send a Pull Request to the `PipelineWise Github Repository <https://github.com/transferwise/pipelinewise>`_.
diff --git a/pipelinewise/cli/constants.py b/pipelinewise/cli/constants.py
index 6593d31d2..d9bab0031 100644
--- a/pipelinewise/cli/constants.py
+++ b/pipelinewise/cli/constants.py
@@ -4,15 +4,31 @@
 class ConnectorType(enum.Enum):
     """
     Enums for various Singer connector type names
-    Todo: add more
     """
 
+    TAP_ADWORDS = 'tap-adwords'
+    TAP_GITHUB = 'tap-github'
+    TAP_GOOGLE_ANALYTICS = 'tap-google-analytics'
+    TAP_JIRA = 'tap-jira'
+    TAP_KAFKA = 'tap-kafka'
+    TAP_MIXPANEL = 'tap-mixpanel'
+    TAP_MONGODB = 'tap-mongodb'
     TAP_MYSQL = 'tap-mysql'
+    TAP_ORACLE = 'tap-oracle'
     TAP_POSTGRES = 'tap-postgres'
-    TAP_MONGODB = 'tap-mongodb'
     TAP_S3_CSV = 'tap-s3-csv'
+    TAP_SALESFORCE = 'tap-salesforce'
+    TAP_SHOPIFY = 'tap-shopify'
+    TAP_SLACK = 'tap-slack'
+    TAP_SNOWFLAKE = 'tap-snowflake'
+    TAP_TWILIO = 'tap-twilio'
+    TAP_ZENDESK = 'tap-zendesk'
+    TAP_ZUORA = 'tap-zuora'
 
     TARGET_BIGQUERY = 'target-bigquery'
     TARGET_POSTGRES = 'target-postgres'
     TARGET_SNOWFLAKE = 'target-snowflake'
     TARGET_REDSHIFT = 'target-redshift'
+    TARGET_S3_CSV = 'target-s3-csv'
+
+    TRANSFORM_FIELD = 'transform-field'

From c939b7c69c7db718dbd762f16e90761d8e646466 Mon Sep 17 00:00:00 2001
From: Dean Morin <morin.dean@gmail.com>
Date: Thu, 23 Sep 2021 00:36:51 -0700
Subject: [PATCH 20/78] Handle dates out of range in PG source (#799)

---
 pipelinewise/fastsync/commons/tap_postgres.py |  5 +-
 tests/db/tap_postgres_data.sql                | 49 ++++++++++---------
 2 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/pipelinewise/fastsync/commons/tap_postgres.py b/pipelinewise/fastsync/commons/tap_postgres.py
index ddfcc40af..25869da3a 100644
--- a/pipelinewise/fastsync/commons/tap_postgres.py
+++ b/pipelinewise/fastsync/commons/tap_postgres.py
@@ -413,7 +413,10 @@ def get_table_columns(self, table_name, max_num=None, date_type='date'):
                 data_type,
                 CASE
                     WHEN data_type = 'ARRAY' THEN 'array_to_json("' || column_name || '") AS ' || column_name
-                    WHEN data_type = 'date' THEN column_name || '::{date_type} AS ' || column_name
+                    WHEN data_type = 'date' THEN
+                       'CASE WHEN "' ||column_name|| E'" < \\'0001-01-01\\' '
+                            'OR "' ||column_name|| E'" > \\'9999-12-31\\' THEN \\'9999-12-31\\' '
+                            'ELSE "' ||column_name|| '"::{date_type} END AS "' ||column_name|| '"'
                     WHEN udt_name = 'time' THEN 'replace("' || column_name || E'"::varchar,\\\'24:00:00\\\',\\\'00:00:00\\\') AS ' || column_name
                     WHEN udt_name = 'timetz' THEN 'replace(("' || column_name || E'" at time zone \'\'UTC\'\')::time::varchar,\\\'24:00:00\\\',\\\'00:00:00\\\') AS ' || column_name
                     WHEN udt_name in ('timestamp', 'timestamptz') THEN
diff --git a/tests/db/tap_postgres_data.sql b/tests/db/tap_postgres_data.sql
index 44a2bdc60..e252631db 100644
--- a/tests/db/tap_postgres_data.sql
+++ b/tests/db/tap_postgres_data.sql
@@ -20,39 +20,40 @@ CREATE TABLE edgydata(
     cjson json,
     cjsonb jsonb,
     cvarchar varchar,
+    "date" date,
     PRIMARY KEY (cid)
 );
 
-insert into edgydata (ctimentz, ctimetz, cjson, cjsonb, cvarchar) values
-    (null, null, null, null, null),
-    ('23:00:15', '23:00:15+00', null, null, null),
-    ('12:00:15', '12:00:15+00:00', null, null, null),
-    ('12:00:15', '12:00:15+0300', null, null, null),
-    ('12:00:15', '12:00:15-0300', null, null, null),
-    ('24:00:00', '24:00:00', null, null, null),
-    ('24:00:00', '24:00:00+0000', null, null, null),
-    ('24:00:00', '24:00:00-0100', null, null, null),
-    ('00:00:00', '00:00:00', null, null, null),
-    (null, null, null, null,'Lorem ipsum dolor sit amet'),
-    (null, null, null, null,'Chinese: 和毛泽东 <<重上井冈山>>. 严永欣, 一九八八年.'),
-    (null, null, null, null,'Thai: แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช'),
-    (null, null, null, null,E'Special Characters: ["/\\,!@£$%^&*()]'),
-    (null, null, '[]', '[]', '[]' ),
-    (null, null, '{}', '{}', '{}'),
-    (null, null, '[{}, {}]', '[{}, {}]',  '[{}, {}]'),
+insert into edgydata (ctimentz, ctimetz, cjson, cjsonb, cvarchar, "date") values
+    (null, null, null, null, null, DATE '20107-05-28'),
+    ('23:00:15', '23:00:15+00', null, null, null, DATE '2011-09-10'),
+    ('12:00:15', '12:00:15+00:00', null, null, null, DATE '2019-12-10'),
+    ('12:00:15', '12:00:15+0300', null, null, null, DATE '0001-09-10'),
+    ('12:00:15', '12:00:15-0300', null, null, null, DATE '1990-09-30'),
+    ('24:00:00', '24:00:00', null, null, null, null),
+    ('24:00:00', '24:00:00+0000', null, null, null, DATE '333333-09-30'),
+    ('24:00:00', '24:00:00-0100', null, null, null, DATE '1990-09-30'),
+    ('00:00:00', '00:00:00', null, null, null, DATE '2021-01-30'),
+    (null, null, null, null,'Lorem ipsum dolor sit amet', DATE '2021-01-30'),
+    (null, null, null, null,'Chinese: 和毛泽东 <<重上井冈山>>. 严永欣, 一九八八年.', DATE '2021-01-30'),
+    (null, null, null, null,'Thai: แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช', DATE '2021-01-30'),
+    (null, null, null, null,E'Special Characters: ["/\\,!@£$%^&*()]', DATE '2021-01-30'),
+    (null, null, '[]', '[]', '[]' , DATE '2021-01-30'),
+    (null, null, '{}', '{}', '{}', DATE '2021-01-30'),
+    (null, null, '[{}, {}]', '[{}, {}]',  '[{}, {}]', DATE '2021-01-30'),
     (null, null, '[{"key": "ValueOne", "actions": []}, {"key": "ValueTwo", "actions": []}]',
         '[{"key": "ValueOne", "actions": []}, {"key": "ValueTwo", "actions": []}]',
-        '[{"key": "ValueOne", "actions": []}, {"key": "ValueTwo", "actions": []}]'),
-    (null, null, E'{"key": "Value\'s One"}', E'{"key": "Value\'s One"}', E'{"key": "Value\'s One"}'),
+        '[{"key": "ValueOne", "actions": []}, {"key": "ValueTwo", "actions": []}]', DATE '2021-01-30'),
+    (null, null, E'{"key": "Value\'s One"}', E'{"key": "Value\'s One"}', E'{"key": "Value\'s One"}', DATE '2021-01-30'),
     (null, null, E'[{"key": "Value\'s One", "actions": []},{"key": "Value\U00000027s Two", "actions": []}]',
         E'[{"key": "Value\'s One", "actions": []},{"key": "Value\U00000027s Two", "actions": []}]',
-        E'[{"key": "Value\'s One", "actions": []},{"key": "Value\U00000027s Two", "actions": []}]'),
-    (null, null, null, null,'	'),
+        E'[{"key": "Value\'s One", "actions": []},{"key": "Value\U00000027s Two", "actions": []}]', DATE '2021-01-30'),
+    (null, null, null, null,'	', DATE '2021-01-30'),
     (null, null, null, null,'Enter	The
-Ninja'),
+Ninja', DATE '2021-01-30'),
     (null, null, null, null,'Liewe
-Maatjies'),
-    (null, null, null, null,'Liewe	Maatjies')
+Maatjies', DATE '2021-01-30'),
+    (null, null, null, null,'Liewe	Maatjies', DATE '2021-01-30')
 ;
 
 COMMIT;

From 11e8bf99e965e56408e8dbd2a6877af11aa0e369 Mon Sep 17 00:00:00 2001
From: nishant8887 <nishant@whiterabbit.ai>
Date: Thu, 23 Sep 2021 17:05:30 +0530
Subject: [PATCH 21/78] Feature: Added new MASK-STRING-SKIP-ENDS transformation
 (#798)

---
 docs/user_guide/transformations.rst           |  2 +
 pipelinewise/cli/schemas/tap.json             | 11 +++-
 .../fastsync/commons/transform_utils.py       | 53 ++++++++++++++++
 .../transform-field/requirements.txt          |  2 +-
 tests/db/tap_mysql_data.sql                   | 38 ++++++++++++
 tests/db/tap_postgres_data.sql                | 30 ++++++++++
 .../test-project/tap_mysql_to_pg.yml.template | 10 ++++
 ...p_mysql_to_pg_buffered_stream.yml.template |  9 +++
 .../test-project/tap_mysql_to_sf.yml.template |  9 +++
 ...p_mysql_to_sf_buffered_stream.yml.template |  9 +++
 .../tap_postgres_to_pg.yml.template           | 10 ++++
 .../tap_postgres_to_sf.yml.template           | 10 ++++
 .../commons/test_fastsync_target_postgres.py  | 27 ++++++++-
 .../commons/test_fastsync_target_snowflake.py | 27 ++++++++-
 .../fastsync/commons/test_transform_utils.py  | 60 +++++++++++++++++++
 15 files changed, 303 insertions(+), 4 deletions(-)

diff --git a/docs/user_guide/transformations.rst b/docs/user_guide/transformations.rst
index 65aed97cc..8f6e867ae 100644
--- a/docs/user_guide/transformations.rst
+++ b/docs/user_guide/transformations.rst
@@ -40,6 +40,8 @@ The following transformations can be added optionally into the :ref:`yaml_config
 
 * **MASK-HIDDEN**: Transforms any string column value to 'hidden'.
 
+* **MASK-STRING-SKIP-ENDS-n**: Transforms string columns to masked version skipping first and last n characters, e.g. MASK-STRING-SKIP-ENDS-3
+
 
 .. _transformation_validation:
 
diff --git a/pipelinewise/cli/schemas/tap.json b/pipelinewise/cli/schemas/tap.json
index 575f52b28..ac7368e66 100644
--- a/pipelinewise/cli/schemas/tap.json
+++ b/pipelinewise/cli/schemas/tap.json
@@ -142,7 +142,16 @@
             "HASH-SKIP-FIRST-9",
             "MASK-DATE",
             "MASK-NUMBER",
-            "MASK-HIDDEN"
+            "MASK-HIDDEN",
+            "MASK-STRING-SKIP-ENDS-1",
+            "MASK-STRING-SKIP-ENDS-2",
+            "MASK-STRING-SKIP-ENDS-3",
+            "MASK-STRING-SKIP-ENDS-4",
+            "MASK-STRING-SKIP-ENDS-5",
+            "MASK-STRING-SKIP-ENDS-6",
+            "MASK-STRING-SKIP-ENDS-7",
+            "MASK-STRING-SKIP-ENDS-8",
+            "MASK-STRING-SKIP-ENDS-9"
           ]
         },
         "when": {
diff --git a/pipelinewise/fastsync/commons/transform_utils.py b/pipelinewise/fastsync/commons/transform_utils.py
index ef831f8a5..727a73232 100644
--- a/pipelinewise/fastsync/commons/transform_utils.py
+++ b/pipelinewise/fastsync/commons/transform_utils.py
@@ -22,6 +22,15 @@ class TransformationType(Enum):
     HASH_SKIP_FIRST_7 = 'HASH-SKIP-FIRST-7'
     HASH_SKIP_FIRST_8 = 'HASH-SKIP-FIRST-8'
     HASH_SKIP_FIRST_9 = 'HASH-SKIP-FIRST-9'
+    MASK_STRING_SKIP_ENDS_1 = 'MASK-STRING-SKIP-ENDS-1'
+    MASK_STRING_SKIP_ENDS_2 = 'MASK-STRING-SKIP-ENDS-2'
+    MASK_STRING_SKIP_ENDS_3 = 'MASK-STRING-SKIP-ENDS-3'
+    MASK_STRING_SKIP_ENDS_4 = 'MASK-STRING-SKIP-ENDS-4'
+    MASK_STRING_SKIP_ENDS_5 = 'MASK-STRING-SKIP-ENDS-5'
+    MASK_STRING_SKIP_ENDS_6 = 'MASK-STRING-SKIP-ENDS-6'
+    MASK_STRING_SKIP_ENDS_7 = 'MASK-STRING-SKIP-ENDS-7'
+    MASK_STRING_SKIP_ENDS_8 = 'MASK-STRING-SKIP-ENDS-8'
+    MASK_STRING_SKIP_ENDS_9 = 'MASK-STRING-SKIP-ENDS-9'
 
 
 @unique
@@ -116,6 +125,17 @@ def get_trans_in_sql_flavor(
                         {'trans': f'{column} = 0', 'conditions': conditions}
                     )
 
+                elif transform_type.value.startswith('MASK-STRING-SKIP-ENDS-'):
+
+                    trans_map.append(
+                        {
+                            'trans': cls.__mask_string_skip_ends_to_sql(
+                                transform_type, column, sql_flavor
+                            ),
+                            'conditions': conditions,
+                        }
+                    )
+
                 elif transform_type == TransformationType.MASK_HIDDEN:
                     trans_map.append(
                         {'trans': f"{column} = 'hidden'", 'conditions': conditions}
@@ -330,3 +350,36 @@ def __mask_date_to_sql(cls, column: str, sql_flavor: SQLFlavor) -> str:
             )
 
         return trans
+
+    @classmethod
+    # pylint: disable=W0238  # False positive when it is used by another classmethod
+    def __mask_string_skip_ends_to_sql(
+        cls, transform_type: TransformationType, column: str, sql_flavor: SQLFlavor
+    ) -> str:
+        """
+        convert MASK-STRING-SKIP-ENDS-n transformation into the right sql string
+        Args:
+            column: column to apply the masking to
+            sql_flavor: the sql flavor to use
+
+        Raises: NotImplementedError if mask-string-skip-ends is not implemented for the given sql flavor
+
+        Returns: sql string equivalent of the mask-string-skip-ends
+        """
+        skip_ends_n = int(transform_type.value[-1])
+
+        if sql_flavor == SQLFlavor.SNOWFLAKE:
+            trans = '{0} = CASE WHEN LENGTH({0}) > 2 * {1} THEN ' \
+                    'CONCAT(SUBSTRING({0}, 1, {1}), REPEAT(\'*\', LENGTH({0})-(2 * {1})), ' \
+                    'SUBSTRING({0}, LENGTH({0})-{1}+1, {1})) ' \
+                    'ELSE REPEAT(\'*\', LENGTH({0})) END'.format(column, skip_ends_n)
+        elif sql_flavor == SQLFlavor.POSTGRES:
+            trans = '{0} = CASE WHEN LENGTH({0}) > 2 * {1} THEN ' \
+                    'CONCAT(SUBSTRING({0}, 1, {1}), REPEAT(\'*\', LENGTH({0})-(2 * {1})), ' \
+                    'SUBSTRING({0}, LENGTH({0})-{1}+1, {1})) ' \
+                    'ELSE REPEAT(\'*\', LENGTH({0})) END'.format(column, skip_ends_n)
+        else:
+            raise NotImplementedError(f'MASK-STRING-SKIP-ENDS transformation in {sql_flavor.value} SQL flavor '
+                                      f'not implemented!')
+
+        return trans
diff --git a/singer-connectors/transform-field/requirements.txt b/singer-connectors/transform-field/requirements.txt
index 428e947f9..d3f27cc8c 100644
--- a/singer-connectors/transform-field/requirements.txt
+++ b/singer-connectors/transform-field/requirements.txt
@@ -1 +1 @@
-pipelinewise-transform-field==2.1.0
+pipelinewise-transform-field==2.2.0
diff --git a/tests/db/tap_mysql_data.sql b/tests/db/tap_mysql_data.sql
index 033d72aaf..deec5c03a 100644
--- a/tests/db/tap_mysql_data.sql
+++ b/tests/db/tap_mysql_data.sql
@@ -1733,6 +1733,44 @@ VALUES ('x',
 UNLOCK TABLES;
 
 
+--
+-- Table structure for table `customers`
+--
+
+DROP TABLE IF EXISTS `customers`;
+CREATE TABLE `customers` (
+  id INT AUTO_INCREMENT PRIMARY KEY,
+  name VARCHAR(100),
+  phone VARCHAR(100),
+  email TEXT
+)
+ENGINE=MyISAM AUTO_INCREMENT=0 DEFAULT CHARSET=utf8;
+
+LOCK TABLES `customers` WRITE;
+/*!40000 ALTER TABLE `customers` DISABLE KEYS */;
+INSERT INTO customers (name, phone, email) VALUES ('Martelle Cristoforetti', '2071506307', 'mcristoforetti0@accuweather.com'),
+	('Joelynn Cawthorne', '9764917766', 'jcawthorne1@bizjournals.com'),
+	('Gerrie Gillie', '7907324372', 'ggillie2@acquirethisname.com'),
+	('Sheilakathryn Muge', '7195638381', 'smuge3@umn.edu'),
+	('Fedora Mellanby', '4842466494', 'fmellanby4@fema.gov'),
+	('Dall Goodbourn', '4977729268', 'dgoodbourn5@dailymail.co.uk'),
+	('Helaina Van Halen', '5148995067', 'hvan6@hp.com'),
+	('Noelle Knight', NULL, 'nknight7@google.co.uk'),
+	('Anetta Doodney', '4669478802', 'adoodney8@1688.com'),
+	('Almira Clinch', '2235449869', 'aclinch9@netvibes.com'),
+	('Aidan Bachnic', '8479230171', 'abachnica@msu.edu'),
+	('Biddy Shirt', '8109841957', 'bshirtb@nps.gov'),
+	('Robinson Matuszinski', '7587994157', 'rmatuszinskic@ed.gov'),
+	('Pall Porcher', '2719539572', 'pporcherd@thetimes.co.uk'),
+	('Briggs Fyall', '5431890133', 'bfyalle@a8.net'),
+	('Shara Eversfield', '1796055914', 'seversfieldf@prlog.org'),
+	('Abbot Scowcraft', '1658501516', 'ascowcraftg@shutterfly.com'),
+	('Ransell Fardo', '9584912534', 'rfardoh@de.vu'),
+	('Leonard Buche', '9391842560', 'lbuchei@netscape.com'),
+	('Xylia Adnet', '6784481146', 'xadnetj@auda.org.au');
+/*!40000 ALTER TABLE `customers` ENABLE KEYS */;
+UNLOCK TABLES;
+
 /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
 /*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
 /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
diff --git a/tests/db/tap_postgres_data.sql b/tests/db/tap_postgres_data.sql
index e252631db..1c075e32b 100644
--- a/tests/db/tap_postgres_data.sql
+++ b/tests/db/tap_postgres_data.sql
@@ -4529,3 +4529,33 @@ values ('A', '2020-01-01 08:53:56.112248-07'),
        ('H', '1000-03-03 10:30:00 BC'),
        ('I', '50000-03-03 10:30:00')
        ;
+
+DROP TABLE IF EXISTS public.customers CASCADE;
+
+CREATE TABLE customers (
+    id serial primary key,
+    name varchar(100) NOT NULL,
+    phone varchar(10),
+    email text
+);
+
+INSERT INTO customers (name, phone, email) VALUES ('Martelle Cristoforetti', '2071506307', 'mcristoforetti0@accuweather.com'),
+	('Joelynn Cawthorne', '9764917766', 'jcawthorne1@bizjournals.com'),
+	('Gerrie Gillie', '7907324372', 'ggillie2@acquirethisname.com'),
+	('Sheilakathryn Muge', '7195638381', 'smuge3@umn.edu'),
+	('Fedora Mellanby', '4842466494', 'fmellanby4@fema.gov'),
+	('Dall Goodbourn', '4977729268', 'dgoodbourn5@dailymail.co.uk'),
+	('Helaina Van Halen', '5148995067', 'hvan6@hp.com'),
+	('Noelle Knight', NULL, 'nknight7@google.co.uk'),
+	('Anetta Doodney', '4669478802', 'adoodney8@1688.com'),
+	('Almira Clinch', '2235449869', 'aclinch9@netvibes.com'),
+	('Aidan Bachnic', '8479230171', 'abachnica@msu.edu'),
+	('Biddy Shirt', '8109841957', 'bshirtb@nps.gov'),
+	('Robinson Matuszinski', '7587994157', 'rmatuszinskic@ed.gov'),
+	('Pall Porcher', '2719539572', 'pporcherd@thetimes.co.uk'),
+	('Briggs Fyall', '5431890133', 'bfyalle@a8.net'),
+	('Shara Eversfield', '1796055914', 'seversfieldf@prlog.org'),
+	('Abbot Scowcraft', '1658501516', 'ascowcraftg@shutterfly.com'),
+	('Ransell Fardo', '9584912534', 'rfardoh@de.vu'),
+	('Leonard Buche', '9391842560', 'lbuchei@netscape.com'),
+	('Xylia Adnet', '6784481146', 'xadnetj@auda.org.au');
diff --git a/tests/end_to_end/test-project/tap_mysql_to_pg.yml.template b/tests/end_to_end/test-project/tap_mysql_to_pg.yml.template
index 9c08c849a..15cb07e18 100644
--- a/tests/end_to_end/test-project/tap_mysql_to_pg.yml.template
+++ b/tests/end_to_end/test-project/tap_mysql_to_pg.yml.template
@@ -100,3 +100,13 @@ schemas:
       ### Table with all possible data types
       - table_name: "all_datatypes"
         replication_method: "LOG_BASED"
+
+      ### Table with LOG_BASED replication
+      - table_name: "customers"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
diff --git a/tests/end_to_end/test-project/tap_mysql_to_pg_buffered_stream.yml.template b/tests/end_to_end/test-project/tap_mysql_to_pg_buffered_stream.yml.template
index 07adeb3c0..6caa231a9 100644
--- a/tests/end_to_end/test-project/tap_mysql_to_pg_buffered_stream.yml.template
+++ b/tests/end_to_end/test-project/tap_mysql_to_pg_buffered_stream.yml.template
@@ -101,3 +101,12 @@ schemas:
       - table_name: "all_datatypes"
         replication_method: "LOG_BASED"
 
+      ### Table with LOG_BASED replication
+      - table_name: "customers"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
diff --git a/tests/end_to_end/test-project/tap_mysql_to_sf.yml.template b/tests/end_to_end/test-project/tap_mysql_to_sf.yml.template
index e31b85d4d..9100d9922 100644
--- a/tests/end_to_end/test-project/tap_mysql_to_sf.yml.template
+++ b/tests/end_to_end/test-project/tap_mysql_to_sf.yml.template
@@ -101,3 +101,12 @@ schemas:
       - table_name: "all_datatypes"
         replication_method: "LOG_BASED"
 
+      ### Table with LOG_BASED replication
+      - table_name: "customers"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
diff --git a/tests/end_to_end/test-project/tap_mysql_to_sf_buffered_stream.yml.template b/tests/end_to_end/test-project/tap_mysql_to_sf_buffered_stream.yml.template
index 0ade4e873..9ba05b188 100644
--- a/tests/end_to_end/test-project/tap_mysql_to_sf_buffered_stream.yml.template
+++ b/tests/end_to_end/test-project/tap_mysql_to_sf_buffered_stream.yml.template
@@ -101,3 +101,12 @@ schemas:
       - table_name: "all_datatypes"
         replication_method: "LOG_BASED"
 
+      ### Table with LOG_BASED replication
+      - table_name: "customers"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
diff --git a/tests/end_to_end/test-project/tap_postgres_to_pg.yml.template b/tests/end_to_end/test-project/tap_postgres_to_pg.yml.template
index d895bf453..fde6ffa3a 100644
--- a/tests/end_to_end/test-project/tap_postgres_to_pg.yml.template
+++ b/tests/end_to_end/test-project/tap_postgres_to_pg.yml.template
@@ -78,6 +78,16 @@ schemas:
       - table_name: "table_with_reserved_words"
         replication_method: "FULL_TABLE"
 
+      ### Table with INCREMENTAL replication
+      - table_name: "customers"
+        replication_method: "INCREMENTAL"
+        replication_key: "id"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
 
   ### SOURCE SCHEMA 2: public2
   - source_schema: "public2"
diff --git a/tests/end_to_end/test-project/tap_postgres_to_sf.yml.template b/tests/end_to_end/test-project/tap_postgres_to_sf.yml.template
index b7a2a2952..449150f6a 100644
--- a/tests/end_to_end/test-project/tap_postgres_to_sf.yml.template
+++ b/tests/end_to_end/test-project/tap_postgres_to_sf.yml.template
@@ -75,6 +75,16 @@ schemas:
       - table_name: "table_with_reserved_words"
         replication_method: "FULL_TABLE"
 
+      ### Table with INCREMENTAL replication
+      - table_name: "customers"
+        replication_method: "INCREMENTAL"
+        replication_key: "id"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
 
   ### SOURCE SCHEMA 2: public2
   - source_schema: "public2"
diff --git a/tests/units/fastsync/commons/test_fastsync_target_postgres.py b/tests/units/fastsync/commons/test_fastsync_target_postgres.py
index 65647cd5a..047008539 100644
--- a/tests/units/fastsync/commons/test_fastsync_target_postgres.py
+++ b/tests/units/fastsync/commons/test_fastsync_target_postgres.py
@@ -312,6 +312,11 @@ def test_obfuscate_columns_case3(self):
                     'tap_stream_name': 'public-my_table',
                     'type': 'HASH-SKIP-FIRST-5',
                 },
+                {
+                    'field_id': 'col_7',
+                    'tap_stream_name': 'public-my_table',
+                    'type': 'MASK-STRING-SKIP-ENDS-3',
+                },
             ]
         }
 
@@ -328,7 +333,11 @@ def test_obfuscate_columns_case3(self):
                 '"col_4" = 0, '
                 '"col_5" = ENCODE(DIGEST("col_5", \'sha256\'), \'hex\'), '
                 '"col_6" = CONCAT(SUBSTRING("col_6", 1, 5), '
-                'ENCODE(DIGEST(SUBSTRING("col_6", 5 + 1), \'sha256\'), \'hex\'));'
+                'ENCODE(DIGEST(SUBSTRING("col_6", 5 + 1), \'sha256\'), \'hex\')), '
+                '"col_7" = CASE WHEN LENGTH("col_7") > 2 * 3 THEN '
+                'CONCAT(SUBSTRING("col_7", 1, 3), REPEAT(\'*\', LENGTH("col_7")-(2 * 3)), '
+                'SUBSTRING("col_7", LENGTH("col_7")-3+1, 3)) '
+                'ELSE REPEAT(\'*\', LENGTH("col_7")) END;'
             ],
         )
 
@@ -383,6 +392,16 @@ def test_obfuscate_columns_case4(self):
                         {'column': 'col_2', 'regex_match': r'[0-9]{3}\.[0-9]{3}'},
                     ],
                 },
+                {
+                    'field_id': 'col_7',
+                    'tap_stream_name': 'public-my_table',
+                    'type': 'MASK-STRING-SKIP-ENDS-3',
+                    'when': [
+                        {'column': 'col_1', 'equals': 30},
+                        {'column': 'col_2', 'regex_match': r'[0-9]{3}\.[0-9]{3}'},
+                        {'column': 'col_4', 'equals': None},
+                    ],
+                },
             ]
         }
 
@@ -400,6 +419,12 @@ def test_obfuscate_columns_case4(self):
                 '"col_6" = CONCAT(SUBSTRING("col_6", 1, 5), '
                 'ENCODE(DIGEST(SUBSTRING("col_6", 5 + 1), \'sha256\'), \'hex\')) WHERE ("col_1" = 30) AND '
                 '("col_2" ~ \'[0-9]{3}\.[0-9]{3}\');',  # pylint: disable=W1401  # noqa: W605
+                'UPDATE "my_schema"."my_table_temp" SET '
+                '"col_7" = CASE WHEN LENGTH("col_7") > 2 * 3 THEN '
+                'CONCAT(SUBSTRING("col_7", 1, 3), REPEAT(\'*\', LENGTH("col_7")-(2 * 3)), '
+                'SUBSTRING("col_7", LENGTH("col_7")-3+1, 3)) '
+                'ELSE REPEAT(\'*\', LENGTH("col_7")) END WHERE ("col_1" = 30) AND '
+                '("col_2" ~ \'[0-9]{3}\.[0-9]{3}\') AND ("col_4" IS NULL);',  # pylint: disable=W1401  # noqa: W605
                 'UPDATE "my_schema"."my_table_temp" SET "col_1" = NULL, '
                 '"col_4" = 0, "col_5" = ENCODE(DIGEST("col_5", \'sha256\'), \'hex\');',
             ],
diff --git a/tests/units/fastsync/commons/test_fastsync_target_snowflake.py b/tests/units/fastsync/commons/test_fastsync_target_snowflake.py
index c66a527ac..3c6fa9ad6 100644
--- a/tests/units/fastsync/commons/test_fastsync_target_snowflake.py
+++ b/tests/units/fastsync/commons/test_fastsync_target_snowflake.py
@@ -488,6 +488,11 @@ def test_obfuscate_columns_case3(self):
                     'tap_stream_name': 'public-my_table',
                     'type': 'HASH-SKIP-FIRST-5',
                 },
+                {
+                    'field_id': 'col_7',
+                    'tap_stream_name': 'public-my_table',
+                    'type': 'MASK-STRING-SKIP-ENDS-3',
+                },
             ]
         }
 
@@ -502,7 +507,11 @@ def test_obfuscate_columns_case3(self):
                 '"COL_3" = TIMESTAMP_NTZ_FROM_PARTS(DATE_FROM_PARTS(YEAR("COL_3"), 1, 1),TO_TIME("COL_3")), '
                 '"COL_4" = 0, '
                 '"COL_5" = SHA2("COL_5", 256), '
-                '"COL_6" = CONCAT(SUBSTRING("COL_6", 1, 5), SHA2(SUBSTRING("COL_6", 5 + 1), 256));'
+                '"COL_6" = CONCAT(SUBSTRING("COL_6", 1, 5), SHA2(SUBSTRING("COL_6", 5 + 1), 256)), '
+                '"COL_7" = CASE WHEN LENGTH("COL_7") > 2 * 3 THEN '
+                'CONCAT(SUBSTRING("COL_7", 1, 3), REPEAT(\'*\', LENGTH("COL_7")-(2 * 3)), '
+                'SUBSTRING("COL_7", LENGTH("COL_7")-3+1, 3)) '
+                'ELSE REPEAT(\'*\', LENGTH("COL_7")) END;'
             ],
         )
 
@@ -557,6 +566,16 @@ def test_obfuscate_columns_case4(self):
                         {'column': 'col_2', 'regex_match': r'[0-9]{3}\.[0-9]{3}'},
                     ],
                 },
+                {
+                    'field_id': 'col_7',
+                    'tap_stream_name': 'public-my_table',
+                    'type': 'MASK-STRING-SKIP-ENDS-3',
+                    'when': [
+                        {'column': 'col_1', 'equals': 30},
+                        {'column': 'col_2', 'regex_match': r'[0-9]{3}\.[0-9]{3}'},
+                        {'column': 'col_4', 'equals': None},
+                    ],
+                },
             ]
         }
 
@@ -572,6 +591,12 @@ def test_obfuscate_columns_case4(self):
                 '"COL_6" = CONCAT(SUBSTRING("COL_6", 1, 5), SHA2(SUBSTRING("COL_6", 5 + 1), 256)) '
                 'WHERE ("COL_1" = 30) AND ("COL_2" '
                 'REGEXP \'[0-9]{3}\.[0-9]{3}\');',  # pylint: disable=W1401  # noqa: W605
+                'UPDATE "MY_SCHEMA"."MY_TABLE_TEMP" SET '
+                '"COL_7" = CASE WHEN LENGTH("COL_7") > 2 * 3 THEN '
+                'CONCAT(SUBSTRING("COL_7", 1, 3), REPEAT(\'*\', LENGTH("COL_7")-(2 * 3)), '
+                'SUBSTRING("COL_7", LENGTH("COL_7")-3+1, 3)) '
+                'ELSE REPEAT(\'*\', LENGTH("COL_7")) END WHERE ("COL_1" = 30) AND ("COL_2" '
+                'REGEXP \'[0-9]{3}\.[0-9]{3}\') AND ("COL_4" IS NULL);',  # pylint: disable=W1401  # noqa: W605
                 'UPDATE "MY_SCHEMA"."MY_TABLE_TEMP" SET "COL_1" = NULL, "COL_4" = 0, "COL_5" = SHA2("COL_5", 256);',
             ],
         )
diff --git a/tests/units/fastsync/commons/test_transform_utils.py b/tests/units/fastsync/commons/test_transform_utils.py
index bdc99f764..68278afed 100644
--- a/tests/units/fastsync/commons/test_transform_utils.py
+++ b/tests/units/fastsync/commons/test_transform_utils.py
@@ -104,6 +104,11 @@ def test_get_trans_in_sql_case4(self):
                 'tap_stream_name': 'public-my_table',
                 'type': 'HASH-SKIP-FIRST-5',
             },
+            {
+                'field_id': 'col_7',
+                'tap_stream_name': 'public-my_table',
+                'type': 'MASK-STRING-SKIP-ENDS-3',
+            },
         ]
 
         trans = TransformationHelper.get_trans_in_sql_flavor(
@@ -126,6 +131,13 @@ def test_get_trans_in_sql_case4(self):
                     'trans': '"COL_6" = CONCAT(SUBSTRING("COL_6", 1, 5), SHA2(SUBSTRING("COL_6", 5 + 1), 256))',
                     'conditions': None,
                 },
+                {
+                    'trans': '"COL_7" = CASE WHEN LENGTH("COL_7") > 2 * 3 THEN '
+                    'CONCAT(SUBSTRING("COL_7", 1, 3), REPEAT(\'*\', LENGTH("COL_7")-(2 * 3)), '
+                    'SUBSTRING("COL_7", LENGTH("COL_7")-3+1, 3)) '
+                    'ELSE REPEAT(\'*\', LENGTH("COL_7")) END',
+                    'conditions': None,
+                },
             ],
         )
 
@@ -169,6 +181,11 @@ def test_get_trans_in_sql_case5(self):
                 'tap_stream_name': 'public-my_table',
                 'type': 'HASH-SKIP-FIRST-5',
             },
+            {
+                'field_id': 'col_7',
+                'tap_stream_name': 'public-my_table',
+                'type': 'MASK-STRING-SKIP-ENDS-3',
+            },
         ]
 
         trans = TransformationHelper.get_trans_in_sql_flavor(
@@ -201,6 +218,13 @@ def test_get_trans_in_sql_case5(self):
                     'ENCODE(DIGEST(SUBSTRING("col_6", 5 + 1), \'sha256\'), \'hex\'))',
                     'conditions': None,
                 },
+                {
+                    'trans': '"col_7" = CASE WHEN LENGTH("col_7") > 2 * 3 THEN '
+                    'CONCAT(SUBSTRING("col_7", 1, 3), REPEAT(\'*\', LENGTH("col_7")-(2 * 3)), '
+                    'SUBSTRING("col_7", LENGTH("col_7")-3+1, 3)) '
+                    'ELSE REPEAT(\'*\', LENGTH("col_7")) END',
+                    'conditions': None,
+                },
             ],
         )
 
@@ -254,6 +278,16 @@ def test_get_trans_in_sql_case6(self):
                     {'column': 'col_2', 'regex_match': r'[0-9]{3}\.[0-9]{3}'},
                 ],
             },
+            {
+                'field_id': 'col_7',
+                'tap_stream_name': 'public-my_table',
+                'type': 'MASK-STRING-SKIP-ENDS-3',
+                'when': [
+                    {'column': 'col_1', 'equals': 30},
+                    {'column': 'col_2', 'regex_match': r'[0-9]{3}\.[0-9]{3}'},
+                    {'column': 'col_4', 'equals': None},
+                ],
+            },
         ]
 
         trans = TransformationHelper.get_trans_in_sql_flavor(
@@ -289,6 +323,14 @@ def test_get_trans_in_sql_case6(self):
                     'conditions': '("COL_1" = 30) AND ("COL_2" '
                     'REGEXP \'[0-9]{3}\.[0-9]{3}\')',  # pylint: disable=W1401  # noqa: W605
                 },
+                {
+                    'trans': '"COL_7" = CASE WHEN LENGTH("COL_7") > 2 * 3 THEN '
+                    'CONCAT(SUBSTRING("COL_7", 1, 3), REPEAT(\'*\', LENGTH("COL_7")-(2 * 3)), '
+                    'SUBSTRING("COL_7", LENGTH("COL_7")-3+1, 3)) '
+                    'ELSE REPEAT(\'*\', LENGTH("COL_7")) END',
+                    'conditions': '("COL_1" = 30) AND ("COL_2" '
+                    'REGEXP \'[0-9]{3}\.[0-9]{3}\') AND ("COL_4" IS NULL)',  # pylint: disable=W1401  # noqa: W605
+                },
             ],
         )
 
@@ -342,6 +384,16 @@ def test_get_trans_in_sql_case7(self):
                     {'column': 'col_2', 'regex_match': r'[0-9]{3}\.[0-9]{3}'},
                 ],
             },
+            {
+                'field_id': 'col_7',
+                'tap_stream_name': 'public-my_table',
+                'type': 'MASK-STRING-SKIP-ENDS-3',
+                'when': [
+                    {'column': 'col_1', 'equals': 30},
+                    {'column': 'col_2', 'regex_match': r'[0-9]{3}\.[0-9]{3}'},
+                    {'column': 'col_4', 'equals': None},
+                ],
+            },
         ]
 
         trans = TransformationHelper.get_trans_in_sql_flavor(
@@ -383,5 +435,13 @@ def test_get_trans_in_sql_case7(self):
                     '\'sha256\'), \'hex\'))',
                     'conditions': '("col_1" = 30) AND ("col_2" ~ \'[0-9]{3}\.[0-9]{3}\')',  # pylint: disable=W1401  # noqa: W605, E501
                 },
+                {
+                    'trans': '"col_7" = CASE WHEN LENGTH("col_7") > 2 * 3 THEN '
+                    'CONCAT(SUBSTRING("col_7", 1, 3), REPEAT(\'*\', LENGTH("col_7")-(2 * 3)), '
+                    'SUBSTRING("col_7", LENGTH("col_7")-3+1, 3)) '
+                    'ELSE REPEAT(\'*\', LENGTH("col_7")) END',
+                    'conditions': '("col_1" = 30) AND ("col_2" ~ \'[0-9]{3}\.[0-9]{3}\') '  # pylint: disable=W1401  # noqa: W605, E501
+                    'AND ("col_4" IS NULL)',
+                },
             ],
         )

From 75a1abad16ffc6a2b7ec24c27a5a99c8eea94b2a Mon Sep 17 00:00:00 2001
From: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
Date: Fri, 24 Sep 2021 13:29:28 +0300
Subject: [PATCH 22/78] Patch tap pg (#804)

---
 singer-connectors/tap-postgres/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singer-connectors/tap-postgres/requirements.txt b/singer-connectors/tap-postgres/requirements.txt
index 18712a76a..ca346f8db 100644
--- a/singer-connectors/tap-postgres/requirements.txt
+++ b/singer-connectors/tap-postgres/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-postgres==1.8.0
+pipelinewise-tap-postgres==1.8.1

From 739e775fb1e822cfe7b8342b22fd814a5e2008a8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 24 Sep 2021 14:23:13 +0300
Subject: [PATCH 23/78] Bump pre-commit from 2.14.0 to 2.15.0 (#794)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 506f4482c..086f2f708 100644
--- a/setup.py
+++ b/setup.py
@@ -31,7 +31,7 @@
           'singer-encodings==0.0.*',
           'messytables==0.15.*',
           'python-pidfile==3.0.0',
-          'pre-commit==2.14.0',
+          'pre-commit==2.15.0',
           'pymongo>=3.10,<3.13',
           'tzlocal>=2.0,<2.2',
           'slackclient>=2.7,<2.10',

From 0be085433f340c3193fb95bc4343a00a108bd884 Mon Sep 17 00:00:00 2001
From: Amir Mofakhar <amir.mofakhar@transferwise.com>
Date: Tue, 28 Sep 2021 08:49:59 +0100
Subject: [PATCH 24/78] [AP-1055] Makefile for installation (#803)

* introducing makefile

* updated docker files

* updated documentations and scripts to use makefile

* updated workflow

* update workflow to install only connectors

* fixed wrong place of character

* removed unnecessary target for all_connectors

* fixed to support ubuntu 18

* fixed list variables to be compatibale with ubuntu 18

* updated ci check

* updated help message
---
 .circleci/config.yml                     |   2 +-
 .github/workflows/connectors.yml         |   2 +-
 Dockerfile                               |   2 +-
 Dockerfile.ci                            |   2 +-
 Makefile                                 | 298 +++++++++++++++++++++++
 README.md                                |  20 +-
 dev-project/entrypoint.sh                |   2 +-
 docs/installation_guide/installation.rst |   6 +-
 install.sh                               | 249 -------------------
 scripts/ci_check_no_file_changes.sh      |   2 +-
 10 files changed, 321 insertions(+), 264 deletions(-)
 create mode 100644 Makefile
 delete mode 100755 install.sh

diff --git a/.circleci/config.yml b/.circleci/config.yml
index db5de8242..958ce2065 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -115,7 +115,7 @@ jobs:
               ./tests/db/tap_mongodb.sh
               ./tests/db/target_postgres.sh
 
-              ./install.sh --acceptlicenses --connectors=target-snowflake,target-postgres,target-bigquery,tap-mysql,tap-postgres,tap-mongodb,transform-field,tap-s3-csv
+              make pipelinewise connectors -e pw_acceptlicenses=y -e pw_connector=target-snowflake,target-postgres,target-bigquery,tap-mysql,tap-postgres,tap-mongodb,transform-field,tap-s3-csv
               . .virtualenvs/pipelinewise/bin/activate
               export PIPELINEWISE_HOME=$PWD
               pytest tests/end_to_end -vx
diff --git a/.github/workflows/connectors.yml b/.github/workflows/connectors.yml
index e7838201a..1c19c56e9 100644
--- a/.github/workflows/connectors.yml
+++ b/.github/workflows/connectors.yml
@@ -27,4 +27,4 @@ jobs:
 
       - name: Check all connectors are installable
         run: |
-          ./install.sh --acceptlicenses --connectors=all
+          make all_connectors -e pw_acceptlicenses=y
diff --git a/Dockerfile b/Dockerfile
index 9ad1de1f1..0ddda2d8a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,7 +16,7 @@ RUN apt-get -qq update \
     # Install Oracle Instant Client for tap-oracle if its in the connectors list
     && bash -c "if grep -q \"tap-oracle\" <<< \"$connectors\"; then wget https://download.oracle.com/otn_software/linux/instantclient/193000/oracle-instantclient19.3-basiclite-19.3.0.0.0-1.x86_64.rpm -O /app/oracle-instantclient.rpm && alien -i /app/oracle-instantclient.rpm --scripts && rm -rf /app/oracle-instantclient.rpm ; fi" \
     && cd /app \
-    && ./install.sh --connectors=$connectors --acceptlicenses --nousage --notestextras \
+    && make pipelinewise_no_test_extras connectors -e pw_connector=$connectors -e pw_acceptlicenses=y \
     && ln -s /root/.pipelinewise /app/.pipelinewise
 
 ENTRYPOINT ["/app/entrypoint.sh"]
diff --git a/Dockerfile.ci b/Dockerfile.ci
index d6f620011..a0137407d 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -7,7 +7,7 @@ RUN apt-get -qq update \
     && rm -rf /var/lib/apt/lists/* \
     && pip install -U --no-cache-dir pip \
     && cd /app \
-    && ./install.sh --connectors=none --acceptlicenses --nousage --notestextras \
+    && make pipelinewise_no_test_extras -e pw_acceptlicenses=y \
     && ln -s /root/.pipelinewise /app/.pipelinewise
 
 ENTRYPOINT ["/app/entrypoint.sh"]
diff --git a/Makefile b/Makefile
new file mode 100644
index 000000000..5d67a7fcc
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,298 @@
+SHELL = bash
+OK_MSG = \x1b[32m ✔\x1b[0m
+FAIL_MSG = \x1b[31m ✖\x1b[0m
+YELLOW = \x1b[33m
+BLUE = \x1b[36m
+RED = \x1b[31m
+RESET_COLOR = \x1b[0m
+PIPELINEWISE_HOME = $(shell pwd)
+VENV_DIR = ${PIPELINEWISE_HOME}/.virtualenvs
+
+start_time:=$(shell date +%s)
+
+PIP_ARGS="[test]"
+
+pw_connector=
+
+define DEFAULT_CONNECTORS
+tap-jira\
+tap-kafka\
+tap-mysql\
+tap-postgres\
+tap-s3-csv\
+tap-salesforce\
+tap-snowflake\
+tap-zendesk\
+tap-mongodb\
+tap-github\
+tap-slack\
+tap-mixpanel\
+tap-twilio\
+target-s3-csv\
+target-snowflake\
+target-redshift\
+target-postgres\
+target-bigquery\
+transform-field
+endef
+
+define EXTRA_CONNECTORS
+tap-adwords\
+tap-oracle\
+tap-zuora\
+tap-google-analytics\
+tap-shopify
+endef
+
+define print_installed_connectors
+	@echo
+	@echo "--------------------------------------------------------------------------"
+	@echo "Installed components:"
+	@echo "--------------------------------------------------------------------------"
+	@echo
+	@echo "Component            Version"
+	@echo "-------------------- -------"
+	@for i in $(shell ls $(VENV_DIR)); do\
+		VERSION=`$(VENV_DIR)/$$i/bin/python3 -m pip list | grep "$$i[[:space:]]" | awk '{print $$2}'`;\
+		printf "%-20s %s\n" $$i "$$VERSION";\
+	done;
+	@echo "-------------------- -------"
+endef
+
+define print_execute_time
+	$(eval end_time:=`date +%s`)
+	@echo
+	@echo "--------------------------------------------------------------------------"
+	@echo "$(1) installed successfully in $$(( $(end_time) - $(start_time) )) seconds"
+	@echo "--------------------------------------------------------------------------"
+endef
+
+define clean_connectors
+	echo -n "Cleaning previous installations in $(VENV_DIR)/$(1)..."
+	rm -rf $(VENV_DIR)/$(1)
+	@echo -e "$(OK_MSG)"
+endef
+
+define check_license
+	@echo "Checking license..."
+	@echo -e "$(YELLOW)"
+	@$(VENV_DIR)/$(1)/bin/python3 -m pip install pip-licenses
+	@echo -e "$(RESET_COLOR)"
+	$(eval PKG_NAME:=`$(VENV_DIR)/$(1)/bin/pip-licenses|grep "$(1)[[:space:]]"| awk '{print $$$$1}'`)
+	$(eval PKG_VERSION:=`$(VENV_DIR)/$(1)/bin/pip-licenses | grep "$(1)[[:space:]]" | awk '{print $$$$2}'`)
+	$(eval PKG_LICENSE:=`$(VENV_DIR)/$(1)/bin/pip-licenses --from mixed | grep "$(1)[[:space:]]" | awk '{for (i=1; i<=NF-2; i++) $$$$i = $$$$(i+2); NF-=2; print}'`)
+
+	$(eval MAIN_LICENSE:="Apache Software License")
+
+	@if [[ "$(PKG_LICENSE)" != $(MAIN_LICENSE) && "$(PKG_LICENSE)" != "UNKNOWN" ]]; then\
+		echo -e "$(RED)";\
+		echo;\
+        echo "  | $(PKG_NAME) ($(PKG_VERSION)) is licensed under $(PKG_LICENSE)";\
+        echo "  |";\
+        echo "  | WARNING. The license of this connector is different than the default PipelineWise license ($(MAIN_LICENSE)).";\
+        if [[ "$(ACCEPT_LICENSES)" != "YES" ]]; then\
+            echo "  | You need to accept the connector's license agreement to proceed.";\
+            echo "  |";\
+            read -r -p "  | Do you accept the [$(PKG_LICENSE)] license agreement of $(PKG_NAME) connector? [y/N] " response;\
+			if [[ $$response != "y" && $$response != "Y" ]]; then\
+				echo;\
+				echo -e "$(RESET_COLOR)";\
+				echo "EXIT. License agreement not accepted!";\
+				exit 1;\
+			fi;\
+		else\
+			echo "  | You automatically accepted this license agreement by running this script with acceptlicenses=YES option.";\
+        fi;\
+        echo;\
+	fi
+	@echo -e "$(RESET_COLOR)"
+	@echo -n "License accepted..."
+	@echo -e "$(OK_MSG)"
+endef
+
+define make_virtualenv
+	@echo -n "Making Virtual Environment for $(1) in $(VENV_DIR)..."
+	@echo -e -n "$(YELLOW)"
+	@test -d $(VENV_DIR)/$(1) || python3 -m venv $(VENV_DIR)/$(1)
+	@source $(VENV_DIR)/$(1)/bin/activate
+	@echo -e "$(OK_MSG)"
+	@echo -e -n "$(YELLOW)"
+	@$(VENV_DIR)/$(1)/bin/python3 -m pip install --upgrade pip setuptools wheel
+	@echo -e "$(RESET_COLOR)"
+	@echo -n "Python setup tools updated..."
+	@echo -e "$(OK_MSG)"
+	@echo -e -n "$(YELLOW)"
+	@test ! -s $(2)pre_requirements.txt ||\
+ 		($(VENV_DIR)/$(1)/bin/pip install --upgrade -r $(2)pre_requirements.txt\
+ 		&& echo -e "$(RESET_COLOR)"\
+ 		&& echo -n "Pre requirements installed..."\
+ 		&& echo -e "$(OK_MSG)")
+	@echo -e -n "$(YELLOW)"
+	@test ! -s $(2)requirements.txt ||\
+		($(VENV_DIR)/$(1)/bin/pip install --upgrade -r $(2)requirements.txt\
+ 		&& echo -e "$(RESET_COLOR)"\
+ 		&& echo -n "Requirements installed..."\
+ 		&& echo -e "$(OK_MSG)")
+	@echo -e -n "$(RESET_COLOR)"
+	@test ! -s $(2)setup.py ||\
+		(echo "Installing the package..."\
+		 && echo -e "$(YELLOW)"\
+		 && $(VENV_DIR)/$(1)/bin/pip install --upgrade -e .$(PIP_ARGS)\
+		 && echo -e "$(RESET_COLOR)"\
+		 && echo -n "Package installation completed..."\
+		 && echo -e "$(OK_MSG)")
+	@echo -e "$(RESET_COLOR)"
+	$(call check_license,$(1))
+endef
+
+
+define install_connectors
+	echo
+	echo "--------------------------------------------------------------------------"
+	echo "Installing $1 connector..."
+	echo "--------------------------------------------------------------------------"
+	if [[ ! -d singer-connectors/$1 ]]; then\
+		echo "ERROR: Directory not exists and does not look like a valid singer connector: singer-connectors: singer-connectors/$1";\
+		exit 1;\
+    fi
+    $(call make_virtualenv,$1,singer-connectors/$1/)
+endef
+
+define print_list_of_connectors
+	echo "   $1"
+endef
+
+help: .check_gettext .pw_logo
+	@echo
+	@echo "  Targets"
+	@echo "  ======="
+	@echo "     pipelinewise                                               Install the main PipelineWise component"
+	@echo "     pipelinewise_no_test_extras                                Install the main Pipelinewise component without test extras"
+	@echo
+	@echo "     all_connectors                                             Install all connectors"
+	@echo "     default_connectors                                         Install default connectors"
+	@echo "     extra_connectors                                           Install only extra connectors"
+	@echo "     connectors -e pw_connector=connector1,connector2,...       Install specific connector(s)"
+	@echo
+	@echo "     list_installed_components                                  Show a list of installed components"
+	@echo "     list_default_connectors                                    Show a list of available default connectors"
+	@echo "     list_extra_connectors                                      Show a list of available extra connectors"
+	@echo
+	@echo "     clean_all                                                  Clean all installed components"
+	@echo "     clean -e pw_connector=connector1,connector2,...            Clean a specific connector(s)"
+	@echo
+	@echo "   Options"
+	@echo "   ======="
+	@echo "      -e pw_connector=connector1,connector2,...                 Define a list of connectors for installing or cleaning"
+	@echo "      -e pw_acceptlicenses=y/Y/Yes/YES                           Forcing to accept the licenses automatically"
+	@echo
+	@echo "   To start CLI"
+	@echo "   ============"
+	@echo "      $$ source $(VENV_DIR)/pipelinewise/bin/activate"
+	@echo "      $$ export PIPELINEWISE_HOME=$(PIPELINEWISE_HOME)"
+	@echo "      $$ pipelinewise status"
+	@echo
+	@echo "--------------------------------------------------------------------------"
+
+
+pipelinewise: .check_gettext .pw_logo
+	$(call make_virtualenv,pipelinewise)
+	$(call print_execute_time,PipelineWise)
+
+pipelinewise_no_test_extras: .set_pip_args pipelinewise
+
+clean_all:
+	@echo -n "Cleaning previous installations in $(VENV_DIR)..."
+	@rm -rf $(VENV_DIR)
+	@echo -e "$(OK_MSG)"
+
+clean:
+ifeq ($(pw_connector),)
+	@echo "use -e pw_connector=connector1,connector2,...."
+	@exit 1
+endif
+	$(eval space:= )
+	$(eval space+= )
+	$(eval comma:=,)
+	$(eval connectors_list:=$(subst $(comma),$(space),$(pw_connector)))
+
+	@$(foreach var,$(connectors_list), $(call clean_connectors,$(var));)
+
+connectors: .check_license_env_var
+ifeq ($(pw_connector),)
+	@echo "use -e pw_connector=connector1,connector2,...."
+	@exit 1
+endif
+	$(eval space:= )
+	$(eval space+= )
+	$(eval comma:=,)
+	$(eval connectors_list:=$(subst $(comma),$(space),$(pw_connector)))
+
+	@$(foreach var,$(connectors_list), $(call install_connectors,$(var));)
+	$(call print_execute_time,Connectors)
+
+
+all_connectors: default_connectors extra_connectors
+	@echo "Install all connectors..."
+	$(call print_execute_time,All connectors)
+
+default_connectors: .check_license_env_var
+	@echo "Installing default connectors..."
+	@$(foreach var,$(DEFAULT_CONNECTORS), $(call install_connectors,$(var));)
+	$(call print_execute_time,Default connectors)
+
+extra_connectors: .check_license_env_var
+	@echo "Installing extra connectors..."
+	@$(foreach var,$(EXTRA_CONNECTORS), $(call install_connectors,$(var));)
+	$(call print_execute_time,Extra connectors)
+
+
+list_installed_components:
+	$(call print_installed_connectors)
+
+list_default_connectors:
+	@echo
+	@echo "   ============================"
+	@echo "   Available Default Connectors"
+	@echo "   ============================"
+	@$(foreach var,$(DEFAULT_CONNECTORS), $(call print_list_of_connectors,$(var));)
+	@echo "   ----------------------------"
+
+list_extra_connectors:
+	@echo
+	@echo "   ============================"
+	@echo "   Available Extra Connectors"
+	@echo "   ============================"
+	@$(foreach var,$(EXTRA_CONNECTORS), $(call print_list_of_connectors,$(var));)
+	@echo "   ----------------------------"
+
+.pw_logo:
+	@echo -e "$(BLUE)"
+	@(CURRENT_YEAR=$(shell date +"%Y") envsubst < motd)
+	@echo -e "$(RESET_COLOR)"
+
+.check_license_env_var:
+	$(eval ACCEPT_LICENSES:=NO)
+ifeq ($(pw_acceptlicenses),y)
+	$(eval ACCEPT_LICENSES:=YES)
+endif
+ifeq ($(pw_acceptlicenses),Y)
+	$(eval ACCEPT_LICENSES:=YES)
+endif
+ifeq ($(pw_acceptlicenses),Yes)
+	$(eval ACCEPT_LICENSES:=YES)
+endif
+ifeq ($(pw_acceptlicenses),YES)
+	$(eval ACCEPT_LICENSES:=YES)
+endif
+
+.check_gettext:
+	@echo -n "Checking gettext..."
+	@if ! ENVSUBST_LOC="$$(type -p "envsubst")" || [[ -z ENVSUBST_LOC ]]; then\
+		echo -e "$(FAIL_MSG)" &&\
+		echo "envsubst not found but it is required to run this script. Try to install gettext or gettext-base package" && exit 1;\
+	fi
+	@echo -e "$(OK_MSG)"
+
+.set_pip_args:
+	$(eval PIP_ARGS:="")
diff --git a/README.md b/README.md
index 682f40b18..da3761343 100644
--- a/README.md
+++ b/README.md
@@ -113,13 +113,21 @@ You can run any pipelinewise command at this point. Tutorials to create and run
     * mongo-tools
     * mbuffer
 
-2. Run the install script that installs the PipelineWise CLI and all supported singer connectors into separate virtual environments:
+2. Run the Makefile that installs the PipelineWise CLI and all supported singer connectors into separate virtual environments:
 
-    ```sh
-    $ ./install.sh --connectors=all
+    ```shell
+    $ make pipelinewise  all_connectors
+    ```
+    Press `Y` to accept the license agreement of the required singer components. To automate the installation and accept every license agreement run:
+    ```shell
+    $ make pipelinewise all_connectors -e pw_acceptlicenses=y
     ```
-    Press `Y` to accept the license agreement of the required singer components. To automate the installation and accept every license agreement run `./install --acceptlicenses`
-    Use the optional `--connectors=...,...` argument to install only a specific list of singer connectors.
+    And to install only a specific list of singer connectors:
+    ```shell
+    $ make connectors -e pw_connector=<connector_1>,<connector_2>
+    ```
+
+   Run `make` or `make -h` to see the help for Makefile and all options.
 
 3. To start the CLI you need to activate the CLI virtual environment and set `PIPELINEWISE_HOME` environment variable:
 
@@ -127,7 +135,7 @@ You can run any pipelinewise command at this point. Tutorials to create and run
     $ source {ACTUAL_ABSOLUTE_PATH}/.virtualenvs/pipelinewise/bin/activate
     $ export PIPELINEWISE_HOME={ACTUAL_ABSOLUTE_PATH}
     ```
-    (The `ACTUAL_ABSOLUTE_PATH` differs on every system, the install script prints the correct commands once the installation completes)
+    (The `ACTUAL_ABSOLUTE_PATH` differs on every system, running `make -h` prints the correct commands for CLI)
 
 4. Check if the installation was successful by running the `pipelinewise status` command:
 
diff --git a/dev-project/entrypoint.sh b/dev-project/entrypoint.sh
index 880b8350d..d5cdb700a 100755
--- a/dev-project/entrypoint.sh
+++ b/dev-project/entrypoint.sh
@@ -42,7 +42,7 @@ cd dev-project
 ../tests/db/target_postgres.sh
 
 # Install PipelineWise and connectors in the container
-../install.sh --acceptlicenses --nousage --connectors=target-snowflake,target-postgres,target-bigquery,tap-mysql,tap-postgres,tap-mongodb,transform-field,tap-s3-csv
+make pipelinewise connectors -e pw_acceptlicenses=y -e pw_connector=target-snowflake,target-postgres,target-bigquery,tap-mysql,tap-postgres,tap-mongodb,transform-field,tap-s3-csv
 if [[ $? != 0 ]]; then
     echo
     echo "ERROR: Docker container not started. Failed to install one or more PipelineWise components."
diff --git a/docs/installation_guide/installation.rst b/docs/installation_guide/installation.rst
index 5646b9f64..e5bdd43f1 100644
--- a/docs/installation_guide/installation.rst
+++ b/docs/installation_guide/installation.rst
@@ -98,14 +98,14 @@ PipelineWise CLI and every supported singer connectors into separated virtual en
 
     $ git clone https://github.com/transferwise/pipelinewise.git
     $ cd ./pipelinewise
-    $ ./install.sh --connectors=all
+    $ make pipelinewise all_connectors
 
 Press ``Y`` to accept the license agreement of the required singer components. To automate
 the installation and accept every license agreement run ``./install --acceptlicenses``.
 
 .. code-block:: bash
 
-    $ ./install.sh --connectors=all
+    $ make pipelinewise all_connectors
 
     (...installation usually takes 5-10 minutes...)
 
@@ -131,7 +131,7 @@ running:
 
 .. code-block:: bash
 
-    $ ./install.sh --connectors=tap-mysql,tap-postgres,target-snowflake
+    $ make pipelinewise connectors -e pw_connector=tap-mysql,tap-postgres,target-snowflake
 
 .. warning::
 
diff --git a/install.sh b/install.sh
deleted file mode 100755
index 6c671573a..000000000
--- a/install.sh
+++ /dev/null
@@ -1,249 +0,0 @@
-#!/bin/bash
-
-# Exit script on first error
-set -e
-
-# Capture start_time
-start_time=`date +%s`
-
-# Source directory defined as location of install.sh
-SRC_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-# Install pipelinewise venvs in the present working directory
-PIPELINEWISE_HOME=$(pwd)
-VENV_DIR=${PIPELINEWISE_HOME}/.virtualenvs
-
-check_license() {
-    python3 -m pip install pip-licenses
-
-    echo
-    echo "Checking license..."
-    PKG_NAME=`pip-licenses | grep "$1[[:space:]]" | awk '{print $1}'`
-    PKG_VERSION=`pip-licenses | grep "$1[[:space:]]" | awk '{print $2}'`
-    PKG_LICENSE=`pip-licenses --from mixed | grep "$1[[:space:]]" | awk '{for (i=1; i<=NF-2; i++) $i = $(i+2); NF-=2; print}'`
-
-    # Any License Agreement that is not Apache Software License (2.0) has to be accepted
-    MAIN_LICENSE="Apache Software License"
-    if [[ $PKG_LICENSE != $MAIN_LICENSE && $PKG_LICENSE != 'UNKNOWN' ]]; then
-        echo
-        echo "  | $PKG_NAME ($PKG_VERSION) is licensed under $PKG_LICENSE"
-        echo "  |"
-        echo "  | WARNING. The license of this connector is different than the default PipelineWise license ($MAIN_LICENSE)."
-
-        if [[ $ACCEPT_LICENSES != "YES" ]]; then
-            echo "  | You need to accept the connector's license agreement to proceed."
-            echo "  |"
-            read -r -p "  | Do you accept the [$PKG_LICENSE] license agreement of $PKG_NAME connector? [y/N] " response
-            case "$response" in
-                [yY][eE][sS]|[yY])
-                    ;;
-                *)
-                    echo
-                    echo "EXIT. License agreement not accepted"
-                    exit 1
-                    ;;
-            esac
-        else
-            echo "  | You automatically accepted this license agreement by running this script with --acceptlicenses option."
-        fi
-
-    fi
-}
-
-clean_virtualenvs() {
-    echo "Cleaning previous installations in $VENV_DIR"
-    rm -rf $VENV_DIR
-}
-
-make_virtualenv() {
-    echo "Making Virtual Environment for [$1] in $VENV_DIR"
-    python3 -m venv $VENV_DIR/$1
-    source $VENV_DIR/$1/bin/activate
-    python3 -m pip install --upgrade pip setuptools wheel
-
-    if [ -f "pre_requirements.txt" ]; then
-        python3 -m pip install --upgrade -r pre_requirements.txt
-    fi
-    if [ -f "requirements.txt" ]; then
-        python3 -m pip install --upgrade -r requirements.txt
-    fi
-    if [ -f "setup.py" ]; then
-        PIP_ARGS=
-        if [[ ! $NO_TEST_EXTRAS == "YES" ]]; then
-            PIP_ARGS=$PIP_ARGS"[test]"
-        fi
-
-        python3 -m pip install --upgrade -e .$PIP_ARGS
-    fi
-
-    echo ""
-
-    check_license $1
-    deactivate
-}
-
-install_connector() {
-    echo
-    echo "--------------------------------------------------------------------------"
-    echo "Installing $1 connector..."
-    echo "--------------------------------------------------------------------------"
-
-    CONNECTOR_DIR=$SRC_DIR/singer-connectors/$1
-    if [[ ! -d $CONNECTOR_DIR ]]; then
-        echo "ERROR: Directory not exists and does not look like a valid singer connector: $CONNECTOR_DIR"
-        exit 1
-    fi
-
-    cd $CONNECTOR_DIR
-    make_virtualenv $1
-}
-
-print_installed_connectors() {
-    cd $SRC_DIR
-
-    echo
-    echo "--------------------------------------------------------------------------"
-    echo "Installed components:"
-    echo "--------------------------------------------------------------------------"
-    echo
-    echo "Component            Version"
-    echo "-------------------- -------"
-
-    for i in `ls $VENV_DIR`; do
-        source $VENV_DIR/$i/bin/activate
-        VERSION=`python3 -m pip list | grep "$i[[:space:]]" | awk '{print $2}'`
-        printf "%-20s %s\n" $i "$VERSION"
-    done
-
-    if [[ $CONNECTORS != "all" ]]; then
-        echo
-        echo "WARNING: Not every singer connector installed. If you are missing something use the --connectors=...,... argument"
-        echo "         with an explicit list of required connectors or use the --connectors=all to install every available"
-        echo "         connector"
-    fi
-}
-
-# Parse command line arguments
-for arg in "$@"; do
-    case $arg in
-        # Auto accept license agreements. Useful if PipelineWise installed by an automated script
-        --acceptlicenses)
-            ACCEPT_LICENSES="YES"
-            ;;
-        # Do not print usage information at the end of the install
-        --nousage)
-            NO_USAGE="YES"
-            ;;
-        # Install with test requirements that allows running tests
-        --notestextras)
-            NO_TEST_EXTRAS="YES"
-            ;;
-        # Install extra connectors
-        --connectors=*)
-            CONNECTORS="${arg#*=}"
-            shift
-            ;;
-        # Clean previous installation
-        --clean)
-            clean_virtualenvs
-            exit 0
-            ;;
-        *)
-            echo "Invalid argument: $arg"
-            exit 1
-            ;;
-    esac
-done
-
-# Welcome message
-if ! ENVSUBST_LOC="$(type -p "envsubst")" || [[ -z ENVSUBST_LOC ]]; then
-  echo "envsubst not found but it's required to run this script. Try to install gettext or gettext-base package"
-  exit 1
-fi
-
-CURRENT_YEAR=$(date +"%Y") envsubst < $SRC_DIR/motd
-
-# Install PipelineWise core components
-cd $SRC_DIR
-make_virtualenv pipelinewise
-
-# Set default and extra singer connectors
-DEFAULT_CONNECTORS=(
-    tap-jira
-    tap-kafka
-    tap-mysql
-    tap-postgres
-    tap-s3-csv
-    tap-salesforce
-    tap-snowflake
-    tap-zendesk
-    tap-mongodb
-    tap-github
-    tap-slack
-    tap-mixpanel
-    tap-twilio
-    target-s3-csv
-    target-snowflake
-    target-redshift
-    target-postgres
-    target-bigquery
-    transform-field
-)
-EXTRA_CONNECTORS=(
-    tap-adwords
-    tap-oracle
-    tap-zuora
-    tap-google-analytics
-    tap-shopify
-)
-
-# Install only the default connectors if --connectors argument not passed
-if [[ -z $CONNECTORS ]]; then
-    for i in ${DEFAULT_CONNECTORS[@]}; do
-        install_connector $i
-    done
-
-# don't install any connectors if --connectors=none passed
-elif [[ $CONNECTORS == "none" ]]; then
-  echo "No connectors will be installed"
-
-# Install every available connectors if --connectors=all passed
-elif [[ $CONNECTORS == "all" ]]; then
-    for i in ${DEFAULT_CONNECTORS[@]}; do
-        install_connector $i
-    done
-    for i in ${EXTRA_CONNECTORS[@]}; do
-        install_connector $i
-    done
-
-# Install the selected connectors if --connectors argument passed
-elif [[ ! -z $CONNECTORS ]]; then
-    OLDIFS=$IFS
-    IFS=,
-    for connector in $CONNECTORS; do
-        install_connector $connector
-    done
-    IFS=$OLDIFS
-fi
-
-# Capture end_time
-end_time=`date +%s`
-echo
-echo "--------------------------------------------------------------------------"
-echo "PipelineWise installed successfully in $((end_time-start_time)) seconds"
-echo "--------------------------------------------------------------------------"
-
-if [[ $CONNECTORS != "none" ]]; then
-  print_installed_connectors
-fi
-
-if [[ $NO_USAGE != "YES" ]]; then
-    echo
-    echo "To start CLI:"
-    echo " $ source $VENV_DIR/pipelinewise/bin/activate"
-    echo " $ export PIPELINEWISE_HOME=$PIPELINEWISE_HOME"
-
-    echo " $ pipelinewise status"
-    echo
-    echo "--------------------------------------------------------------------------"
-fi
diff --git a/scripts/ci_check_no_file_changes.sh b/scripts/ci_check_no_file_changes.sh
index 673a73f20..28a1c4a85 100755
--- a/scripts/ci_check_no_file_changes.sh
+++ b/scripts/ci_check_no_file_changes.sh
@@ -42,7 +42,7 @@ REGEXES=()
 for CHECK in "$@"
 do
   if [[ ${CHECK} == "python" ]]; then
-    REGEX="(^tests\/|^pipelinewise\/|^singer-connectors\/|^setup\.py)"
+    REGEX="(^tests\/|^pipelinewise\/|^singer-connectors\/|^setup\.py|^Makefile)"
     echo "Searching for changes in python files"
   elif [[ ${CHECK} == "doc" ]]; then
     REGEX="(^docs\/|.circleci/publish_docs.sh)"

From b8a003694d588b2a16d9daaf7ad2a28aa2ca2c61 Mon Sep 17 00:00:00 2001
From: Amir Mofakhar <amir.mofakhar@transferwise.com>
Date: Wed, 29 Sep 2021 09:03:48 +0100
Subject: [PATCH 25/78] fixed handling build arguments (#808)

---
 Dockerfile | 6 +++++-
 Makefile   | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 0ddda2d8a..c5704c4e0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,7 +16,11 @@ RUN apt-get -qq update \
     # Install Oracle Instant Client for tap-oracle if its in the connectors list
     && bash -c "if grep -q \"tap-oracle\" <<< \"$connectors\"; then wget https://download.oracle.com/otn_software/linux/instantclient/193000/oracle-instantclient19.3-basiclite-19.3.0.0.0-1.x86_64.rpm -O /app/oracle-instantclient.rpm && alien -i /app/oracle-instantclient.rpm --scripts && rm -rf /app/oracle-instantclient.rpm ; fi" \
     && cd /app \
-    && make pipelinewise_no_test_extras connectors -e pw_connector=$connectors -e pw_acceptlicenses=y \
+    && make pipelinewise_no_test_extras -e pw_acceptlicenses=y\
+    && if [ "$connectors" = "all" ]; then make all_connectors -e pw_acceptlicenses=y; fi\
+    && if [ "$connectors" = "default" ]; then make default_connectors -e pw_acceptlicenses=y; fi\
+    && if [ "$connectors" = "extra" ]; then make extra_connectors -e pw_acceptlicenses=y; fi\
+    && if [ "$connectors" != "all" ] && [ "$connectors" != "extra" ] && [ "$connectors" != "default" ] && [ "$connectors" != "none" ] && [ ! -z $connectors ]; then make connectors -e pw_connector=$connectors -e pw_acceptlicenses=y; fi\
     && ln -s /root/.pipelinewise /app/.pipelinewise
 
 ENTRYPOINT ["/app/entrypoint.sh"]
diff --git a/Makefile b/Makefile
index 5d67a7fcc..14bd42965 100644
--- a/Makefile
+++ b/Makefile
@@ -184,7 +184,7 @@ help: .check_gettext .pw_logo
 	@echo "   Options"
 	@echo "   ======="
 	@echo "      -e pw_connector=connector1,connector2,...                 Define a list of connectors for installing or cleaning"
-	@echo "      -e pw_acceptlicenses=y/Y/Yes/YES                           Forcing to accept the licenses automatically"
+	@echo "      -e pw_acceptlicenses=y/Y/Yes/YES                          Forcing to accept the licenses automatically"
 	@echo
 	@echo "   To start CLI"
 	@echo "   ============"

From bde4566e02d42e8769659112820e4692d1eb422d Mon Sep 17 00:00:00 2001
From: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
Date: Thu, 30 Sep 2021 15:06:03 +0300
Subject: [PATCH 26/78] Release 0.36.0 (#809)

---
 CHANGELOG.md | 23 +++++++++++++++++++++++
 setup.py     |  2 +-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 63f60dc7f..b2c9e6342 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,26 @@
+0.36.0 (2021-09-30)
+-------------------
+
+*New*
+- Add new transformation type: **MASK-STRING-SKIP-ENDS**
+- Bump `pipelinewise-target-snowflake` from `1.13.1` to `1.14.0` ([Changelog](https://github.com/transferwise/pipelinewise-target-snowflake/blob/master/CHANGELOG.md#1140-2021-09-30))
+    - Support `date` property format
+    - Don't log record on failure to avoind exposing data   
+
+*Changes*
+- Use Makefile for installation
+- Enforce PEP8
+
+*Fixes*
+- Dates out of range (with year > 9999) in FastSync from PG. 
+- Bump `pipelinewise-tap-postgres` from `1.8.0` to `1.8.1` ([Changelog](https://github.com/transferwise/pipelinewise-tap-postgres/blob/master/CHANGELOG.md#181-2021-09-23))
+    -  LOG_BASED: Handle dates with year > 9999.
+    -  INCREMENTAL & FULL_TABLE: Avoid processing timestamps arrays as timestamp
+
+- `Decimal` not JSON serializable in FastSync MongoDB
+- Don't use non-existent FastSync for MongoDB-Redshift pipelines.
+
+
 0.35.2 (2021-08-17)
 -------------------
 - Bump `pipelinewise-tap-github` from `1.0.1` to `1.0.2`
diff --git a/setup.py b/setup.py
index 086f2f708..26efe839a 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      version='0.35.2',
+      version='0.36.0',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',

From 792ea21981f47f3ae7c2d49540338541052090b4 Mon Sep 17 00:00:00 2001
From: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
Date: Thu, 30 Sep 2021 15:40:23 +0300
Subject: [PATCH 27/78] Bump target SF (#811)

---
 singer-connectors/target-snowflake/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singer-connectors/target-snowflake/requirements.txt b/singer-connectors/target-snowflake/requirements.txt
index 3db380e2a..fa712508f 100644
--- a/singer-connectors/target-snowflake/requirements.txt
+++ b/singer-connectors/target-snowflake/requirements.txt
@@ -1 +1 @@
-pipelinewise-target-snowflake==1.13.1
+pipelinewise-target-snowflake==1.14.0

From 5e856350e95c2f6eb7783b1ec63984c38decc925 Mon Sep 17 00:00:00 2001
From: Amir Mofakhar <amir.mofakhar@transferwise.com>
Date: Mon, 11 Oct 2021 11:51:26 +0100
Subject: [PATCH 28/78] fixed proc.info parcing in a case cmdline is None!
 (#817)

---
 tests/units/cli/test_cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/units/cli/test_cli.py b/tests/units/cli/test_cli.py
index d6b769bb3..3d29719da 100644
--- a/tests/units/cli/test_cli.py
+++ b/tests/units/cli/test_cli.py
@@ -559,7 +559,7 @@ def test_command_stop_tap(self):
 
         # Should not have any remaining Pipelinewise related linux process
         for proc in psutil.process_iter(['cmdline']):
-            full_command = ' '.join(proc.info['cmdline'])
+            full_command = ' '.join(proc.info['cmdline']) if proc.info['cmdline'] else ''
             assert re.match('scheduler|pipelinewise|tap|target', full_command) is None
 
     def test_command_sync_tables(self):

From 05e67715aac2c666ef7b242adf2df29c4fa368db Mon Sep 17 00:00:00 2001
From: Amir Mofakhar <amir.mofakhar@transferwise.com>
Date: Tue, 12 Oct 2021 13:31:37 +0100
Subject: [PATCH 29/78] fixed make process in docker-compose file (#819)

---
 Makefile                  |  2 +-
 dev-project/entrypoint.sh | 15 ++++++---------
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index 14bd42965..4da3be490 100644
--- a/Makefile
+++ b/Makefile
@@ -76,7 +76,7 @@ endef
 define check_license
 	@echo "Checking license..."
 	@echo -e "$(YELLOW)"
-	@$(VENV_DIR)/$(1)/bin/python3 -m pip install pip-licenses
+	@$(VENV_DIR)/$(1)/bin/python3 -m pip install pip-licenses==3.5.3
 	@echo -e "$(RESET_COLOR)"
 	$(eval PKG_NAME:=`$(VENV_DIR)/$(1)/bin/pip-licenses|grep "$(1)[[:space:]]"| awk '{print $$$$1}'`)
 	$(eval PKG_VERSION:=`$(VENV_DIR)/$(1)/bin/pip-licenses | grep "$(1)[[:space:]]" | awk '{print $$$$2}'`)
diff --git a/dev-project/entrypoint.sh b/dev-project/entrypoint.sh
index d5cdb700a..673660599 100755
--- a/dev-project/entrypoint.sh
+++ b/dev-project/entrypoint.sh
@@ -27,19 +27,16 @@ rm -rf /var/lib/apt/lists/* \
 # alien -i oracle-instantclient.rpm --scripts
 # rm -f oracle-instantclient.rpm
 
-
-# Change to dev-project folder
-cd dev-project
-
 # Install PipelineWise in the container
 
 # Build test databasese
-../tests/db/tap_mysql_db.sh
-../tests/db/tap_postgres_db.sh
 
-./mongo/init_rs.sh
-../tests/db/tap_mongodb.sh
-../tests/db/target_postgres.sh
+tests/db/tap_mysql_db.sh
+tests/db/tap_postgres_db.sh
+
+dev-project/mongo/init_rs.sh
+tests/db/tap_mongodb.sh
+tests/db/target_postgres.sh
 
 # Install PipelineWise and connectors in the container
 make pipelinewise connectors -e pw_acceptlicenses=y -e pw_connector=target-snowflake,target-postgres,target-bigquery,tap-mysql,tap-postgres,tap-mongodb,transform-field,tap-s3-csv

From 0831c2a7d112dd94d5efcacb9a8b1253f5e9c551 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 14 Oct 2021 16:35:26 +0200
Subject: [PATCH 30/78] Bump pytest from 6.2.4 to 6.2.5 (#815)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 26efe839a..168a5d047 100644
--- a/setup.py
+++ b/setup.py
@@ -41,7 +41,7 @@
       extras_require={
           'test': [
               'flake8==3.9.2',
-              'pytest==6.2.4',
+              'pytest==6.2.5',
               'pytest-dependency==0.4.0',
               'pytest-cov==2.12.1',
               'python-dotenv==0.19.0',

From 365a711a152738f22e19481d5821811fd40cecdf Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 15 Oct 2021 10:22:04 +0200
Subject: [PATCH 31/78] Bump ansible from 4.4.0 to 4.7.0 (#821)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 168a5d047..41f51a969 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@
           'argparse==1.4.0',
           'tabulate==0.8.9',
           'PyYAML==5.4.1',
-          'ansible==4.4.0',
+          'ansible==4.7.0',
           'Jinja2==3.0.1',
           'joblib==1.0.0',
           'PyMySQL==0.7.11',

From b22e0ff72ff9082274f240c700b941b657371640 Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Fri, 15 Oct 2021 10:29:28 +0100
Subject: [PATCH 32/78] Bump target-snowflake (#822)

---
 singer-connectors/target-snowflake/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singer-connectors/target-snowflake/requirements.txt b/singer-connectors/target-snowflake/requirements.txt
index fa712508f..66a29fd00 100644
--- a/singer-connectors/target-snowflake/requirements.txt
+++ b/singer-connectors/target-snowflake/requirements.txt
@@ -1 +1 @@
-pipelinewise-target-snowflake==1.14.0
+pipelinewise-target-snowflake==1.14.1

From 0113da2a30b2539d6c3d5ef425a7dc9117d76c41 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 15 Oct 2021 13:08:19 +0200
Subject: [PATCH 33/78] Bump python-dotenv from 0.19.0 to 0.19.1 (#823)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 41f51a969..ed1578752 100644
--- a/setup.py
+++ b/setup.py
@@ -44,7 +44,7 @@
               'pytest==6.2.5',
               'pytest-dependency==0.4.0',
               'pytest-cov==2.12.1',
-              'python-dotenv==0.19.0',
+              'python-dotenv==0.19.1',
               'pylint==2.10.2',
               'unify==0.5'
           ]

From 665366dc437b6997acf28a2aaff7e02439bf21c2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 15 Oct 2021 16:43:57 +0200
Subject: [PATCH 34/78] Bump jinja2 from 3.0.1 to 3.0.2 (#824)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index ed1578752..2461cc6d3 100644
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,7 @@
           'tabulate==0.8.9',
           'PyYAML==5.4.1',
           'ansible==4.7.0',
-          'Jinja2==3.0.1',
+          'Jinja2==3.0.2',
           'joblib==1.0.0',
           'PyMySQL==0.7.11',
           'psycopg2-binary==2.8.6',

From 223198a176f2e2a670a9c525a3661cf0bb090aac Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 15 Oct 2021 17:34:22 +0100
Subject: [PATCH 35/78] Bump flake8 from 3.9.2 to 4.0.1 (#820)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 2461cc6d3..a5861fdeb 100644
--- a/setup.py
+++ b/setup.py
@@ -40,7 +40,7 @@
       ],
       extras_require={
           'test': [
-              'flake8==3.9.2',
+              'flake8==4.0.1',
               'pytest==6.2.5',
               'pytest-dependency==0.4.0',
               'pytest-cov==2.12.1',

From a4207f8624fbbd51a822b5ddd47c493c74422723 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 15 Oct 2021 19:43:14 +0200
Subject: [PATCH 36/78] Bump joblib from 1.0.0 to 1.1.0 (#826)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index a5861fdeb..fa650adfe 100644
--- a/setup.py
+++ b/setup.py
@@ -22,7 +22,7 @@
           'PyYAML==5.4.1',
           'ansible==4.7.0',
           'Jinja2==3.0.2',
-          'joblib==1.0.0',
+          'joblib==1.1.0',
           'PyMySQL==0.7.11',
           'psycopg2-binary==2.8.6',
           'snowflake-connector-python[pandas]==2.4.6',

From 33dae8d62b44bb210acf9e00454ae8d721f68e1e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 15 Oct 2021 22:41:31 +0100
Subject: [PATCH 37/78] Bump pytest-cov from 2.12.1 to 3.0.0 (#816)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index fa650adfe..58dd15691 100644
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,7 @@
               'flake8==4.0.1',
               'pytest==6.2.5',
               'pytest-dependency==0.4.0',
-              'pytest-cov==2.12.1',
+              'pytest-cov==3.0.0',
               'python-dotenv==0.19.1',
               'pylint==2.10.2',
               'unify==0.5'

From 034e77299570539fea57208937bedd0055a4c148 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 19 Oct 2021 09:00:28 +0100
Subject: [PATCH 38/78] Update tzlocal requirement from <2.2,>=2.0 to
 >=2.0,<4.1 (#827)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 58dd15691..f0c69cd1b 100644
--- a/setup.py
+++ b/setup.py
@@ -33,7 +33,7 @@
           'python-pidfile==3.0.0',
           'pre-commit==2.15.0',
           'pymongo>=3.10,<3.13',
-          'tzlocal>=2.0,<2.2',
+          'tzlocal>=2.0,<4.1',
           'slackclient>=2.7,<2.10',
           'psutil==5.8.0',
           'ujson==4.1.*'

From 2b772c8a37796840a60cb59fab7155f6cdd7a4a8 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 22 Oct 2021 11:08:18 +0100
Subject: [PATCH 39/78] Update ujson requirement from ==4.1.* to >=4.1,<4.3
 (#813)

Updates the requirements on [ujson](https://github.com/ultrajson/ultrajson) to permit the latest version.
- [Release notes](https://github.com/ultrajson/ultrajson/releases)
- [Commits](https://github.com/ultrajson/ultrajson/compare/4.1.0...4.2.0)

---
updated-dependencies:
- dependency-name: ujson
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Co-authored-by: Amir Mofakhar <amir.mofakhar@transferwise.com>
---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index f0c69cd1b..2087aa89b 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,7 @@
           'tzlocal>=2.0,<4.1',
           'slackclient>=2.7,<2.10',
           'psutil==5.8.0',
-          'ujson==4.1.*'
+          'ujson>=4.1,<4.3'
       ],
       extras_require={
           'test': [

From 3fd4eab5ae66cfb6f84a1e2e0276e6f2efb58543 Mon Sep 17 00:00:00 2001
From: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
Date: Thu, 4 Nov 2021 13:59:12 +0200
Subject: [PATCH 40/78] AP-1067 Moving last bits away from CircleCI (#831)

---
 .circleci/config.yml                   | 227 -------------------------
 .github/workflows/connectors.yml       |   4 +
 .github/workflows/dockerhub.yml        |  31 +++-
 .github/workflows/linter.yml           |   4 +
 .github/workflows/publish_doc.yml      |  43 +++++
 .github/workflows/tests.yml            | 115 +++++++++++++
 .github/workflows/unit_tests.yml       |  47 -----
 Dockerfile.ci => Dockerfile.barebone   |   0
 dev-project/docker-compose.yml         |   3 +
 dev-project/entrypoint.sh              |   4 +-
 scripts/ci_check_no_file_changes.sh    |   4 +-
 {.circleci => scripts}/publish_docs.sh |  13 +-
 12 files changed, 204 insertions(+), 291 deletions(-)
 delete mode 100644 .circleci/config.yml
 create mode 100644 .github/workflows/publish_doc.yml
 create mode 100644 .github/workflows/tests.yml
 delete mode 100644 .github/workflows/unit_tests.yml
 rename Dockerfile.ci => Dockerfile.barebone (100%)
 rename {.circleci => scripts}/publish_docs.sh (90%)

diff --git a/.circleci/config.yml b/.circleci/config.yml
deleted file mode 100644
index 958ce2065..000000000
--- a/.circleci/config.yml
+++ /dev/null
@@ -1,227 +0,0 @@
-version: 2
-
-references:
-
-  docker_k8s_deployer: &docker_k8s_deployer
-    working_directory: ~/pipelinewise
-    docker:
-      - image: docker.tw.ee/k8s-deployer:2
-
-  attach_workspace: &attach_workspace
-    attach_workspace:
-      at: ~/pipelinewise
-
-  container_config: &container_config
-    working_directory: ~/pipelinewise
-    docker:
-      # Main python container
-      # Use circleci next-gen image
-      - image: cimg/python:3.7
-        environment:
-          TAP_MYSQL_HOST: db_mysql_source
-          TAP_MYSQL_PORT: 3306
-          TAP_MYSQL_ROOT_PASSWORD: test
-          TAP_MYSQL_USER: test
-          TAP_MYSQL_PASSWORD: test
-          TAP_MYSQL_DB: mysql_source_db
-
-          TAP_POSTGRES_HOST: db_postgres_source
-          TAP_POSTGRES_PORT: 5432
-          TAP_POSTGRES_USER: test
-          TAP_POSTGRES_PASSWORD: test
-          TAP_POSTGRES_DB: postgres_source_db
-
-          TARGET_POSTGRES_HOST: db_postgres_dwh
-          TARGET_POSTGRES_PORT: 5432
-          TARGET_POSTGRES_USER: test
-          TARGET_POSTGRES_PASSWORD: test
-          TARGET_POSTGRES_DB: postgres_dwh
-
-          TAP_MONGODB_HOST: mongodb_source
-          TAP_MONGODB_PORT: 27017
-          TAP_MONGODB_USER: mongoUser
-          TAP_MONGODB_PASSWORD: Password1
-          TAP_MONGODB_DB: test
-          TAP_MONGODB_ROOT_USER: mongoUser
-          TAP_MONGODB_ROOT_PASSWORD: Password1
-
-      # PostgreSQL service container image used as test source database (for tap-postgres)
-      - image: debezium/postgres:12-alpine
-        name: db_postgres_source
-        # enable logical decoding
-        command: -c "wal_level=logical" -c "max_replication_slots=5" -c "max_wal_senders=5"
-        environment:
-          POSTGRES_USER: test
-          POSTGRES_PASSWORD: test
-          POSTGRES_DB: postgres_source_db
-
-       # MariaDB service container image used as test source database (for tap-mysql)
-      - image: mariadb:10.2.26
-        name: db_mysql_source
-        command: --default-authentication-plugin=mysql_native_password --log-bin=mysql-bin --binlog-format=ROW
-        environment:
-          MYSQL_ROOT_PASSWORD: test
-          MYSQL_USER: test
-          MYSQL_PASSWORD: test
-          MYSQL_DATABASE: mysql_source_db
-
-      # PostgreSQL service container image used as test target (DWH) database (for target-postgres)
-      - image: postgres:11.4
-        name: db_postgres_dwh
-        environment:
-          POSTGRES_USER: test
-          POSTGRES_PASSWORD: test
-          POSTGRES_DB: postgres_dwh
-
-      - image: "mongo:4.2-bionic"
-        name: "mongodb_source"
-        environment:
-          MONGO_INITDB_ROOT_USERNAME: mongoUser
-          MONGO_INITDB_ROOT_PASSWORD: Password1
-          MONGO_INITDB_DATABASE: test
-        command: [mongod, --replSet, rs0]
-
-jobs:
-  e2e_tests:
-    <<: *container_config
-    steps:
-      - checkout
-      - setup_remote_docker:
-          docker_layer_caching: true
-      - run:
-          when:
-
-          name: 'Run E2E tests'
-          command: |
-            ./scripts/ci_check_no_file_changes.sh python && rc=$? || rc=$?
-
-            if [[ $rc -eq 1 ]]
-            then
-              sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 4B7C549A058F8B6B
-              echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb.list
-              sudo apt-get update
-              sudo apt install --no-install-recommends \
-                  mariadb-client \
-                  postgresql-client \
-                  mongo-tools \
-                  mbuffer \
-                  gettext-base \
-                  mongodb-org-shell=4.2.7
-
-              ./dev-project/mongo/init_rs.sh
-
-              ./tests/db/tap_mysql_db.sh
-              ./tests/db/tap_postgres_db.sh
-              ./tests/db/tap_mongodb.sh
-              ./tests/db/target_postgres.sh
-
-              make pipelinewise connectors -e pw_acceptlicenses=y -e pw_connector=target-snowflake,target-postgres,target-bigquery,tap-mysql,tap-postgres,tap-mongodb,transform-field,tap-s3-csv
-              . .virtualenvs/pipelinewise/bin/activate
-              export PIPELINEWISE_HOME=$PWD
-              pytest tests/end_to_end -vx
-            fi
-          no_output_timeout: 30m
-
-  # For documentation deployment. You'll need the following environment vars
-  # in your Circle CI settings, otherwise this will not work.
-  #
-  # GH_NAME (your git username)
-  # GH_EMAIL (your git email)
-  # GH_TOKEN (the personal Git token with pushes enabled)
-  deploy-doc:
-    docker:
-      - image: cimg/python:3.7
-    working_directory: ~/gh_doc_automation
-    steps:
-      - checkout
-      - run:
-          name: "Publish doc if it has changed"
-          command: |
-            ./scripts/ci_check_no_file_changes.sh doc && rc=$? || rc=$?
-
-            if [[ $rc -eq 1 ]]
-            then
-              .circleci/publish_docs.sh
-            fi
-
-  upload_docker_image:
-    <<: *docker_k8s_deployer
-    steps:
-      - checkout
-      - setup_remote_docker
-      - *attach_workspace
-      - run:
-          name: Push and release the new docker image to artifactory
-          command: k8s-deployment docker-build jfrog-cli-docker-push
-          environment:
-            DEPLOY_SLACK_CHANNEL: "#analytics-platform-builds"
-            DEPLOY_IMAGE_NAME: "pipelinewise"
-            DEPLOY_DOCKERFILE: "./Dockerfile"
-            DEPLOY_IMAGE_ADDITIONAL_TAGS: "latest"
-
-  upload_ci_docker_image:
-    <<: *docker_k8s_deployer
-    steps:
-      - checkout
-      - setup_remote_docker
-      - *attach_workspace
-      - run:
-          name: Push and release the new ci docker image to artifactory
-          command: k8s-deployment docker-build jfrog-cli-docker-push
-          environment:
-            DEPLOY_SLACK_CHANNEL: "#analytics-platform-builds"
-            DEPLOY_IMAGE_NAME: "pipelinewise-ci"
-            DEPLOY_DOCKERFILE: "./Dockerfile.ci"
-            DEPLOY_IMAGE_ADDITIONAL_TAGS: "latest"
-
-  promote_docker_image:
-    <<: *docker_k8s_deployer
-    steps:
-      - checkout
-      - setup_remote_docker
-      - *attach_workspace
-      - run:
-          name: Promote built artifact for PipelineWise
-          command: k8s-deployment trigger-image-promotion
-          environment:
-            DEPLOY_IMAGE_NAME: "pipelinewise"
-            DEPLOY_SLACK_CHANNEL: "#analytics-platform-builds"
-
-
-workflows:
-  version: 2
-
-  build:
-    jobs:
-      - e2e_tests
-
-      - upload_docker_image:
-          context: kubernetes-staging
-          requires:
-            - e2e_tests
-          filters:
-            branches:
-              only:
-                - master
-
-      - upload_ci_docker_image:
-          context: kubernetes-staging
-          requires:
-            - e2e_tests
-          filters:
-            branches:
-              only:
-                - master
-
-      - promote_docker_image:
-          context: promote-build
-          requires:
-            - upload_docker_image
-          filters:
-            branches:
-              only:
-                - master
-
-  deploy_doc:
-    jobs:
-      - deploy-doc
diff --git a/.github/workflows/connectors.yml b/.github/workflows/connectors.yml
index 1c19c56e9..8ab11b045 100644
--- a/.github/workflows/connectors.yml
+++ b/.github/workflows/connectors.yml
@@ -9,6 +9,10 @@ on:
 
   workflow_dispatch:
 
+concurrency:
+  group: singer-connectors-${{ github.head_ref }}
+  cancel-in-progress: true
+
 jobs:
   check:
     runs-on: ubuntu-20.04
diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml
index ae80a4a20..f20d068d8 100644
--- a/.github/workflows/dockerhub.yml
+++ b/.github/workflows/dockerhub.yml
@@ -1,10 +1,12 @@
-name: Docker Image to DockerHub
+name: Docker Images to DockerHub
 
 on:
   release:
     types:
       - published
 
+  workflow_dispatch:
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -14,7 +16,7 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Generate tag
-      uses: frabert/replace-string-action@master
+      uses: frabert/replace-string-action@v2.0
       id: genTag
       with:
         pattern: '.*(\d+\.\d+\.\d+).*'
@@ -27,14 +29,27 @@ jobs:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-    - name: Build and push
-      id: docker_build
+    - name: Build and push main image
+      id: docker_build_main
       uses: docker/build-push-action@v2
       with:
         file: ./Dockerfile
         push: true
-        tags: transferwiseworkspace/pipelinewise:${{ steps.genTag.outputs.replaced }}
-
-    - name: Image digest
-      run: echo ${{ steps.docker_build.outputs.digest }}
+        tags: |
+          transferwiseworkspace/pipelinewise:${{ steps.genTag.outputs.replaced }}
+          transferwiseworkspace/pipelinewise:latest
 
+    - name: Build and push barebone image
+      id: docker_build_barebone
+      uses: docker/build-push-action@v2
+      with:
+        file: ./Dockerfile.barebone
+        push: true
+        tags: |
+          transferwiseworkspace/pipelinewise-barebone:${{ steps.genTag.outputs.replaced }}
+          transferwiseworkspace/pipelinewise-barebone:latest
+
+    - name: Image digests
+      run: |
+        echo ${{ steps.docker_build_main.outputs.digest }}
+        echo ${{ steps.docker_build_barebone.outputs.digest }}
diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
index e287748ca..1d0e0107f 100644
--- a/.github/workflows/linter.yml
+++ b/.github/workflows/linter.yml
@@ -8,6 +8,10 @@ on:
 
   workflow_dispatch:
 
+concurrency:
+  group: linter-${{ github.head_ref }}
+  cancel-in-progress: true
+
 jobs:
   check:
     runs-on: ubuntu-20.04
diff --git a/.github/workflows/publish_doc.yml b/.github/workflows/publish_doc.yml
new file mode 100644
index 000000000..18507716b
--- /dev/null
+++ b/.github/workflows/publish_doc.yml
@@ -0,0 +1,43 @@
+name: Publish documentation
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+concurrency:
+  group: doc-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: github-pages
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          path: gh_doc_automation
+
+      - name: check structure
+        run: ls -l gh_doc_automation
+
+      - name: Check if doc has changed
+        working-directory: gh_doc_automation
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh doc
+
+      - name: Publish doc
+        if: steps.check.outcome == 'failure'
+        working-directory: gh_doc_automation
+        env:
+          GH_NAME: ap-github                        # can be anything
+          GH_EMAIL: ap-github@wise.com              # can be anything
+          GH_TOKEN:  ${{ secrets.GITHUB_TOKEN }}     # Comes from GH itself, available at runtime, not stored in secrets
+        run: ./scripts/publish_docs.sh
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 000000000..c739c251a
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,115 @@
+name: Tests
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+  workflow_dispatch:
+
+concurrency:
+  group: tests-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit_test:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+
+    steps:
+      - name: Checking out repo
+        uses: actions/checkout@v2
+
+      - name: Check if python changes are present
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh python
+
+      - name: Set up Python ${{ matrix.python-version }}
+        if: steps.check.outcome == 'failure'
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        if: steps.check.outcome == 'failure'
+        run: |
+          pip install -U pip setuptools wheel
+          pip install -Ue .[test]
+
+      - name: Run Unit tests
+        if: steps.check.outcome == 'failure'
+        run: |
+          pytest --cov=pipelinewise --cov-fail-under=72 -v tests/units
+
+  e2e_tests:
+    runs-on: ubuntu-20.04
+    environment: ci_tests
+
+    steps:
+      - name: Checking out repo
+        uses: actions/checkout@v2
+
+      - name: Check if python changes are present
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh python
+
+      - name: Setup test containers
+        if: steps.check.outcome == 'failure'
+        run: docker-compose -f dev-project/docker-compose.yml up -d
+
+      - name: Wait for test containers to be ready
+        if: steps.check.outcome == 'failure'
+        run: |
+          until docker logs pipelinewise_dev | grep "PipelineWise Dev environment is ready"
+          do
+            echo 'Sleeping for 1min';
+            sleep 60;
+          done
+
+      - name: Run target end-to-end tests
+        if: steps.check.outcome == 'failure'
+        env:
+          TARGET_SNOWFLAKE_ACCOUNT: ${{ secrets.TARGET_SNOWFLAKE_ACCOUNT }}
+          TARGET_SNOWFLAKE_AWS_ACCESS_KEY: ${{ secrets.TARGET_SNOWFLAKE_AWS_ACCESS_KEY }}
+          TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY: ${{ secrets.TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY }}
+          TARGET_SNOWFLAKE_DBNAME: ${{ secrets.TARGET_SNOWFLAKE_DBNAME }}
+          TARGET_SNOWFLAKE_FILE_FORMAT: ${{ secrets.TARGET_SNOWFLAKE_FILE_FORMAT }}
+          TARGET_SNOWFLAKE_PASSWORD: ${{ secrets.TARGET_SNOWFLAKE_PASSWORD }}
+          TARGET_SNOWFLAKE_S3_BUCKET: ${{ secrets.TARGET_SNOWFLAKE_S3_BUCKET }}
+          TARGET_SNOWFLAKE_S3_KEY_PREFIX: ${{ secrets.TARGET_SNOWFLAKE_S3_KEY_PREFIX }}
+          TARGET_SNOWFLAKE_SCHEMA: ${{ secrets.TARGET_SNOWFLAKE_SCHEMA }}
+          TARGET_SNOWFLAKE_STAGE: ${{ secrets.TARGET_SNOWFLAKE_STAGE }}
+          TARGET_SNOWFLAKE_USER: ${{ secrets.TARGET_SNOWFLAKE_USER }}
+          TARGET_SNOWFLAKE_WAREHOUSE: ${{ secrets.TARGET_SNOWFLAKE_WAREHOUSE }}
+          TAP_S3_CSV_AWS_KEY: ${{ secrets.TAP_S3_CSV_AWS_KEY }}
+          TAP_S3_CSV_AWS_SECRET_ACCESS_KEY: ${{ secrets.TAP_S3_CSV_AWS_SECRET_ACCESS_KEY }}
+          TAP_S3_CSV_BUCKET: ${{ secrets.TAP_S3_CSV_BUCKET }}
+        run: |
+          docker exec -t \
+          -e TAP_S3_CSV_AWS_KEY=$TAP_S3_CSV_AWS_KEY \
+          -e TAP_S3_CSV_AWS_SECRET_ACCESS_KEY=$TAP_S3_CSV_AWS_SECRET_ACCESS_KEY \
+          -e TAP_S3_CSV_BUCKET=$TAP_S3_CSV_BUCKET \
+          -e TARGET_SNOWFLAKE_ACCOUNT=$TARGET_SNOWFLAKE_ACCOUNT \
+          -e TARGET_SNOWFLAKE_AWS_ACCESS_KEY=$TARGET_SNOWFLAKE_AWS_ACCESS_KEY \
+          -e TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY=$TARGET_SNOWFLAKE_AWS_SECRET_ACCESS_KEY \
+          -e TARGET_SNOWFLAKE_DBNAME=$TARGET_SNOWFLAKE_DBNAME \
+          -e TARGET_SNOWFLAKE_FILE_FORMAT=$TARGET_SNOWFLAKE_FILE_FORMAT \
+          -e TARGET_SNOWFLAKE_PASSWORD=$TARGET_SNOWFLAKE_PASSWORD \
+          -e TARGET_SNOWFLAKE_S3_BUCKET=$TARGET_SNOWFLAKE_S3_BUCKET \
+          -e TARGET_SNOWFLAKE_S3_KEY_PREFIX=$TARGET_SNOWFLAKE_S3_KEY_PREFIX \
+          -e TARGET_SNOWFLAKE_SCHEMA=$TARGET_SNOWFLAKE_SCHEMA \
+          -e TARGET_SNOWFLAKE_STAGE=$TARGET_SNOWFLAKE_STAGE \
+          -e TARGET_SNOWFLAKE_USER=$TARGET_SNOWFLAKE_USER \
+          -e TARGET_SNOWFLAKE_WAREHOUSE=$TARGET_SNOWFLAKE_WAREHOUSE \
+          pipelinewise_dev pytest tests/end_to_end -vx
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
deleted file mode 100644
index ed2f890ec..000000000
--- a/.github/workflows/unit_tests.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-name: Unit Tests
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-    branches: [master]
-
-  workflow_dispatch:
-
-jobs:
-  test:
-
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: [3.6, 3.7, 3.8]
-
-    steps:
-      - name: Checking out repo
-        uses: actions/checkout@v2
-
-      - name: Check if python changes are present
-        id: check
-        env:
-          GITHUB_REPO: ${{ github.repository }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-        continue-on-error: true
-        run: ./scripts/ci_check_no_file_changes.sh python
-
-      - name: Set up Python ${{ matrix.python-version }}
-        if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
-        run: |
-          pip install -U pip setuptools wheel
-          pip install -Ue .[test]
-
-      - name: Run Unit tests
-        if: steps.check.outcome == 'failure'
-        run: |
-          export PIPELINEWISE_HOME=$PWD
-          pytest --cov=pipelinewise --cov-fail-under=72 -v tests/units
diff --git a/Dockerfile.ci b/Dockerfile.barebone
similarity index 100%
rename from Dockerfile.ci
rename to Dockerfile.barebone
diff --git a/dev-project/docker-compose.yml b/dev-project/docker-compose.yml
index 4f1347bc6..e68ae00dd 100644
--- a/dev-project/docker-compose.yml
+++ b/dev-project/docker-compose.yml
@@ -38,6 +38,9 @@ services:
       TAP_MONGODB_DB: ${TAP_MONGODB_DB}
       TAP_MONGODB_ROOT_USER: ${TAP_MONGODB_ROOT_USER}
       TAP_MONGODB_ROOT_PASSWORD: ${TAP_MONGODB_ROOT_PASSWORD}
+
+      PATH: /opt/pipelinewise/dev-project/.virtualenvs/pipelinewise/bin:${PATH}
+
     volumes:
       - ../:/opt/pipelinewise
       # don't create container specific .virtualenvs files on host
diff --git a/dev-project/entrypoint.sh b/dev-project/entrypoint.sh
index 673660599..64b867364 100755
--- a/dev-project/entrypoint.sh
+++ b/dev-project/entrypoint.sh
@@ -27,9 +27,7 @@ rm -rf /var/lib/apt/lists/* \
 # alien -i oracle-instantclient.rpm --scripts
 # rm -f oracle-instantclient.rpm
 
-# Install PipelineWise in the container
-
-# Build test databasese
+# Build test databases
 
 tests/db/tap_mysql_db.sh
 tests/db/tap_postgres_db.sh
diff --git a/scripts/ci_check_no_file_changes.sh b/scripts/ci_check_no_file_changes.sh
index 28a1c4a85..a130cebc6 100755
--- a/scripts/ci_check_no_file_changes.sh
+++ b/scripts/ci_check_no_file_changes.sh
@@ -44,9 +44,11 @@ do
   if [[ ${CHECK} == "python" ]]; then
     REGEX="(^tests\/|^pipelinewise\/|^singer-connectors\/|^setup\.py|^Makefile)"
     echo "Searching for changes in python files"
+
   elif [[ ${CHECK} == "doc" ]]; then
-    REGEX="(^docs\/|.circleci/publish_docs.sh)"
+    REGEX="(^docs\/|^scripts/publish_docs.sh)"
     echo "Searching for changes in documentation files"
+
   else
     echo "Invalid check: \"${CHECK}\". Falling back to exiting with FAILURE code"
     exit 1
diff --git a/.circleci/publish_docs.sh b/scripts/publish_docs.sh
similarity index 90%
rename from .circleci/publish_docs.sh
rename to scripts/publish_docs.sh
index 77644569f..4fcda9287 100755
--- a/.circleci/publish_docs.sh
+++ b/scripts/publish_docs.sh
@@ -11,7 +11,7 @@ set -e
 # from the top level of the package and not in the subdirectory...
 # Shouldn't ever be an issue the way we've got this setup, and you'll
 # want to change it a bit to make it work with your project structure.
-if [[ ! -f .circleci/config.yml ]]; then
+if [[ ! -d .github/workflows ]]; then
     echo "This must be run from the gh_doc_automation project directory"
     exit 1
 fi
@@ -51,11 +51,13 @@ mv docs/_build/html ./
 git stash
 
 # Checkout our gh-pages branch, remove everything but .git
+echo "Checking out gh-pages branch ..."
 git checkout gh-pages
 git pull origin gh-pages
 
 # Make sure to set the credentials! You'll need these environment vars
-# set in the "Environment Variables" section in Circle CI
+# set in the "Environment Variables" section in CI
+echo "Configuring git creds ..."
 git config user.email "$GH_EMAIL" > /dev/null 2>&1
 git config user.name "$GH_NAME" > /dev/null 2>&1
 
@@ -92,13 +94,14 @@ rm -r html/
 # Add everything, get ready for commit. But only do it if we're on
 # master. If you want to deploy on different branches, you can change
 # this.
-if [[ "$CIRCLE_BRANCH" =~ ^master$|^[0-9]+\.[0-9]+\.X$ ]]; then
+echo "Current branch ref: $GITHUB_REF"
+if [[ "$GITHUB_REF" =~ ^refs/heads/master$|^[0-9]+\.[0-9]+\.X$ ]]; then
     git add --all
     # Make sure "|| echo" is at the end to avoid error codes when no changes to commit
     git commit -m "[ci skip] publishing updated documentation..." || echo 
 
     # We have to re-add the origin with the GH_TOKEN credentials. You
-    # will need this SSH key in your environment variables on Circle.
+    # will need this SSH key in your environment variables on CI.
     # Make sure you change the <project>.git pattern at the end!
     git remote rm origin
     git remote add origin https://"$GH_NAME":"$GH_TOKEN"@github.com/transferwise/pipelinewise.git
@@ -106,5 +109,5 @@ if [[ "$CIRCLE_BRANCH" =~ ^master$|^[0-9]+\.[0-9]+\.X$ ]]; then
     # NOW we should be able to push it
     git push origin gh-pages
 else
-    echo "Not on master, so won't push doc"
+    echo "Not on master branch, so won't push doc"
 fi

From 6d967dceeb61eec49d3f7d3d63f9946a98253e03 Mon Sep 17 00:00:00 2001
From: Amir Mofakhar <amir.mofakhar@transferwise.com>
Date: Fri, 5 Nov 2021 11:03:03 +0000
Subject: [PATCH 41/78] [AP-1056] Added cleanup method for state file (#832)

* added cleanup method for state file

* fixed formatting

* fixed pylint

* make filtering more simple

* using helper for removing the file

Co-authored-by: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
---
 pipelinewise/cli/pipelinewise.py | 30 +++++++++++
 tests/units/cli/test_cli.py      | 88 ++++++++++++++++++++++++++++++++
 2 files changed, 118 insertions(+)

diff --git a/pipelinewise/cli/pipelinewise.py b/pipelinewise/cli/pipelinewise.py
index 0216efa7a..b07d2bbd8 100644
--- a/pipelinewise/cli/pipelinewise.py
+++ b/pipelinewise/cli/pipelinewise.py
@@ -1381,6 +1381,8 @@ def sync_tables(self):
             )
             sys.exit(1)
 
+        self._cleanup_tap_state_file()
+
         # Generate and run the command to run the tap directly
         tap_config = self.tap['files']['config']
         tap_inheritable_config = self.tap['files']['inheritable_config']
@@ -1729,6 +1731,34 @@ def _run_post_import_tap_checks(
 
         return errors
 
+    def _cleanup_tap_state_file(self) -> None:
+        tables = self.args.tables
+        state_file = self.tap['files']['state']
+        if tables:
+            self._clean_tables_from_bookmarks_in_state_file(state_file, tables)
+        else:
+            utils.silentremove(state_file)
+
+    @staticmethod
+    def _clean_tables_from_bookmarks_in_state_file(state_file_to_clean: str, tables: str) -> None:
+        try:
+            with open(state_file_to_clean, 'r+', encoding='UTF-8') as state_file:
+                state_data = json.load(state_file)
+                bookmarks = state_data.get('bookmarks')
+                list_of_tables = tables.split(',')
+                if bookmarks:
+                    for table_name in list_of_tables:
+                        bookmarks.pop(table_name, None)
+
+                state_file.seek(0)
+                json.dump(state_data, state_file)
+                state_file.truncate()
+
+        except FileNotFoundError:
+            pass
+        except json.JSONDecodeError:
+            pass
+
     def __validate_transformations(
         self, transformation_file: str, catalog: Dict, tap_id: str, target_id: str
     ) -> Optional[str]:
diff --git a/tests/units/cli/test_cli.py b/tests/units/cli/test_cli.py
index 3d29719da..1435cde78 100644
--- a/tests/units/cli/test_cli.py
+++ b/tests/units/cli/test_cli.py
@@ -1,3 +1,4 @@
+import json
 import os
 import re
 import time
@@ -9,6 +10,7 @@
 
 from pathlib import Path
 from unittest.mock import patch
+from typing import Callable, Optional
 
 from tests.units.cli.cli_args import CliArgs
 from pipelinewise import cli
@@ -39,6 +41,8 @@ def setup_method(self):
         self.pipelinewise = PipelineWise(
             self.args, CONFIG_DIR, VIRTUALENVS_DIR, PROFILING_DIR
         )
+        if os.path.exists('/tmp/pwtest'):
+            shutil.rmtree('/tmp/pwtest')
 
     def teardown_method(self):
         """Delete test directories"""
@@ -48,6 +52,42 @@ def teardown_method(self):
         except Exception:
             pass
 
+    @staticmethod
+    def _init_for_sync_tables_states_cleanup(tables_arg: str = None) -> PipelineWise:
+        temp_path = '/tmp/pwtest/'
+        args = CliArgs(target='target_one', tap='tap_one', tables=tables_arg)
+        pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
+        pipelinewise.tap['files']['state'] = f'{temp_path}state.json'
+        pipelinewise.venv_dir = temp_path
+
+        # Making a test tap bin file
+        os.makedirs(f'{temp_path}pipelinewise/bin')
+        with open(f'{temp_path}pipelinewise/bin/mysql-to-snowflake', 'a', encoding='UTF-8'):
+            pass
+
+        return pipelinewise
+
+    @staticmethod
+    def _make_sample_state_file(test_state_file: str) -> None:
+        sample_state_data = {
+            'currently_syncing': None,
+            'bookmarks': {
+                'table1': {'foo': 'bar'},
+                'table2': {'foo': 'bar'},
+                'table3': {'foo': 'bar'}
+            }
+        }
+        with open(test_state_file, 'w', encoding='UTF-8') as state_file:
+            json.dump(sample_state_data, state_file)
+
+    @staticmethod
+    def _assert_calling_sync_tables(pipelinewise: PipelineWise, side_effect_method: Optional[Callable] = None) -> None:
+        with patch('pipelinewise.cli.pipelinewise.PipelineWise.run_tap_fastsync') as mocked_fastsync:
+            if side_effect_method:
+                mocked_fastsync.side_effect = side_effect_method
+            pipelinewise.sync_tables()
+        mocked_fastsync.assert_called_once()
+
     def test_target_dir(self):
         """Singer target connector config path must be relative to the project config dir"""
         assert self.pipelinewise.get_target_dir(
@@ -575,6 +615,54 @@ def test_command_sync_tables(self):
         assert pytest_wrapped_e.type == SystemExit
         assert pytest_wrapped_e.value.code == 1
 
+    def test_command_sync_tables_cleanup_state_if_file_not_exists_and_no_tables_argument(self):
+        """Testing sync_tables cleanup state if file not exists and there is no tables argument"""
+        pipelinewise = self._init_for_sync_tables_states_cleanup()
+        self._assert_calling_sync_tables(pipelinewise)
+
+    def test_command_sync_tables_cleanup_state_if_file_not_exists_and_tables_argument(self):
+        """Testing sync_tables cleanup state if file not exists and there is tables argument"""
+        pipelinewise = self._init_for_sync_tables_states_cleanup(tables_arg='table1,table3')
+        self._assert_calling_sync_tables(pipelinewise)
+
+    def test_command_sync_tables_cleanup_state_if_file_exists_and_no_table_argument(self):
+        """Testing sync_tables cleanup state if file exists and there is no table argument"""
+        def _assert_state_file_is_deleted(*args, **kwargs):
+            # pylint: disable=unused-argument
+            assert os.path.isfile(test_state_file) is False
+
+        pipelinewise = self._init_for_sync_tables_states_cleanup()
+        test_state_file = pipelinewise.tap['files']['state']
+        self._make_sample_state_file(test_state_file)
+        self._assert_calling_sync_tables(pipelinewise, _assert_state_file_is_deleted)
+
+    def test_command_sync_tables_cleanup_state_if_file_exists_and_table_argument(self):
+        """Testing sync_tables cleanup state if file exists and there is table argument"""
+        def _assert_state_file_is_cleaned(*args, **kwargs):
+            # pylint: disable=unused-argument
+            expected_state_data = {
+                'currently_syncing': None,
+                'bookmarks': {
+                    'table2': {'foo': 'bar'},
+                }
+            }
+            with open(test_state_file, encoding='UTF-8') as state_file:
+                state_data = json.load(state_file)
+            assert state_data == expected_state_data
+
+        pipelinewise = self._init_for_sync_tables_states_cleanup(tables_arg='table1,table3')
+        test_state_file = pipelinewise.tap['files']['state']
+        self._make_sample_state_file(test_state_file)
+        self._assert_calling_sync_tables(pipelinewise, _assert_state_file_is_cleaned)
+
+    def test_command_sync_tables_cleanup_state_if_file_empty_and_table_argument(self):
+        """Testing sync_tables cleanup state if file empty and there is table argument"""
+        pipelinewise = self._init_for_sync_tables_states_cleanup(tables_arg='table1,table3')
+        test_state_file = pipelinewise.tap['files']['state']
+        with open(test_state_file, 'a', encoding='UTF-8'):
+            pass
+        self._assert_calling_sync_tables(pipelinewise)
+
     # pylint: disable=protected-access
     def test_exit_gracefully(self):
         """Gracefully shoudl run tap command"""

From 1c3ad835e39ac8a58bff6c2a2979694228ecf0c9 Mon Sep 17 00:00:00 2001
From: Amir Mofakhar <amir.mofakhar@transferwise.com>
Date: Fri, 19 Nov 2021 12:09:01 +0100
Subject: [PATCH 42/78] Release 0.37.0 (#841)

* Update changelog and setup for new release

* Update CHANGELOG.md

Co-authored-by: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>

* Update CHANGELOG.md

Co-authored-by: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>

Co-authored-by: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
---
 CHANGELOG.md | 24 ++++++++++++++++++++++++
 setup.py     |  2 +-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b2c9e6342..763affbaf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,27 @@
+0.37.0 (2021-11-19)
+-------------------
+
+*New*
+- Added cleanup method for state file.
+- Bump `pytest-cov` from `2.12.1` to `3.0.0` ([Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst#300-2021-10-04))
+- Bump `joblib` from `1.0.0` to `1.1.0`
+- Bump `flake8` from `3.9.2` to `4.0.1` 
+- Bump `jinja2` from `3.0.1` to `3.0.2`
+- Bump `python-dotenv` from `0.19.0` to `0.19.1`
+- Bump `target-snowflake` from `1.14.0` to `1.14.1`
+- Bump `ansible` from `4.4.0` to `4.7.0`
+- Bump `pytest` from `6.2.4` to `6.2.5`
+
+*Changes*
+- Fully migrate CI to Github Actions.
+- Update `ujson` requirement from `==4.1.*` to `>=4.1,<4.3`
+- Update `tzlocal` requirement from `<2.2,>=2.0` to `>=2.0,<4.1`
+
+*Fixes*
+- Make process in docker-compose file.
+- proc.info parsing in a case cmdline is None!
+
+
 0.36.0 (2021-09-30)
 -------------------
 
diff --git a/setup.py b/setup.py
index 2087aa89b..b3da360c0 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      version='0.36.0',
+      version='0.37.0',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',

From 37e2d29d6ae9ae0b76af85442e573d525737215f Mon Sep 17 00:00:00 2001
From: Amir Mofakhar <amir.mofakhar@transferwise.com>
Date: Thu, 25 Nov 2021 11:48:17 +0000
Subject: [PATCH 43/78] [AP-992] make a postfix for sf schemas in e2e (#839)

* make a postfix for sf schemas in e2e

* fixed docstring

* fixed pep8
---
 tests/end_to_end/helpers/assertions.py        | 25 +++--
 tests/end_to_end/helpers/env.py               | 20 ++--
 .../tap_mongodb_to_sf.yml.template            |  2 +-
 .../test-project/tap_mysql_to_sf.yml.template |  2 +-
 ...p_mysql_to_sf_buffered_stream.yml.template |  2 +-
 ...mysql_to_sf_split_large_files.yml.template |  2 +-
 .../tap_postgres_to_sf.yml.template           |  8 +-
 ...gres_to_sf_archive_load_files.yml.template |  4 +-
 ...tgres_to_sf_split_large_files.yml.template |  2 +-
 .../tap_s3_csv_to_sf.yml.template             |  2 +-
 tests/end_to_end/test_target_snowflake.py     | 96 +++++++++++--------
 11 files changed, 98 insertions(+), 67 deletions(-)

diff --git a/tests/end_to_end/helpers/assertions.py b/tests/end_to_end/helpers/assertions.py
index 470bf6a7a..b720588b4 100644
--- a/tests/end_to_end/helpers/assertions.py
+++ b/tests/end_to_end/helpers/assertions.py
@@ -102,7 +102,7 @@ def assert_state_file_valid(target_name, tap_name, log_path=None):
 
 
 def assert_cols_in_table(
-    query_runner_fn: callable, table_schema: str, table_name: str, columns: List[str]
+    query_runner_fn: callable, table_schema: str, table_name: str, columns: List[str], schema_postfix: str = ''
 ):
     """Fetches the given table's columns from information_schema and
     tests if every given column is in the result
@@ -111,8 +111,9 @@ def assert_cols_in_table(
     :param table_schema: search table in this schema
     :param table_name: table with the columns
     :param columns: list of columns to check if there are in the table's columns
+    :param schema_postfix: schema postfix for snowflake target
     """
-    funcs = _map_tap_to_target_functions(None, query_runner_fn)
+    funcs = _map_tap_to_target_functions(None, query_runner_fn, schema_postfix)
     sql_get_columns_for_table_fn = funcs.get(
         'target_sql_get_table_cols_fn', db.sql_get_columns_for_table
     )
@@ -136,7 +137,7 @@ def _run_sql(query_runner_fn: callable, sql_query: str) -> List:
 
 
 def _map_tap_to_target_functions(
-    tap_query_runner_fn: callable, target_query_runner_fn: callable
+    tap_query_runner_fn: callable, target_query_runner_fn: callable, schema_postfix: str = ''
 ) -> dict:
     """Takes two query runner methods and creates a map with the compatible database
     specific functions that required to run assertions.
@@ -149,14 +150,15 @@ def _map_tap_to_target_functions(
         # tap-mysql specific attributes and functions
         'run_query_tap_mysql': {
             'source_schemas': ['mysql_source_db'],
-            'target_schemas': ['ppw_e2e_tap_mysql'],
+            'target_schemas': [f'ppw_e2e_tap_mysql{schema_postfix}'],
             'source_sql_get_cols_fn': db.sql_get_columns_mysql,
             'source_sql_dynamic_row_count_fn': db.sql_dynamic_row_count_mysql,
         },
         # tap-postgres specific attributes and functions
         'run_query_tap_postgres': {
             'source_schemas': ['public', 'public2'],
-            'target_schemas': ['ppw_e2e_tap_postgres', 'ppw_e2e_tap_postgres_public2'],
+            'target_schemas': [f'ppw_e2e_tap_postgres{schema_postfix}',
+                               f'ppw_e2e_tap_postgres_public2{schema_postfix}'],
             'source_sql_get_cols_fn': db.sql_get_columns_postgres,
             'source_sql_dynamic_row_count_fn': db.sql_dynamic_row_count_postgres,
         },
@@ -194,15 +196,16 @@ def _map_tap_to_target_functions(
 
 
 def assert_row_counts_equal(
-    tap_query_runner_fn: callable, target_query_runner_fn: callable
+    tap_query_runner_fn: callable, target_query_runner_fn: callable, schema_postfix: str = ''
 ) -> None:
     """Takes two query runner methods, counts the row numbers in every table in both the
     source and target databases and tests if the row counts are matching.
 
     :param tap_query_runner_fn: method to run queries in the first connection
-    :param target_query_runner_fn: method to run queries in the second connection"""
+    :param target_query_runner_fn: method to run queries in the second connection
+    :param schema_postfix: schema postfix for snowflake target"""
     # Generate a map of source and target specific functions
-    funcs = _map_tap_to_target_functions(tap_query_runner_fn, target_query_runner_fn)
+    funcs = _map_tap_to_target_functions(tap_query_runner_fn, target_query_runner_fn, schema_postfix)
 
     # Get source and target schemas
     source_schemas = funcs['source_schemas']
@@ -246,6 +249,7 @@ def assert_all_columns_exist(
     target_query_runner_fn: callable,
     column_type_mapper_fn: callable = None,
     ignore_cols: Union[Set, List] = None,
+    schema_postfix: str = '',
 ) -> None:
     """Takes two query runner methods, gets the columns list for every table in both the
     source and target database and tests if every column in source exists in the target database.
@@ -255,9 +259,10 @@ def assert_all_columns_exist(
     :param tap_query_runner_fn: method to run queries in the first connection
     :param target_query_runner_fn: method to run queries in the second connection
     :param column_type_mapper_fn: method to convert source to target column types
-    :param ignore_cols: List or set of columns to ignore if we know target table won't have them"""
+    :param ignore_cols: List or set of columns to ignore if we know target table won't have them
+    :param schema_postfix: Schema postfix for Snowflake targets"""
     # Generate a map of source and target specific functions
-    funcs = _map_tap_to_target_functions(tap_query_runner_fn, target_query_runner_fn)
+    funcs = _map_tap_to_target_functions(tap_query_runner_fn, target_query_runner_fn, schema_postfix)
 
     # Get source and target schemas
     source_schemas = funcs['source_schemas']
diff --git a/tests/end_to_end/helpers/env.py b/tests/end_to_end/helpers/env.py
index 9327bff73..42377ae77 100644
--- a/tests/end_to_end/helpers/env.py
+++ b/tests/end_to_end/helpers/env.py
@@ -4,6 +4,7 @@
 import boto3
 import shutil
 import subprocess
+import uuid
 
 from dotenv import load_dotenv
 from . import db
@@ -21,6 +22,7 @@ class E2EEnv:
     on the supported databases"""
 
     def __init__(self, project_dir):
+        self.sf_schema_postfix = f'_{str(uuid.uuid4())[:8]}'
         self._load_env()
 
         # Generate test project YAMLs from templates
@@ -195,6 +197,10 @@ def _load_env(self):
                         ),
                         'optional': True,
                     },
+                    'SCHEMA_POSTFIX': {
+                        'value': os.environ.get('TARGET_SNOWFLAKE_SCHEMA_POSTFIX', self.sf_schema_postfix),
+                        'optional': True,
+                    }
                 },
             },
             # ------------------------------------------------------------------
@@ -588,25 +594,25 @@ def setup_target_redshift(self):
     def setup_target_snowflake(self):
         """Clean snowflake target database and prepare for test run"""
         self.run_query_target_snowflake(
-            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres CASCADE'
+            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres{self.sf_schema_postfix} CASCADE'
         )
         self.run_query_target_snowflake(
-            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_public2 CASCADE'
+            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_public2{self.sf_schema_postfix} CASCADE'
         )
         self.run_query_target_snowflake(
-            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical1 CASCADE'
+            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical1{self.sf_schema_postfix} CASCADE'
         )
         self.run_query_target_snowflake(
-            'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical2 CASCADE'
+            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical2{self.sf_schema_postfix} CASCADE'
         )
         self.run_query_target_snowflake(
-            'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql CASCADE'
+            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql{self.sf_schema_postfix} CASCADE'
         )
         self.run_query_target_snowflake(
-            'DROP SCHEMA IF EXISTS ppw_e2e_tap_s3_csv CASCADE'
+            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_s3_csv{self.sf_schema_postfix} CASCADE'
         )
         self.run_query_target_snowflake(
-            'DROP SCHEMA IF EXISTS ppw_e2e_tap_mongodb CASCADE'
+            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_mongodb{self.sf_schema_postfix} CASCADE'
         )
 
         # Clean config directory
diff --git a/tests/end_to_end/test-project/tap_mongodb_to_sf.yml.template b/tests/end_to_end/test-project/tap_mongodb_to_sf.yml.template
index 5f9384c68..1ab6e6664 100644
--- a/tests/end_to_end/test-project/tap_mongodb_to_sf.yml.template
+++ b/tests/end_to_end/test-project/tap_mongodb_to_sf.yml.template
@@ -37,7 +37,7 @@ stream_buffer_size: 0                  # In-memory buffer size (MB) between taps
 # ------------------------------------------------------------------------------
 schemas:
   - source_schema: "${TAP_MONGODB_DB}"
-    target_schema: "ppw_e2e_tap_mongodb"
+    target_schema: "ppw_e2e_tap_mongodb${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"
 
     tables:
       - table_name: "listings"
diff --git a/tests/end_to_end/test-project/tap_mysql_to_sf.yml.template b/tests/end_to_end/test-project/tap_mysql_to_sf.yml.template
index 9100d9922..027d3b28d 100644
--- a/tests/end_to_end/test-project/tap_mysql_to_sf.yml.template
+++ b/tests/end_to_end/test-project/tap_mysql_to_sf.yml.template
@@ -34,7 +34,7 @@ stream_buffer_size: 0                  # In-memory buffer size (MB) between taps
 # ------------------------------------------------------------------------------
 schemas:
   - source_schema: "mysql_source_db"
-    target_schema: "ppw_e2e_tap_mysql"
+    target_schema: "ppw_e2e_tap_mysql${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"
 
     tables:
       ### Table with LOG_BASED replication
diff --git a/tests/end_to_end/test-project/tap_mysql_to_sf_buffered_stream.yml.template b/tests/end_to_end/test-project/tap_mysql_to_sf_buffered_stream.yml.template
index 9ba05b188..3080c8b28 100644
--- a/tests/end_to_end/test-project/tap_mysql_to_sf_buffered_stream.yml.template
+++ b/tests/end_to_end/test-project/tap_mysql_to_sf_buffered_stream.yml.template
@@ -34,7 +34,7 @@ stream_buffer_size: 20                 # In-memory buffer size (MB) between taps
 # ------------------------------------------------------------------------------
 schemas:
   - source_schema: "mysql_source_db"
-    target_schema: "ppw_e2e_tap_mysql"
+    target_schema: "ppw_e2e_tap_mysql${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"
 
     tables:
       ### Table with LOG_BASED replication
diff --git a/tests/end_to_end/test-project/tap_mysql_to_sf_split_large_files.yml.template b/tests/end_to_end/test-project/tap_mysql_to_sf_split_large_files.yml.template
index e89ebbd64..5464ff3a8 100644
--- a/tests/end_to_end/test-project/tap_mysql_to_sf_split_large_files.yml.template
+++ b/tests/end_to_end/test-project/tap_mysql_to_sf_split_large_files.yml.template
@@ -39,7 +39,7 @@ split_file_max_chunks: 5
 # ------------------------------------------------------------------------------
 schemas:
   - source_schema: "mysql_source_db"
-    target_schema: "ppw_e2e_tap_mysql"
+    target_schema: "ppw_e2e_tap_mysql${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"
 
     tables:
       - table_name: "address"
diff --git a/tests/end_to_end/test-project/tap_postgres_to_sf.yml.template b/tests/end_to_end/test-project/tap_postgres_to_sf.yml.template
index 449150f6a..fdcf1b023 100644
--- a/tests/end_to_end/test-project/tap_postgres_to_sf.yml.template
+++ b/tests/end_to_end/test-project/tap_postgres_to_sf.yml.template
@@ -37,7 +37,7 @@ schemas:
 
   ### SOURCE SCHEMA 1: public
   - source_schema: "public"
-    target_schema: "ppw_e2e_tap_postgres"
+    target_schema: "ppw_e2e_tap_postgres${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"
 
     tables:
 
@@ -88,7 +88,7 @@ schemas:
 
   ### SOURCE SCHEMA 2: public2
   - source_schema: "public2"
-    target_schema: "ppw_e2e_tap_postgres_public2"
+    target_schema: "ppw_e2e_tap_postgres_public2${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"
 
     tables:
       ### Table with FULL_TABLE replication
@@ -102,7 +102,7 @@ schemas:
 
   ### SOURCE SCHEMA 3: logical 1
   #- source_schema: "logical1"
-  #  target_schema: "ppw_e2e_tap_postgres_logical1"
+  #  target_schema: "ppw_e2e_tap_postgres_logical1${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"
   #
   #  tables:
   #    - table_name: "logical1_table1"
@@ -112,6 +112,6 @@ schemas:
 
   ### SOURCE SCHEMA 4: logical2
   #- source_schema: "logical2"
-  #  target_schema: "ppw_e2e_tap_postgres_logical2"
+  #  target_schema: "ppw_e2e_tap_postgres_logical2${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"
   #  tables:
   #    - table_name: "logical2_table1"
\ No newline at end of file
diff --git a/tests/end_to_end/test-project/tap_postgres_to_sf_archive_load_files.yml.template b/tests/end_to_end/test-project/tap_postgres_to_sf_archive_load_files.yml.template
index 71c47a625..bc07f32a8 100644
--- a/tests/end_to_end/test-project/tap_postgres_to_sf_archive_load_files.yml.template
+++ b/tests/end_to_end/test-project/tap_postgres_to_sf_archive_load_files.yml.template
@@ -39,7 +39,7 @@ schemas:
 
   ### SOURCE SCHEMA 1: public
   - source_schema: "public"
-    target_schema: "ppw_e2e_tap_postgres"
+    target_schema: "ppw_e2e_tap_postgres${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"
 
     tables:
 
@@ -54,7 +54,7 @@ schemas:
 
   ### SOURCE SCHEMA 2: public2
   - source_schema: "public2"
-    target_schema: "ppw_e2e_tap_postgres_public2"
+    target_schema: "ppw_e2e_tap_postgres_public2${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"
 
     tables:
       ### Table with FULL_TABLE replication
diff --git a/tests/end_to_end/test-project/tap_postgres_to_sf_split_large_files.yml.template b/tests/end_to_end/test-project/tap_postgres_to_sf_split_large_files.yml.template
index e2b2b8e8a..78528d65d 100644
--- a/tests/end_to_end/test-project/tap_postgres_to_sf_split_large_files.yml.template
+++ b/tests/end_to_end/test-project/tap_postgres_to_sf_split_large_files.yml.template
@@ -40,7 +40,7 @@ split_file_max_chunks: 5
 # ------------------------------------------------------------------------------
 schemas:
   - source_schema: "public"
-    target_schema: "ppw_e2e_tap_postgres"
+    target_schema: "ppw_e2e_tap_postgres${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"
 
     tables:
       - table_name: "city"
diff --git a/tests/end_to_end/test-project/tap_s3_csv_to_sf.yml.template b/tests/end_to_end/test-project/tap_s3_csv_to_sf.yml.template
index fdb72bbb6..6f3d60edc 100644
--- a/tests/end_to_end/test-project/tap_s3_csv_to_sf.yml.template
+++ b/tests/end_to_end/test-project/tap_s3_csv_to_sf.yml.template
@@ -41,7 +41,7 @@ default_target_schema: "ppw_e2e_tap_s3_csv"
 # ------------------------------------------------------------------------------
 schemas:
   - source_schema: "s3_feeds" # This is mandatory, but can be anything in this tap type
-    target_schema: "ppw_e2e_tap_s3_csv" # Target schema in the destination Data Warehouse
+    target_schema: "ppw_e2e_tap_s3_csv${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}" # Target schema in the destination Data Warehouse
     
     # List of CSV files to destination tables
     tables:
diff --git a/tests/end_to_end/test_target_snowflake.py b/tests/end_to_end/test_target_snowflake.py
index 468f9f0b1..6e612b7b3 100644
--- a/tests/end_to_end/test_target_snowflake.py
+++ b/tests/end_to_end/test_target_snowflake.py
@@ -36,14 +36,23 @@ class TestTargetSnowflake:
     def setup_method(self):
         """Initialise test project by generating YAML files from
         templates for all the configured connectors"""
-        self.project_dir = os.path.join(DIR, 'test-project')
 
         # Init query runner methods
-        self.e2e = E2EEnv(self.project_dir)
         self.run_query_tap_mysql = self.e2e.run_query_tap_mysql
         self.run_query_tap_postgres = self.e2e.run_query_tap_postgres
         self.run_query_target_snowflake = self.e2e.run_query_target_snowflake
         self.mongodb_con = self.e2e.get_tap_mongodb_connection()
+        self.snowflake_schema_postfix = self.e2e.sf_schema_postfix
+
+    def setup_class(self):
+        """Initialise test suite"""
+        self.project_dir = os.path.join(DIR, 'test-project')
+        self.e2e = E2EEnv(self.project_dir)
+
+    def teardown_class(self):
+        """Teardown test suite"""
+        # Cleanup the Snowflake test schemas
+        self.e2e.setup_target_snowflake()
 
     def teardown_method(self):
         """Delete test directories and database objects"""
@@ -79,12 +88,13 @@ def test_replicate_mariadb_to_sf(self, tap_mariadb_id=TAP_MARIADB_ID):
             tap_mariadb_id, TARGET_ID, ['fastsync', 'singer']
         )
         assertions.assert_row_counts_equal(
-            self.run_query_tap_mysql, self.run_query_target_snowflake
+            self.run_query_tap_mysql, self.run_query_target_snowflake, self.snowflake_schema_postfix
         )
         assertions.assert_all_columns_exist(
             self.run_query_tap_mysql,
             self.e2e.run_query_target_snowflake,
             mysql_to_snowflake.tap_type_to_target_type,
+            schema_postfix=self.snowflake_schema_postfix
         )
 
         # 2. Make changes in MariaDB source database
@@ -126,35 +136,36 @@ def test_replicate_mariadb_to_sf(self, tap_mariadb_id=TAP_MARIADB_ID):
             tap_mariadb_id, TARGET_ID, ['fastsync', 'singer']
         )
         assertions.assert_row_counts_equal(
-            self.run_query_tap_mysql, self.run_query_target_snowflake
+            self.run_query_tap_mysql, self.run_query_target_snowflake, schema_postfix=self.snowflake_schema_postfix
         )
         assertions.assert_all_columns_exist(
             self.run_query_tap_mysql,
             self.e2e.run_query_target_snowflake,
             mysql_to_snowflake.tap_type_to_target_type,
             {'blob_col'},
+            schema_postfix=self.snowflake_schema_postfix
         )
 
         # Checking if mask-date transformation is working
         result = self.run_query_target_snowflake(
-            'SELECT count(1) FROM ppw_e2e_tap_mysql.address '
-            'where MONTH(date_created) != 1 or DAY(date_created)::int != 1;'
+            f'SELECT count(1) FROM ppw_e2e_tap_mysql{self.snowflake_schema_postfix}.address '
+            f'where MONTH(date_created) != 1 or DAY(date_created)::int != 1;'
         )[0][0]
 
         assert result == 0
 
         # Checking if conditional MASK-NUMBER transformation is working
         result = self.run_query_target_snowflake(
-            'SELECT count(1) FROM ppw_e2e_tap_mysql.address '
-            'where zip_code_zip_code_id != 0 and street_number REGEXP \'[801]\';'
+            f'SELECT count(1) FROM ppw_e2e_tap_mysql{self.snowflake_schema_postfix}.address '
+            f'where zip_code_zip_code_id != 0 and street_number REGEXP \'[801]\';'
         )[0][0]
 
         assert result == 0
 
         # Checking if conditional SET-NULL transformation is working
         result = self.run_query_target_snowflake(
-            'SELECT count(1) FROM ppw_e2e_tap_mysql.edgydata '
-            'where "GROUP" is not null and "CASE" = \'B\';'
+            f'SELECT count(1) FROM ppw_e2e_tap_mysql{self.snowflake_schema_postfix}.edgydata '
+            f'where "GROUP" is not null and "CASE" = \'B\';'
         )[0][0]
 
         assert result == 0
@@ -166,12 +177,13 @@ def test_resync_mariadb_to_sf(self, tap_mariadb_id=TAP_MARIADB_ID):
             tap_mariadb_id, TARGET_ID, profiling=True
         )
         assertions.assert_row_counts_equal(
-            self.run_query_tap_mysql, self.run_query_target_snowflake
+            self.run_query_tap_mysql, self.run_query_target_snowflake, schema_postfix=self.snowflake_schema_postfix
         )
         assertions.assert_all_columns_exist(
             self.run_query_tap_mysql,
             self.run_query_target_snowflake,
             mysql_to_snowflake.tap_type_to_target_type,
+            schema_postfix=self.snowflake_schema_postfix
         )
 
     # pylint: disable=invalid-name
@@ -184,12 +196,13 @@ def test_resync_mariadb_to_sf_with_split_large_files(
             tap_mariadb_id, TARGET_ID, profiling=True
         )
         assertions.assert_row_counts_equal(
-            self.run_query_tap_mysql, self.run_query_target_snowflake
+            self.run_query_tap_mysql, self.run_query_target_snowflake, schema_postfix=self.snowflake_schema_postfix
         )
         assertions.assert_all_columns_exist(
             self.run_query_tap_mysql,
             self.run_query_target_snowflake,
             mysql_to_snowflake.tap_type_to_target_type,
+            schema_postfix=self.snowflake_schema_postfix
         )
 
     # pylint: disable=invalid-name
@@ -202,12 +215,13 @@ def test_resync_pg_to_sf_with_split_large_files(
             tap_postgres_id, TARGET_ID, profiling=True
         )
         assertions.assert_row_counts_equal(
-            self.run_query_tap_mysql, self.run_query_target_snowflake
+            self.run_query_tap_mysql, self.run_query_target_snowflake, schema_postfix=self.snowflake_schema_postfix
         )
         assertions.assert_all_columns_exist(
             self.run_query_tap_mysql,
             self.run_query_target_snowflake,
             mysql_to_snowflake.tap_type_to_target_type,
+            schema_postfix=self.snowflake_schema_postfix
         )
 
     # pylint: disable=invalid-name
@@ -225,29 +239,29 @@ def test_replicate_pg_to_sf(self):
             TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer']
         )
         assertions.assert_row_counts_equal(
-            self.run_query_tap_postgres, self.run_query_target_snowflake
+            self.run_query_tap_postgres, self.run_query_target_snowflake, schema_postfix=self.snowflake_schema_postfix
         )
         assertions.assert_all_columns_exist(
-            self.run_query_tap_postgres, self.run_query_target_snowflake
+            self.run_query_tap_postgres, self.run_query_target_snowflake, schema_postfix=self.snowflake_schema_postfix
         )
         assertions.assert_date_column_naive_in_target(
             self.run_query_target_snowflake,
             'updated_at',
-            'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE"',
+            f'ppw_e2e_tap_postgres{self.snowflake_schema_postfix}."TABLE_WITH_SPACE AND UPPERCASE"',
         )
 
         result = self.run_query_target_snowflake(
-            'SELECT updated_at FROM '
-            'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE" '
-            'where cvarchar=\'H\';'
+            f'SELECT updated_at FROM '
+            f'ppw_e2e_tap_postgres{self.snowflake_schema_postfix}."TABLE_WITH_SPACE AND UPPERCASE" '
+            f'where cvarchar=\'H\';'
         )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 998993)
 
         result = self.run_query_target_snowflake(
-            'SELECT updated_at FROM '
-            'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE" '
-            'where cvarchar=\'I\';'
+            f'SELECT updated_at FROM '
+            f'ppw_e2e_tap_postgres{self.snowflake_schema_postfix}."TABLE_WITH_SPACE AND UPPERCASE" '
+            f'where cvarchar=\'I\';'
         )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 998993)
@@ -282,35 +296,37 @@ def test_replicate_pg_to_sf(self):
             TAP_POSTGRES_ID, TARGET_ID, ['fastsync', 'singer'], profiling=True
         )
         assertions.assert_row_counts_equal(
-            self.run_query_tap_postgres, self.run_query_target_snowflake
+            self.run_query_tap_postgres, self.run_query_target_snowflake, schema_postfix=self.snowflake_schema_postfix
         )
         assertions.assert_all_columns_exist(
-            self.run_query_tap_postgres, self.run_query_target_snowflake
+            self.run_query_tap_postgres, self.run_query_target_snowflake, schema_postfix=self.snowflake_schema_postfix
         )
         assertions.assert_date_column_naive_in_target(
             self.run_query_target_snowflake,
             'updated_at',
-            'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE"',
+            f'ppw_e2e_tap_postgres{self.snowflake_schema_postfix}."TABLE_WITH_SPACE AND UPPERCASE"',
         )
 
         result = self.run_query_target_snowflake(
-            'SELECT updated_at FROM ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE" where cvarchar=\'X\';'
+            f'SELECT updated_at FROM '
+            f'ppw_e2e_tap_postgres{self.snowflake_schema_postfix}."TABLE_WITH_SPACE AND UPPERCASE"'
+            f' where cvarchar=\'X\';'
         )[0][0]
 
         assert result == datetime(2019, 12, 31, 22, 53, 56, 800000)
 
         result = self.run_query_target_snowflake(
-            'SELECT updated_at FROM '
-            'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE" '
-            'where cvarchar=\'faaaar future\';'
+            f'SELECT updated_at FROM '
+            f'ppw_e2e_tap_postgres{self.snowflake_schema_postfix}."TABLE_WITH_SPACE AND UPPERCASE" '
+            f'where cvarchar=\'faaaar future\';'
         )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 998993)
 
         result = self.run_query_target_snowflake(
-            'SELECT updated_at FROM '
-            'ppw_e2e_tap_postgres."TABLE_WITH_SPACE AND UPPERCASE" '
-            'where cvarchar=\'BC\';'
+            f'SELECT updated_at FROM '
+            f'ppw_e2e_tap_postgres{self.snowflake_schema_postfix}."TABLE_WITH_SPACE AND UPPERCASE" '
+            f'where cvarchar=\'BC\';'
         )[0][0]
 
         assert result == datetime(9999, 12, 31, 23, 59, 59, 998993)
@@ -423,6 +439,7 @@ def assert_columns_exist():
                 'ppw_e2e_tap_s3_csv',
                 'countries',
                 ['CITY', 'COUNTRY', 'CURRENCY', 'ID', 'LANGUAGE'],
+                schema_postfix=self.snowflake_schema_postfix
             )
             assertions.assert_cols_in_table(
                 self.run_query_target_snowflake,
@@ -439,6 +456,7 @@ def assert_columns_exist():
                     'IS_PENSIONEER',
                     'LAST_NAME',
                 ],
+                schema_postfix=self.snowflake_schema_postfix
             )
 
         # 1. Run tap first time - both fastsync and a singer should be triggered
@@ -461,7 +479,7 @@ def assert_columns_exist(table):
             """Helper inner function to test if every table and column exists in the target"""
             assertions.assert_cols_in_table(
                 self.run_query_target_snowflake,
-                'ppw_e2e_tap_mongodb',
+                f'ppw_e2e_tap_mongodb{self.snowflake_schema_postfix}',
                 table,
                 [
                     '_ID',
@@ -470,6 +488,7 @@ def assert_columns_exist(table):
                     '_SDC_BATCHED_AT',
                     '_SDC_DELETED_AT',
                 ],
+                schema_postfix=self.snowflake_schema_postfix,
             )
 
         def assert_row_counts_equal(target_schema, table, count_in_source):
@@ -492,10 +511,10 @@ def assert_row_counts_equal(target_schema, table, count_in_source):
         my_coll_count = self.mongodb_con['my_collection'].count_documents({})
         all_datatypes_count = self.mongodb_con['all_datatypes'].count_documents({})
 
-        assert_row_counts_equal('ppw_e2e_tap_mongodb', 'listings', listing_count)
-        assert_row_counts_equal('ppw_e2e_tap_mongodb', 'my_collection', my_coll_count)
+        assert_row_counts_equal(f'ppw_e2e_tap_mongodb{self.snowflake_schema_postfix}', 'listings', listing_count)
+        assert_row_counts_equal(f'ppw_e2e_tap_mongodb{self.snowflake_schema_postfix}', 'my_collection', my_coll_count)
         assert_row_counts_equal(
-            'ppw_e2e_tap_mongodb', 'all_datatypes', all_datatypes_count
+            f'ppw_e2e_tap_mongodb{self.snowflake_schema_postfix}', 'all_datatypes', all_datatypes_count
         )
 
         result_insert = self.mongodb_con.my_collection.insert_many(
@@ -542,8 +561,9 @@ def assert_row_counts_equal(target_schema, table, count_in_source):
         assert (
             result_update.modified_count
             == self.run_query_target_snowflake(
-                'select count(_id) from ppw_e2e_tap_mongodb.my_collection where document:id = 0'
+                f'select count(_id) from ppw_e2e_tap_mongodb{self.snowflake_schema_postfix}.my_collection'
+                f' where document:id = 0'
             )[0][0]
         )
 
-        assert_row_counts_equal('ppw_e2e_tap_mongodb', 'my_collection', my_coll_count)
+        assert_row_counts_equal(f'ppw_e2e_tap_mongodb{self.snowflake_schema_postfix}', 'my_collection', my_coll_count)

From 7b55acb18366c22bd665ab935a493ad26cc86fc3 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 9 Dec 2021 16:40:22 +0000
Subject: [PATCH 44/78] Bump google-cloud-bigquery from 1.24.0 to 2.31.0 (#845)

* Bump google-cloud-bigquery from 1.24.0 to 2.31.0

Bumps [google-cloud-bigquery](https://github.com/googleapis/python-bigquery) from 1.24.0 to 2.31.0.
- [Release notes](https://github.com/googleapis/python-bigquery/releases)
- [Changelog](https://github.com/googleapis/python-bigquery/blob/main/CHANGELOG.md)
- [Commits](https://github.com/googleapis/python-bigquery/compare/v1.24.0...v2.31.0)

---
updated-dependencies:
- dependency-name: google-cloud-bigquery
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

* fix pylint 3.8 error

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Jeet Parekh <jeet.parekh@transferwise.com>
---
 setup.py                       | 2 +-
 tests/end_to_end/helpers/db.py | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index b3da360c0..8bd403764 100644
--- a/setup.py
+++ b/setup.py
@@ -26,7 +26,7 @@
           'PyMySQL==0.7.11',
           'psycopg2-binary==2.8.6',
           'snowflake-connector-python[pandas]==2.4.6',
-          'google-cloud-bigquery==1.24.0',
+          'google-cloud-bigquery==2.31.0',
           'pipelinewise-singer-python==1.*',
           'singer-encodings==0.0.*',
           'messytables==0.15.*',
diff --git a/tests/end_to_end/helpers/db.py b/tests/end_to_end/helpers/db.py
index bd7ef8200..700b58305 100644
--- a/tests/end_to_end/helpers/db.py
+++ b/tests/end_to_end/helpers/db.py
@@ -78,8 +78,7 @@ def run_query_bigquery(query, project):
     """Run and SQL query in a BigQuery database"""
     client = bigquery.Client(project=project)
     query_job = client.query(query)
-    query_job.result()
-    return [r.values() for r in query_job]
+    return [r.values() for r in query_job.result()]
 
 
 def run_query_redshift(query, host, port, user, password, database):

From bfa174b24f8dfe174fe59934d68f84c53b564d28 Mon Sep 17 00:00:00 2001
From: jeet-parekh-wise <94441288+jeet-parekh-wise@users.noreply.github.com>
Date: Fri, 10 Dec 2021 11:11:00 +0000
Subject: [PATCH 45/78] Release v0.37.1 (#847)

---
 CHANGELOG.md | 6 ++++++
 setup.py     | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 763affbaf..f6b80aecc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+0.37.1 (2021-12-10)
+-------------------
+- Make a postfix for Snowflake schemas in end-to-end tests.
+- Bump `google-cloud-bigquery` from `1.24.0` to `2.31.0` ([Changelog](https://github.com/googleapis/python-bigquery/blob/main/CHANGELOG.md#2310-2021-11-24))
+
+
 0.37.0 (2021-11-19)
 -------------------
 
diff --git a/setup.py b/setup.py
index 8bd403764..f284c1417 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      version='0.37.0',
+      version='0.37.1',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',

From 003d7e1dbd3f966341d751646ded7b065dfe2091 Mon Sep 17 00:00:00 2001
From: jeet-parekh-wise <94441288+jeet-parekh-wise@users.noreply.github.com>
Date: Fri, 10 Dec 2021 12:15:10 +0000
Subject: [PATCH 46/78] bump tap github to v1.0.3 (#848)

---
 singer-connectors/tap-github/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singer-connectors/tap-github/requirements.txt b/singer-connectors/tap-github/requirements.txt
index 1830c2385..05ec4f45f 100644
--- a/singer-connectors/tap-github/requirements.txt
+++ b/singer-connectors/tap-github/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-github==1.0.2
+pipelinewise-tap-github==1.0.3

From e39e33099ef3fcd91a1c7cecd0f40d1f3fecde0a Mon Sep 17 00:00:00 2001
From: jeet-parekh-wise <94441288+jeet-parekh-wise@users.noreply.github.com>
Date: Fri, 10 Dec 2021 13:00:01 +0000
Subject: [PATCH 47/78] v0.37.2 (#849)

---
 CHANGELOG.md | 5 +++++
 setup.py     | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f6b80aecc..13a0b50f9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+0.37.2 (2021-12-10)
+-------------------
+- Bump `pipelinewise-tap-github` from `1.0.2` to `1.0.3`
+
+
 0.37.1 (2021-12-10)
 -------------------
 - Make a postfix for Snowflake schemas in end-to-end tests.
diff --git a/setup.py b/setup.py
index f284c1417..e9b06b31e 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      version='0.37.1',
+      version='0.37.2',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',

From fd6a6ab556a44a3d690088a7649c099472207ef1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Dec 2021 11:11:35 +0000
Subject: [PATCH 48/78] Bump pyyaml from 5.4.1 to 6.0 (#825)

* Bump pyyaml from 5.4.1 to 6.0

Bumps [pyyaml](https://github.com/yaml/pyyaml) from 5.4.1 to 6.0.
- [Release notes](https://github.com/yaml/pyyaml/releases)
- [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES)
- [Commits](https://github.com/yaml/pyyaml/compare/5.4.1...6.0)

---
updated-dependencies:
- dependency-name: pyyaml
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

* switch to yaml.safe_load

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Co-authored-by: Amir Mofakhar <amir.mofakhar@transferwise.com>
Co-authored-by: Jeet Parekh <jeet.parekh@transferwise.com>
Co-authored-by: jeet-parekh-wise <94441288+jeet-parekh-wise@users.noreply.github.com>
---
 pipelinewise/cli/utils.py | 2 +-
 setup.py                  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pipelinewise/cli/utils.py b/pipelinewise/cli/utils.py
index 9015b4dcf..67b3abe90 100644
--- a/pipelinewise/cli/utils.py
+++ b/pipelinewise/cli/utils.py
@@ -196,7 +196,7 @@ def load_yaml(yaml_file, vault_secret=None):
             try:
                 if is_encrypted_file(stream):
                     file_data = stream.read()
-                    data = yaml.load(vault.decrypt(file_data, None))
+                    data = yaml.safe_load(vault.decrypt(file_data, None))
                 else:
                     loader = AnsibleLoader(stream, None, vault.secrets)
                     try:
diff --git a/setup.py b/setup.py
index e9b06b31e..8ee90bce6 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,7 @@
       install_requires=[
           'argparse==1.4.0',
           'tabulate==0.8.9',
-          'PyYAML==5.4.1',
+          'PyYAML==6.0',
           'ansible==4.7.0',
           'Jinja2==3.0.2',
           'joblib==1.1.0',

From f751503747878ee87db7c915530d3c0e19623b82 Mon Sep 17 00:00:00 2001
From: Andrey Tatarinov <elephantropy@gmail.com>
Date: Wed, 5 Jan 2022 11:17:43 +0300
Subject: [PATCH 49/78] Implement support for MongoDB Atlas fastsync (#858)

---
 Dockerfile                                    |  27 ++++-
 Dockerfile.barebone                           |  15 ++-
 dev-project/entrypoint.sh                     |   8 +-
 docs/connectors/taps/mongodb.rst              |  15 +++
 pipelinewise/fastsync/commons/tap_mongodb.py  | 105 ++++++++++--------
 setup.py                                      |   3 +-
 .../tap-mongodb/requirements.txt              |   2 +-
 .../commons/test_fastsync_tap_mongodb.py      |   4 +-
 8 files changed, 117 insertions(+), 62 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index c5704c4e0..711f681d4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,26 +1,43 @@
 FROM python:3.7-slim-buster
 
 ARG connectors=all
-COPY . /app
 
 RUN apt-get -qq update \
     && apt-get -qqy --no-install-recommends install \
         apt-utils \
         alien \
+        gnupg \
         libaio1 \
-        mongo-tools \
         mbuffer \
         wget \
     && rm -rf /var/lib/apt/lists/* \
-    && pip install -U --no-cache-dir pip \
+    && pip install -U --no-cache-dir pip
+
+# Add Mongodb ppa
+RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - \
+    && echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb.list \
+    && apt-get -qq update \
+    && apt-get -qqy --no-install-recommends install \
+        mongodb-database-tools \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY singer-connectors/ /app/singer-connectors/
+COPY Makefile /app
+
+RUN echo "setup connectors" \
     # Install Oracle Instant Client for tap-oracle if its in the connectors list
     && bash -c "if grep -q \"tap-oracle\" <<< \"$connectors\"; then wget https://download.oracle.com/otn_software/linux/instantclient/193000/oracle-instantclient19.3-basiclite-19.3.0.0.0-1.x86_64.rpm -O /app/oracle-instantclient.rpm && alien -i /app/oracle-instantclient.rpm --scripts && rm -rf /app/oracle-instantclient.rpm ; fi" \
     && cd /app \
-    && make pipelinewise_no_test_extras -e pw_acceptlicenses=y\
     && if [ "$connectors" = "all" ]; then make all_connectors -e pw_acceptlicenses=y; fi\
     && if [ "$connectors" = "default" ]; then make default_connectors -e pw_acceptlicenses=y; fi\
     && if [ "$connectors" = "extra" ]; then make extra_connectors -e pw_acceptlicenses=y; fi\
-    && if [ "$connectors" != "all" ] && [ "$connectors" != "extra" ] && [ "$connectors" != "default" ] && [ "$connectors" != "none" ] && [ ! -z $connectors ]; then make connectors -e pw_connector=$connectors -e pw_acceptlicenses=y; fi\
+    && if [ "$connectors" != "all" ] && [ "$connectors" != "extra" ] && [ "$connectors" != "default" ] && [ "$connectors" != "none" ] && [ ! -z $connectors ]; then make connectors -e pw_connector=$connectors -e pw_acceptlicenses=y; fi
+
+COPY . /app
+
+RUN echo "setup pipelinewise" \
+    && cd /app \
+    && make pipelinewise_no_test_extras -e pw_acceptlicenses=y\
     && ln -s /root/.pipelinewise /app/.pipelinewise
 
 ENTRYPOINT ["/app/entrypoint.sh"]
diff --git a/Dockerfile.barebone b/Dockerfile.barebone
index a0137407d..167f91709 100644
--- a/Dockerfile.barebone
+++ b/Dockerfile.barebone
@@ -1,11 +1,18 @@
 FROM python:3.7-slim-buster
 
-COPY . /app
-
 RUN apt-get -qq update \
-    && apt-get -qqy --no-install-recommends install apt-utils alien libaio1 wget \
+    && apt-get -qqy --no-install-recommends install \
+        apt-utils \
+        alien \
+        gnupg \
+        libaio1 \
+        wget \
     && rm -rf /var/lib/apt/lists/* \
-    && pip install -U --no-cache-dir pip \
+    && pip install -U --no-cache-dir pip
+
+COPY . /app
+
+RUN echo "setup pipelinewise" \
     && cd /app \
     && make pipelinewise_no_test_extras -e pw_acceptlicenses=y \
     && ln -s /root/.pipelinewise /app/.pipelinewise
diff --git a/dev-project/entrypoint.sh b/dev-project/entrypoint.sh
index 64b867364..25ca6e432 100755
--- a/dev-project/entrypoint.sh
+++ b/dev-project/entrypoint.sh
@@ -3,8 +3,8 @@
 set -e
 
 # Add Mongodb ppa
-apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 4B7C549A058F8B6B
-echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | tee /etc/apt/sources.list.d/mongodb.list
+wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add -
+echo "deb [ arch=amd64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.4 multiverse" | tee /etc/apt/sources.list.d/mongodb.list
 
 # Install OS dependencies
 apt-get update
@@ -14,8 +14,8 @@ apt-get install -y --no-install-recommends \
   libaio1 \
   mariadb-client \
   mbuffer \
-  mongo-tools \
-  mongodb-org-shell=4.2.7 \
+  mongodb-database-tools \
+  mongodb-org-shell \
   postgresql-client
 
 rm -rf /var/lib/apt/lists/* \
diff --git a/docs/connectors/taps/mongodb.rst b/docs/connectors/taps/mongodb.rst
index 65859304f..4861b9505 100644
--- a/docs/connectors/taps/mongodb.rst
+++ b/docs/connectors/taps/mongodb.rst
@@ -97,6 +97,7 @@ Example YAML for ``tap-mongodb``:
 	db_conn:
 		host: "mongodb_host1,mongodb_host2,mongodb_host3" 	# Mongodb host(s)
 		port: 27017                           				# Mongodb port
+		srv: "false"										# For MongoDB Atlas `srv` should be "true" and `port` will be ignored
 		user: "PipelineWiseUser"                  			# Mongodb user
 		password: "mY_VerY_StRonG_PaSSwoRd"                 # Mongodb plain string or vault encrypted
 		auth_database: "admin"            					# Mongodb database to authenticate on
@@ -143,3 +144,17 @@ Example YAML for ``tap-mongodb``:
 
 		  	# default replication method is LOG_BASED
 		  	- table_name: "my_other_collection"
+
+
+Example connection to MongoDB Atlas
+"""""""""""""""""""""""""""""""""""
+
+.. code-block:: bash
+
+	db_conn:
+		srv: "true"
+		host: "xxxxxxxxx.xxxxx.mongodb.net"
+		auth_database: "admin"			# the Mongodb database name to authenticate on
+		dbname: "db-name"				# Mongodb database name to sync from
+		user: "user-name"				# User with read roles
+		password: "password"			# Plain string or vault encrypted
diff --git a/pipelinewise/fastsync/commons/tap_mongodb.py b/pipelinewise/fastsync/commons/tap_mongodb.py
index ee4498a97..1a6c951a8 100644
--- a/pipelinewise/fastsync/commons/tap_mongodb.py
+++ b/pipelinewise/fastsync/commons/tap_mongodb.py
@@ -5,12 +5,12 @@
 import ujson
 import logging
 import os
-import ssl
 import subprocess
 import uuid
 import bson
 import pytz
 import tzlocal
+from urllib import parse
 
 from typing import Tuple, Optional, Dict, Callable, Any
 from pymongo import MongoClient
@@ -153,6 +153,50 @@ def transform_value(value: Any, path) -> Any:
     return value
 
 
+def get_connection_string(config: Dict):
+    """
+    Generates a MongoClientConnectionString based on configuration
+    Args:
+        config: DB config
+
+    Returns: A MongoClient connection string
+    """
+    srv = config.get('srv') == 'true'
+
+    # Default SSL verify mode to true, give option to disable
+    verify_mode = config.get('verify_mode', 'true') == 'true'
+    use_ssl = config.get('ssl') == 'true'
+
+    connection_query = {
+        'readPreference': 'secondaryPreferred',
+        'authSource': config['auth_database'],
+    }
+
+    if config.get('replica_set'):
+        connection_query['replicaSet'] = config['replica_set']
+
+    if use_ssl:
+        connection_query['ssl'] = 'true'
+
+    # NB: "sslAllowInvalidCertificates" must ONLY be supplied if `SSL` is true.
+    if not verify_mode and use_ssl:
+        connection_query['tlsAllowInvalidCertificates'] = 'true'
+
+    query_string = parse.urlencode(connection_query)
+
+    connection_string = '{protocol}://{user}:{password}@{host}{port}/{database}?{query_string}'.format(
+        protocol='mongodb+srv' if srv else 'mongodb',
+        user=config['user'],
+        password=config['password'],
+        host=config['host'],
+        port='' if srv else ':{port}'.format(port=int(config['port'])),
+        database=config['database'],
+        query_string=query_string
+    )
+
+    return connection_string
+
+
 class FastSyncTapMongoDB:
     """
     Common functions for fastsync from a MongoDB database
@@ -170,6 +214,8 @@ def __init__(self, connection_config: Dict, tap_type_to_target_type: Callable):
             'write_batch_rows', DEFAULT_WRITE_BATCH_ROWS
         )
 
+        self.connection_config['connection_string'] = get_connection_string(self.connection_config)
+
         self.tap_type_to_target_type = tap_type_to_target_type
         self.database: Optional[Database] = None
 
@@ -177,26 +223,8 @@ def open_connection(self):
         """
         Open connection
         """
-        # Default SSL verify mode to true, give option to disable
-        verify_mode = self.connection_config.get('verify_mode', 'true') == 'true'
-        use_ssl = self.connection_config.get('ssl') == 'true'
-
-        connection_params = dict(
-            host=self.connection_config['host'],
-            port=int(self.connection_config['port']),
-            username=self.connection_config['user'],
-            password=self.connection_config['password'],
-            authSource=self.connection_config['auth_database'],
-            ssl=use_ssl,
-            replicaSet=self.connection_config.get('replica_set', None),
-            readPreference='secondaryPreferred',
-        )
 
-        # NB: "ssl_cert_reqs" must ONLY be supplied if `SSL` is true.
-        if not verify_mode and use_ssl:
-            connection_params['ssl_cert_reqs'] = ssl.CERT_NONE
-
-        self.database = MongoClient(**connection_params)[
+        self.database = MongoClient(self.connection_config['connection_string'])[
             self.connection_config['database']
         ]
 
@@ -395,32 +423,19 @@ def _export_collection(self, export_dir: str, collection_name) -> str:
         """
         LOGGER.info('Starting export of table "%s"', collection_name)
 
-        url = (
-            f'mongodb://{self.connection_config["user"]}:{self.connection_config["password"]}'
-            f'@{self.connection_config["host"]}:{self.connection_config["port"]}/'
-            f'{self.connection_config["database"]}?authSource={self.connection_config["auth_database"]}'
-            f'&readPreference=secondaryPreferred'
-        )
+        cmd = [
+            'mongodump',
+            '--uri',
+            f'"{self.connection_config["connection_string"]}"',
+            '--forceTableScan',
+            '--gzip',
+            '-c',
+            collection_name,
+            '-o',
+            export_dir,
+        ]
 
-        if self.connection_config.get('replica_set', None) is not None:
-            url += f'&replicaSet={self.connection_config["replica_set"]}'
-
-        if self.connection_config.get('ssl', None) is not None:
-            url += f'&ssl={self.connection_config["ssl"]}'
-
-        return_code = subprocess.call(
-            [
-                'mongodump',
-                '--uri',
-                f'"{url}"',
-                '--forceTableScan',
-                '--gzip',
-                '-c',
-                collection_name,
-                '-o',
-                export_dir,
-            ]
-        )
+        return_code = subprocess.call(cmd)
 
         LOGGER.debug('Export command return code %s', return_code)
 
diff --git a/setup.py b/setup.py
index 8ee90bce6..5c0dffa23 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,8 @@
           'tzlocal>=2.0,<4.1',
           'slackclient>=2.7,<2.10',
           'psutil==5.8.0',
-          'ujson>=4.1,<4.3'
+          'ujson>=4.1,<4.3',
+          'dnspython==2.1.*',
       ],
       extras_require={
           'test': [
diff --git a/singer-connectors/tap-mongodb/requirements.txt b/singer-connectors/tap-mongodb/requirements.txt
index 90593b95c..162130640 100644
--- a/singer-connectors/tap-mongodb/requirements.txt
+++ b/singer-connectors/tap-mongodb/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-mongodb==1.2.0
+pipelinewise-tap-mongodb==1.3.0
\ No newline at end of file
diff --git a/tests/units/fastsync/commons/test_fastsync_tap_mongodb.py b/tests/units/fastsync/commons/test_fastsync_tap_mongodb.py
index 081506c73..405f4b6b0 100644
--- a/tests/units/fastsync/commons/test_fastsync_tap_mongodb.py
+++ b/tests/units/fastsync/commons/test_fastsync_tap_mongodb.py
@@ -179,7 +179,7 @@ def test_copy_table_with_collection_found_but_export_failed_expect_exception(sel
                     'mongodump',
                     '--uri',
                     '"mongodb://my_user:secret@foo.com:3306/my_db'
-                    '?authSource=admin&readPreference=secondaryPreferred&ssl=true"',
+                    '?readPreference=secondaryPreferred&authSource=admin&ssl=true"',
                     '--forceTableScan',
                     '--gzip',
                     '-c',
@@ -239,7 +239,7 @@ def test_copy_table_with_collection_found_success(self):
                                 'mongodump',
                                 '--uri',
                                 '"mongodb://my_user:secret@foo.com:3306/my_db'
-                                '?authSource=admin&readPreference=secondaryPreferred&ssl=true"',
+                                '?readPreference=secondaryPreferred&authSource=admin&ssl=true"',
                                 '--forceTableScan',
                                 '--gzip',
                                 '-c',

From 206e75e630933d1a2b2ab9afb36261a55ccf0e4c Mon Sep 17 00:00:00 2001
From: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
Date: Mon, 10 Jan 2022 16:11:27 +0200
Subject: [PATCH 50/78] AP-1098 Upgrade base image to Python 3.8 (#853)

---
 Dockerfile                                         | 2 +-
 Dockerfile.barebone                                | 2 +-
 dev-project/docker-compose.yml                     | 2 +-
 pytest.ini                                         | 2 ++
 singer-connectors/target-postgres/requirements.txt | 2 +-
 5 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 711f681d4..6b595b1a8 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.7-slim-buster
+FROM python:3.8-slim-buster
 
 ARG connectors=all
 
diff --git a/Dockerfile.barebone b/Dockerfile.barebone
index 167f91709..af48710a4 100644
--- a/Dockerfile.barebone
+++ b/Dockerfile.barebone
@@ -1,4 +1,4 @@
-FROM python:3.7-slim-buster
+FROM python:3.8-slim-buster
 
 RUN apt-get -qq update \
     && apt-get -qqy --no-install-recommends install \
diff --git a/dev-project/docker-compose.yml b/dev-project/docker-compose.yml
index e68ae00dd..0619d99a8 100644
--- a/dev-project/docker-compose.yml
+++ b/dev-project/docker-compose.yml
@@ -2,7 +2,7 @@ version: '3'
 services:
   ### Primary container with PipelineWise CLI
   pipelinewise:
-    image: python:3.7.4-buster
+    image: python:3.8-buster
     container_name: pipelinewise_dev
     working_dir: /opt/pipelinewise
     entrypoint: /opt/pipelinewise/dev-project/entrypoint.sh
diff --git a/pytest.ini b/pytest.ini
index 231bc76da..1df0b254c 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -4,3 +4,5 @@ filterwarnings =
     ignore::DeprecationWarning:botocore
     ignore::DeprecationWarning:ansible
     ignore::DeprecationWarning:tabulate
+    ignore::DeprecationWarning:_yaml
+    ignore::DeprecationWarning:messytables
diff --git a/singer-connectors/target-postgres/requirements.txt b/singer-connectors/target-postgres/requirements.txt
index 6178a16aa..0c0edb51e 100644
--- a/singer-connectors/target-postgres/requirements.txt
+++ b/singer-connectors/target-postgres/requirements.txt
@@ -1 +1 @@
-pipelinewise-target-postgres==2.1.0
+pipelinewise-target-postgres==2.1.1

From 0ac6fb28b56f3e8d774681414de92ce070adb136 Mon Sep 17 00:00:00 2001
From: Jeet Parekh <94441288+jeet-parekh-wise@users.noreply.github.com>
Date: Fri, 14 Jan 2022 13:14:08 +0000
Subject: [PATCH 51/78] AP-1078 - fix mysql tap fastsync - connect to replica
 server (#859)

* AP-1078 - fix mysql tap - connect to replica server

* AP-1078 - fix pylint errors

* AP-1078 - fix flake8 error

* AP-1078 - add mysql replica test template

* AP-1078 - manually load the env dict

* AP-1078 - fix drop schema statement

* AP-1078 - add template for mariadb replica to snowflake
---
 dev-project/.env                              |  11 ++
 dev-project/docker-compose.yml                |  25 +++-
 pipelinewise/fastsync/commons/tap_mysql.py    | 117 ++++++++++--------
 tests/db/tap_mysql_db.sh                      |  56 ++++++++-
 tests/end_to_end/helpers/assertions.py        |   7 ++
 tests/end_to_end/helpers/env.py               |  23 ++++
 .../tap_mysql_replica_to_pg.yml.template      | 117 ++++++++++++++++++
 .../tap_mysql_replica_to_sf.yml.template      | 116 +++++++++++++++++
 tests/end_to_end/test_target_postgres.py      |  18 +++
 tests/end_to_end/test_target_snowflake.py     |  23 +++-
 .../commons/test_fastsync_tap_mysql.py        |  48 ++++++-
 11 files changed, 502 insertions(+), 59 deletions(-)
 create mode 100644 tests/end_to_end/test-project/tap_mysql_replica_to_pg.yml.template
 create mode 100644 tests/end_to_end/test-project/tap_mysql_replica_to_sf.yml.template

diff --git a/dev-project/.env b/dev-project/.env
index c2856e7dc..670d647bf 100644
--- a/dev-project/.env
+++ b/dev-project/.env
@@ -21,6 +21,17 @@ TAP_MYSQL_USER=pipelinewise
 TAP_MYSQL_PASSWORD=secret
 TAP_MYSQL_DB=mysql_source_db
 
+# ------------------------------------------------------------------------------
+# Test MySQL database credentials used as test source replica database
+# ------------------------------------------------------------------------------
+
+TAP_MYSQL_REPLICA_PORT=3306
+TAP_MYSQL_REPLICA_PORT_ON_HOST=14406
+TAP_MYSQL_REPLICA_ROOT_PASSWORD=secret
+TAP_MYSQL_REPLICA_USER=pipelinewise
+TAP_MYSQL_REPLICA_PASSWORD=secret
+TAP_MYSQL_REPLICA_DB=mysql_source_db_2
+
 # ------------------------------------------------------------------------------
 # Test Mongodb database credentials used as test source database
 # ------------------------------------------------------------------------------
diff --git a/dev-project/docker-compose.yml b/dev-project/docker-compose.yml
index 0619d99a8..50c3d05f8 100644
--- a/dev-project/docker-compose.yml
+++ b/dev-project/docker-compose.yml
@@ -8,6 +8,7 @@ services:
     entrypoint: /opt/pipelinewise/dev-project/entrypoint.sh
     environment:
       PIPELINEWISE_HOME: /opt/pipelinewise/dev-project
+
       TAP_MYSQL_HOST: db_mysql_source
       TAP_MYSQL_PORT: ${TAP_MYSQL_PORT}
       TAP_MYSQL_PORT_ON_HOST: ${TAP_MYSQL_PORT_ON_HOST}
@@ -16,6 +17,14 @@ services:
       TAP_MYSQL_PASSWORD: ${TAP_MYSQL_PASSWORD}
       TAP_MYSQL_DB: ${TAP_MYSQL_DB}
 
+      TAP_MYSQL_REPLICA_HOST: db_mysql_source_replica
+      TAP_MYSQL_REPLICA_PORT: ${TAP_MYSQL_REPLICA_PORT}
+      TAP_MYSQL_REPLICA_PORT_ON_HOST: ${TAP_MYSQL_REPLICA_PORT_ON_HOST}
+      TAP_MYSQL_REPLICA_ROOT_PASSWORD: ${TAP_MYSQL_REPLICA_ROOT_PASSWORD}
+      TAP_MYSQL_REPLICA_USER: ${TAP_MYSQL_REPLICA_USER}
+      TAP_MYSQL_REPLICA_PASSWORD: ${TAP_MYSQL_REPLICA_PASSWORD}
+      TAP_MYSQL_REPLICA_DB: ${TAP_MYSQL_REPLICA_DB}
+
       TAP_POSTGRES_HOST: db_postgres_source
       TAP_POSTGRES_PORT: ${TAP_POSTGRES_PORT}
       TAP_POSTGRES_PORT_ON_HOST: ${TAP_POSTGRES_PORT_ON_HOST}
@@ -80,7 +89,7 @@ services:
     container_name: pipelinewise_dev_mysql_source
     ports:
       - ${TAP_MYSQL_PORT_ON_HOST}:${TAP_MYSQL_PORT}
-    command: --default-authentication-plugin=mysql_native_password --log-bin=mysql-bin --binlog-format=ROW
+    command: --default-authentication-plugin=mysql_native_password --server-id=1 --log-bin=mysql-bin --binlog-format=ROW --binlog-do-db=${TAP_MYSQL_DB} --binlog-do-db=${TAP_MYSQL_REPLICA_DB}
     environment:
       MYSQL_ROOT_PASSWORD: ${TAP_MYSQL_ROOT_PASSWORD}
       MYSQL_USER: ${TAP_MYSQL_USER}
@@ -89,6 +98,20 @@ services:
     networks:
       - pipelinewise_network
 
+  db_mysql_source_replica:
+    image: mariadb:10.2.26
+    container_name: pipelinewise_dev_mysql_source_replica
+    ports:
+      - ${TAP_MYSQL_REPLICA_PORT_ON_HOST}:${TAP_MYSQL_REPLICA_PORT}
+    command: --default-authentication-plugin=mysql_native_password --server-id=2 --log-bin=mysql-bin --relay-log=mysql-relay-log --replicate-do-db=${TAP_MYSQL_REPLICA_DB}
+    environment:
+      MYSQL_ROOT_PASSWORD: ${TAP_MYSQL_REPLICA_ROOT_PASSWORD}
+      MYSQL_USER: ${TAP_MYSQL_REPLICA_USER}
+      MYSQL_PASSWORD: ${TAP_MYSQL_REPLICA_PASSWORD}
+      MYSQL_DATABASE: ${TAP_MYSQL_REPLICA_DB}
+    networks:
+      - pipelinewise_network
+
   db_mongo_source:
     container_name: "pipelinewise_dev_mongodb_source"
     image: "mongo:4.2-bionic"
diff --git a/pipelinewise/fastsync/commons/tap_mysql.py b/pipelinewise/fastsync/commons/tap_mysql.py
index f8c2ab344..4d50de714 100644
--- a/pipelinewise/fastsync/commons/tap_mysql.py
+++ b/pipelinewise/fastsync/commons/tap_mysql.py
@@ -2,12 +2,13 @@
 import datetime
 import decimal
 import logging
-import pymysql
+from typing import Tuple
 
+import pymysql
 from pymysql import InterfaceError, OperationalError
 
-from . import utils, split_gzip
 from ...utils import safe_column_name
+from . import split_gzip, utils
 
 LOGGER = logging.getLogger(__name__)
 
@@ -26,7 +27,7 @@ class FastSyncTapMySql:
     Common functions for fastsync from a MySQL database
     """
 
-    def __init__(self, connection_config, tap_type_to_target_type, target_quote=None):
+    def __init__(self, connection_config: dict, tap_type_to_target_type, target_quote=None):
         self.connection_config = connection_config
         self.connection_config['charset'] = connection_config.get(
             'charset', DEFAULT_CHARSET
@@ -41,55 +42,59 @@ def __init__(self, connection_config, tap_type_to_target_type, target_quote=None
         self.target_quote = target_quote
         self.conn = None
         self.conn_unbuffered = None
+        self.is_replica = False
+
+    def get_connection_parameters(self) -> Tuple[dict, bool]:
+        """
+        Method to get connection parameters
+        Connection is either to the primary or a replica if its credentials are given
+
+        Args:
+            connection_config: dictionary containing the db connection details
+        Returns:
+            dict with credentials
+        """
+
+        is_replica = False
+
+        if 'replica_host' in self.connection_config:
+            is_replica = True
+
+        host = self.connection_config.get('replica_host', self.connection_config['host'])
+        port = int(self.connection_config.get('replica_port', self.connection_config['port']))
+        user = self.connection_config.get('replica_user', self.connection_config['user'])
+        password = self.connection_config.get('replica_password', self.connection_config['password'])
+        charset = self.connection_config['charset']
+
+        return ({
+            'host': host,
+            'port': port,
+            'user': user,
+            'password': password,
+            'charset': charset,
+        }, is_replica)
 
     def open_connections(self):
         """
         Open connection
         """
+
+        # Fastsync is using replica_{host|port|user|password} values from the config by default
+        # to avoid making heavy load on the primary source database when syncing large tables
+        #
+        # If replica_{host|port|user|password} values are not defined in the config then it's
+        # using the normal credentials to connect
+
+        conn_params, is_replica = self.get_connection_parameters()
+
+        self.is_replica = is_replica
+
         self.conn = pymysql.connect(
-            # Fastsync is using bulk_sync_{host|port|user|password} values from the config by default
-            # to avoid making heavy load on the primary source database when syncing large tables
-            #
-            # If bulk_sync_{host|port|user|password} values are not defined in the config then it's
-            # using the normal credentials to connect
-            host=self.connection_config.get(
-                'bulk_sync_host', self.connection_config['host']
-            ),
-            port=int(
-                self.connection_config.get(
-                    'bulk_sync_port', self.connection_config['port']
-                )
-            ),
-            user=self.connection_config.get(
-                'bulk_sync_user', self.connection_config['user']
-            ),
-            password=self.connection_config.get(
-                'bulk_sync_password', self.connection_config['password']
-            ),
-            charset=self.connection_config['charset'],
+            **conn_params,
             cursorclass=pymysql.cursors.DictCursor,
         )
         self.conn_unbuffered = pymysql.connect(
-            # Fastsync is using bulk_sync_{host|port|user|password} values from the config by default
-            # to avoid making heavy load on the primary source database when syncing large tables
-            #
-            # If bulk_sync_{host|port|user|password} values are not defined in the config then it's
-            # using the normal credentials to connect
-            host=self.connection_config.get(
-                'bulk_sync_host', self.connection_config['host']
-            ),
-            port=int(
-                self.connection_config.get(
-                    'bulk_sync_port', self.connection_config['port']
-                )
-            ),
-            user=self.connection_config.get(
-                'bulk_sync_user', self.connection_config['user']
-            ),
-            password=self.connection_config.get(
-                'bulk_sync_password', self.connection_config['password']
-            ),
-            charset=self.connection_config['charset'],
+            **conn_params,
             cursorclass=pymysql.cursors.SSCursor,
         )
 
@@ -175,16 +180,28 @@ def fetch_current_log_pos(self):
         """
         Get the actual binlog position in MySQL
         """
-        result = self.query('SHOW MASTER STATUS')
-        if len(result) == 0:
-            raise Exception('MySQL binary logging is not enabled.')
+        if self.is_replica:
+            result = self.query('SHOW SLAVE STATUS')
+            if len(result) == 0:
+                raise Exception('MySQL binary logging is not enabled.')
+            binlog_pos = result[0]
+            log_file = binlog_pos.get('Master_Log_File')
+            log_pos = binlog_pos.get('Read_Master_Log_Pos')
+            version = binlog_pos.get('version', 1)
 
-        binlog_pos = result[0]
+        else:
+            result = self.query('SHOW MASTER STATUS')
+            if len(result) == 0:
+                raise Exception('MySQL binary logging is not enabled.')
+            binlog_pos = result[0]
+            log_file = binlog_pos.get('File')
+            log_pos = binlog_pos.get('Position')
+            version = binlog_pos.get('version', 1)
 
         return {
-            'log_file': binlog_pos.get('File'),
-            'log_pos': binlog_pos.get('Position'),
-            'version': binlog_pos.get('version', 1),
+            'log_file': log_file,
+            'log_pos': log_pos,
+            'version': version,
         }
 
     # pylint: disable=invalid-name
diff --git a/tests/db/tap_mysql_db.sh b/tests/db/tap_mysql_db.sh
index 60d07d3c7..d00d5a5e3 100755
--- a/tests/db/tap_mysql_db.sh
+++ b/tests/db/tap_mysql_db.sh
@@ -22,12 +22,56 @@ fi
 
 # Grant Replication client and replication slave privileges that
 # requires for LOG_BASED CDC replication
-export MYSQL_PWD=${TAP_MYSQL_ROOT_PASSWORD}
-mysql --protocol TCP --host ${TAP_MYSQL_HOST} --port ${TAP_MYSQL_PORT} --user root -e "GRANT REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO ${TAP_MYSQL_USER}"
 
-# Grant insert privileges for testing
-mysql --protocol TCP --host ${TAP_MYSQL_HOST} --port ${TAP_MYSQL_PORT} --user root -e "GRANT INSERT ON *.* TO ${TAP_MYSQL_USER}"
+echo "SETTING UP MYSQL PRIMARY SERVER FOR REPLICATION"
+
+mysql --protocol TCP \
+--host ${TAP_MYSQL_HOST} \
+--port ${TAP_MYSQL_PORT} \
+--user root \
+--password=${TAP_MYSQL_ROOT_PASSWORD} \
+-e "CREATE DATABASE IF NOT EXISTS ${TAP_MYSQL_REPLICA_DB}; GRANT ALL PRIVILEGES ON ${TAP_MYSQL_REPLICA_DB}.* TO ${TAP_MYSQL_USER}; GRANT REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO ${TAP_MYSQL_USER}; FLUSH PRIVILEGES;"
+
+echo "SET UP MYSQL REPLICA SERVER FOR REPLICATION"
+
+mysql --protocol TCP \
+--host ${TAP_MYSQL_REPLICA_HOST} \
+--port ${TAP_MYSQL_REPLICA_PORT} \
+--user root \
+--password=${TAP_MYSQL_REPLICA_ROOT_PASSWORD} \
+-e "GRANT REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO ${TAP_MYSQL_REPLICA_USER}; GRANT SUPER ON *.* TO ${TAP_MYSQL_REPLICA_USER}; FLUSH PRIVILEGES;"
+
+echo "GETTING MYSQL PRIMARY SERVER LOG INFO"
+
+MASTER_LOG_STATUS=`mysql --protocol TCP --host ${TAP_MYSQL_HOST} --port ${TAP_MYSQL_PORT} --user root --password=${TAP_MYSQL_ROOT_PASSWORD} -e "SHOW MASTER STATUS;"`
+CURRENT_LOG=`echo $MASTER_LOG_STATUS | awk '{print $5}'`
+CURRENT_POS=`echo $MASTER_LOG_STATUS | awk '{print $6}'`
+
+echo "STARTING MYSQL REPLICATION"
+
+mysql --protocol TCP \
+--host=${TAP_MYSQL_REPLICA_HOST} \
+--port ${TAP_MYSQL_REPLICA_PORT} \
+--user ${TAP_MYSQL_REPLICA_USER} \
+--password=${TAP_MYSQL_REPLICA_PASSWORD} \
+-e "STOP SLAVE; CHANGE MASTER TO MASTER_HOST='${TAP_MYSQL_HOST}',MASTER_USER='${TAP_MYSQL_USER}',MASTER_PASSWORD='${TAP_MYSQL_PASSWORD}',MASTER_LOG_FILE='${CURRENT_LOG}',MASTER_LOG_POS=${CURRENT_POS}; START SLAVE;"
 
 # Download the sample database and build it
-export MYSQL_PWD=${TAP_MYSQL_PASSWORD}
-mysql --protocol TCP --host ${TAP_MYSQL_HOST} --port ${TAP_MYSQL_PORT} --user ${TAP_MYSQL_USER} ${TAP_MYSQL_DB} < ${TEST_DB_SQL}
\ No newline at end of file
+
+echo "DUMPING DATA INTO PRIMARY MYSQL DATABASE"
+
+mysql --protocol TCP \
+--host ${TAP_MYSQL_HOST} \
+--port ${TAP_MYSQL_PORT} \
+--user ${TAP_MYSQL_USER} \
+--password=${TAP_MYSQL_PASSWORD} \
+${TAP_MYSQL_DB} < ${TEST_DB_SQL}
+
+echo "DUMPING DATA INTO PRIMARY MYSQL DATABASE2"
+
+mysql --protocol TCP \
+--host ${TAP_MYSQL_HOST} \
+--port ${TAP_MYSQL_PORT} \
+--user ${TAP_MYSQL_USER} \
+--password=${TAP_MYSQL_PASSWORD} \
+${TAP_MYSQL_REPLICA_DB} < ${TEST_DB_SQL}
diff --git a/tests/end_to_end/helpers/assertions.py b/tests/end_to_end/helpers/assertions.py
index b720588b4..ce989b613 100644
--- a/tests/end_to_end/helpers/assertions.py
+++ b/tests/end_to_end/helpers/assertions.py
@@ -154,6 +154,13 @@ def _map_tap_to_target_functions(
             'source_sql_get_cols_fn': db.sql_get_columns_mysql,
             'source_sql_dynamic_row_count_fn': db.sql_dynamic_row_count_mysql,
         },
+        # tap-mysql specific attributes and functions
+        'run_query_tap_mysql_2': {
+            'source_schemas': ['mysql_source_db_2'],
+            'target_schemas': [f'ppw_e2e_tap_mysql_2{schema_postfix}'],
+            'source_sql_get_cols_fn': db.sql_get_columns_mysql,
+            'source_sql_dynamic_row_count_fn': db.sql_dynamic_row_count_mysql,
+        },
         # tap-postgres specific attributes and functions
         'run_query_tap_postgres': {
             'source_schemas': ['public', 'public2'],
diff --git a/tests/end_to_end/helpers/env.py b/tests/end_to_end/helpers/env.py
index 42377ae77..3eaf46c9e 100644
--- a/tests/end_to_end/helpers/env.py
+++ b/tests/end_to_end/helpers/env.py
@@ -90,6 +90,12 @@ def _load_env(self):
                     'USER': {'value': os.environ.get('TAP_MYSQL_USER')},
                     'PASSWORD': {'value': os.environ.get('TAP_MYSQL_PASSWORD')},
                     'DB': {'value': os.environ.get('TAP_MYSQL_DB')},
+                    'DB_2': {'value': os.environ.get('TAP_MYSQL_REPLICA_DB')},
+                    'REPLICA_HOST': {'value': os.environ.get('TAP_MYSQL_REPLICA_HOST')},
+                    'REPLICA_PORT': {'value': os.environ.get('TAP_MYSQL_REPLICA_PORT')},
+                    'REPLICA_USER': {'value': os.environ.get('TAP_MYSQL_REPLICA_USER')},
+                    'REPLICA_PASSWORD': {'value': os.environ.get('TAP_MYSQL_REPLICA_PASSWORD')},
+                    'REPLICA_DB': {'value': os.environ.get('TAP_MYSQL_REPLICA_DB')},
                 },
             },
             # ------------------------------------------------------------------
@@ -465,6 +471,17 @@ def run_query_tap_mysql(self, query):
             database=self._get_conn_env_var('TAP_MYSQL', 'DB'),
         )
 
+    def run_query_tap_mysql_2(self, query):
+        """Run and SQL query in tap mysql database"""
+        return db.run_query_mysql(
+            query,
+            host=self._get_conn_env_var('TAP_MYSQL', 'HOST'),
+            port=int(self._get_conn_env_var('TAP_MYSQL', 'PORT')),
+            user=self._get_conn_env_var('TAP_MYSQL', 'USER'),
+            password=self._get_conn_env_var('TAP_MYSQL', 'PASSWORD'),
+            database=self._get_conn_env_var('TAP_MYSQL', 'DB_2'),
+        )
+
     def run_query_target_snowflake(self, query):
         """Run and SQL query in target snowflake database"""
         return db.run_query_snowflake(
@@ -551,6 +568,9 @@ def setup_target_postgres(self):
         self.run_query_target_postgres(
             'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql CASCADE'
         )
+        self.run_query_target_postgres(
+            'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql_2 CASCADE'
+        )
         self.run_query_target_postgres(
             'DROP SCHEMA IF EXISTS ppw_e2e_tap_s3_csv CASCADE'
         )
@@ -608,6 +628,9 @@ def setup_target_snowflake(self):
         self.run_query_target_snowflake(
             f'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql{self.sf_schema_postfix} CASCADE'
         )
+        self.run_query_target_postgres(
+            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql_2{self.sf_schema_postfix} CASCADE'
+        )
         self.run_query_target_snowflake(
             f'DROP SCHEMA IF EXISTS ppw_e2e_tap_s3_csv{self.sf_schema_postfix} CASCADE'
         )
diff --git a/tests/end_to_end/test-project/tap_mysql_replica_to_pg.yml.template b/tests/end_to_end/test-project/tap_mysql_replica_to_pg.yml.template
new file mode 100644
index 000000000..0b7803aa9
--- /dev/null
+++ b/tests/end_to_end/test-project/tap_mysql_replica_to_pg.yml.template
@@ -0,0 +1,117 @@
+---
+
+# ------------------------------------------------------------------------------
+# General Properties
+# ------------------------------------------------------------------------------
+id: "mariadb_replica_to_pg"
+name: "MariaDB source test database"
+type: "tap-mysql"
+owner: "test-runner"
+
+
+# ------------------------------------------------------------------------------
+# Source (Tap) - MySQL connection details
+# ------------------------------------------------------------------------------
+db_conn:
+  host: "${TAP_MYSQL_HOST}"                           # MySQL host
+  port: ${TAP_MYSQL_PORT}                             # MySQL port
+  user: "${TAP_MYSQL_USER}"                           # MySQL user
+  password: "${TAP_MYSQL_PASSWORD}"                   # Plain string or vault encrypted
+  replica_host: "${TAP_MYSQL_REPLICA_HOST}"           # MySQL replica host
+  replica_port: ${TAP_MYSQL_REPLICA_PORT}             # MySQL replica port
+  replica_user: "${TAP_MYSQL_REPLICA_USER}"           # MySQL replica user
+  replica_password: "${TAP_MYSQL_REPLICA_PASSWORD}"   # Plain string or vault encrypted
+  dbname: "${TAP_MYSQL_REPLICA_DB}"                   # MySQL database name
+
+
+
+# ------------------------------------------------------------------------------
+# Destination (Target) - Target properties
+# Connection details should be in the relevant target YAML file
+# ------------------------------------------------------------------------------
+target: "postgres_dwh"                 # ID of the target connector where the data will be loaded
+batch_size_rows: 20000                 # Batch size for the stream to optimise load performance
+stream_buffer_size: 0                  # In-memory buffer size (MB) between taps and targets for asynchronous data pipes
+
+
+# ------------------------------------------------------------------------------
+# Source to target Schema mapping
+# ------------------------------------------------------------------------------
+schemas:
+  - source_schema: "${TAP_MYSQL_REPLICA_DB}"
+    target_schema: "ppw_e2e_tap_mysql_2"
+
+    tables:
+      ### Table with LOG_BASED replication
+      - table_name: "weight_unit"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "weight_unit_name"
+            type: "HASH-SKIP-FIRST-2"
+
+      ### Table with INCREMENTAL replication
+      - table_name: "address"
+        replication_method: "INCREMENTAL"
+        replication_key: "date_updated"
+        transformations:
+          - column: "zip_code_zip_code_id"
+            type: "MASK-NUMBER"
+            when:
+              - column: 'street_number'
+                regex_match: '[801]'
+
+          - column: "date_created"
+            type: "MASK-DATE"
+
+      ### Table with FULL_TABLE replication
+      - table_name: "order"
+        replication_method: "FULL_TABLE"
+
+      ### Table with no primary key
+      - table_name: "no_pk_table"
+        replication_method: "FULL_TABLE"
+
+      ### Table with binary and varbinary columns
+      - table_name: "table_with_binary"
+        replication_method: "LOG_BASED"
+
+      ### Table with reserved words and columns and special characters
+      - table_name: "edgydata"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "case"
+            type: "HASH"
+
+          - column: "group"
+            type: "MASK-NUMBER"
+            when:
+              - column: 'case'
+                equals: 'A'
+          - column: "group"
+            type: "SET-NULL"
+            when:
+              - column: 'case'
+                equals: 'B'
+
+      ### Table with reserved word
+      - table_name: "full"
+        replication_method: "INCREMENTAL"
+        replication_key: "begin"
+
+      ### Table with space and mixed upper and lowercase characters
+      - table_name: "table_with_space and UPPERCase"
+        replication_method: "LOG_BASED"
+
+      ### Table with all possible data types
+      - table_name: "all_datatypes"
+        replication_method: "LOG_BASED"
+
+      ### Table with LOG_BASED replication
+      - table_name: "customers"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
diff --git a/tests/end_to_end/test-project/tap_mysql_replica_to_sf.yml.template b/tests/end_to_end/test-project/tap_mysql_replica_to_sf.yml.template
new file mode 100644
index 000000000..1b4763dae
--- /dev/null
+++ b/tests/end_to_end/test-project/tap_mysql_replica_to_sf.yml.template
@@ -0,0 +1,116 @@
+---
+
+# ------------------------------------------------------------------------------
+# General Properties
+# ------------------------------------------------------------------------------
+id: "mariadb_replica_to_sf"
+name: "MariaDB source test database"
+type: "tap-mysql"
+owner: "test-runner"
+
+
+# ------------------------------------------------------------------------------
+# Source (Tap) - MySQL connection details
+# ------------------------------------------------------------------------------
+db_conn:
+  host: "${TAP_MYSQL_HOST}"                           # MySQL host
+  port: ${TAP_MYSQL_PORT}                             # MySQL port
+  user: "${TAP_MYSQL_USER}"                           # MySQL user
+  password: "${TAP_MYSQL_PASSWORD}"                   # Plain string or vault encrypted
+  replica_host: "${TAP_MYSQL_REPLICA_HOST}"           # MySQL replica host
+  replica_port: ${TAP_MYSQL_REPLICA_PORT}             # MySQL replica port
+  replica_user: "${TAP_MYSQL_REPLICA_USER}"           # MySQL replica user
+  replica_password: "${TAP_MYSQL_REPLICA_PASSWORD}"   # Plain string or vault encrypted
+  dbname: "${TAP_MYSQL_REPLICA_DB}"                   # MySQL database name
+
+
+# ------------------------------------------------------------------------------
+# Destination (Target) - Target properties
+# Connection details should be in the relevant target YAML file
+# ------------------------------------------------------------------------------
+target: "snowflake"                    # ID of the target connector where the data will be loaded
+batch_size_rows: 20000                 # Batch size for the stream to optimise load performance
+stream_buffer_size: 0                  # In-memory buffer size (MB) between taps and targets for asynchronous data pipes
+
+
+# ------------------------------------------------------------------------------
+# Source to target Schema mapping
+# ------------------------------------------------------------------------------
+schemas:
+  - source_schema: "${TAP_MYSQL_REPLICA_DB}"
+    target_schema: "ppw_e2e_tap_mysql_2${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"
+
+    tables:
+      ### Table with LOG_BASED replication
+      - table_name: "weight_unit"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "weight_unit_name"
+            type: "HASH-SKIP-FIRST-2"
+
+      ### Table with INCREMENTAL replication
+      - table_name: "address"
+        replication_method: "INCREMENTAL"
+        replication_key: "date_updated"
+        transformations:
+          - column: "zip_code_zip_code_id"
+            type: "MASK-NUMBER"
+            when:
+              - column: 'street_number'
+                regex_match: '[801]'
+
+          - column: "date_created"
+            type: "MASK-DATE"
+
+      ### Table with FULL_TABLE replication
+      - table_name: "order"
+        replication_method: "FULL_TABLE"
+
+      ### Table with no primary key
+      - table_name: "no_pk_table"
+        replication_method: "FULL_TABLE"
+
+      ### Table with binary and varbinary columns
+      - table_name: "table_with_binary"
+        replication_method: "LOG_BASED"
+
+      ### Table with reserved words and columns and special characters
+      - table_name: "edgydata"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "case"
+            type: "HASH"
+
+          - column: "group"
+            type: "MASK-NUMBER"
+            when:
+              - column: 'case'
+                equals: 'A'
+          - column: "group"
+            type: "SET-NULL"
+            when:
+              - column: 'case'
+                equals: 'B'
+
+      ### Table with reserved word
+      - table_name: "full"
+        replication_method: "INCREMENTAL"
+        replication_key: "begin"
+
+      ### Table with space and mixed upper and lowercase characters
+      - table_name: "table_with_space and UPPERCase"
+        replication_method: "LOG_BASED"
+
+      ### Table with all possible data types
+      - table_name: "all_datatypes"
+        replication_method: "LOG_BASED"
+
+      ### Table with LOG_BASED replication
+      - table_name: "customers"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
diff --git a/tests/end_to_end/test_target_postgres.py b/tests/end_to_end/test_target_postgres.py
index 182f5edfd..e97212323 100644
--- a/tests/end_to_end/test_target_postgres.py
+++ b/tests/end_to_end/test_target_postgres.py
@@ -16,6 +16,7 @@
 DIR = os.path.dirname(__file__)
 TAP_MARIADB_ID = 'mariadb_to_pg'
 TAP_MARIADB_BUFFERED_STREAM_ID = 'mariadb_to_pg_buffered_stream'
+TAP_MARIADB_REPLICA_ID = 'mariadb_replica_to_pg'
 TAP_MONGODB_ID = 'mongo_to_pg'
 TAP_POSTGRES_ID = 'postgres_to_pg'
 TAP_S3_CSV_ID = 's3_csv_to_pg'
@@ -36,6 +37,7 @@ def setup_method(self):
         # Init query runner methods
         self.e2e = E2EEnv(self.project_dir)
         self.run_query_tap_mysql = self.e2e.run_query_tap_mysql
+        self.run_query_tap_mysql_2 = self.e2e.run_query_tap_mysql_2
         self.run_query_tap_postgres = self.e2e.run_query_tap_postgres
         self.run_query_target_postgres = self.e2e.run_query_target_postgres
         self.mongodb_con = self.e2e.get_tap_mongodb_connection()
@@ -195,6 +197,22 @@ def test_replicate_mariadb_to_pg_with_custom_buffer_size(self):
         Same tests cases as test_replicate_mariadb_to_pg but using another tap with custom stream buffer size"""
         self.test_resync_mariadb_to_pg(tap_mariadb_id=TAP_MARIADB_BUFFERED_STREAM_ID)
 
+    @pytest.mark.dependency(depends=['import_config'])
+    def test_replicate_mariadb_replica_to_pg(self):
+        """Replicate data from MariaDB to Postgres DWH"""
+        # 1. Run tap first time - both fastsync and a singer should be triggered
+        assertions.assert_run_tap_success(
+            TAP_MARIADB_REPLICA_ID, TARGET_ID, ['fastsync', 'singer'], profiling=True
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql_2, self.run_query_target_postgres
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql_2,
+            self.run_query_target_postgres,
+            mysql_to_postgres.tap_type_to_target_type,
+        )
+
     @pytest.mark.dependency(depends=['import_config'])
     def test_replicate_pg_to_pg(self):
         """Replicate data from Postgres to Postgres DWH"""
diff --git a/tests/end_to_end/test_target_snowflake.py b/tests/end_to_end/test_target_snowflake.py
index 6e612b7b3..22e031354 100644
--- a/tests/end_to_end/test_target_snowflake.py
+++ b/tests/end_to_end/test_target_snowflake.py
@@ -19,6 +19,7 @@
 TAP_MARIADB_ID = 'mariadb_to_sf'
 TAP_MARIADB_SPLIT_LARGE_FILES_ID = 'mariadb_to_sf_split_large_files'
 TAP_MARIADB_BUFFERED_STREAM_ID = 'mariadb_to_sf_buffered_stream'
+TAP_MARIADB_REPLICA_ID = 'mariadb_replica_to_sf'
 TAP_POSTGRES_ID = 'postgres_to_sf'
 TAP_POSTGRES_SPLIT_LARGE_FILES_ID = 'postgres_to_sf_split_large_files'
 TAP_POSTGRES_ARCHIVE_LOAD_FILES_ID = 'postgres_to_sf_archive_load_files'
@@ -27,7 +28,7 @@
 TARGET_ID = 'snowflake'
 
 
-# pylint: disable=attribute-defined-outside-init
+# pylint: disable=attribute-defined-outside-init,too-many-instance-attributes
 class TestTargetSnowflake:
     """
     End to end tests for Target Snowflake
@@ -39,6 +40,7 @@ def setup_method(self):
 
         # Init query runner methods
         self.run_query_tap_mysql = self.e2e.run_query_tap_mysql
+        self.run_query_tap_mysql_2 = self.e2e.run_query_tap_mysql_2
         self.run_query_tap_postgres = self.e2e.run_query_tap_postgres
         self.run_query_target_snowflake = self.e2e.run_query_target_snowflake
         self.mongodb_con = self.e2e.get_tap_mongodb_connection()
@@ -205,6 +207,25 @@ def test_resync_mariadb_to_sf_with_split_large_files(
             schema_postfix=self.snowflake_schema_postfix
         )
 
+    @pytest.mark.dependency(depends=['import_config'])
+    def test_replicate_mariadb_replica_to_sf(self):
+        """Replicate data from MariaDB to Snowflake"""
+        # 1. Run tap first time - both fastsync and a singer should be triggered
+        assertions.assert_run_tap_success(
+            TAP_MARIADB_REPLICA_ID, TARGET_ID, ['fastsync', 'singer']
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_mysql_2,
+            self.run_query_target_snowflake,
+            self.snowflake_schema_postfix
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_mysql_2,
+            self.e2e.run_query_target_snowflake,
+            mysql_to_snowflake.tap_type_to_target_type,
+            schema_postfix=self.snowflake_schema_postfix
+        )
+
     # pylint: disable=invalid-name
     @pytest.mark.dependency(depends=['import_config'])
     def test_resync_pg_to_sf_with_split_large_files(
diff --git a/tests/units/fastsync/commons/test_fastsync_tap_mysql.py b/tests/units/fastsync/commons/test_fastsync_tap_mysql.py
index d1b67af9b..3bd94d7b1 100644
--- a/tests/units/fastsync/commons/test_fastsync_tap_mysql.py
+++ b/tests/units/fastsync/commons/test_fastsync_tap_mysql.py
@@ -1,7 +1,7 @@
 from unittest import TestCase
 from unittest.mock import patch
-import pymysql
 
+import pymysql
 from pipelinewise.fastsync.commons import tap_mysql
 from pipelinewise.fastsync.commons.tap_mysql import FastSyncTapMySql
 
@@ -58,6 +58,52 @@ def test_open_connections_with_default_session_sqls(self):
         assert self.mysql.executed_queries == tap_mysql.DEFAULT_SESSION_SQLS
         assert self.mysql.executed_queries_unbuffered == self.mysql.executed_queries
 
+    def test_get_connection_to_primary(self):
+        """
+        Check that get connection uses the right credentials to connect to primary
+        """
+        creds = {
+            'host': 'my_primary_host',
+            'port': 3306,
+            'user': 'my_primary_user',
+            'password': 'my_primary_user',
+        }
+
+        conn_params, is_replica = FastSyncTapMySql(
+            connection_config=creds,
+            tap_type_to_target_type='testing'
+        ).get_connection_parameters()
+        self.assertFalse(is_replica)
+        self.assertEqual(conn_params['host'], creds['host'])
+        self.assertEqual(conn_params['port'], creds['port'])
+        self.assertEqual(conn_params['user'], creds['user'])
+        self.assertEqual(conn_params['password'], creds['password'])
+
+    def test_get_connection_to_replica(self):
+        """
+        Check that get connection uses the right credentials to connect to secondary if present
+        """
+        creds = {
+            'host': 'my_primary_host',
+            'replica_host': 'my_replica_host',
+            'port': 3306,
+            'replica_port': 4406,
+            'user': 'my_primary_user',
+            'replica_user': 'my_replica_user',
+            'password': 'my_primary_user',
+            'replica_password': 'my_replica_user',
+        }
+
+        conn_params, is_replica = FastSyncTapMySql(
+            connection_config=creds,
+            tap_type_to_target_type='testing'
+        ).get_connection_parameters()
+        self.assertTrue(is_replica)
+        self.assertEqual(conn_params['host'], creds['replica_host'])
+        self.assertEqual(conn_params['port'], creds['replica_port'])
+        self.assertEqual(conn_params['user'], creds['replica_user'])
+        self.assertEqual(conn_params['password'], creds['replica_password'])
+
     def test_open_connections_with_session_sqls(self):
         """Custom session parameters should be applied if defined"""
         session_sqls = [

From dbd74e6c2411e70169ddd0e10a230e04f2ed1948 Mon Sep 17 00:00:00 2001
From: Amir Mofakhar <amir.mofakhar@transferwise.com>
Date: Fri, 14 Jan 2022 17:03:33 +0000
Subject: [PATCH 52/78] dumps: (#866)

pipelinewise-target-snowflake to 1.15.0
pipelinewise-tap-s3-csv to 1.2.3
pipelinewise-tap-postgres to 1.8.2
---
 singer-connectors/tap-postgres/requirements.txt     | 2 +-
 singer-connectors/tap-s3-csv/requirements.txt       | 2 +-
 singer-connectors/target-snowflake/requirements.txt | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/singer-connectors/tap-postgres/requirements.txt b/singer-connectors/tap-postgres/requirements.txt
index ca346f8db..9d032ea9b 100644
--- a/singer-connectors/tap-postgres/requirements.txt
+++ b/singer-connectors/tap-postgres/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-postgres==1.8.1
+pipelinewise-tap-postgres==1.8.2
diff --git a/singer-connectors/tap-s3-csv/requirements.txt b/singer-connectors/tap-s3-csv/requirements.txt
index cefd110c4..d92ece2be 100644
--- a/singer-connectors/tap-s3-csv/requirements.txt
+++ b/singer-connectors/tap-s3-csv/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-s3-csv==1.2.2
+pipelinewise-tap-s3-csv==1.2.3
diff --git a/singer-connectors/target-snowflake/requirements.txt b/singer-connectors/target-snowflake/requirements.txt
index 66a29fd00..bd53e5232 100644
--- a/singer-connectors/target-snowflake/requirements.txt
+++ b/singer-connectors/target-snowflake/requirements.txt
@@ -1 +1 @@
-pipelinewise-target-snowflake==1.14.1
+pipelinewise-target-snowflake==1.15.0

From 6934b3851512dbdd4280790bf253a0a13ab65684 Mon Sep 17 00:00:00 2001
From: Jeet Parekh <94441288+jeet-parekh-wise@users.noreply.github.com>
Date: Fri, 14 Jan 2022 17:31:28 +0000
Subject: [PATCH 53/78] v0.38.0 (#867)

* v0.38.0
---
 CHANGELOG.md | 10 ++++++++++
 setup.py     |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 13a0b50f9..f6afb3c46 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,13 @@
+0.38.0 (2022-01-14)
+-------------------
+- MySQL tap now connects to replica instance during fastsync if credentials are provided
+- Added fastsync support for MongoDB Atlas
+- Docker base image to Python 3.8
+- Bump `pyyaml` from `5.4.1` to `6.0`
+- Bump `pipelinewise-target-snowflake` from `1.14.1` to `1.15.0`
+- Bump `pipelinewise-tap-s3-csv` from `1.2.2` to `1.2.3`
+- Bump `pipelinewise-tap-postgres` from `1.8.1` to `1.8.2`
+
 0.37.2 (2021-12-10)
 -------------------
 - Bump `pipelinewise-tap-github` from `1.0.2` to `1.0.3`
diff --git a/setup.py b/setup.py
index 5c0dffa23..a62b8907e 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      version='0.37.2',
+      version='0.38.0',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',

From 84976a739da79accefe5613a191f71a1359d45cd Mon Sep 17 00:00:00 2001
From: Judah Rand <17158624+judahrand@users.noreply.github.com>
Date: Wed, 19 Jan 2022 09:32:57 +0000
Subject: [PATCH 54/78] Update `pipelinewise-target-bigquery` (#860)

---
 singer-connectors/target-bigquery/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singer-connectors/target-bigquery/requirements.txt b/singer-connectors/target-bigquery/requirements.txt
index 24832ec8b..541908aee 100644
--- a/singer-connectors/target-bigquery/requirements.txt
+++ b/singer-connectors/target-bigquery/requirements.txt
@@ -1 +1 @@
-pipelinewise-target-bigquery==1.1.1
+pipelinewise-target-bigquery==1.2.0

From 4c8c2ddc5a8be4c760e1f27bac87c61a55dd7b4b Mon Sep 17 00:00:00 2001
From: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
Date: Wed, 19 Jan 2022 15:01:45 +0200
Subject: [PATCH 55/78] AP-1080 Prevent usage of new transformation feature whe
 FastSync exists (#851)

---
 docs/user_guide/transformations.rst           |  18 ++-
 pipelinewise/cli/config.py                    |  99 +++++++++++-----
 pipelinewise/cli/errors.py                    |  12 ++
 pipelinewise/cli/pipelinewise.py              | 106 ++++++++++++------
 pipelinewise/cli/schemas/tap.json             |  27 ++++-
 pipelinewise/cli/utils.py                     |   7 +-
 .../transform-field/requirements.txt          |   2 +-
 tests/end_to_end/test_target_bigquery.py      |  12 ++
 tests/end_to_end/test_target_postgres.py      |  12 ++
 tests/end_to_end/test_target_redshift.py      |  12 ++
 tests/end_to_end/test_target_snowflake.py     |  12 ++
 .../tap_test.yml                              |  46 ++++++++
 .../target_test.yml                           |  18 +++
 tests/units/cli/test_cli.py                   |  23 +++-
 tests/units/cli/test_cli_utils.py             |   7 +-
 tests/units/cli/test_cli_utils_tap_github.py  |   6 +-
 tests/units/cli/test_config.py                |   6 +-
 17 files changed, 340 insertions(+), 85 deletions(-)
 create mode 100644 tests/units/cli/resources/test_validate_command/json_transformation_in_fastsync/tap_test.yml
 create mode 100644 tests/units/cli/resources/test_validate_command/json_transformation_in_fastsync/target_test.yml

diff --git a/docs/user_guide/transformations.rst b/docs/user_guide/transformations.rst
index 8f6e867ae..1e6e72b0b 100644
--- a/docs/user_guide/transformations.rst
+++ b/docs/user_guide/transformations.rst
@@ -92,7 +92,7 @@ in the :ref:`yaml_configuration`:
               - column: "property_name"
                 equals: 'passwordHash'
 
-                # Tip: Use 'regex_match' instead of 'equal' if you need
+                # Tip: Use 'regex_match' instead of 'equals' if you need
                 # more complex matching criteria. For example:
                 # regex_match: 'password|salt|passwordHash'
 
@@ -103,6 +103,22 @@ in the :ref:`yaml_configuration`:
                 equals: 'com.transferwise.fx.user.User'
               - column: "property_name"
                 equals: 'passwordHash'
+
+          - column: "column_3"
+            type: "HASH"
+            when:
+              - column: "json_column"
+                field_path: 'metadata/property_name'
+                equals: 'passwordHash'
+
+      - table_name: "users"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "json_column"
+            field_paths:
+              - "user/info/phone"
+              - "user/info/addresses/0"
+            type: "SET-NULL"
     ...
     ...
 
diff --git a/pipelinewise/cli/config.py b/pipelinewise/cli/config.py
index 0530aa5fb..5e31ce259 100644
--- a/pipelinewise/cli/config.py
+++ b/pipelinewise/cli/config.py
@@ -6,7 +6,7 @@
 import sys
 import json
 
-from typing import Dict
+from typing import Dict, List
 
 from pipelinewise.utils import safe_column_name
 from . import utils
@@ -247,12 +247,9 @@ def save_tap_jsons(self, target, tap, extra_config_keys=None):
         """
         Generating JSON config files for a singer tap connector:
             1. config.json             :(Singer spec):  Tap connection details
-            2. properties.json         :(Singer spec):  Tap schema properties (generated)
-            3. state.json              :(Singer spec):  Bookmark for incremental and log_based
-                                                        replications
 
             4. selection.json          :(Pipelinewise): List of streams/tables to replicate
-            5. inheritabel_config.json :(Pipelinewise): Extra config keys for the linked
+            5. inheritable_config.json :(Pipelinewise): Extra config keys for the linked
                                                         singer target connector that
                                                         pipelinewise will pass at run time
             6. transformation.json     :(Pipelinewise): Column transformations between the
@@ -260,6 +257,19 @@ def save_tap_jsons(self, target, tap, extra_config_keys=None):
         """
         if extra_config_keys is None:
             extra_config_keys = {}
+
+        # Generate tap config dict
+        tap_config = self.generate_tap_connection_config(tap, extra_config_keys)
+
+        # Generate tap selection
+        tap_selection = {'selection': self.generate_selection(tap)}
+
+        # Generate tap transformation
+        tap_transformation = {'transformations': self.generate_transformations(tap)}
+
+        # Generate tap inheritable_config dict
+        tap_inheritable_config = self.generate_inheritable_config(tap)
+
         tap_dir = self.get_tap_dir(target.get('id'), tap.get('id'))
         self.logger.info('SAVING TAP JSONS to %s', tap_dir)
 
@@ -273,14 +283,34 @@ def save_tap_jsons(self, target, tap, extra_config_keys=None):
         if not os.path.exists(tap_dir):
             os.mkdir(tap_dir)
 
-        # Generate tap config dict: a merged dictionary of db_connection and optional extra_keys
-        tap_config = {**tap.get('db_conn'), **extra_config_keys}
+        # Save the generated JSON files
+        utils.save_json(tap_config, tap_config_path)
+        utils.save_json(tap_inheritable_config, tap_inheritable_config_path)
+        utils.save_json(tap_transformation, tap_transformation_path)
+        utils.save_json(tap_selection, tap_selection_path)
+
+    @classmethod
+    def generate_tap_connection_config(cls, tap: Dict, extra_config_keys: Dict) -> Dict:
+        """
+        Generate tap connection config which is a merged dictionary of db_connection and optional extra_keys
+        Args:
+            tap: tap config
+            extra_config_keys:  extra keys to add to the db conn config
+        Returns: Dictionary of tap connection config
+        """
+        return {**tap.get('db_conn'), **extra_config_keys}
 
-        # Get additional properties will be needed later to generate tap_stream_id
-        tap_dbname = tap_config.get('dbname')
+    @classmethod
+    def generate_selection(cls, tap: Dict) -> List[Dict]:
+        """
+        Generate the selection data which is the list of selected streams and their replication method
+        Args:
+            tap: the tap config dictionary
 
-        # Generate tap selection
+        Returns: List of dictionaries of selected streams
+        """
         selection = []
+
         for schema in tap.get('schemas', []):
             schema_name = schema.get('source_schema')
             for table in schema.get('tables', []):
@@ -292,20 +322,29 @@ def save_tap_jsons(self, target, tap, extra_config_keys=None):
                     utils.delete_empty_keys(
                         {
                             'tap_stream_id': utils.get_tap_stream_id(
-                                tap, tap_dbname, schema_name, table_name
+                                tap, tap['db_conn'].get('dbname'), schema_name, table_name
                             ),
                             'replication_method': replication_method,
                             # Add replication_key only if replication_method is INCREMENTAL
                             'replication_key': table.get('replication_key')
-                            if replication_method == 'INCREMENTAL'
-                            else None,
+                            if replication_method == 'INCREMENTAL' else None,
                         }
                     )
                 )
-        tap_selection = {'selection': selection}
 
-        # Generate tap transformation
+        return selection
+
+    @classmethod
+    def generate_transformations(cls, tap: Dict) -> List[Dict]:
+        """
+        Generate the transformations data from the given tap config
+        Args:
+            tap: the tap config dictionary
+
+        Returns: List of transformations
+        """
         transformations = []
+
         for schema in tap.get('schemas', []):
             schema_name = schema.get('source_schema')
             for table in schema.get('tables', []):
@@ -314,20 +353,29 @@ def save_tap_jsons(self, target, tap, extra_config_keys=None):
                     transformations.append(
                         {
                             'tap_stream_name': utils.get_tap_stream_name(
-                                tap, tap_dbname, schema_name, table_name
-                            ),
+                                tap, tap['db_conn'].get('dbname'), schema_name, table_name),
                             'field_id': trans['column'],
                             # Make column name safe by wrapping it in quotes, it's useful when a field_id is a reserved
                             # word to be used by target snowflake in fastsync
                             'safe_field_id': safe_column_name(trans['column']),
+                            'field_paths': trans.get('field_paths'),
                             'type': trans['type'],
                             'when': trans.get('when'),
                         }
                     )
-        tap_transformation = {'transformations': transformations}
 
-        # Generate stream to schema mapping
+        return transformations
+
+    def generate_inheritable_config(self, tap: Dict) -> Dict:
+        """
+        Generate the inheritable config which is the custom config that should be fed to the target at runtime
+        Args:
+            tap: tap config
+
+        Returns: Dictionary of config
+        """
         schema_mapping = {}
+
         for schema in tap.get('schemas', []):
             source_schema = schema.get('source_schema')
             target_schema = schema.get('target_schema')
@@ -389,16 +437,15 @@ def save_tap_jsons(self, target, tap, extra_config_keys=None):
                 # flatten the schema and data by creating columns automatically. When 'data_flattening_max_level'
                 # is set to 0 then flattening functionality is turned off.
                 #
-                #  The value can be set in mutliple place and evaluated in the following order:
+                # The value can be set in multiple place and evaluated in the following order:
                 # ------------
                 #   1: First we try to find it in the tap YAML
                 #   2: Second we try to get the tap type specific default value
                 #   3: Otherwise we set flattening level to 0 (disabled)
                 'data_flattening_max_level': tap.get(
                     'data_flattening_max_level',
-                    utils.get_tap_property(tap, 'default_data_flattening_max_level')
-                    or 0,
-                ),
+                    utils.get_tap_property(tap, 'default_data_flattening_max_level') or 0,
+                    ),
                 'validate_records': tap.get('validate_records', False),
                 'add_metadata_columns': tap.get('add_metadata_columns', False),
                 'split_large_files': tap.get('split_large_files', False),
@@ -414,8 +461,4 @@ def save_tap_jsons(self, target, tap, extra_config_keys=None):
             }
         )
 
-        # Save the generated JSON files
-        utils.save_json(tap_config, tap_config_path)
-        utils.save_json(tap_inheritable_config, tap_inheritable_config_path)
-        utils.save_json(tap_transformation, tap_transformation_path)
-        utils.save_json(tap_selection, tap_selection_path)
+        return tap_inheritable_config
diff --git a/pipelinewise/cli/errors.py b/pipelinewise/cli/errors.py
index e9a0f4872..08df9bb55 100644
--- a/pipelinewise/cli/errors.py
+++ b/pipelinewise/cli/errors.py
@@ -15,3 +15,15 @@ def __init__(self, buffer_size, max_buffer_size):
             f'{max_buffer_size}M'
         )
         super().__init__(msg)
+
+
+class InvalidTransformationException(Exception):
+    """Raised if invalid transformation config is found"""
+
+
+class DuplicateConfigException(Exception):
+    """Raised if a duplicate tap/target config is found"""
+
+
+class InvalidConfigException(Exception):
+    """Raised if an invalid config is found"""
diff --git a/pipelinewise/cli/pipelinewise.py b/pipelinewise/cli/pipelinewise.py
index b07d2bbd8..7ad31186d 100644
--- a/pipelinewise/cli/pipelinewise.py
+++ b/pipelinewise/cli/pipelinewise.py
@@ -24,6 +24,7 @@
 from .config import Config
 from .alert_sender import AlertSender
 from .alert_handlers.base_alert_handler import BaseAlertHandler
+from .errors import InvalidTransformationException, DuplicateConfigException, InvalidConfigException
 
 FASTSYNC_PAIRS = {
     ConnectorType.TAP_MYSQL: {
@@ -1467,6 +1468,7 @@ def validate(self):
         yaml_dir = self.args.dir
         self.logger.info('Searching YAML config files in %s', yaml_dir)
         tap_yamls, target_yamls = utils.get_tap_target_names(yaml_dir)
+
         self.logger.info('Detected taps: %s', tap_yamls)
         self.logger.info('Detected targets: %s', target_yamls)
 
@@ -1475,47 +1477,74 @@ def validate(self):
 
         vault_secret = self.args.secret
 
-        target_ids = set()
-        # pylint: disable=E1136,E1137  # False positive when loading vault encrypted YAML
+        # dictionary of targets ID and type
+        targets = {}
+
         # Validate target json schemas and that no duplicate IDs exist
         for yaml_file in target_yamls:
-            self.logger.info('Started validating %s', yaml_file)
-            loaded_yaml = utils.load_yaml(
-                os.path.join(yaml_dir, yaml_file), vault_secret
-            )
-            utils.validate(loaded_yaml, target_schema)
+            self.logger.info('Started validating target file: %s', yaml_file)
 
-            if loaded_yaml['id'] in target_ids:
-                self.logger.error('Duplicate target found "%s"', loaded_yaml['id'])
-                sys.exit(1)
+            # pylint: disable=E1136  # False positive when loading vault encrypted YAML
+            target_yml = utils.load_yaml(os.path.join(yaml_dir, yaml_file), vault_secret)
+            utils.validate(target_yml, target_schema)
 
-            target_ids.add(loaded_yaml['id'])
-            self.logger.info('Finished validating %s', yaml_file)
+            if target_yml['id'] in targets:
+                raise DuplicateConfigException(f'Duplicate target found "{target_yml["id"]}"')
+
+            targets[target_yml['id']] = target_yml['type']
+
+            self.logger.info('Finished validating target file: %s', yaml_file)
 
         tap_ids = set()
-        # pylint: disable=E1136,E1137  # False positive when loading vault encrypted YAML
+
         # Validate tap json schemas, check that every tap has valid 'target' and that no duplicate IDs exist
         for yaml_file in tap_yamls:
-            self.logger.info('Started validating %s', yaml_file)
-            loaded_yaml = utils.load_yaml(
-                os.path.join(yaml_dir, yaml_file), vault_secret
-            )
-            utils.validate(loaded_yaml, tap_schema)
-
-            if loaded_yaml['id'] in tap_ids:
-                self.logger.error('Duplicate tap found "%s"', loaded_yaml['id'])
-                sys.exit(1)
-
-            if loaded_yaml['target'] not in target_ids:
-                self.logger.error(
-                    "Can'f find the target with the ID '%s' referenced in '%s'. Available target IDs: %s",
-                    loaded_yaml['target'],
-                    yaml_file,
-                    target_ids,
-                )
-                sys.exit(1)
+            self.logger.info('Started validating %s ...', yaml_file)
+
+            # pylint: disable=E1136  # False positive when loading vault encrypted YAML
+            tap_yml = utils.load_yaml(os.path.join(yaml_dir, yaml_file), vault_secret)
+            utils.validate(tap_yml, tap_schema)
+
+            if tap_yml['id'] in tap_ids:
+                raise DuplicateConfigException(f'Duplicate tap found "{tap_yml["id"]}"')
+
+            if tap_yml['target'] not in targets:
+                raise InvalidConfigException(
+                    f"Can't find the target with the ID '{tap_yml['target']}' referenced in '{yaml_file}'."
+                    f'Available target IDs: {list(targets.keys())}',
+                    )
+
+            tap_ids.add(tap_yml['id'])
+
+            # If there is a fastsync component for this tap-target combo and transformations on json properties are
+            # configured then fail the validation.
+            # The reason being that at the time of writing this, transformations in Fastsync are done on the
+            # target side using mostly SQL UPDATE, and transformations on properties in json fields are not
+            # implemented due to the need of converting XPATH syntax to SQL which has been deemed as not worth it
+            if self.__does_fastsync_component_exist(targets[tap_yml['target']], tap_yml['type']):
+                self.logger.debug('FastSync component found for tap %s', tap_yml['id'])
+
+                # Load the transformations
+                transformations = Config.generate_transformations(tap_yml)
+
+                # check if transformations are using "field_paths" or "field_path" config, fail if so
+                for transformation in transformations:
+                    if transformation.get('field_paths') is not None:
+                        raise InvalidTransformationException(
+                            'This tap-target combo has FastSync component and is configuring a transformation on json '
+                            'properties which are not supported by FastSync!\n'
+                            f'Please omit "field_paths" from the transformation config of tap "{tap_yml["id"]}"'
+                        )
+
+                    if transformation['when'] is not None:
+                        for condition in transformation['when']:
+                            if condition.get('field_path') is not None:
+                                raise InvalidTransformationException(
+                                    'This tap-target combo has FastSync component and is configuring a transformation '
+                                    'conditions on json properties which are not supported by FastSync!\n'
+                                    f'Please omit "field_path" from the transformation config of tap "{tap_yml["id"]}"'
+                                )
 
-            tap_ids.add(loaded_yaml['id'])
             self.logger.info('Finished validating %s', yaml_file)
 
         self.logger.info('Validation successful')
@@ -1801,3 +1830,16 @@ def __validate_transformations(
 
             if returncode != 0:
                 return stderr
+
+    @classmethod
+    def __does_fastsync_component_exist(cls, target_type: str, tap_type: str) -> bool:
+        """
+        Checks if the given tap-target combo have FastSync
+        Args:
+            target_type: type of the target
+            tap_type: type of tap
+
+        Returns:
+            Boolean, True if FastSync exists, False otherwise.
+        """
+        return ConnectorType(target_type) in FASTSYNC_PAIRS.get(ConnectorType(tap_type), {})
diff --git a/pipelinewise/cli/schemas/tap.json b/pipelinewise/cli/schemas/tap.json
index ac7368e66..84e075e2f 100644
--- a/pipelinewise/cli/schemas/tap.json
+++ b/pipelinewise/cli/schemas/tap.json
@@ -38,13 +38,19 @@
           "type": "object",
           "properties": {
             "column": {
-              "type": "string"
+              "type": "string",
+              "minLength": 1
+            },
+            "field_path": {
+              "type": "string",
+              "minLength": 1
             },
             "equals": {
               "type": ["null", "integer", "string", "boolean", "number"]
             }
           },
-          "required": ["column", "equals"]
+          "required": ["column", "equals"],
+          "additionalProperties": false
         },
         {
           "type": "object",
@@ -54,6 +60,10 @@
             },
             "regex_match": {
               "type": "string"
+            },
+            "field_path": {
+              "type": "string",
+              "minLength": 1
             }
           },
           "required": ["column", "regex_match"]
@@ -122,9 +132,19 @@
     },
     "transformation": {
       "type": "object",
+      "additionalProperties": false,
       "properties": {
         "column": {
-          "type": "string"
+          "type": "string",
+          "minLength": 1
+        },
+        "field_paths": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "minLength": 1
+          },
+          "minItems": 1
         },
         "type": {
           "type": "string",
@@ -169,6 +189,7 @@
     },
     "s3_csv_mapping": {
       "type": "object",
+      "additionalProperties": false,
       "properties": {
         "search_prefix": {
           "type": "string"
diff --git a/pipelinewise/cli/utils.py b/pipelinewise/cli/utils.py
index 67b3abe90..b6a3eadc6 100644
--- a/pipelinewise/cli/utils.py
+++ b/pipelinewise/cli/utils.py
@@ -12,7 +12,6 @@
 import sys
 import tempfile
 import warnings
-
 import jsonschema
 import yaml
 
@@ -28,6 +27,7 @@
 from ansible.parsing.yaml.objects import AnsibleMapping, AnsibleVaultEncryptedUnicode
 
 from . import tap_properties
+from .errors import InvalidConfigException
 
 LOGGER = logging.getLogger(__name__)
 
@@ -290,9 +290,8 @@ def validate(instance, schema):
         # Serialise vault encrypted objects to string
         schema_safe_inst = json.loads(json.dumps(instance, cls=AnsibleJSONEncoder))
         jsonschema.validate(instance=schema_safe_inst, schema=schema)
-    except jsonschema.exceptions.ValidationError:
-        LOGGER.critical('json object doesn\'t match schema %s', schema)
-        sys.exit(1)
+    except jsonschema.exceptions.ValidationError as ex:
+        raise InvalidConfigException(f'json object doesn\'t match schema {schema}') from ex
 
 
 def delete_empty_keys(dic):
diff --git a/singer-connectors/transform-field/requirements.txt b/singer-connectors/transform-field/requirements.txt
index d3f27cc8c..4bb4c2de4 100644
--- a/singer-connectors/transform-field/requirements.txt
+++ b/singer-connectors/transform-field/requirements.txt
@@ -1 +1 @@
-pipelinewise-transform-field==2.2.0
+pipelinewise-transform-field==2.3.0
diff --git a/tests/end_to_end/test_target_bigquery.py b/tests/end_to_end/test_target_bigquery.py
index fd1c9a2a9..70a71daf5 100644
--- a/tests/end_to_end/test_target_bigquery.py
+++ b/tests/end_to_end/test_target_bigquery.py
@@ -45,6 +45,18 @@ def setup_method(self):
     def teardown_method(self):
         """Delete test directories and database objects"""
 
+    @pytest.mark.dependency(name='validate')
+    def test_validate(self):
+        """Validate the YAML project with taps and target """
+
+        # validate project
+        return_code, stdout, stderr = tasks.run_command(f'pipelinewise validate --dir {self.project_dir}')
+
+        print('--------------- stdout ----------', stdout)
+        print('--------------- stderr ----------', stderr)
+        assert return_code == 0
+
+    @pytest.mark.dependency(depends=['validate'])
     @pytest.mark.dependency(name='import_config')
     def test_import_project(self):
         """Import the YAML project with taps and target and do discovery mode
diff --git a/tests/end_to_end/test_target_postgres.py b/tests/end_to_end/test_target_postgres.py
index e97212323..36842a346 100644
--- a/tests/end_to_end/test_target_postgres.py
+++ b/tests/end_to_end/test_target_postgres.py
@@ -45,6 +45,18 @@ def setup_method(self):
     def teardown_method(self):
         """Delete test directories and database objects"""
 
+    @pytest.mark.dependency(name='validate')
+    def test_validate(self):
+        """Validate the YAML project with taps and target """
+
+        # validate project
+        return_code, stdout, stderr = tasks.run_command(f'pipelinewise validate --dir {self.project_dir}')
+
+        print('--------------- stdout ----------', stdout)
+        print('--------------- stderr ----------', stderr)
+        assert return_code == 0
+
+    @pytest.mark.dependency(depends=['validate'])
     @pytest.mark.dependency(name='import_config')
     def test_import_project(self):
         """Import the YAML project with taps and target and do discovery mode
diff --git a/tests/end_to_end/test_target_redshift.py b/tests/end_to_end/test_target_redshift.py
index 76be9131b..22a1f1f6d 100644
--- a/tests/end_to_end/test_target_redshift.py
+++ b/tests/end_to_end/test_target_redshift.py
@@ -35,6 +35,18 @@ def setup_method(self):
     def teardown_method(self):
         """Delete test directories and database objects"""
 
+    @pytest.mark.dependency(name='validate')
+    def test_validate(self):
+        """Validate the YAML project with taps and target """
+
+        # validate project
+        return_code, stdout, stderr = tasks.run_command(f'pipelinewise validate --dir {self.project_dir}')
+
+        print('--------------- stdout ----------', stdout)
+        print('--------------- stderr ----------', stderr)
+        assert return_code == 0
+
+    @pytest.mark.dependency(depends=['validate'])
     @pytest.mark.dependency(name='import_config')
     def test_import_project(self):
         """Import the YAML project with taps and target and do discovery mode
diff --git a/tests/end_to_end/test_target_snowflake.py b/tests/end_to_end/test_target_snowflake.py
index 22e031354..258b42bc5 100644
--- a/tests/end_to_end/test_target_snowflake.py
+++ b/tests/end_to_end/test_target_snowflake.py
@@ -59,6 +59,18 @@ def teardown_class(self):
     def teardown_method(self):
         """Delete test directories and database objects"""
 
+    @pytest.mark.dependency(name='validate')
+    def test_validate(self):
+        """Validate the YAML project with taps and target """
+
+        # validate project
+        return_code, stdout, stderr = tasks.run_command(f'pipelinewise validate --dir {self.project_dir}')
+
+        print('--------------- stdout ----------', stdout)
+        print('--------------- stderr ----------', stderr)
+        assert return_code == 0
+
+    @pytest.mark.dependency(depends=['validate'])
     @pytest.mark.dependency(name='import_config')
     def test_import_project(self):
         """Import the YAML project with taps and target and do discovery mode
diff --git a/tests/units/cli/resources/test_validate_command/json_transformation_in_fastsync/tap_test.yml b/tests/units/cli/resources/test_validate_command/json_transformation_in_fastsync/tap_test.yml
new file mode 100644
index 000000000..0df3015ab
--- /dev/null
+++ b/tests/units/cli/resources/test_validate_command/json_transformation_in_fastsync/tap_test.yml
@@ -0,0 +1,46 @@
+---
+
+# ------------------------------------------------------------------------------
+# General Properties
+# ------------------------------------------------------------------------------
+id: "pg_sample"                     # Unique identifier of the tap
+name: "Sample PG Database"          # Name of the tap
+type: "tap-postgres"                # !! THIS SHOULD NOT CHANGE !!
+owner: "somebody@foo.com"           # Data owner to contact
+
+
+# ------------------------------------------------------------------------------
+# Source (Tap) - / Postgres connection details
+# ------------------------------------------------------------------------------
+db_conn:
+  host: "<HOST>"                       # Postgres host
+  port: 3306                           # Postgres port
+  user: "<USER>"                       # Postgres user
+  password: "<PASSWORD>"               # Plain string or vault encrypted
+  dbname: "<DB_NAME>"                  # Postgres database name
+
+
+# ------------------------------------------------------------------------------
+# Destination (Target) - Target properties
+# Connection details should be in the relevant target YAML file
+# ------------------------------------------------------------------------------
+target: "sf_target"               # ID of the target connector where the data will be loaded
+batch_size_rows: 20000         # Batch size for the stream to optimise load performance
+
+
+# ------------------------------------------------------------------------------
+# Source to target Schema mapping
+# ------------------------------------------------------------------------------
+schemas:
+
+  - source_schema: "my_db"             # Source schema (aka. database) in / Postgres with tables
+    target_schema: "repl_my_db"        # Target schema in the destination Data Warehouse
+
+    tables:
+      - table_name: "table_two"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: json_col
+            type: "SET-NULL"
+            field_paths:
+              - 'array_field/2'
diff --git a/tests/units/cli/resources/test_validate_command/json_transformation_in_fastsync/target_test.yml b/tests/units/cli/resources/test_validate_command/json_transformation_in_fastsync/target_test.yml
new file mode 100644
index 000000000..7a97d6c57
--- /dev/null
+++ b/tests/units/cli/resources/test_validate_command/json_transformation_in_fastsync/target_test.yml
@@ -0,0 +1,18 @@
+---
+# This is a minimalistic target configuration that used only for testing purposes
+id: "sf_target"
+name: "Test Target Connector"
+type: "target-snowflake"
+db_conn:
+  account: "account"
+  dbname: "foo_db"
+  user: "user"
+  password: "secret"
+  warehouse: "MY_WAREHOUSE"
+  s3_bucket: "s3_bucket"
+  s3_key_prefix: "s3_prefix/"
+  aws_access_key_id: "access_key_id"
+  stage: "foo_stage"
+  file_format: "foo_file_format"
+  aws_secret_access_key: "secret_access_key"
+  client_side_encryption_master_key: "master_key"
diff --git a/tests/units/cli/test_cli.py b/tests/units/cli/test_cli.py
index 1435cde78..31b09fc97 100644
--- a/tests/units/cli/test_cli.py
+++ b/tests/units/cli/test_cli.py
@@ -17,6 +17,7 @@
 from pipelinewise.cli.constants import ConnectorType
 from pipelinewise.cli.config import Config
 from pipelinewise.cli.pipelinewise import PipelineWise
+from pipelinewise.cli.errors import DuplicateConfigException, InvalidConfigException, InvalidTransformationException
 
 RESOURCES_DIR = '{}/resources'.format(os.path.dirname(__file__))
 CONFIG_DIR = '{}/sample_json_config'.format(RESOURCES_DIR)
@@ -519,7 +520,7 @@ def test_command_init(self):
         ):
             assert os.path.isfile(os.path.join(TEST_PROJECT_DIR, sample_yaml))
 
-        # Re-creating project should reaise exception of directory not empty
+        # Re-creating project should raise exception of directory not empty
         with pytest.raises(SystemExit) as pytest_wrapped_e:
             pipelinewise.init()
         assert pytest_wrapped_e.type == SystemExit
@@ -693,7 +694,7 @@ def test_validate_command_1(self):
         args = CliArgs(dir=test_validate_command_dir)
         pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
 
-        with pytest.raises(SystemExit):
+        with pytest.raises(InvalidConfigException):
             pipelinewise.validate()
 
     def test_validate_command_2(self):
@@ -713,7 +714,7 @@ def test_validate_command_3(self):
         args = CliArgs(dir=test_validate_command_dir)
         pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
 
-        with pytest.raises(SystemExit):
+        with pytest.raises(InvalidConfigException):
             pipelinewise.validate()
 
     def test_validate_command_4(self):
@@ -724,7 +725,21 @@ def test_validate_command_4(self):
         args = CliArgs(dir=test_validate_command_dir)
         pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
 
-        with pytest.raises(SystemExit):
+        with pytest.raises(DuplicateConfigException):
+            pipelinewise.validate()
+
+    def test_validate_command_5(self):
+        """
+        Test validate command should fail because of transformation on json properties for a tap-target combo that
+        has Fastsync
+        """
+        test_validate_command_dir = \
+            f'{os.path.dirname(__file__)}/resources/test_validate_command/json_transformation_in_fastsync'
+
+        args = CliArgs(dir=test_validate_command_dir)
+        pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
+
+        with pytest.raises(InvalidTransformationException):
             pipelinewise.validate()
 
     # pylint: disable=protected-access
diff --git a/tests/units/cli/test_cli_utils.py b/tests/units/cli/test_cli_utils.py
index 069bfae58..42d5a8842 100644
--- a/tests/units/cli/test_cli_utils.py
+++ b/tests/units/cli/test_cli_utils.py
@@ -1,8 +1,9 @@
 import os
 import re
+import pytest
 
 from pipelinewise import cli
-import pytest
+from pipelinewise.cli.errors import InvalidConfigException
 
 VIRTUALENVS_DIR = './virtualenvs-dummy'
 
@@ -14,10 +15,8 @@ class TestUtils:
     """
     def assert_json_is_invalid(self, schema, invalid_target):
         """Simple assertion to check if validate function exits with error"""
-        with pytest.raises(SystemExit) as pytest_wrapped_e:
+        with pytest.raises(InvalidConfigException):
             cli.utils.validate(invalid_target, schema)
-        assert pytest_wrapped_e.type == SystemExit
-        assert pytest_wrapped_e.value.code == 1
 
     def test_json_detectors(self):
         """Testing JSON detector functions"""
diff --git a/tests/units/cli/test_cli_utils_tap_github.py b/tests/units/cli/test_cli_utils_tap_github.py
index 54dcb4f22..c17ec9513 100644
--- a/tests/units/cli/test_cli_utils_tap_github.py
+++ b/tests/units/cli/test_cli_utils_tap_github.py
@@ -3,6 +3,7 @@
 
 from unittest import TestCase
 from pipelinewise import cli
+from pipelinewise.cli.errors import InvalidConfigException
 
 TAP_GITHUB_YAML = '{}/resources/tap-github.yml'.format(os.path.dirname(__file__))
 
@@ -16,12 +17,9 @@ class TestUtils(TestCase):
 
     def assert_json_is_invalid(self, schema, invalid_yaml):
         """Simple assertion to check if validate function exits with error"""
-        with pytest.raises(SystemExit) as pytest_wrapped_e:
+        with pytest.raises(InvalidConfigException):
             cli.utils.validate(invalid_yaml, schema)
 
-        self.assertEqual(pytest_wrapped_e.type, SystemExit)
-        self.assertEqual(pytest_wrapped_e.value.code, 1)
-
     def test_should_pass_with_valid_json_schema(self):
         """
         Test Should pass with valid json schema
diff --git a/tests/units/cli/test_config.py b/tests/units/cli/test_config.py
index a68b1064c..23f27ac21 100644
--- a/tests/units/cli/test_config.py
+++ b/tests/units/cli/test_config.py
@@ -4,6 +4,7 @@
 
 from pipelinewise import cli
 from pipelinewise.cli.config import Config
+from pipelinewise.cli.errors import InvalidConfigException
 
 PIPELINEWISE_TEST_HOME = '/tmp/.pipelinewise'
 
@@ -155,12 +156,9 @@ def test_from_invalid_mongodb_yamls(self):
         )
         vault_secret = '{}/resources/vault-secret.txt'.format(os.path.dirname(__file__))
         print(yaml_config_dir)
-        with pytest.raises(SystemExit) as pytest_wrapped_e:
+        with pytest.raises(InvalidConfigException):
             Config.from_yamls(PIPELINEWISE_TEST_HOME, yaml_config_dir, vault_secret)
 
-        assert pytest_wrapped_e.type == SystemExit
-        assert pytest_wrapped_e.value.code == 1
-
     def test_from_invalid_yamls(self):
         """Test creating Config object using invalid YAML configuration directory"""
 

From bf0031c11ad5999421562f03109d63a4bfd5d3d4 Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Mon, 24 Jan 2022 11:28:39 +0000
Subject: [PATCH 56/78] Bump tap-kafka from 4.0.1 to 5.0.0 (#870)

---
 singer-connectors/tap-kafka/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singer-connectors/tap-kafka/requirements.txt b/singer-connectors/tap-kafka/requirements.txt
index 2d30a0cf5..6a32fbb51 100644
--- a/singer-connectors/tap-kafka/requirements.txt
+++ b/singer-connectors/tap-kafka/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-kafka==4.0.1
+pipelinewise-tap-kafka==5.0.0

From 88d680d6031c3cd7938a577f23ba3009bcd80329 Mon Sep 17 00:00:00 2001
From: Judah Rand <17158624+judahrand@users.noreply.github.com>
Date: Tue, 25 Jan 2022 12:04:58 +0000
Subject: [PATCH 57/78] Fix broken Postgres to BigQuery `fastsync` (#861)

---
 pipelinewise/fastsync/commons/split_gzip.py   | 11 ++-
 pipelinewise/fastsync/commons/tap_postgres.py |  4 +-
 .../fastsync/commons/transform_utils.py       |  5 ++
 pipelinewise/fastsync/mongodb_to_bigquery.py  |  2 +-
 pipelinewise/fastsync/mongodb_to_postgres.py  |  2 +-
 pipelinewise/fastsync/mongodb_to_snowflake.py |  2 +-
 pipelinewise/fastsync/postgres_to_bigquery.py |  2 +-
 pipelinewise/fastsync/postgres_to_postgres.py |  2 +-
 pipelinewise/fastsync/postgres_to_redshift.py |  2 +-
 .../fastsync/postgres_to_snowflake.py         |  2 +-
 pipelinewise/fastsync/s3_csv_to_bigquery.py   |  2 +-
 pipelinewise/fastsync/s3_csv_to_postgres.py   |  2 +-
 pipelinewise/fastsync/s3_csv_to_redshift.py   |  2 +-
 pipelinewise/fastsync/s3_csv_to_snowflake.py  |  2 +-
 tests/end_to_end/helpers/env.py               |  7 ++
 .../test-project/tap_mysql_to_bq.yml.template | 10 +++
 ...p_mysql_to_bq_buffered_stream.yml.template | 10 +++
 ...mysql_to_bq_split_large_files.yml.template | 73 +++++++++++++++++
 .../tap_postgres_to_bq.yml.template           | 11 +++
 ...tgres_to_bq_split_large_files.yml.template | 78 +++++++++++++++++++
 tests/end_to_end/test_target_bigquery.py      | 49 ++++++------
 .../units/fastsync/commons/test_split_gzip.py | 35 ++++++++-
 22 files changed, 275 insertions(+), 40 deletions(-)

diff --git a/pipelinewise/fastsync/commons/split_gzip.py b/pipelinewise/fastsync/commons/split_gzip.py
index 3f49684fd..f6b557982 100644
--- a/pipelinewise/fastsync/commons/split_gzip.py
+++ b/pipelinewise/fastsync/commons/split_gzip.py
@@ -126,9 +126,14 @@ def _activate_chunk_file(self):
             if self.compress:
                 self.chunk_file = gzip.open(self.chunk_filename, self.mode)
             else:
-                self.chunk_file = builtins.open(
-                    self.chunk_filename, self.mode, encoding='utf-8'
-                )
+                if 'b' in self.mode:
+                    self.chunk_file = builtins.open(  # pylint: disable=unspecified-encoding
+                        self.chunk_filename, self.mode
+                    )
+                else:
+                    self.chunk_file = builtins.open(
+                        self.chunk_filename, self.mode, encoding='utf-8'
+                    )
 
     @staticmethod
     def _bytes_to_megabytes(size: int) -> float:
diff --git a/pipelinewise/fastsync/commons/tap_postgres.py b/pipelinewise/fastsync/commons/tap_postgres.py
index 25869da3a..66b094e72 100644
--- a/pipelinewise/fastsync/commons/tap_postgres.py
+++ b/pipelinewise/fastsync/commons/tap_postgres.py
@@ -388,8 +388,8 @@ def get_table_columns(self, table_name, max_num=None, date_type='date'):
         if max_num:
             decimals = len(max_num.split('.')[1]) if '.' in max_num else 0
             decimal_format = f"""
-              'CASE WHEN "' || column_name || '" IS NULL THEN NULL ELSE GREATEST(LEAST({max_num}, ROUND("' || column_name || '"::numeric , {decimals})), -{max_num}) END'  # noqa E501
-            """
+              'CASE WHEN "' || column_name || '" IS NULL THEN NULL ELSE GREATEST(LEAST({max_num}, ROUND("' || column_name || '"::numeric , {decimals})), -{max_num}) END'
+            """ # noqa E501
             integer_format = """
               '"' || column_name || '"'
             """
diff --git a/pipelinewise/fastsync/commons/transform_utils.py b/pipelinewise/fastsync/commons/transform_utils.py
index 727a73232..d0cbd1950 100644
--- a/pipelinewise/fastsync/commons/transform_utils.py
+++ b/pipelinewise/fastsync/commons/transform_utils.py
@@ -378,6 +378,11 @@ def __mask_string_skip_ends_to_sql(
                     'CONCAT(SUBSTRING({0}, 1, {1}), REPEAT(\'*\', LENGTH({0})-(2 * {1})), ' \
                     'SUBSTRING({0}, LENGTH({0})-{1}+1, {1})) ' \
                     'ELSE REPEAT(\'*\', LENGTH({0})) END'.format(column, skip_ends_n)
+        elif sql_flavor == SQLFlavor.BIGQUERY:
+            trans = '{0} = CASE WHEN LENGTH({0}) > 2 * {1} THEN ' \
+                    'CONCAT(SUBSTRING({0}, 1, {1}), REPEAT(\'*\', LENGTH({0})-(2 * {1})), ' \
+                    'SUBSTRING({0}, LENGTH({0})-{1}+1, {1})) ' \
+                    'ELSE REPEAT(\'*\', LENGTH({0})) END'.format(column, skip_ends_n)
         else:
             raise NotImplementedError(f'MASK-STRING-SKIP-ENDS transformation in {sql_flavor.value} SQL flavor '
                                       f'not implemented!')
diff --git a/pipelinewise/fastsync/mongodb_to_bigquery.py b/pipelinewise/fastsync/mongodb_to_bigquery.py
index 4e7b724ad..c82dd3cbc 100644
--- a/pipelinewise/fastsync/mongodb_to_bigquery.py
+++ b/pipelinewise/fastsync/mongodb_to_bigquery.py
@@ -31,7 +31,7 @@
 LOCK = multiprocessing.Lock()
 
 
-def tap_type_to_target_type(mongo_type):
+def tap_type_to_target_type(mongo_type, *_):
     """Data type mapping from MongoDB to Bigquery"""
     return {
         'string': 'STRING',
diff --git a/pipelinewise/fastsync/mongodb_to_postgres.py b/pipelinewise/fastsync/mongodb_to_postgres.py
index fe4643afb..b8039dbb6 100644
--- a/pipelinewise/fastsync/mongodb_to_postgres.py
+++ b/pipelinewise/fastsync/mongodb_to_postgres.py
@@ -30,7 +30,7 @@
 LOCK = multiprocessing.Lock()
 
 
-def tap_type_to_target_type(mongo_type):
+def tap_type_to_target_type(mongo_type, *_):
     """Data type mapping from MongoDB to Postgres"""
     return {
         'string': 'CHARACTER VARYING',
diff --git a/pipelinewise/fastsync/mongodb_to_snowflake.py b/pipelinewise/fastsync/mongodb_to_snowflake.py
index 96fa606c1..9a662cc4c 100644
--- a/pipelinewise/fastsync/mongodb_to_snowflake.py
+++ b/pipelinewise/fastsync/mongodb_to_snowflake.py
@@ -39,7 +39,7 @@
 LOCK = multiprocessing.Lock()
 
 
-def tap_type_to_target_type(mongo_type):
+def tap_type_to_target_type(mongo_type, *_):
     """Data type mapping from MongoDB to Snowflake"""
     return {
         'string': 'TEXT',
diff --git a/pipelinewise/fastsync/postgres_to_bigquery.py b/pipelinewise/fastsync/postgres_to_bigquery.py
index 8bc207886..0539c91b7 100644
--- a/pipelinewise/fastsync/postgres_to_bigquery.py
+++ b/pipelinewise/fastsync/postgres_to_bigquery.py
@@ -26,7 +26,7 @@
 LOCK = multiprocessing.Lock()
 
 
-def tap_type_to_target_type(pg_type):
+def tap_type_to_target_type(pg_type, *_):
     """Data type mapping from Postgres to Bigquery"""
     return {
         'char': 'STRING',
diff --git a/pipelinewise/fastsync/postgres_to_postgres.py b/pipelinewise/fastsync/postgres_to_postgres.py
index 728d0c140..3e74d8a64 100644
--- a/pipelinewise/fastsync/postgres_to_postgres.py
+++ b/pipelinewise/fastsync/postgres_to_postgres.py
@@ -30,7 +30,7 @@
 LOCK = multiprocessing.Lock()
 
 
-def tap_type_to_target_type(pg_type):
+def tap_type_to_target_type(pg_type, *_):
     """Data type mapping from Postgres to Postgres"""
     return {
         'char': 'CHARACTER VARYING',
diff --git a/pipelinewise/fastsync/postgres_to_redshift.py b/pipelinewise/fastsync/postgres_to_redshift.py
index ab45de889..42234f3f4 100644
--- a/pipelinewise/fastsync/postgres_to_redshift.py
+++ b/pipelinewise/fastsync/postgres_to_redshift.py
@@ -27,7 +27,7 @@
 LOCK = multiprocessing.Lock()
 
 
-def tap_type_to_target_type(pg_type):
+def tap_type_to_target_type(pg_type, *_):
     """Data type mapping from MySQL to Redshift"""
     return {
         'char': 'CHARACTER VARYING({})'.format(DEFAULT_VARCHAR_LENGTH),
diff --git a/pipelinewise/fastsync/postgres_to_snowflake.py b/pipelinewise/fastsync/postgres_to_snowflake.py
index 867f2b96a..1ac904515 100644
--- a/pipelinewise/fastsync/postgres_to_snowflake.py
+++ b/pipelinewise/fastsync/postgres_to_snowflake.py
@@ -41,7 +41,7 @@
 LOCK = multiprocessing.Lock()
 
 
-def tap_type_to_target_type(pg_type):
+def tap_type_to_target_type(pg_type, *_):
     """Data type mapping from Postgres to Snowflake"""
     return {
         'char': 'VARCHAR',
diff --git a/pipelinewise/fastsync/s3_csv_to_bigquery.py b/pipelinewise/fastsync/s3_csv_to_bigquery.py
index 7c81c30fe..995c90017 100644
--- a/pipelinewise/fastsync/s3_csv_to_bigquery.py
+++ b/pipelinewise/fastsync/s3_csv_to_bigquery.py
@@ -25,7 +25,7 @@
 LOCK = multiprocessing.Lock()
 
 
-def tap_type_to_target_type(csv_type):
+def tap_type_to_target_type(csv_type, *_):
     """Data type mapping from S3 csv to Bigquery"""
 
     return {
diff --git a/pipelinewise/fastsync/s3_csv_to_postgres.py b/pipelinewise/fastsync/s3_csv_to_postgres.py
index 6ac9639f6..3ca4a5dae 100644
--- a/pipelinewise/fastsync/s3_csv_to_postgres.py
+++ b/pipelinewise/fastsync/s3_csv_to_postgres.py
@@ -22,7 +22,7 @@
 LOCK = multiprocessing.Lock()
 
 
-def tap_type_to_target_type(csv_type):
+def tap_type_to_target_type(csv_type, *_):
     """Data type mapping from S3 csv to Snowflake"""
 
     return {
diff --git a/pipelinewise/fastsync/s3_csv_to_redshift.py b/pipelinewise/fastsync/s3_csv_to_redshift.py
index b65ac6fe0..e5be651e3 100644
--- a/pipelinewise/fastsync/s3_csv_to_redshift.py
+++ b/pipelinewise/fastsync/s3_csv_to_redshift.py
@@ -22,7 +22,7 @@
 LOCK = multiprocessing.Lock()
 
 
-def tap_type_to_target_type(csv_type):
+def tap_type_to_target_type(csv_type, *_):
     """Data type mapping from S3 csv to Snowflake"""
 
     return {
diff --git a/pipelinewise/fastsync/s3_csv_to_snowflake.py b/pipelinewise/fastsync/s3_csv_to_snowflake.py
index 84c0268e4..f91abdf15 100644
--- a/pipelinewise/fastsync/s3_csv_to_snowflake.py
+++ b/pipelinewise/fastsync/s3_csv_to_snowflake.py
@@ -32,7 +32,7 @@
 LOCK = multiprocessing.Lock()
 
 
-def tap_type_to_target_type(csv_type):
+def tap_type_to_target_type(csv_type, *_):
     """Data type mapping from S3 csv to Snowflake"""
 
     return {
diff --git a/tests/end_to_end/helpers/env.py b/tests/end_to_end/helpers/env.py
index 3eaf46c9e..d0edcecb9 100644
--- a/tests/end_to_end/helpers/env.py
+++ b/tests/end_to_end/helpers/env.py
@@ -5,6 +5,7 @@
 import shutil
 import subprocess
 import uuid
+from pathlib import Path
 
 from dotenv import load_dotenv
 from . import db
@@ -650,3 +651,9 @@ def setup_target_bigquery(self):
         self.delete_dataset_target_bigquery('ppw_e2e_tap_mysql')
         self.delete_dataset_target_bigquery('ppw_e2e_tap_s3_csv')
         self.delete_dataset_target_bigquery('ppw_e2e_tap_mongodb')
+
+    @staticmethod
+    def remove_all_state_files():
+        """Clean up state files to ensure tests behave the same every time"""
+        for state_file in Path(CONFIG_DIR).glob('**/state.json'):
+            state_file.unlink()
diff --git a/tests/end_to_end/test-project/tap_mysql_to_bq.yml.template b/tests/end_to_end/test-project/tap_mysql_to_bq.yml.template
index c7a8046ff..a49862adc 100644
--- a/tests/end_to_end/test-project/tap_mysql_to_bq.yml.template
+++ b/tests/end_to_end/test-project/tap_mysql_to_bq.yml.template
@@ -101,3 +101,13 @@ schemas:
       - table_name: "all_datatypes"
         replication_method: "LOG_BASED"
 
+      ### Table with INCREMENTAL replication
+      - table_name: "customers"
+        replication_method: "INCREMENTAL"
+        replication_key: "id"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
diff --git a/tests/end_to_end/test-project/tap_mysql_to_bq_buffered_stream.yml.template b/tests/end_to_end/test-project/tap_mysql_to_bq_buffered_stream.yml.template
index d1c5af6db..60b8bcbdf 100644
--- a/tests/end_to_end/test-project/tap_mysql_to_bq_buffered_stream.yml.template
+++ b/tests/end_to_end/test-project/tap_mysql_to_bq_buffered_stream.yml.template
@@ -101,3 +101,13 @@ schemas:
       - table_name: "all_datatypes"
         replication_method: "LOG_BASED"
 
+      ### Table with INCREMENTAL replication
+      - table_name: "customers"
+        replication_method: "INCREMENTAL"
+        replication_key: "id"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
diff --git a/tests/end_to_end/test-project/tap_mysql_to_bq_split_large_files.yml.template b/tests/end_to_end/test-project/tap_mysql_to_bq_split_large_files.yml.template
index 12a69bf01..b5bb01c85 100644
--- a/tests/end_to_end/test-project/tap_mysql_to_bq_split_large_files.yml.template
+++ b/tests/end_to_end/test-project/tap_mysql_to_bq_split_large_files.yml.template
@@ -42,4 +42,77 @@ schemas:
     target_schema: "ppw_e2e_tap_mysql"
 
     tables:
+      ### Table with LOG_BASED replication
+      - table_name: "weight_unit"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "weight_unit_name"
+            type: "HASH-SKIP-FIRST-2"
+
+      ### Table with INCREMENTAL replication
       - table_name: "address"
+        replication_method: "INCREMENTAL"
+        replication_key: "date_updated"
+        transformations:
+          - column: "zip_code_zip_code_id"
+            type: "MASK-NUMBER"
+            when:
+              - column: 'street_number'
+                regex_match: '[801]'
+
+          - column: "date_created"
+            type: "MASK-DATE"
+
+      ### Table with FULL_TABLE replication
+      - table_name: "order"
+        replication_method: "FULL_TABLE"
+
+      ### Table with no primary key
+      - table_name: "no_pk_table"
+        replication_method: "FULL_TABLE"
+
+      ### Table with binary and varbinary columns
+      - table_name: "table_with_binary"
+        replication_method: "LOG_BASED"
+
+      ### Table with reserved words and columns and special characters
+      - table_name: "edgydata"
+        replication_method: "LOG_BASED"
+        transformations:
+          - column: "case"
+            type: "HASH"
+
+          - column: "group"
+            type: "MASK-NUMBER"
+            when:
+              - column: 'case'
+                equals: 'A'
+          - column: "group"
+            type: "SET-NULL"
+            when:
+              - column: 'case'
+                equals: 'B'
+
+      ### Table with reserved word
+      - table_name: "full"
+        replication_method: "INCREMENTAL"
+        replication_key: "begin"
+
+      ### Table with space and mixed upper and lowercase characters
+      - table_name: "table_with_space and UPPERCase"
+        replication_method: "LOG_BASED"
+
+      ### Table with all possible data types
+      - table_name: "all_datatypes"
+        replication_method: "LOG_BASED"
+
+      ### Table with INCREMENTAL replication
+      - table_name: "customers"
+        replication_method: "INCREMENTAL"
+        replication_key: "id"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
diff --git a/tests/end_to_end/test-project/tap_postgres_to_bq.yml.template b/tests/end_to_end/test-project/tap_postgres_to_bq.yml.template
index 7d8697c4d..4b55d3a09 100644
--- a/tests/end_to_end/test-project/tap_postgres_to_bq.yml.template
+++ b/tests/end_to_end/test-project/tap_postgres_to_bq.yml.template
@@ -75,6 +75,17 @@ schemas:
       - table_name: "table_with_reserved_words"
         replication_method: "FULL_TABLE"
 
+      ### Table with INCREMENTAL replication
+      - table_name: "customers"
+        replication_method: "INCREMENTAL"
+        replication_key: "id"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
+
 
   ### SOURCE SCHEMA 2: public2
   - source_schema: "public2"
diff --git a/tests/end_to_end/test-project/tap_postgres_to_bq_split_large_files.yml.template b/tests/end_to_end/test-project/tap_postgres_to_bq_split_large_files.yml.template
index 4e5e81f86..b42d7218a 100644
--- a/tests/end_to_end/test-project/tap_postgres_to_bq_split_large_files.yml.template
+++ b/tests/end_to_end/test-project/tap_postgres_to_bq_split_large_files.yml.template
@@ -39,8 +39,86 @@ split_file_max_chunks: 5
 # Source to target Schema mapping
 # ------------------------------------------------------------------------------
 schemas:
+
+  ### SOURCE SCHEMA 1: public
   - source_schema: "public"
     target_schema: "ppw_e2e_tap_postgres"
 
     tables:
+
+      ### Table with INCREMENTAL replication
       - table_name: "city"
+        replication_method: "INCREMENTAL"
+        replication_key: "id"
+
+      ### Table with FULL_TABLE replication
+      - table_name: "country"
+        replication_method: "FULL_TABLE"
+
+      ### Table with no primary key
+      - table_name: "no_pk_table"
+        replication_method: "FULL_TABLE"
+
+      ### Table with reserved words and columns and special characters
+      - table_name: "edgydata"
+        replication_method: "INCREMENTAL"
+        replication_key: "cid"
+        transformations:
+          - column: "cvarchar"
+            type: "HASH-SKIP-FIRST-3"
+
+      ### Table with reserved word
+      - table_name: "order"
+        replication_method: "INCREMENTAL"
+        replication_key: "id"
+
+      ### Table with space and mixed upper and lowercase characters
+      - table_name: "table_with_space and UPPERCase"
+        replication_method: "LOG_BASED"
+
+      ### Table with space and mixed upper and lowercase characters
+      - table_name: "table_with_reserved_words"
+        replication_method: "FULL_TABLE"
+
+      ### Table with INCREMENTAL replication
+      - table_name: "customers"
+        replication_method: "INCREMENTAL"
+        replication_key: "id"
+        transformations:
+          - column: "phone"
+            type: "MASK-STRING-SKIP-ENDS-2"
+
+          - column: "email"
+            type: "MASK-STRING-SKIP-ENDS-6"
+
+
+  ### SOURCE SCHEMA 2: public2
+  - source_schema: "public2"
+    target_schema: "ppw_e2e_tap_postgres_public2"
+
+    tables:
+      ### Table with FULL_TABLE replication
+      - table_name: "wearehere"
+        replication_method: "FULL_TABLE"
+
+      ### Table with reserved words and columns and special characters
+      - table_name: "public2_edgydata"
+        replication_method: "INCREMENTAL"
+        replication_key: "cid"
+
+
+  ### SOURCE SCHEMA 3: logical 1
+  #- source_schema: "logical1"
+  #  target_schema: "ppw_e2e_tap_postgres_logical1"
+  #
+  #  tables:
+  #    - table_name: "logical1_table1"
+  #      replication_method: "LOG_BASED"
+  #    - table_name: "logical1_table2"
+  #    - table_name: "logical1_edgydata"
+
+  ### SOURCE SCHEMA 4: logical2
+  #- source_schema: "logical2"
+  #  target_schema: "ppw_e2e_tap_postgres_logical2"
+  #  tables:
+  #    - table_name: "logical2_table1"
diff --git a/tests/end_to_end/test_target_bigquery.py b/tests/end_to_end/test_target_bigquery.py
index 70a71daf5..43c08b5fa 100644
--- a/tests/end_to_end/test_target_bigquery.py
+++ b/tests/end_to_end/test_target_bigquery.py
@@ -7,7 +7,7 @@
 import bson
 import pytest
 from bson import Timestamp
-from pipelinewise.fastsync import mysql_to_bigquery
+from pipelinewise.fastsync import mysql_to_bigquery, postgres_to_bigquery
 
 from .helpers import tasks
 from .helpers import assertions
@@ -41,6 +41,9 @@ def setup_method(self):
         self.run_query_tap_postgres = self.e2e.run_query_tap_postgres
         self.run_query_target_bigquery = self.e2e.run_query_target_bigquery
         self.mongodb_con = self.e2e.get_tap_mongodb_connection()
+        self.e2e.remove_all_state_files()
+        if self.e2e.env['TARGET_BIGQUERY']['is_configured']:
+            self.e2e.setup_target_bigquery()
 
     def teardown_method(self):
         """Delete test directories and database objects"""
@@ -201,24 +204,6 @@ def test_resync_mariadb_to_bq_with_split_large_files(
             mysql_to_bigquery.tap_type_to_target_type,
         )
 
-    # pylint: disable=invalid-name
-    @pytest.mark.dependency(depends=['import_config'])
-    def test_resync_pg_to_bq_with_split_large_files(
-        self, tap_postgres_id=TAP_POSTGRES_SPLIT_LARGE_FILES_ID
-    ):
-        """Resync tables from Postgres to Bigquery using splitting large files option"""
-        assertions.assert_resync_tables_success(
-            tap_postgres_id, TARGET_ID, profiling=True
-        )
-        assertions.assert_row_counts_equal(
-            self.run_query_tap_mysql, self.run_query_target_bigquery
-        )
-        assertions.assert_all_columns_exist(
-            self.run_query_tap_mysql,
-            self.run_query_target_bigquery,
-            mysql_to_bigquery.tap_type_to_target_type,
-        )
-
     # pylint: disable=invalid-name
     @pytest.mark.dependency(depends=['import_config'])
     def test_replicate_mariadb_to_bq_with_custom_buffer_size(self):
@@ -246,7 +231,7 @@ def test_replicate_pg_to_bq(self):
             'where cvarchar=\'H\';'
         )[0][0]
 
-        assert result == datetime(9999, 12, 31, 23, 59, 59, 999008, tzinfo=timezone.utc)
+        assert result == datetime(9999, 12, 31, 23, 59, 59, 999000, tzinfo=timezone.utc)
 
         result = self.run_query_target_bigquery(
             'SELECT updated_at FROM '
@@ -254,7 +239,7 @@ def test_replicate_pg_to_bq(self):
             'where cvarchar=\'I\';'
         )[0][0]
 
-        assert result == datetime(9999, 12, 31, 23, 59, 59, 999008, tzinfo=timezone.utc)
+        assert result == datetime(9999, 12, 31, 23, 59, 59, 999000, tzinfo=timezone.utc)
 
         # 2. Make changes in PG source database
         #  LOG_BASED
@@ -304,7 +289,7 @@ def test_replicate_pg_to_bq(self):
             'where cvarchar=\'faaaar future\';'
         )[0][0]
 
-        assert result == datetime(9999, 12, 31, 23, 59, 59, 999008, tzinfo=timezone.utc)
+        assert result == datetime(9999, 12, 31, 23, 59, 59, 999000, tzinfo=timezone.utc)
 
         result = self.run_query_target_bigquery(
             'SELECT updated_at FROM '
@@ -312,7 +297,25 @@ def test_replicate_pg_to_bq(self):
             'where cvarchar=\'BC\';'
         )[0][0]
 
-        assert result == datetime(9999, 12, 31, 23, 59, 59, 999008, tzinfo=timezone.utc)
+        assert result == datetime(9999, 12, 31, 23, 59, 59, 999000, tzinfo=timezone.utc)
+
+    # pylint: disable=invalid-name
+    @pytest.mark.dependency(depends=['import_config'])
+    def test_resync_pg_to_bq_with_split_large_files(
+        self, tap_postgres_id=TAP_POSTGRES_SPLIT_LARGE_FILES_ID
+    ):
+        """Resync tables from Postgres to Bigquery using splitting large files option"""
+        assertions.assert_resync_tables_success(
+            tap_postgres_id, TARGET_ID, profiling=True
+        )
+        assertions.assert_row_counts_equal(
+            self.run_query_tap_postgres, self.run_query_target_bigquery
+        )
+        assertions.assert_all_columns_exist(
+            self.run_query_tap_postgres,
+            self.run_query_target_bigquery,
+            postgres_to_bigquery.tap_type_to_target_type,
+        )
 
     @pytest.mark.dependency(depends=['import_config'])
     def test_replicate_s3_to_bq(self):
diff --git a/tests/units/fastsync/commons/test_split_gzip.py b/tests/units/fastsync/commons/test_split_gzip.py
index 2766e6163..7fd4aa85e 100644
--- a/tests/units/fastsync/commons/test_split_gzip.py
+++ b/tests/units/fastsync/commons/test_split_gzip.py
@@ -1,3 +1,4 @@
+import builtins
 import gzip
 import itertools
 import glob
@@ -111,7 +112,7 @@ def test_write_with_no_split(self):
         """
         Write gzip without splitting it and reading it
         """
-        # max_chunk = 0 should create a file with no splitting
+        # max_chunk = 0 should create a file with no splitting and binary mode
         with split_gzip.SplitGzipFile(self.filename, 'wb', max_chunks=0) as f_write:
             f_write.write(DATA_WITH_100_BYTES * 50)
 
@@ -120,6 +121,38 @@ def test_write_with_no_split(self):
 
         self.assertEqual(file_content, DATA_WITH_100_BYTES * 50)
 
+        # max_chunk = 0 should create a file with no splitting and text mode
+        with split_gzip.SplitGzipFile(self.filename, 'wt', max_chunks=0) as f_write:
+            f_write.write(str(DATA_WITH_100_BYTES * 50))
+
+        with gzip.open(self.filename, 'rt') as f_read:
+            file_content = f_read.read()
+
+        self.assertEqual(file_content, str(DATA_WITH_100_BYTES * 50))
+
+    def test_write_with_no_split_no_compress(self):
+        """
+        Write gzip without splitting it and reading it
+        """
+        # max_chunk = 0 should create a file with no splitting and binary mode
+        with split_gzip.SplitGzipFile(
+                self.filename, 'wb', max_chunks=0, compress=False) as f_write:
+            f_write.write(DATA_WITH_100_BYTES * 50)
+
+        with builtins.open(self.filename, 'rb') as f_read:
+            file_content = f_read.read()
+
+        self.assertEqual(file_content, DATA_WITH_100_BYTES * 50)
+
+        # max_chunk = 0 should create a file with no splitting and text mode
+        with split_gzip.SplitGzipFile(self.filename, 'wt', max_chunks=0) as f_write:
+            f_write.write(str(DATA_WITH_100_BYTES * 50))
+
+        with gzip.open(self.filename, 'rt') as f_read:
+            file_content = f_read.read()
+
+        self.assertEqual(file_content, str(DATA_WITH_100_BYTES * 50))
+
     def test_write_with_single_chunk(self):
         """
         Write all data into one chunk

From ea2e031ea1e95f3160217e1d8d3f0fde16f42cd9 Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Tue, 25 Jan 2022 15:02:53 +0000
Subject: [PATCH 58/78] v0.39.0 (#873)

---
 CHANGELOG.md | 9 +++++++++
 setup.py     | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f6afb3c46..7340e06a6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,12 @@
+0.39.0 (2022-02-25)
+-------------------
+- Bump `pipelinewise-tap-kafka` from `4.0.1` to `5.0.0`
+- Bump `pipelinewise-target-bigquery` from `1.1.1` to `1.2.0`
+- Bump `pipelinewise-transform-field` from `2.2.0` to `2.3.0`
+- Prevent usage of extended transformation feature when FastSync exists
+- Fixed fastsync from postgres to bigquery
+- Fixed an issue when `SplitGzipFile` doesn't work with binary mode
+
 0.38.0 (2022-01-14)
 -------------------
 - MySQL tap now connects to replica instance during fastsync if credentials are provided
diff --git a/setup.py b/setup.py
index a62b8907e..80b1d41d9 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      version='0.38.0',
+      version='0.39.0',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',

From 4314c67791881d6aa4bc0b1ca68a040382ea422b Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Wed, 26 Jan 2022 12:38:07 +0000
Subject: [PATCH 59/78]  Bump tap-kafka from 5.0.0 to 5.0.1 (#876)

---
 singer-connectors/tap-kafka/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singer-connectors/tap-kafka/requirements.txt b/singer-connectors/tap-kafka/requirements.txt
index 6a32fbb51..7322f70f6 100644
--- a/singer-connectors/tap-kafka/requirements.txt
+++ b/singer-connectors/tap-kafka/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-kafka==5.0.0
+pipelinewise-tap-kafka==5.0.1

From adffed517bd27378a1806b6d8fc49d0ddcdb3f91 Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Wed, 26 Jan 2022 13:07:56 +0000
Subject: [PATCH 60/78] v0.39.1 (#877)

---
 CHANGELOG.md | 4 ++++
 setup.py     | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7340e06a6..1dda71db9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+0.39.1 (2022-02-26)
+-------------------
+- Bump `pipelinewise-tap-kafka` from `5.0.0` to `5.0.1`
+
 0.39.0 (2022-02-25)
 -------------------
 - Bump `pipelinewise-tap-kafka` from `4.0.1` to `5.0.0`
diff --git a/setup.py b/setup.py
index 80b1d41d9..bbcb89268 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      version='0.39.0',
+      version='0.39.1',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',

From 35f4c89e0226d5047517bb0bed62e1de14691286 Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Thu, 27 Jan 2022 14:15:57 +0000
Subject: [PATCH 61/78] Bump pipelinewise-tap-kafka from 5.0.1 to 5.1.0 (#878)

---
 singer-connectors/tap-kafka/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/singer-connectors/tap-kafka/requirements.txt b/singer-connectors/tap-kafka/requirements.txt
index 7322f70f6..ba2ab2c0e 100644
--- a/singer-connectors/tap-kafka/requirements.txt
+++ b/singer-connectors/tap-kafka/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-kafka==5.0.1
+pipelinewise-tap-kafka==5.1.0

From 3a8a93a3be4ea2f8eb677531f379e532eab30f28 Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Thu, 27 Jan 2022 15:30:34 +0000
Subject: [PATCH 62/78] [DOC] Document tap kafka YAML format after 5.x (#879)

---
 docs/connectors/taps/kafka.rst                | 30 ++++++++++++-------
 pipelinewise/cli/samples/tap_kafka.yml.sample | 23 +++++++++++---
 2 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/docs/connectors/taps/kafka.rst b/docs/connectors/taps/kafka.rst
index f468f63e1..b1c31e2a2 100644
--- a/docs/connectors/taps/kafka.rst
+++ b/docs/connectors/taps/kafka.rst
@@ -12,12 +12,7 @@ Messages from kafka topics are extracted into the following fields:
 * ``MESSAGE``: The original and full kafka message
 * `Dynamic primary key columns`: (Optional) Fields extracted from the Kafka JSON messages by JSONPath selector(s).
 
-Consuming Kafka messages
-''''''''''''''''''''''''
-
-Tap Kafka saves consumed messages into a local disk storage and sends commit messages to Kafka after every
-consumed message. A batching mechanism keeps maintaining of deleting and flushing messages from the local storage
-and sends singer compatible messages in small batches to standard output.
+Supported message formats: JSON and Protobuf (experimental).
 
 
 Configuring what to replicate
@@ -60,19 +55,34 @@ Example YAML for ``tap-kafka``:
       primary_keys:
          transfer_id: "/transferMetadata/transferId"
 
+      #initial_start_time:                      # (Default: latest) Start time reference of the message consumption if
+                                                # no bookmarked position in state.json. One of: latest, earliest or an
+                                                # ISO-8601 formatted timestamp string.
+
       # --------------------------------------------------------------------------
-      # Kafka Consumer optional parameters
+      # Kafka Consumer optional parameters. Commented values are default values.
       # --------------------------------------------------------------------------
       #max_runtime_ms: 300000                   # The maximum time for the tap to collect new messages from Kafka topic.
       #consumer_timeout_ms: 10000               # KafkaConsumer setting. Number of milliseconds to block during message iteration before raising StopIteration
       #session_timeout_ms: 30000                # KafkaConsumer setting. The timeout used to detect failures when using Kafka’s group management facilities.
       #heartbeat_interval_ms: 10000             # KafkaConsumer setting. The expected time in milliseconds between heartbeats to the consumer coordinator when using Kafka’s group management facilities.
       #max_poll_interval_ms: 300000             # KafkaConsumer setting. The maximum delay between invocations of poll() when using consumer group management.
-      #max_poll_records: 500                    # KafkaConsumer setting. The maximum number of records returned in a single call to poll().
 
       #commit_interval_ms: 5000                 # Number of milliseconds between two commits. This is different than the kafka auto commit feature. Tap-kafka sends commit messages automatically but only when the data consumed successfully and persisted to local store.
-      #local_store_dir: ./tap-kafka-local-store # Path to the local store with consumed kafka messages
-      #local_store_batch_size_rows: 1000        # Number of messages to write to disk in one go. This can avoid high I/O issues when messages written to local store disk too frequently.
+
+      # --------------------------------------------------------------------------
+      # Protobuf support - Experimental
+      # --------------------------------------------------------------------------
+      #message_format: protobuf                 # (Default: json) Supported message formats are json and protobuf.
+      #proto_schema: |                          # Protobuf message format in .proto syntax. Required if the message_format is protobuf.
+      #     syntax = "proto3";
+      #
+      #     message ProtoMessage {
+      #       string query = 1;
+      #       int32 page_number = 2;
+      #       int32 result_per_page = 3;
+      #     }
+      #proto_classess_dir:                      # (Default: current working dir) Directory where to store runtime compiled proto classes
 
 
     # ------------------------------------------------------------------------------
diff --git a/pipelinewise/cli/samples/tap_kafka.yml.sample b/pipelinewise/cli/samples/tap_kafka.yml.sample
index 341d93164..8e156df5d 100644
--- a/pipelinewise/cli/samples/tap_kafka.yml.sample
+++ b/pipelinewise/cli/samples/tap_kafka.yml.sample
@@ -27,19 +27,34 @@ db_conn:
   primary_keys:
      transfer_id: "/transferMetadata/transferId"
 
+  #initial_start_time:                      # (Default: latest) Start time reference of the message consumption if
+                                            # no bookmarked position in state.json. One of: latest, earliest or an
+                                            # ISO-8601 formatted timestamp string.
+
   # --------------------------------------------------------------------------
-  # Kafka Consumer optional parameters
+  # Kafka Consumer optional parameters. Commented values are default values.
   # --------------------------------------------------------------------------
   #max_runtime_ms: 300000                   # The maximum time for the tap to collect new messages from Kafka topic.
   #consumer_timeout_ms: 10000               # KafkaConsumer setting. Number of milliseconds to block during message iteration before raising StopIteration
   #session_timeout_ms: 30000                # KafkaConsumer setting. The timeout used to detect failures when using Kafka’s group management facilities.
   #heartbeat_interval_ms: 10000             # KafkaConsumer setting. The expected time in milliseconds between heartbeats to the consumer coordinator when using Kafka’s group management facilities.
   #max_poll_interval_ms: 300000             # KafkaConsumer setting. The maximum delay between invocations of poll() when using consumer group management.
-  #max_poll_records: 500                    # KafkaConsumer setting. The maximum number of records returned in a single call to poll().
 
   #commit_interval_ms: 5000                 # Number of milliseconds between two commits. This is different than the kafka auto commit feature. Tap-kafka sends commit messages automatically but only when the data consumed successfully and persisted to local store.
-  #local_store_dir: ./tap-kafka-local-store # Path to the local store with consumed kafka messages
-  #local_store_batch_size_rows: 1000        # Number of messages to write to disk in one go. This can avoid high I/O issues when messages written to local store disk too frequently.
+
+  # --------------------------------------------------------------------------
+  # Protobuf support - Experimental
+  # --------------------------------------------------------------------------
+  #message_format: protobuf                 # (Default: json) Supported message formats are json and protobuf.
+  #proto_schema: |                          # Protobuf message format in .proto syntax. Required if the message_format is protobuf.
+  #     syntax = "proto3";
+  #
+  #     message ProtoMessage {
+  #       string query = 1;
+  #       int32 page_number = 2;
+  #       int32 result_per_page = 3;
+  #     }
+  #proto_classess_dir:                      # (Default: current working dir) Directory where to store runtime compiled proto classes
 
 # ------------------------------------------------------------------------------
 # Destination (Target) - Target properties

From 7d4c9a980872cef703ecffa119f101daf5a72ef6 Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Thu, 27 Jan 2022 16:35:05 +0000
Subject: [PATCH 63/78] v0.40.0 (#880)

---
 CHANGELOG.md | 4 ++++
 setup.py     | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1dda71db9..b4dc76dd8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+0.40.0 (2022-02-27)
+-------------------
+- Bump `pipelinewise-tap-kafka` from `5.0.1` to `5.1.0`
+
 0.39.1 (2022-02-26)
 -------------------
 - Bump `pipelinewise-tap-kafka` from `5.0.0` to `5.0.1`
diff --git a/setup.py b/setup.py
index bbcb89268..127c6fa4b 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      version='0.39.1',
+      version='0.40.0',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',

From 49dd7ea8cc1c24d3877346b336a22695e9f36690 Mon Sep 17 00:00:00 2001
From: Amir Mofakhar <amir.mofakhar@wise.com>
Date: Mon, 7 Feb 2022 10:21:53 +0000
Subject: [PATCH 64/78] [AP-851] Fix for config json files (#852)

* added json properties validator

* fixed pep8

* fixed mocked object

* refactoring

* refactoring testcase

* fixed lint

* Update tests/units/cli/test_commands.py

Co-authored-by: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>

* using unittest mock

* separate tests

* refactored unit test

* refactored tests

* fixed annotations

Co-authored-by: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
---
 pipelinewise/cli/commands.py              |  94 ++++++++++-
 tests/units/cli/test_commands.py          |  24 ++-
 tests/units/cli/test_config_validation.py | 196 ++++++++++++++++++++++
 3 files changed, 307 insertions(+), 7 deletions(-)
 create mode 100644 tests/units/cli/test_config_validation.py

diff --git a/pipelinewise/cli/commands.py b/pipelinewise/cli/commands.py
index 9537eb7c0..0b2c662b9 100644
--- a/pipelinewise/cli/commands.py
+++ b/pipelinewise/cli/commands.py
@@ -4,6 +4,9 @@
 import os
 import shlex
 import logging
+import json
+import time
+
 from subprocess import PIPE, STDOUT, Popen
 from collections import namedtuple
 
@@ -16,26 +19,104 @@
 MIN_STREAM_BUFFER_SIZE = 10
 MAX_STREAM_BUFFER_SIZE = 2500
 
+PARAMS_VALIDATION_RETRY_PERIOD_SEC = 2
+PARAMS_VALIDATION_RETRY_TIMES = 3
+
 STATUS_RUNNING = 'running'
 STATUS_FAILED = 'failed'
 STATUS_SUCCESS = 'success'
 
-TapParams = namedtuple(
+OriginalTapParams = namedtuple(
     'TapParams', ['id', 'type', 'bin', 'python_bin', 'config', 'properties', 'state']
 )
-TargetParams = namedtuple('TargetParams', ['id', 'type', 'bin', 'python_bin', 'config'])
+OriginalTargetParams = namedtuple('TargetParams', ['id', 'type', 'bin', 'python_bin', 'config'])
 TransformParams = namedtuple(
     'TransformParams', ['bin', 'python_bin', 'config', 'tap_id', 'target_id']
 )
 
 
-class RunCommandException(Exception):
+def _verify_json_file(json_file_path: str, file_must_exists: bool, allowed_empty: bool) -> bool:
+    """Checking if input file is a valid json or not, in some cases it is allowed to have an empty file,
+     or it is allowed file not exists!
     """
-    Custom exception to raise when run command fails
+    try:
+        with open(json_file_path, 'r', encoding='utf-8') as json_file:
+            json.load(json_file)
+    except FileNotFoundError:
+        return not file_must_exists
+    except json.decoder.JSONDecodeError:
+        if not allowed_empty or os.stat(json_file_path).st_size != 0:
+            return False
+    return True
+
+
+def do_json_conf_validation(json_file: str, file_property: dict) -> bool:
+    """
+    Validating a json format config property and retry if it is invalid
     """
+    for _ in range(PARAMS_VALIDATION_RETRY_TIMES):
+        if _verify_json_file(json_file_path=json_file,
+                             file_must_exists=file_property['file_must_exists'],
+                             allowed_empty=file_property['allowed_empty']):
+            return True
 
+        time.sleep(PARAMS_VALIDATION_RETRY_PERIOD_SEC)
+    return False
+
+
+# pylint: disable=function-redefined)
+class TapParams(OriginalTapParams):
+    """
+    for overriding TapParam init and validating json properties
+    """
+    # pylint: disable=unused-argument
     def __init__(self, *args, **kwargs):
-        Exception.__init__(self, *args, **kwargs)
+        super().__init__()
+        if not getattr(self, 'config', None):
+            raise RunCommandException(
+                f'Invalid json file for config: {getattr(self, "config", None)}')
+
+        list_of_params_in_json_file = {
+            'config': {'file_must_exists': True, 'allowed_empty': False},
+            'properties': {'file_must_exists': True, 'allowed_empty': False},
+            'state': {'file_must_exists': False, 'allowed_empty': True}
+        }
+
+        for param, file_property in list_of_params_in_json_file.items():
+            valid_json = do_json_conf_validation(
+                json_file=getattr(self, param, None),
+                file_property=file_property
+               ) if getattr(self, param, None) else True
+
+            if not valid_json:
+                raise RunCommandException(
+                    f'Invalid json file for {param}: {getattr(self, param, None)}')
+
+
+# pylint: disable=function-redefined)
+class TargetParams(OriginalTargetParams):
+    """
+    for overriding TargetParam init and validating json properties
+    """
+    # pylint: disable=unused-argument
+    def __init__(self, *args, **kwargs):
+        super().__init__()
+        json_file = getattr(self, 'config', None)
+
+        valid_json = do_json_conf_validation(
+            json_file=getattr(self, 'config', None),
+            file_property={'file_must_exists': True, 'allowed_empty': False}) if json_file else False
+
+        if not valid_json:
+            raise RunCommandException(f'Invalid json file for config: {getattr(self, "config", None)}')
+
+
+# pylint: disable=unnecessary-pass
+class RunCommandException(Exception):
+    """
+    Custom exception to raise when run command fails
+    """
+    pass
 
 
 def exists_and_executable(bin_path: str) -> bool:
@@ -51,13 +132,14 @@ def exists_and_executable(bin_path: str) -> bool:
         boolean: True if file exists and executable, otherwise False
 
     """
+
     if not os.access(bin_path, os.X_OK):
+
         try:
             paths = f"{os.environ['PATH']}".split(':')
             (p for p in paths if os.access(f'{p}/{bin_path}', os.X_OK)).__next__()
         except StopIteration:
             return False
-
     return True
 
 
diff --git a/tests/units/cli/test_commands.py b/tests/units/cli/test_commands.py
index 50046f5eb..a15472577 100644
--- a/tests/units/cli/test_commands.py
+++ b/tests/units/cli/test_commands.py
@@ -2,6 +2,10 @@
 import sys
 import pytest
 
+
+from tempfile import TemporaryDirectory
+from unittest import mock
+
 from pipelinewise.cli import commands
 from pipelinewise.cli.errors import StreamBufferTooLargeException
 
@@ -23,11 +27,18 @@ def test_exists_and_executable(self):
         # Should be false if file not exists
         assert commands.exists_and_executable('invalid_executable') is False
         # Should be false if file exists but not executable
-        assert commands.exists_and_executable(__file__) is False
+        with TemporaryDirectory() as temp_dir:
+            with open(f'{temp_dir}/test.tmp', 'w', encoding='utf-8') as tmp_file:
+                tmp_file.write('foo')
+            assert commands.exists_and_executable(f'{temp_dir}/test.tmp') is False
 
+    @mock.patch('pipelinewise.cli.commands._verify_json_file', mock.MagicMock(return_value=True))
     def test_build_tap_command(self):
         """Tests the function that generates tap executable command"""
+
+        # we are using some config files which does not exist, so we patch the method that verifies the json files
         # State file should not be included if state file path not passed
+
         tap = commands.TapParams(
             id='my_tap_mysql',
             type='tap_mysql',
@@ -125,8 +136,11 @@ def test_build_tap_command(self):
             f'--config .ppw/config.json --catalog .ppw/properties.json --state {state_mock}'
         )
 
+    @mock.patch('pipelinewise.cli.commands._verify_json_file', mock.MagicMock(return_value=True))
     def test_build_target_command(self):
         """Tests the function that generates target executable command"""
+
+        # we are using some config files which does not exist, so we patch the method that verifies the json files
         # Should return a input piped command with an executable target command
 
         target = commands.TargetParams(
@@ -253,9 +267,13 @@ def test_build_stream_buffer_command(self):
             == 'mbuffer -m 100M -q -l stream_buffer.log.running'
         )
 
+    @mock.patch('pipelinewise.cli.commands._verify_json_file', mock.MagicMock(return_value=True))
     def test_build_singer_command(self):
         """Tests the function that generates the full singer singer command
         that connects the required components with linux pipes"""
+
+        # we are using some config files which does not exist, so we patch the method that verifies the json files
+
         transform_config = '{}/resources/transform-config.json'.format(
             os.path.dirname(__file__)
         )
@@ -486,8 +504,12 @@ def test_build_singer_command(self):
             '--config .ppw/config.json'
         )
 
+    @mock.patch('pipelinewise.cli.commands._verify_json_file', mock.MagicMock(return_value=True))
     def test_build_fastsync_command(self):
         """Tests the function that generates the fastsync command"""
+
+        # we are using some config files which does not exist, so we patch the method that verifies the json files
+
         transform_config = '{}/resources/transform-config.json'.format(
             os.path.dirname(__file__)
         )
diff --git a/tests/units/cli/test_config_validation.py b/tests/units/cli/test_config_validation.py
new file mode 100644
index 000000000..d17886ac0
--- /dev/null
+++ b/tests/units/cli/test_config_validation.py
@@ -0,0 +1,196 @@
+import json
+import threading
+import time
+
+from tempfile import TemporaryDirectory
+from unittest import TestCase
+
+from pipelinewise.cli import commands
+
+
+class TestConfigValidation(TestCase):
+    """Unit tests for json properties validation"""
+    def setUp(self):
+        commands.PARAMS_VALIDATION_RETRY_PERIOD_SEC = 0.1
+        commands.PARAMS_VALIDATION_RETRY_TIMES = 3
+        self.sec_to_repair_json_file = 0.15  #: pylint: disable=attribute-defined-outside-init
+        self.temp_dir = TemporaryDirectory()  #: pylint: disable=consider-using-with
+        self.invalid_json_file = f'{self.temp_dir.name}/invalid_file.json'
+        self.valid_json_file = f'{self.temp_dir.name}/valid_file.json'
+        self.empty_file = f'{self.temp_dir.name}/empty_file.json'
+
+        with open(self.valid_json_file, 'w', encoding='utf-8') as valid_file:
+            json.dump({'foo': 'bar'}, valid_file)
+
+        with open(self.empty_file, 'w', encoding='utf-8') as empty_file:
+            empty_file.write('')
+
+    def tearDown(self):
+        self.temp_dir.cleanup()
+
+    class AsyncWriteJsonFile(threading.Thread):
+        """Helper class for asynchronous writing on a file"""
+        def __init__(self, file_name: str, content: str, waiting_time: int):
+            threading.Thread.__init__(self)
+            self.file_name = file_name
+            self.content = content
+            self.waiting_time = waiting_time
+
+        def run(self):
+            time.sleep(self.waiting_time)
+            with open(self.file_name, 'w', encoding='utf-8') as json_file:
+                json_file.write(self.content)
+
+    @staticmethod
+    def _assert_tap_config(config: str, properties: str, state: str) -> None:
+        commands.TapParams(
+            id='foo',
+            type='bar',
+            bin='foo_bin',
+            python_bin='foo/python',
+            config=config,
+            properties=properties,
+            state=state,
+        )
+
+    @staticmethod
+    def _assert_target_config(config: str) -> None:
+        commands.TargetParams(
+            id='foo',
+            type='bar',
+            bin='foo_bin',
+            python_bin='foo/python',
+            config=config,
+        )
+
+    def _assert_retry_validation_of_json_file(self, config: str, properties: str, state: str) -> None:
+        invalid_file_to_be_fixed_later = self.invalid_json_file
+
+        # Starts with an invalid file and since the main method is retrying we fix the file after some seconds
+        fixed_file = self.AsyncWriteJsonFile(invalid_file_to_be_fixed_later,
+                                             '{"foo": "bar"}',
+                                             self.sec_to_repair_json_file)
+        fixed_file.start()
+        fixed_file.join()
+
+        self._assert_tap_config(config=config, properties=properties, state=state)
+
+    def _assert_raise_exception_on_invalid_file_content(self, test_case_invalid: str, invalid_file_contents: tuple):
+        for invalid_content in invalid_file_contents:
+            with open(self.invalid_json_file, 'w', encoding='utf-8') as invalid_file:
+                invalid_file.write(invalid_content)
+
+            with self.assertRaises(commands.RunCommandException) as command_exception:
+                self._assert_tap_config(
+                    config=self.invalid_json_file if test_case_invalid == 'config' else self.valid_json_file,
+                    properties=self.invalid_json_file if test_case_invalid == 'properties' else self.valid_json_file,
+                    state=self.invalid_json_file if test_case_invalid == 'state' else self.valid_json_file,
+                )
+            self.assertEqual(f'Invalid json file for {test_case_invalid}: {self.invalid_json_file}',
+                             str(command_exception.exception))
+
+    # -----------------  TEST CASES FOR TAP --------------------------
+
+    def test_tap_config_json_validation_retry_with_invalid_config_and_then_fix(self):
+        """Test it retries to validate the file if config file is invalid and then during the process it is fixed"""
+        self._assert_retry_validation_of_json_file(config=self.invalid_json_file,
+                                                   properties=self.valid_json_file,
+                                                   state=self.valid_json_file)
+
+    def test_tap_config_json_validation_retry_with_invalid_properties_and_then_fix(self):
+        """Test it retries to validate the file if properties file is invalid and then during the process it is fixed"""
+        self._assert_retry_validation_of_json_file(config=self.valid_json_file,
+                                                   properties=self.invalid_json_file,
+                                                   state=self.valid_json_file)
+
+    def test_tap_config_json_validation_retry_with_invalid_state_and_then_fix(self):
+        """Test it retries to validate the file if state file is invalid and then during the process it is fixed"""
+        self._assert_retry_validation_of_json_file(config=self.valid_json_file,
+                                                   properties=self.valid_json_file,
+                                                   state=self.invalid_json_file)
+
+    def test_tap_config_json_valid_if_state_file_does_not_exist(self):
+        """Test it is valid if state file does not exists at all"""
+        not_exists_state_file = f'{self.temp_dir.name}/not_exists.json'
+        self._assert_tap_config(config=self.valid_json_file,
+                                properties=self.valid_json_file,
+                                state=not_exists_state_file)
+
+    def test_tap_config_json_valid_if_state_file_is_empty(self):
+        """Test it is valid if state file is empty"""
+        self._assert_tap_config(
+            config=self.valid_json_file,
+            properties=self.valid_json_file,
+            state=self.empty_file
+        )
+
+    def test_tap_config_raises_exception_if_config_is_none(self):
+        """Test it raises an exception if not any file is defined for config"""
+        with self.assertRaises(commands.RunCommandException) as command_exception:
+            self._assert_tap_config(config=None, properties=self.valid_json_file, state=self.valid_json_file)
+
+        self.assertEqual('Invalid json file for config: None', str(command_exception.exception))
+
+    def test_tap_config_valid_if_properties_is_none(self):
+        """Test TapConfig is valid if not a file defined for properties"""
+        self._assert_tap_config(config=self.valid_json_file, properties=None, state=self.valid_json_file)
+
+    def test_tap_config_valid_if_state_is_none(self):
+        """Test TapConfig is valid if not a file defined for state"""
+        self._assert_tap_config(config=self.valid_json_file, properties=self.valid_json_file, state=None)
+
+    def test_tap_config_raise_exception_if_invalid_config_yet_after_retries(self):
+        """Test it raises an exception if invalid json file for config yet after many retries"""
+        self._assert_raise_exception_on_invalid_file_content(
+            test_case_invalid='config',
+            invalid_file_contents=('', ' ', 'foo', '{"foo": 1')
+        )
+
+    def test_tap_config_raise_exception_if_invalid_properties_yet_after_retries(self):
+        """Test it raises an exception if invalid json file for properties yet after many retries"""
+        self._assert_raise_exception_on_invalid_file_content(
+            test_case_invalid='properties',
+            invalid_file_contents=('', ' ', 'foo', '{"foo": 1')
+        )
+
+    def test_tap_config_json_raise_exception_on_invalid_content_for_state_file(self):
+        """Test if exception is raised exception on invalid content"""
+        self._assert_raise_exception_on_invalid_file_content(
+            test_case_invalid='state',
+            invalid_file_contents=(' ', 'foo', '{"foo": 1')
+        )
+
+    # -----------------  TEST CASES FOR TARGET --------------------------
+
+    def test_target_config_json_validation_retires(self):
+        """Test it retries if config json file is invalid"""
+        invalid_file_to_be_fixed_later = self.invalid_json_file
+
+        # It starts with an invalid file and since the main method is retrying we fix the file after some seconds
+        fixed_file = self.AsyncWriteJsonFile(invalid_file_to_be_fixed_later,
+                                             '{"foo": "bar"}',
+                                             self.sec_to_repair_json_file)
+        fixed_file.start()
+        fixed_file.join()
+
+        self._assert_target_config(config=invalid_file_to_be_fixed_later)
+
+    def test_target_config_raises_exception_if_json_property_is_none(self):
+        """Test it raises an exception if not any file is defined for config"""
+        with self.assertRaises(commands.RunCommandException) as command_exception:
+            self._assert_target_config(config=None)
+        self.assertEqual('Invalid json file for config: None', str(command_exception.exception))
+
+    def test_target_config_raises_exception_if_not_valid_json_after_retries(self):
+        """Test if it raises and exception if invalid json files yet after many retries"""
+        invalid_file_contents = ['', 'foo', '{"foo": 1']
+
+        for invalid_content in invalid_file_contents:
+            with open(self.invalid_json_file, 'w', encoding='utf-8') as invalid_file:
+                invalid_file.write(invalid_content)
+
+            with self.assertRaises(commands.RunCommandException) as command_exception:
+                self._assert_target_config(config=self.invalid_json_file)
+
+            self.assertEqual(f'Invalid json file for config: {self.invalid_json_file}',
+                             str(command_exception.exception))

From 4f0544202bf8e5a5eed233b4e713ddad280c5ee3 Mon Sep 17 00:00:00 2001
From: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
Date: Thu, 10 Feb 2022 12:30:59 +0200
Subject: [PATCH 65/78] Fix: e2e tests fail when SF credentials are not present
 (#891)

---
 pytest.ini                                |  2 +
 tests/end_to_end/helpers/env.py           | 50 ++++++++++++-----------
 tests/end_to_end/test_target_snowflake.py | 20 +++++----
 3 files changed, 40 insertions(+), 32 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 1df0b254c..c8e77ebbc 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -6,3 +6,5 @@ filterwarnings =
     ignore::DeprecationWarning:tabulate
     ignore::DeprecationWarning:_yaml
     ignore::DeprecationWarning:messytables
+    ignore::DeprecationWarning:setuptools
+    ignore::DeprecationWarning:pandas
diff --git a/tests/end_to_end/helpers/env.py b/tests/end_to_end/helpers/env.py
index d0edcecb9..2841454bd 100644
--- a/tests/end_to_end/helpers/env.py
+++ b/tests/end_to_end/helpers/env.py
@@ -614,30 +614,32 @@ def setup_target_redshift(self):
 
     def setup_target_snowflake(self):
         """Clean snowflake target database and prepare for test run"""
-        self.run_query_target_snowflake(
-            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres{self.sf_schema_postfix} CASCADE'
-        )
-        self.run_query_target_snowflake(
-            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_public2{self.sf_schema_postfix} CASCADE'
-        )
-        self.run_query_target_snowflake(
-            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical1{self.sf_schema_postfix} CASCADE'
-        )
-        self.run_query_target_snowflake(
-            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical2{self.sf_schema_postfix} CASCADE'
-        )
-        self.run_query_target_snowflake(
-            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql{self.sf_schema_postfix} CASCADE'
-        )
-        self.run_query_target_postgres(
-            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql_2{self.sf_schema_postfix} CASCADE'
-        )
-        self.run_query_target_snowflake(
-            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_s3_csv{self.sf_schema_postfix} CASCADE'
-        )
-        self.run_query_target_snowflake(
-            f'DROP SCHEMA IF EXISTS ppw_e2e_tap_mongodb{self.sf_schema_postfix} CASCADE'
-        )
+
+        if self.env['TARGET_SNOWFLAKE']['is_configured']:
+            self.run_query_target_snowflake(
+                f'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres{self.sf_schema_postfix} CASCADE'
+            )
+            self.run_query_target_snowflake(
+                f'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_public2{self.sf_schema_postfix} CASCADE'
+            )
+            self.run_query_target_snowflake(
+                f'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical1{self.sf_schema_postfix} CASCADE'
+            )
+            self.run_query_target_snowflake(
+                f'DROP SCHEMA IF EXISTS ppw_e2e_tap_postgres_logical2{self.sf_schema_postfix} CASCADE'
+            )
+            self.run_query_target_snowflake(
+                f'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql{self.sf_schema_postfix} CASCADE'
+            )
+            self.run_query_target_postgres(
+                f'DROP SCHEMA IF EXISTS ppw_e2e_tap_mysql_2{self.sf_schema_postfix} CASCADE'
+            )
+            self.run_query_target_snowflake(
+                f'DROP SCHEMA IF EXISTS ppw_e2e_tap_s3_csv{self.sf_schema_postfix} CASCADE'
+            )
+            self.run_query_target_snowflake(
+                f'DROP SCHEMA IF EXISTS ppw_e2e_tap_mongodb{self.sf_schema_postfix} CASCADE'
+            )
 
         # Clean config directory
         shutil.rmtree(os.path.join(CONFIG_DIR, 'snowflake'), ignore_errors=True)
diff --git a/tests/end_to_end/test_target_snowflake.py b/tests/end_to_end/test_target_snowflake.py
index 258b42bc5..087aff125 100644
--- a/tests/end_to_end/test_target_snowflake.py
+++ b/tests/end_to_end/test_target_snowflake.py
@@ -33,7 +33,6 @@ class TestTargetSnowflake:
     """
     End to end tests for Target Snowflake
     """
-
     def setup_method(self):
         """Initialise test project by generating YAML files from
         templates for all the configured connectors"""
@@ -59,6 +58,18 @@ def teardown_class(self):
     def teardown_method(self):
         """Delete test directories and database objects"""
 
+    @pytest.fixture(autouse=True)
+    def skip_if_no_sf_credentials(self):
+        """
+        Test fixture to be used by all tests to check if SF is configured to decide whether to run the test or not.
+        """
+        # Skip every test if required env vars not provided
+        print('skip_if_no_sf_credentials executed')
+        if self.e2e.env['TARGET_SNOWFLAKE']['is_configured']:
+            yield
+        else:
+            pytest.skip('Target Snowflake environment variables are not provided')
+
     @pytest.mark.dependency(name='validate')
     def test_validate(self):
         """Validate the YAML project with taps and target """
@@ -76,10 +87,6 @@ def test_import_project(self):
         """Import the YAML project with taps and target and do discovery mode
         to write the JSON files for singer connectors"""
 
-        # Skip every target_postgres related test if required env vars not provided
-        if not self.e2e.env['TARGET_SNOWFLAKE']['is_configured']:
-            pytest.skip('Target Snowflake environment variables are not provided')
-
         # Setup and clean source and target databases
         self.e2e.setup_tap_mysql()
         self.e2e.setup_tap_postgres()
@@ -368,9 +375,6 @@ def test_replicate_pg_to_sf(self):
     @pytest.mark.dependency(depends=['import_config'])
     def test_replicate_pg_to_sf_with_archive_load_files(self):
         """Fastsync tables from Postgres to Snowflake with archive load files enabled"""
-        if not self.e2e.env['TARGET_SNOWFLAKE']['is_configured']:
-            pytest.skip('Target Snowflake environment variables are not provided')
-
         archive_s3_prefix = 'archive_folder'  # Custom s3 prefix defined in TAP_POSTGRES_ARCHIVE_LOAD_FILES_ID
 
         s3_bucket = os.environ.get('TARGET_SNOWFLAKE_S3_BUCKET')

From 2fc52b2408342205b0899a493d08232a113ff114 Mon Sep 17 00:00:00 2001
From: Peter Kosztolanyi <peter.kosztolanyi@transferwise.com>
Date: Thu, 10 Feb 2022 11:26:15 +0000
Subject: [PATCH 66/78] Bump ujson from 4.x to 5.1.0 (#872)

Co-authored-by: Jeet Parekh <94441288+jeet-parekh-wise@users.noreply.github.com>
Co-authored-by: Samira El Aabidi <54845154+Samira-El@users.noreply.github.com>
---
 .github/workflows/linter.yml | 2 +-
 .github/workflows/tests.yml  | 2 +-
 setup.py                     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
index 1d0e0107f..546ff9d99 100644
--- a/.github/workflows/linter.yml
+++ b/.github/workflows/linter.yml
@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.7, 3.8]
 
     steps:
       - name: Checking out repo
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index c739c251a..2c1ac2078 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.7, 3.8]
 
     steps:
       - name: Checking out repo
diff --git a/setup.py b/setup.py
index 127c6fa4b..6fdc205ae 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,7 @@
           'tzlocal>=2.0,<4.1',
           'slackclient>=2.7,<2.10',
           'psutil==5.8.0',
-          'ujson>=4.1,<4.3',
+          'ujson==5.1.0',
           'dnspython==2.1.*',
       ],
       extras_require={

From bfe6230b75104e18b62d628b1b2949d74601a87a Mon Sep 17 00:00:00 2001
From: Jeet Parekh <94441288+jeet-parekh-wise@users.noreply.github.com>
Date: Thu, 10 Feb 2022 13:53:30 +0000
Subject: [PATCH 67/78] release v0.41.0 (#892)

* release v0.41.0

* Update CHANGELOG.md

* parameterise python in makefile

* Update CHANGELOG.md

Co-authored-by: Amir Mofakhar <amir.mofakhar@wise.com>

* Update Makefile

Co-authored-by: Amir Mofakhar <amir.mofakhar@wise.com>
---
 CHANGELOG.md                                  | 15 ++++++++++++---
 Makefile                                      |  4 +++-
 setup.py                                      |  3 ++-
 singer-connectors/tap-s3-csv/requirements.txt |  2 +-
 4 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b4dc76dd8..eb0a3c2e0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,12 +1,21 @@
-0.40.0 (2022-02-27)
+0.41.0 (2022-02-10)
+-------------------
+
+- Dropped support for python 3.6
+- Bump `ujson` from `4.3.0` to `5.1.0`
+- Bump `pipelinewise-tap-s3-csv` to `2.0.0`
+- Fix for config json files
+- Fix: e2e tests fail when SF credentials are not present
+
+0.40.0 (2022-01-27)
 -------------------
 - Bump `pipelinewise-tap-kafka` from `5.0.1` to `5.1.0`
 
-0.39.1 (2022-02-26)
+0.39.1 (2022-01-26)
 -------------------
 - Bump `pipelinewise-tap-kafka` from `5.0.0` to `5.0.1`
 
-0.39.0 (2022-02-25)
+0.39.0 (2022-01-25)
 -------------------
 - Bump `pipelinewise-tap-kafka` from `4.0.1` to `5.0.0`
 - Bump `pipelinewise-target-bigquery` from `1.1.1` to `1.2.0`
diff --git a/Makefile b/Makefile
index 4da3be490..fc11f4571 100644
--- a/Makefile
+++ b/Makefile
@@ -8,6 +8,8 @@ RESET_COLOR = \x1b[0m
 PIPELINEWISE_HOME = $(shell pwd)
 VENV_DIR = ${PIPELINEWISE_HOME}/.virtualenvs
 
+python ?= "python3"
+
 start_time:=$(shell date +%s)
 
 PIP_ARGS="[test]"
@@ -113,7 +115,7 @@ endef
 define make_virtualenv
 	@echo -n "Making Virtual Environment for $(1) in $(VENV_DIR)..."
 	@echo -e -n "$(YELLOW)"
-	@test -d $(VENV_DIR)/$(1) || python3 -m venv $(VENV_DIR)/$(1)
+	@test -d $(VENV_DIR)/$(1) || $(python) -m venv $(VENV_DIR)/$(1)
 	@source $(VENV_DIR)/$(1)/bin/activate
 	@echo -e "$(OK_MSG)"
 	@echo -e -n "$(YELLOW)"
diff --git a/setup.py b/setup.py
index 6fdc205ae..e92d8de0b 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,8 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      version='0.40.0',
+      python_requires='>=3.7',
+      version='0.41.0',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,
       long_description_content_type='text/markdown',
diff --git a/singer-connectors/tap-s3-csv/requirements.txt b/singer-connectors/tap-s3-csv/requirements.txt
index d92ece2be..3e27e7d03 100644
--- a/singer-connectors/tap-s3-csv/requirements.txt
+++ b/singer-connectors/tap-s3-csv/requirements.txt
@@ -1 +1 @@
-pipelinewise-tap-s3-csv==1.2.3
+pipelinewise-tap-s3-csv==2.0.0

From 8b3ecb8588ac27bda6a368ebd51d04b5bbd9ce65 Mon Sep 17 00:00:00 2001
From: Judah Rand <17158624+judahrand@users.noreply.github.com>
Date: Fri, 11 Feb 2022 07:55:22 +0000
Subject: [PATCH 68/78] Convert `NamedTuple` to `dataclass` (#889)

---
 pipelinewise/cli/commands.py              | 70 +++++++++++++----------
 pipelinewise/cli/pipelinewise.py          | 10 ++--
 tests/units/cli/test_commands.py          | 22 +++----
 tests/units/cli/test_config_validation.py |  4 +-
 4 files changed, 59 insertions(+), 47 deletions(-)

diff --git a/pipelinewise/cli/commands.py b/pipelinewise/cli/commands.py
index 0b2c662b9..ed90bf6bd 100644
--- a/pipelinewise/cli/commands.py
+++ b/pipelinewise/cli/commands.py
@@ -7,8 +7,8 @@
 import json
 import time
 
+from dataclasses import dataclass
 from subprocess import PIPE, STDOUT, Popen
-from collections import namedtuple
 
 from . import utils
 from .errors import StreamBufferTooLargeException
@@ -26,14 +26,6 @@
 STATUS_FAILED = 'failed'
 STATUS_SUCCESS = 'success'
 
-OriginalTapParams = namedtuple(
-    'TapParams', ['id', 'type', 'bin', 'python_bin', 'config', 'properties', 'state']
-)
-OriginalTargetParams = namedtuple('TargetParams', ['id', 'type', 'bin', 'python_bin', 'config'])
-TransformParams = namedtuple(
-    'TransformParams', ['bin', 'python_bin', 'config', 'tap_id', 'target_id']
-)
-
 
 def _verify_json_file(json_file_path: str, file_must_exists: bool, allowed_empty: bool) -> bool:
     """Checking if input file is a valid json or not, in some cases it is allowed to have an empty file,
@@ -64,17 +56,23 @@ def do_json_conf_validation(json_file: str, file_property: dict) -> bool:
     return False
 
 
-# pylint: disable=function-redefined)
-class TapParams(OriginalTapParams):
+@dataclass
+class TapParams:
     """
-    for overriding TapParam init and validating json properties
+    TapParams validates json properties.
     """
-    # pylint: disable=unused-argument
-    def __init__(self, *args, **kwargs):
-        super().__init__()
-        if not getattr(self, 'config', None):
+    tap_id: str
+    type: str
+    bin: str
+    python_bin: str
+    config: str
+    properties: str
+    state: str
+
+    def __post_init__(self):
+        if not self.config:
             raise RunCommandException(
-                f'Invalid json file for config: {getattr(self, "config", None)}')
+                f'Invalid json file for config: {self.config}')
 
         list_of_params_in_json_file = {
             'config': {'file_must_exists': True, 'allowed_empty': False},
@@ -93,22 +91,36 @@ def __init__(self, *args, **kwargs):
                     f'Invalid json file for {param}: {getattr(self, param, None)}')
 
 
-# pylint: disable=function-redefined)
-class TargetParams(OriginalTargetParams):
+@dataclass
+class TargetParams:
     """
-    for overriding TargetParam init and validating json properties
+    TargetParams validates json properties.
     """
-    # pylint: disable=unused-argument
-    def __init__(self, *args, **kwargs):
-        super().__init__()
-        json_file = getattr(self, 'config', None)
+    target_id: str
+    type: str
+    bin: str
+    python_bin: str
+    config: str
+
+    def __post_init__(self):
+        json_file = self.config
 
         valid_json = do_json_conf_validation(
-            json_file=getattr(self, 'config', None),
+            json_file=json_file,
             file_property={'file_must_exists': True, 'allowed_empty': False}) if json_file else False
 
         if not valid_json:
-            raise RunCommandException(f'Invalid json file for config: {getattr(self, "config", None)}')
+            raise RunCommandException(f'Invalid json file for config: {self.config}')
+
+
+@dataclass
+class TransformParams:
+    """TransformParams."""
+    bin: str
+    python_bin: str
+    config: str
+    tap_id: str
+    target_id: str
 
 
 # pylint: disable=unnecessary-pass
@@ -171,7 +183,7 @@ def build_tap_command(
     tap_command = f'{tap.bin} --config {tap.config} {catalog_argument} {tap.properties} {state_arg}'
 
     if profiling_mode:
-        dump_file = os.path.join(profiling_dir, f'tap_{tap.id}.pstat')
+        dump_file = os.path.join(profiling_dir, f'tap_{tap.tap_id}.pstat')
         tap_command = f'{tap.python_bin} -m cProfile -o {dump_file} {tap_command}'
 
     return tap_command
@@ -195,7 +207,7 @@ def build_target_command(
     target_command = f'{target.bin} --config {target.config}'
 
     if profiling_mode:
-        dump_file = os.path.join(profiling_dir, f'target_{target.id}.pstat')
+        dump_file = os.path.join(profiling_dir, f'target_{target.target_id}.pstat')
         target_command = (
             f'{target.python_bin} -m cProfile -o {dump_file} {target_command}'
         )
@@ -399,7 +411,7 @@ def build_fastsync_command(
     command = f'{fastsync_bin} {command_args}'
 
     if profiling_mode:
-        dump_file = os.path.join(profiling_dir, f'fastsync_{tap.id}_{target.id}.pstat')
+        dump_file = os.path.join(profiling_dir, f'fastsync_{tap.tap_id}_{target.target_id}.pstat')
         command = f'{ppw_python_bin} -m cProfile -o {dump_file} {command}'
 
     LOGGER.debug('FastSync command: %s', command)
diff --git a/pipelinewise/cli/pipelinewise.py b/pipelinewise/cli/pipelinewise.py
index 7ad31186d..b1747c249 100644
--- a/pipelinewise/cli/pipelinewise.py
+++ b/pipelinewise/cli/pipelinewise.py
@@ -1202,7 +1202,7 @@ def run_tap(self):
         try:
             with pidfile.PIDFile(self.tap['files']['pidfile']):
                 target_params = TargetParams(
-                    id=target_id,
+                    target_id=target_id,
                     type=target_type,
                     bin=self.target_bin,
                     python_bin=self.target_python_bin,
@@ -1226,7 +1226,7 @@ def run_tap(self):
                         log_dir, f'{target_id}-{tap_id}-{current_time}.fastsync.log'
                     )
                     tap_params = TapParams(
-                        id=tap_id,
+                        tap_id=tap_id,
                         type=tap_type,
                         bin=self.tap_bin,
                         python_bin=self.tap_python_bin,
@@ -1252,7 +1252,7 @@ def run_tap(self):
                         log_dir, f'{target_id}-{tap_id}-{current_time}.singer.log'
                     )
                     tap_params = TapParams(
-                        id=tap_id,
+                        tap_id=tap_id,
                         type=tap_type,
                         bin=self.tap_bin,
                         python_bin=self.tap_python_bin,
@@ -1415,7 +1415,7 @@ def sync_tables(self):
 
                 # Create parameters as NamedTuples
                 tap_params = TapParams(
-                    id=tap_id,
+                    tap_id=tap_id,
                     type=tap_type,
                     bin=self.tap_bin,
                     python_bin=self.tap_python_bin,
@@ -1425,7 +1425,7 @@ def sync_tables(self):
                 )
 
                 target_params = TargetParams(
-                    id=target_id,
+                    target_id=target_id,
                     type=target_type,
                     bin=self.target_bin,
                     python_bin=self.target_python_bin,
diff --git a/tests/units/cli/test_commands.py b/tests/units/cli/test_commands.py
index a15472577..873c83031 100644
--- a/tests/units/cli/test_commands.py
+++ b/tests/units/cli/test_commands.py
@@ -40,7 +40,7 @@ def test_build_tap_command(self):
         # State file should not be included if state file path not passed
 
         tap = commands.TapParams(
-            id='my_tap_mysql',
+            tap_id='my_tap_mysql',
             type='tap_mysql',
             bin='/bin/tap_mysql.py',
             python_bin='/bin/python3',
@@ -72,7 +72,7 @@ def test_build_tap_command(self):
 
         # State file should not be included if state file passed but file not exists
         tap = commands.TapParams(
-            id='my_tap_mysql',
+            tap_id='my_tap_mysql',
             type='tap_mysql',
             bin='/bin/tap_mysql.py',
             python_bin='/bin/python3',
@@ -106,7 +106,7 @@ def test_build_tap_command(self):
         state_mock = __file__
 
         tap = commands.TapParams(
-            id='my_tap_mysql',
+            tap_id='my_tap_mysql',
             type='tap_mysql',
             bin='/bin/tap_mysql.py',
             python_bin='/bin/python3',
@@ -144,7 +144,7 @@ def test_build_target_command(self):
         # Should return a input piped command with an executable target command
 
         target = commands.TargetParams(
-            id='my_target',
+            target_id='my_target',
             type='target-snowflake',
             bin='/bin/target_postgres.py',
             python_bin='/bin/python',
@@ -284,7 +284,7 @@ def test_build_singer_command(self):
 
         # Should generate a command with tap state and transformation
         tap_params = commands.TapParams(
-            id='my_tap',
+            tap_id='my_tap',
             type='tap-mysql',
             bin='/bin/tap_mysql.py',
             python_bin='/bin/python',
@@ -294,7 +294,7 @@ def test_build_singer_command(self):
         )
 
         target_params = commands.TargetParams(
-            id='my_target',
+            target_id='my_target',
             type='target-postgres',
             bin='/bin/target_postgres.py',
             python_bin='/bin/python',
@@ -379,7 +379,7 @@ def test_build_singer_command(self):
 
         # Should generate a command without state and with transformation
         tap_params = commands.TapParams(
-            id='my_tap',
+            tap_id='my_tap',
             type='tap-mysql',
             bin='/bin/tap_mysql.py',
             python_bin='/bin/python',
@@ -420,7 +420,7 @@ def test_build_singer_command(self):
 
         # Should generate a command with state and without transformation
         tap_params = commands.TapParams(
-            id='my_tap',
+            tap_id='my_tap',
             type='tap-mysql',
             bin='/bin/tap_mysql.py',
             python_bin='/bin/python',
@@ -468,7 +468,7 @@ def test_build_singer_command(self):
 
         # Should generate a command without state and transformation
         tap_params = commands.TapParams(
-            id='my_tap',
+            tap_id='my_tap',
             type='tap-mysql',
             bin='/bin/tap_mysql.py',
             python_bin='/bin/python',
@@ -520,7 +520,7 @@ def test_build_fastsync_command(self):
         # Should generate a fastsync command with transformation
 
         tap_params = commands.TapParams(
-            id='my_tap',
+            tap_id='my_tap',
             type='tap-mysql',
             bin='/bin/tap_mysql.py',
             python_bin='/tap-mysql/bin/python',
@@ -530,7 +530,7 @@ def test_build_fastsync_command(self):
         )
 
         target_params = commands.TargetParams(
-            id='my_target',
+            target_id='my_target',
             type='target-postgres',
             bin='/bin/target_postgres.py',
             python_bin='/target-postgres/bin/python',
diff --git a/tests/units/cli/test_config_validation.py b/tests/units/cli/test_config_validation.py
index d17886ac0..a35e413cf 100644
--- a/tests/units/cli/test_config_validation.py
+++ b/tests/units/cli/test_config_validation.py
@@ -44,7 +44,7 @@ def run(self):
     @staticmethod
     def _assert_tap_config(config: str, properties: str, state: str) -> None:
         commands.TapParams(
-            id='foo',
+            tap_id='foo',
             type='bar',
             bin='foo_bin',
             python_bin='foo/python',
@@ -56,7 +56,7 @@ def _assert_tap_config(config: str, properties: str, state: str) -> None:
     @staticmethod
     def _assert_target_config(config: str) -> None:
         commands.TargetParams(
-            id='foo',
+            target_id='foo',
             type='bar',
             bin='foo_bin',
             python_bin='foo/python',

From 0e2e204bb17540f791f65eab9490b737b843f60f Mon Sep 17 00:00:00 2001
From: Judah Rand <17158624+judahrand@users.noreply.github.com>
Date: Mon, 14 Feb 2022 13:19:02 +0000
Subject: [PATCH 69/78] Update tested Python versions (#888)

---
 .github/workflows/linter.yml | 2 +-
 .github/workflows/tests.yml  | 2 +-
 setup.py                     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
index 546ff9d99..495e56d3e 100644
--- a/.github/workflows/linter.yml
+++ b/.github/workflows/linter.yml
@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: ["3.7", "3.8", "3.9"]
 
     steps:
       - name: Checking out repo
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 2c1ac2078..29b4cfdf8 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: ["3.7", "3.8", "3.9"]
 
     steps:
       - name: Checking out repo
diff --git a/setup.py b/setup.py
index e92d8de0b..cb8a05948 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@
     LONG_DESCRIPTION = f.read()
 
 setup(name='pipelinewise',
-      python_requires='>=3.7',
+      python_requires='>=3.7<3.10',
       version='0.41.0',
       description='PipelineWise',
       long_description=LONG_DESCRIPTION,

From 388dc700367f371b111990510eccff2dc1f52897 Mon Sep 17 00:00:00 2001
From: Judah Rand <17158624+judahrand@users.noreply.github.com>
Date: Wed, 16 Feb 2022 10:58:09 +0000
Subject: [PATCH 70/78] Allow non-default configuration directory (#890)

---
 docs/user_guide/cli.rst      | 14 ++++++++++++++
 pipelinewise/cli/__init__.py |  3 ++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/docs/user_guide/cli.rst b/docs/user_guide/cli.rst
index f53596f13..a75658155 100644
--- a/docs/user_guide/cli.rst
+++ b/docs/user_guide/cli.rst
@@ -130,3 +130,17 @@ validate
 Validates a project directory with YAML tap and target files.
 
 :--dir: relative path to the project directory with YAML taps and targets.
+
+
+Environment variables
+---------------------
+
+`PIPELINEWISE_HOME`
+"""""""""""""""""""
+
+Configures the directory at which PipelineWise expects to find installed tap and targets.
+
+`PIPELINEWISE_CONFIG_DIRECTORY`
+"""""""""""""""""""""""""""""""
+
+Overrides the default directory at which PipelineWise expects to find configuration files generated by `pipelinewise import`.
diff --git a/pipelinewise/cli/__init__.py b/pipelinewise/cli/__init__.py
index 221205993..9f42b2d42 100644
--- a/pipelinewise/cli/__init__.py
+++ b/pipelinewise/cli/__init__.py
@@ -19,7 +19,8 @@
 
 __version__ = get_distribution('pipelinewise').version
 USER_HOME = os.path.expanduser('~')
-CONFIG_DIR = os.path.join(USER_HOME, '.pipelinewise')
+DEFAULT_CONFIG_DIR = os.path.join(USER_HOME, '.pipelinewise')
+CONFIG_DIR = os.environ.get('PIPELINEWISE_CONFIG_DIRECTORY', DEFAULT_CONFIG_DIR)
 PROFILING_DIR = os.path.join(CONFIG_DIR, 'profiling')
 PIPELINEWISE_DEFAULT_HOME = os.path.join(USER_HOME, 'pipelinewise')
 PIPELINEWISE_HOME = os.path.abspath(

From 5d125f873f6a6c86139b49d3dbdeae28cfcefed1 Mon Sep 17 00:00:00 2001
From: Jeet Parekh <94441288+jeet-parekh-wise@users.noreply.github.com>
Date: Thu, 17 Feb 2022 13:41:03 +0000
Subject: [PATCH 71/78] improve OS signal handling for graceful termination
 (#895)

* improve OS signal handling

* change tests

* remove commented code for terminating main process
---
 pipelinewise/cli/pipelinewise.py | 41 ++++++++++++++------------------
 tests/units/cli/test_cli.py      | 31 +++++++-----------------
 2 files changed, 26 insertions(+), 46 deletions(-)

diff --git a/pipelinewise/cli/pipelinewise.py b/pipelinewise/cli/pipelinewise.py
index b1747c249..5e7ae625e 100644
--- a/pipelinewise/cli/pipelinewise.py
+++ b/pipelinewise/cli/pipelinewise.py
@@ -101,7 +101,7 @@ def __init__(self, args, config_dir, venv_dir, profiling_dir=None):
 
         # Catch SIGINT and SIGTERM to exit gracefully
         for sig in [signal.SIGINT, signal.SIGTERM]:
-            signal.signal(sig, self._exit_gracefully)
+            signal.signal(sig, self.stop_tap)
 
     def send_alert(
         self, message: str, level: str = BaseAlertHandler.ERROR, exc: Exception = None
@@ -1300,14 +1300,15 @@ def run_tap(self):
         utils.silentremove(tap_properties_singer)
         self._print_tap_run_summary(self.STATUS_SUCCESS, start_time, datetime.now())
 
-    def stop_tap(self):
+    # pylint: disable=unused-argument
+    def stop_tap(self, sig=None, frame=None):
         """
         Stop running tap
 
         The command finds the tap specific pidfile that was created by run_tap command and sends
-        a SIGINT to the process. The SIGINT signal triggers _exit_gracefully function automatically and
-        the tap stops running.
+        a SIGTERM to the process.
         """
+        self.logger.info('Trying to stop tap gracefully...')
         pidfile_path = self.tap['files']['pidfile']
         try:
             with open(pidfile_path, encoding='utf-8') as pidf:
@@ -1316,12 +1317,19 @@ def stop_tap(self):
 
                 # Terminate child processes
                 for child in parent.children(recursive=True):
-                    self.logger.info('Sending SIGINT to child pid %s...', child.pid)
-                    child.send_signal(signal.SIGINT)
+                    self.logger.info('Sending SIGTERM to child pid %s...', child.pid)
+                    child.send_signal(signal.SIGTERM)
+
+            # Rename log files from running to terminated status
+            if self.tap_run_log_file:
+                tap_run_log_file_running = f'{self.tap_run_log_file}.running'
+                tap_run_log_file_terminated = f'{self.tap_run_log_file}.terminated'
+
+                if os.path.isfile(tap_run_log_file_running):
+                    os.rename(tap_run_log_file_running, tap_run_log_file_terminated)
+
+            sys.exit(1)
 
-                # Terminate main process
-                self.logger.info('Sending SIGINT to main pid %s...', parent.pid)
-                parent.send_signal(signal.SIGINT)
         except ProcessLookupError:
             self.logger.error(
                 'Pid %s not found. Is the tap running on this machine? '
@@ -1329,6 +1337,7 @@ def stop_tap(self):
                 pid,
             )
             sys.exit(1)
+
         except FileNotFoundError:
             self.logger.error(
                 'No pidfile found at %s. Tap does not seem to be running.', pidfile_path
@@ -1663,20 +1672,6 @@ def _is_initial_sync_required(
             )
         )
 
-    # pylint: disable=unused-argument
-    def _exit_gracefully(self, sig, frame, exit_code=1):
-        self.logger.info('Stopping gracefully...')
-
-        # Rename log files from running to terminated status
-        if self.tap_run_log_file:
-            tap_run_log_file_running = f'{self.tap_run_log_file}.running'
-            tap_run_log_file_terminated = f'{self.tap_run_log_file}.terminated'
-
-            if os.path.isfile(tap_run_log_file_running):
-                os.rename(tap_run_log_file_running, tap_run_log_file_terminated)
-
-        sys.exit(exit_code)
-
     def _print_tap_run_summary(self, status, start_time, end_time):
         summary = f"""
 -------------------------------------------------------
diff --git a/tests/units/cli/test_cli.py b/tests/units/cli/test_cli.py
index 31b09fc97..bb323a81e 100644
--- a/tests/units/cli/test_cli.py
+++ b/tests/units/cli/test_cli.py
@@ -3,7 +3,6 @@
 import re
 import time
 import shutil
-import signal
 import psutil
 import pidfile
 import pytest
@@ -575,6 +574,8 @@ def test_command_stop_tap(self):
         """Test stop tap command"""
         args = CliArgs(target='target_one', tap='tap_one')
         pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
+        pipelinewise.tap_run_log_file = 'test-tap-run-dummy.log'
+        Path('{}.running'.format(pipelinewise.tap_run_log_file)).touch()
 
         # Tap is not running, pid file not exist, should exit with error
         with pytest.raises(SystemExit) as pytest_wrapped_e:
@@ -603,6 +604,12 @@ def test_command_stop_tap(self):
             full_command = ' '.join(proc.info['cmdline']) if proc.info['cmdline'] else ''
             assert re.match('scheduler|pipelinewise|tap|target', full_command) is None
 
+        # Graceful exit should rename log file from running status to terminated
+        assert os.path.isfile('{}.terminated'.format(pipelinewise.tap_run_log_file))
+
+        # Delete test log file
+        os.remove('{}.terminated'.format(pipelinewise.tap_run_log_file))
+
     def test_command_sync_tables(self):
         """Test run tap command"""
         args = CliArgs(target='target_one', tap='tap_one')
@@ -664,28 +671,6 @@ def test_command_sync_tables_cleanup_state_if_file_empty_and_table_argument(self
             pass
         self._assert_calling_sync_tables(pipelinewise)
 
-    # pylint: disable=protected-access
-    def test_exit_gracefully(self):
-        """Gracefully shoudl run tap command"""
-        args = CliArgs(target='target_one', tap='tap_one')
-        pipelinewise = PipelineWise(args, CONFIG_DIR, VIRTUALENVS_DIR)
-
-        # Create a test log file, simulating a running tap
-        pipelinewise.tap_run_log_file = 'test-tap-run-dummy.log'
-        Path('{}.running'.format(pipelinewise.tap_run_log_file)).touch()
-
-        # Graceful exit should return 1 by default
-        with pytest.raises(SystemExit) as pytest_wrapped_e:
-            pipelinewise._exit_gracefully(signal.SIGINT, frame=None)
-        assert pytest_wrapped_e.type == SystemExit
-        assert pytest_wrapped_e.value.code == 1
-
-        # Graceful exit should rename log file from running status to terminated
-        assert os.path.isfile('{}.terminated'.format(pipelinewise.tap_run_log_file))
-
-        # Delete test log file
-        os.remove('{}.terminated'.format(pipelinewise.tap_run_log_file))
-
     def test_validate_command_1(self):
         """Test validate command should fail because of missing replication key for incremental"""
         test_validate_command_dir =\

From fb0bd3633318a7641ab61895a1333ede96dd6115 Mon Sep 17 00:00:00 2001
From: Judah Rand <17158624+judahrand@users.noreply.github.com>
Date: Thu, 17 Feb 2022 15:55:45 +0000
Subject: [PATCH 72/78] Relax limit on `split_file_max_chunks` (#897)

---
 pipelinewise/cli/schemas/tap.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pipelinewise/cli/schemas/tap.json b/pipelinewise/cli/schemas/tap.json
index 84e075e2f..a437e4b8f 100644
--- a/pipelinewise/cli/schemas/tap.json
+++ b/pipelinewise/cli/schemas/tap.json
@@ -383,7 +383,7 @@
     "split_file_max_chunks": {
       "type": "integer",
       "min": 1,
-      "max": 100
+      "max": 99999
     },
     "schemas": {
       "type": "array",

From a21e688b277f806d34dc43a910d3974765087722 Mon Sep 17 00:00:00 2001
From: Judah Rand <17158624+judahrand@users.noreply.github.com>
Date: Fri, 18 Feb 2022 08:04:55 +0000
Subject: [PATCH 73/78] Fix version in documentation (#894)

---
 docs/conf.py            | 3 ++-
 scripts/publish_docs.sh | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index a97e64c18..5aba8a4e1 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -14,6 +14,7 @@
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
 import datetime
+import importlib.metadata
 
 
 def setup(app):
@@ -25,7 +26,7 @@ def setup(app):
 project = 'PipelineWise'
 copyright = f'{datetime.datetime.now().year}, Wise Ltd.'
 author = 'Wise'
-version = '0.35.2'
+version = importlib.metadata.version('pipelinewise')
 
 
 # -- General configuration ---------------------------------------------------
diff --git a/scripts/publish_docs.sh b/scripts/publish_docs.sh
index 4fcda9287..aa58025bd 100755
--- a/scripts/publish_docs.sh
+++ b/scripts/publish_docs.sh
@@ -37,6 +37,7 @@ python3 -m venv ~/venv-doc
 . ~/venv-doc/bin/activate
 pip install --upgrade pip
 pip install sphinx sphinx-rtd-theme
+pip install -e .
 
 # CD into docs, make them. If you're not using Sphinx, you'll probably
 # have a different build script.
@@ -98,7 +99,7 @@ echo "Current branch ref: $GITHUB_REF"
 if [[ "$GITHUB_REF" =~ ^refs/heads/master$|^[0-9]+\.[0-9]+\.X$ ]]; then
     git add --all
     # Make sure "|| echo" is at the end to avoid error codes when no changes to commit
-    git commit -m "[ci skip] publishing updated documentation..." || echo 
+    git commit -m "[ci skip] publishing updated documentation..." || echo
 
     # We have to re-add the origin with the GH_TOKEN credentials. You
     # will need this SSH key in your environment variables on CI.

From f4a0ec696e819d4e23a7f3861d9956886e8bcfef Mon Sep 17 00:00:00 2001
From: ayhanaltunkaynak <39399746+ayhanaltunkaynak@users.noreply.github.com>
Date: Fri, 18 Feb 2022 09:28:28 +0100
Subject: [PATCH 74/78] Updates tap_mysql.py for compatibility with MySQL 8
 (#862)

---
 pipelinewise/fastsync/commons/tap_mysql.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/pipelinewise/fastsync/commons/tap_mysql.py b/pipelinewise/fastsync/commons/tap_mysql.py
index 4d50de714..f8c32f16d 100644
--- a/pipelinewise/fastsync/commons/tap_mysql.py
+++ b/pipelinewise/fastsync/commons/tap_mysql.py
@@ -146,7 +146,7 @@ def query(self, query, conn=None, params=None, return_as_cursor=False, n_retry=1
             conn = self.conn
 
         try:
-            with conn as cur:
+            with conn.cursor() as cur:
                 cur.execute(query, params)
 
                 if return_as_cursor:
@@ -277,10 +277,10 @@ def get_table_columns(self, table_name, max_num=None, date_type='date'):
         table_name = table_dict.get('table_name')
 
         sql = f"""
-                SELECT column_name,
-                    data_type,
-                    column_type,
-                    safe_sql_value
+                SELECT column_name AS column_name,
+                    data_type AS data_type,
+                    column_type AS column_type,
+                    safe_sql_value AS safe_sql_value
                 FROM (SELECT column_name,
                             data_type,
                             column_type,
@@ -378,7 +378,7 @@ def copy_table(
         )
         export_batch_rows = self.connection_config['export_batch_rows']
         exported_rows = 0
-        with self.conn_unbuffered as cur:
+        with self.conn_unbuffered.cursor() as cur:
             cur.execute(sql)
             gzip_splitter = split_gzip.open(
                 path,

From e480bb0a7497c3fa26f462236f440a1d43632180 Mon Sep 17 00:00:00 2001
From: Judah Rand <17158624+judahrand@users.noreply.github.com>
Date: Tue, 22 Feb 2022 15:00:06 +0000
Subject: [PATCH 75/78] Replace `ansible` with `ansible-core` (#900)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index cb8a05948..c8be13e51 100644
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,7 @@
           'argparse==1.4.0',
           'tabulate==0.8.9',
           'PyYAML==6.0',
-          'ansible==4.7.0',
+          'ansible-core==2.11.8',
           'Jinja2==3.0.2',
           'joblib==1.1.0',
           'PyMySQL==0.7.11',

From e573db140a48e1b896ea8a57db6335b4e334823e Mon Sep 17 00:00:00 2001
From: Judah Rand <17158624+judahrand@users.noreply.github.com>
Date: Wed, 23 Feb 2022 10:31:06 +0000
Subject: [PATCH 76/78] Improve signal handling (#902)

* Improve signal handling

Only the `bash` process which PipelineWise creates needs to be
terminated. Any other processes are not PipelineWise's responsibility to
kill.

* Move log rename outside `try...except`

* Make sure pidfile is removed

* Guarentee the correct process is killed

* Remove temp files on exit

* Use `pgid` to determine which children to kill

* Add recursive killing back in
---
 pipelinewise/cli/pipelinewise.py | 59 +++++++++++++++-----------------
 1 file changed, 28 insertions(+), 31 deletions(-)

diff --git a/pipelinewise/cli/pipelinewise.py b/pipelinewise/cli/pipelinewise.py
index 5e7ae625e..c02e5beb0 100644
--- a/pipelinewise/cli/pipelinewise.py
+++ b/pipelinewise/cli/pipelinewise.py
@@ -1274,30 +1274,21 @@ def run_tap(self):
 
         except pidfile.AlreadyRunningError:
             self.logger.error('Another instance of the tap is already running.')
-            utils.silentremove(cons_target_config)
-            utils.silentremove(tap_properties_fastsync)
-            utils.silentremove(tap_properties_singer)
             sys.exit(1)
         # Delete temp files if there is any
         except commands.RunCommandException as exc:
             self.logger.exception(exc)
-            utils.silentremove(cons_target_config)
-            utils.silentremove(tap_properties_fastsync)
-            utils.silentremove(tap_properties_singer)
             self._print_tap_run_summary(self.STATUS_FAILED, start_time, datetime.now())
             self.send_alert(message=f'{tap_id} tap failed', exc=exc)
             sys.exit(1)
         except Exception as exc:
-            utils.silentremove(cons_target_config)
-            utils.silentremove(tap_properties_fastsync)
-            utils.silentremove(tap_properties_singer)
             self._print_tap_run_summary(self.STATUS_FAILED, start_time, datetime.now())
             self.send_alert(message=f'{tap_id} tap failed', exc=exc)
             raise exc
-
-        utils.silentremove(cons_target_config)
-        utils.silentremove(tap_properties_fastsync)
-        utils.silentremove(tap_properties_singer)
+        finally:
+            utils.silentremove(cons_target_config)
+            utils.silentremove(tap_properties_fastsync)
+            utils.silentremove(tap_properties_singer)
         self._print_tap_run_summary(self.STATUS_SUCCESS, start_time, datetime.now())
 
     # pylint: disable=unused-argument
@@ -1313,22 +1304,18 @@ def stop_tap(self, sig=None, frame=None):
         try:
             with open(pidfile_path, encoding='utf-8') as pidf:
                 pid = int(pidf.read())
+                pgid = os.getpgid(pid)
                 parent = psutil.Process(pid)
 
-                # Terminate child processes
+                # Terminate all the processes in the current process' process group.
                 for child in parent.children(recursive=True):
-                    self.logger.info('Sending SIGTERM to child pid %s...', child.pid)
-                    child.send_signal(signal.SIGTERM)
-
-            # Rename log files from running to terminated status
-            if self.tap_run_log_file:
-                tap_run_log_file_running = f'{self.tap_run_log_file}.running'
-                tap_run_log_file_terminated = f'{self.tap_run_log_file}.terminated'
-
-                if os.path.isfile(tap_run_log_file_running):
-                    os.rename(tap_run_log_file_running, tap_run_log_file_terminated)
-
-            sys.exit(1)
+                    if os.getpgid(child.pid) == pgid:
+                        self.logger.info('Sending SIGTERM to child pid %s...', child.pid)
+                        child.terminate()
+                        try:
+                            child.wait(timeout=5)
+                        except psutil.TimeoutExpired:
+                            child.kill()
 
         except ProcessLookupError:
             self.logger.error(
@@ -1344,6 +1331,19 @@ def stop_tap(self, sig=None, frame=None):
             )
             sys.exit(1)
 
+        # Remove pidfile.
+        os.remove(pidfile_path)
+
+        # Rename log files from running to terminated status
+        if self.tap_run_log_file:
+            tap_run_log_file_running = f'{self.tap_run_log_file}.running'
+            tap_run_log_file_terminated = f'{self.tap_run_log_file}.terminated'
+
+            if os.path.isfile(tap_run_log_file_running):
+                os.rename(tap_run_log_file_running, tap_run_log_file_terminated)
+
+        sys.exit(1)
+
     # pylint: disable=too-many-locals
     def sync_tables(self):
         """
@@ -1455,20 +1455,17 @@ def sync_tables(self):
 
         except pidfile.AlreadyRunningError:
             self.logger.error('Another instance of the tap is already running.')
-            utils.silentremove(cons_target_config)
             sys.exit(1)
         # Delete temp file if there is any
         except commands.RunCommandException as exc:
             self.logger.exception(exc)
-            utils.silentremove(cons_target_config)
             self.send_alert(message=f'Failed to sync tables in {tap_id} tap', exc=exc)
             sys.exit(1)
         except Exception as exc:
-            utils.silentremove(cons_target_config)
             self.send_alert(message=f'Failed to sync tables in {tap_id} tap', exc=exc)
             raise exc
-
-        utils.silentremove(cons_target_config)
+        finally:
+            utils.silentremove(cons_target_config)
 
     def validate(self):
         """

From a763c0aab02b69903fd96c14865c97b65c463642 Mon Sep 17 00:00:00 2001
From: Jeet Parekh <94441288+jeet-parekh-wise@users.noreply.github.com>
Date: Fri, 25 Feb 2022 11:44:22 +0000
Subject: [PATCH 77/78] [AP-1109] Fix docker build process in github actions
 (#881)

* [AP-1109] Fix docker build process in github actions

* Update dockerhub.yml
---
 .github/workflows/dockerhub.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml
index f20d068d8..1a43717d9 100644
--- a/.github/workflows/dockerhub.yml
+++ b/.github/workflows/dockerhub.yml
@@ -34,6 +34,7 @@ jobs:
       uses: docker/build-push-action@v2
       with:
         file: ./Dockerfile
+        context: .
         push: true
         tags: |
           transferwiseworkspace/pipelinewise:${{ steps.genTag.outputs.replaced }}
@@ -44,12 +45,27 @@ jobs:
       uses: docker/build-push-action@v2
       with:
         file: ./Dockerfile.barebone
+        context: .
         push: true
         tags: |
           transferwiseworkspace/pipelinewise-barebone:${{ steps.genTag.outputs.replaced }}
           transferwiseworkspace/pipelinewise-barebone:latest
+          
+    - name: Build and push main image with only default connectors
+      id: docker_build_default
+      uses: docker/build-push-action@v2
+      with:
+        file: ./Dockerfile
+        context: .
+        build_args: |
+          "connectors=default"
+        push: true
+        tags: |
+          transferwiseworkspace/pipelinewise:${{ steps.genTag.outputs.replaced }}-default
+          transferwiseworkspace/pipelinewise:latest-default
 
     - name: Image digests
       run: |
         echo ${{ steps.docker_build_main.outputs.digest }}
         echo ${{ steps.docker_build_barebone.outputs.digest }}
+        echo ${{ steps.docker_build_default.outputs.digest }}

From d44dde5e4e0ed6ca1f3ba5bc76199092cb386338 Mon Sep 17 00:00:00 2001
From: Jeet Parekh <94441288+jeet-parekh-wise@users.noreply.github.com>
Date: Mon, 28 Feb 2022 11:34:45 +0000
Subject: [PATCH 78/78] [AP-1123] run linting in docker (#904)

* [AP-1123] run linting in docker

* [AP-1123] add __name__ check
---
 .github/workflows/linter.yml      | 60 ++++++++++++++++++++++++-------
 Dockerfile.CI                     | 12 +++++++
 scripts/check_any_file_changed.py | 27 ++++++++++++++
 3 files changed, 86 insertions(+), 13 deletions(-)
 create mode 100644 Dockerfile.CI
 create mode 100644 scripts/check_any_file_changed.py

diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
index 495e56d3e..e28b9be18 100644
--- a/.github/workflows/linter.yml
+++ b/.github/workflows/linter.yml
@@ -31,29 +31,63 @@ jobs:
         continue-on-error: true
         run: ./scripts/ci_check_no_file_changes.sh python
 
-      - name: Set up Python ${{ matrix.python-version }}
-        if: steps.check.outcome == 'failure'
-        uses: actions/setup-python@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
         with:
-          python-version: ${{ matrix.python-version }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Install dependencies
-        if: steps.check.outcome == 'failure'
+      - name: Check whether or not docker image for CI exists remotely
+        id: docker-image-exists-remotely
+        continue-on-error: true
+        run: docker manifest inspect transferwiseworkspace/pipelinewise:ci-py${{ matrix.python-version  }} > /dev/null
+
+      - name: Check if requirements or Dockerfile.CI has changed
+        id: dockerfile-setup-changed
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: python3 scripts/check_any_file_changed.py setup.py Dockerfile.CI
+
+      - name: Rebuild docker image if required
+        if: steps.docker-image-exists-remotely.outcome == 'failure' || steps.dockerfile-setup-changed.outcome == 'success'
         run: |
-          pip install -U pip setuptools wheel
-          pip install -Ue .[test]
+          docker build \
+          --build-arg PYTHON_VERSION=${{ matrix.python-version }} \
+          -t transferwiseworkspace/pipelinewise:ci-py${{ matrix.python-version  }} \
+          -f Dockerfile.CI .
+
+      - name: Push docker image if rebuilt
+        if: steps.docker-image-exists-remotely.outcome == 'failure' || steps.dockerfile-setup-changed.outcome == 'success'
+        run: |
+          docker push transferwiseworkspace/pipelinewise:ci-py${{ matrix.python-version  }}
 
       - name: Check code formatting
         if: steps.check.outcome == 'failure'
-        run: find pipelinewise tests -type f -name '*.py' | xargs unify --check-only
+        run: |
+          docker run --rm \
+          -v $PWD:/app \
+          transferwiseworkspace/pipelinewise:ci-py${{ matrix.python-version  }} \
+          bash -c "find pipelinewise tests -type f -name '*.py' | xargs unify --check-only"
 
       - name: Pylinting
         if: steps.check.outcome == 'failure'
-        run: pylint pipelinewise tests
+        run: |
+          docker run --rm -v $PWD:/app \
+          transferwiseworkspace/pipelinewise:ci-py${{ matrix.python-version  }} \
+          bash -c "pylint pipelinewise tests"
 
       - name: Pep8
         if: steps.check.outcome == 'failure'
         run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          flake8 . --count --max-complexity=15 --max-line-length=120 --statistics
+          docker run --rm -v $PWD:/app \
+          transferwiseworkspace/pipelinewise:ci-py${{ matrix.python-version  }} \
+          bash -c "flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics"
+
+      - name: Pep8 complexity
+        if: steps.check.outcome == 'failure'
+        run: |
+          docker run --rm \
+          -v $PWD:/app \
+          transferwiseworkspace/pipelinewise:ci-py${{ matrix.python-version  }} \
+          bash -c "flake8 . --count --max-complexity=15 --max-line-length=120 --statistics"
diff --git a/Dockerfile.CI b/Dockerfile.CI
new file mode 100644
index 000000000..4c05ea3c4
--- /dev/null
+++ b/Dockerfile.CI
@@ -0,0 +1,12 @@
+ARG PYTHON_VERSION
+
+FROM python:${PYTHON_VERSION}-bullseye
+
+WORKDIR /app
+
+COPY ./setup.py ./README.md ./
+
+RUN pip install -U pip setuptools wheel && \
+    pip install -Ue .[test]
+
+VOLUME /app
diff --git a/scripts/check_any_file_changed.py b/scripts/check_any_file_changed.py
new file mode 100644
index 000000000..6362239b1
--- /dev/null
+++ b/scripts/check_any_file_changed.py
@@ -0,0 +1,27 @@
+import json
+import os
+import sys
+
+import requests
+
+
+def main():
+    with open(os.environ["GITHUB_EVENT_PATH"], mode="r", encoding="utf-8") as f:
+        gh_event_data = json.load(f)
+        PR_URL = gh_event_data["pull_request"]["url"]
+
+    with requests.get(
+        f"{PR_URL}/files",
+        headers={"Authorization": f"Bearer {os.environ['GITHUB_TOKEN']}"},
+    ) as resp:
+        files_changed = [f["filename"] for f in resp.json()]
+
+    for f in sys.argv[1:]:
+        if f in files_changed:
+            sys.exit(0)
+
+    sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()