From 2bafee18f75865b62c2e0e2161ded6f38f264c96 Mon Sep 17 00:00:00 2001 From: Josselin Date: Tue, 5 Nov 2024 16:23:08 -0800 Subject: [PATCH 1/2] CDP-8328: deploy-agent: enable unit tests in bazel --- .gitignore | 10 ++- deploy-agent/README.md | 34 +++++++++- deploy-agent/tests/BUILD.bazel | 66 +++++++++++++++++-- .../tests/unit/deploy/common/test_config.py | 8 +-- .../tests/unit/deploy/common/test_helper.py | 6 +- .../deploy/download/test_download_helper.py | 2 +- .../tests/unit/deploy/server/test_agent.py | 17 +++-- ...{test_helper.py => test_staging_helper.py} | 8 +-- 8 files changed, 126 insertions(+), 25 deletions(-) rename deploy-agent/tests/unit/deploy/staging/{test_helper.py => test_staging_helper.py} (98%) diff --git a/.gitignore b/.gitignore index cb47ed646a..43bf1857bf 100644 --- a/.gitignore +++ b/.gitignore @@ -79,4 +79,12 @@ MANIFEST # Virtual Environments .venv -docs/docs_generator/bin/ \ No newline at end of file +docs/docs_generator/bin/ + +# Bazel +bazel-bin +bazel-out +bazel-testlogs +bazel-weave +bazel-deploy-agent +MODULE.bazel* diff --git a/deploy-agent/README.md b/deploy-agent/README.md index c4697e478b..75cf171f0a 100644 --- a/deploy-agent/README.md +++ b/deploy-agent/README.md @@ -1,7 +1,39 @@ Deploy-agent is a python script runs on every host and execute deploy scripts. See https://github.com/pinterest/teletraan/wiki for more details. -To run unit tests (debian comapatible): +# Before you commit new code + +1. Install [pre-commit](https://pre-commit.com/#install) +```bash +pip install pre-commit +pre-commit install +``` + +# Using Bazel +## Prerequisites +Ensure that your python version is at least python3.8. + +## Building +```bash +sudo bazel build //deployd:deploy-agent +``` + +## Testing +```bash +cd teletraan/deploy-agent/ +sudo bazel test //tests:* + +``` + +# FAQ + +## Run pre-commit on the last commit only +```bash +cd teletraan +pre-commit run --files $(git diff --name-only HEAD^1...HEAD) +``` + +## To run unit tests (debian comapatible): 1. Install `tox` with apt. 2. `> tox` diff --git a/deploy-agent/tests/BUILD.bazel b/deploy-agent/tests/BUILD.bazel index 67988bfa71..06890d2cab 100644 --- a/deploy-agent/tests/BUILD.bazel +++ b/deploy-agent/tests/BUILD.bazel @@ -7,6 +7,13 @@ py_test( python_version = "PY3", ) +py_test( + name = "test_client", + srcs = ['unit/deploy/client/test_client.py'], + deps = ["test_lib"], + python_version = "PY3", +) + py_test( name = "test_serverless_client", srcs = ['unit/deploy/client/test_serverless_client.py'], @@ -15,8 +22,22 @@ py_test( ) py_test( - name = "test_agent", - srcs = ['unit/deploy/server/test_agent.py'], + name = "test_config", + srcs = ['unit/deploy/common/test_config.py'], + deps = ["test_lib"], + python_version = "PY3", +) + +py_test( + name = "test_helper", + srcs = ['unit/deploy/common/test_helper.py'], + deps = ["test_lib"], + python_version = "PY3", +) + +py_test( + name = "test_stats", + srcs = ['unit/deploy/common/test_stats.py'], deps = ["test_lib"], python_version = "PY3", ) @@ -29,8 +50,8 @@ py_test( ) py_test( - name = "test_s3_download_helper", - srcs = ['unit/deploy/download/test_s3_download_helper.py'], + name = "test_download_helper", + srcs = ['unit/deploy/download/test_download_helper.py'], deps = ["test_lib"], python_version = "PY3", ) @@ -42,8 +63,43 @@ py_test( python_version = "PY3", ) +py_test( + name = "test_local_download_helper", + srcs = ['unit/deploy/download/test_local_download_helper.py'], + deps = ["test_lib"], + python_version = "PY3", +) + +py_test( + name = "test_s3_download_helper", + srcs = ['unit/deploy/download/test_s3_download_helper.py'], + deps = ["test_lib"], + python_version = "PY3", +) + +py_test( + name = "test_agent", + srcs = ['unit/deploy/server/test_agent.py'], + deps = ["test_lib"], + python_version = "PY3", +) + +py_test( + name = "test_staging_helper", + srcs = ['unit/deploy/staging/test_staging_helper.py'], + deps = ["test_lib"], + python_version = "PY3", +) + +py_test( + name = "test_transformer", + srcs = ['unit/deploy/staging/test_transformer.py'], + deps = ["test_lib"], + python_version = "PY3", +) + py_library( name = "test_lib", srcs = ["__init__.py"], deps = ["//deployd:lib"], -) \ No newline at end of file +) diff --git a/deploy-agent/tests/unit/deploy/common/test_config.py b/deploy-agent/tests/unit/deploy/common/test_config.py index 826fc2b8d7..6fb6c260ed 100644 --- a/deploy-agent/tests/unit/deploy/common/test_config.py +++ b/deploy-agent/tests/unit/deploy/common/test_config.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import os import os.path import tempfile @@ -78,10 +78,10 @@ def test_init(self): def test_get_config_filename(self): filename = os.path.join(self.dirname, "test_file1.conf") open(os.path.join(self.dirname, "test_file1.conf"), "w") - - config = Config(filenames=filename) + + config = Config(filenames=filename) self.assertEqual(config.get_config_filename(), filename) - + def test_get_deploy_type_from_op_code(self): config = Config() self.assertEqual(config._get_deploy_type_from_opcode(opCode="NOOP"), DeployType.REGULAR) diff --git a/deploy-agent/tests/unit/deploy/common/test_helper.py b/deploy-agent/tests/unit/deploy/common/test_helper.py index 2df764b3a9..42cf60ac40 100644 --- a/deploy-agent/tests/unit/deploy/common/test_helper.py +++ b/deploy-agent/tests/unit/deploy/common/test_helper.py @@ -3,9 +3,9 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -13,7 +13,7 @@ # limitations under the License. import getpass -import mock +from unittest import mock import os.path import shutil import tests diff --git a/deploy-agent/tests/unit/deploy/download/test_download_helper.py b/deploy-agent/tests/unit/deploy/download/test_download_helper.py index a24af755ed..0bba649349 100644 --- a/deploy-agent/tests/unit/deploy/download/test_download_helper.py +++ b/deploy-agent/tests/unit/deploy/download/test_download_helper.py @@ -14,7 +14,7 @@ from deployd.download.s3_download_helper import S3DownloadHelper import os -import mock +from unittest import mock import shutil import tempfile import unittest diff --git a/deploy-agent/tests/unit/deploy/server/test_agent.py b/deploy-agent/tests/unit/deploy/server/test_agent.py index 2262aa5a5b..7ea94d1de2 100644 --- a/deploy-agent/tests/unit/deploy/server/test_agent.py +++ b/deploy-agent/tests/unit/deploy/server/test_agent.py @@ -3,9 +3,9 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -499,7 +499,6 @@ def test_send_deploy_status(self, mock_create_sc): status.report = PingReport(jsonValue=self.deploy_goal1) envs = {'abc': status} - client = mock.Mock() estatus = mock.Mock() estatus.load_envs = mock.Mock(return_value=envs) ping_response_list = [ @@ -513,11 +512,17 @@ def test_send_deploy_status(self, mock_create_sc): agent = DeployAgent(client=client, estatus=estatus, conf=self.config, executor=self.executor, helper=self.helper) agent.serve_build() - mock_create_sc.assert_called_once_with('deployd.stats.deploy.status.sum', tags={ - 'first_run': False, 'deploy_stage': 'PRE_DOWNLOAD', 'env_name': 'abc', 'stage_name': 'beta', 'status_code': 'SUCCEEDED'}) + mock_create_sc.assert_called_once_with( + 'deployd.stats.deploy.status.sum', + tags={ + 'first_run': False, + 'deploy_stage': 'PRE_DOWNLOAD', + 'env_name': 'abc', + 'stage_name': 'beta', + 'status_code': 'SUCCEEDED'}) self.assertEqual(agent._curr_report.report.deployStage, DeployStage.PRE_DOWNLOAD) self.assertEqual(agent._curr_report.report.status, AgentStatus.SUCCEEDED) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/deploy-agent/tests/unit/deploy/staging/test_helper.py b/deploy-agent/tests/unit/deploy/staging/test_staging_helper.py similarity index 98% rename from deploy-agent/tests/unit/deploy/staging/test_helper.py rename to deploy-agent/tests/unit/deploy/staging/test_staging_helper.py index d0987967f4..00ca98a544 100644 --- a/deploy-agent/tests/unit/deploy/staging/test_helper.py +++ b/deploy-agent/tests/unit/deploy/staging/test_staging_helper.py @@ -3,16 +3,16 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock import os.path import shutil import unittest @@ -23,7 +23,7 @@ from deployd.staging.stager import Stager -class TestHelper(unittest.TestCase): +class TestStagingHelper(unittest.TestCase): @classmethod def setUpClass(cls): cls.base_dir = tempfile.mkdtemp() From dc08d7dbbce144c97f0cf5e44b6e479a0009a684 Mon Sep 17 00:00:00 2001 From: Josselin Date: Tue, 5 Nov 2024 09:36:41 -0800 Subject: [PATCH 2/2] CDP-8328: add normandie status and knox status to ping requests and hosts_and_agents sql table --- .gitignore | 6 +- README.md | 10 +++ deploy-agent/README.md | 2 + deploy-agent/deployd/client/client.py | 52 ++++++++++++- deploy-agent/deployd/types/ping_request.py | 21 ++++-- .../tests/unit/deploy/client/test_client.py | 9 +++ deploy-service/common/pom.xml | 6 ++ .../deployservice/bean/HostAgentBean.java | 73 +++++-------------- .../deployservice/bean/PingRequestBean.java | 12 +++ .../deployservice/handler/PingHandler.java | 12 ++- .../main/resources/sql/schema-update-20.sql | 1 + .../pinterest/deployservice/db/DBDAOTest.java | 67 +++++++++++++++++ tools/mysql/schema-update-20.sql | 12 +++ 13 files changed, 210 insertions(+), 73 deletions(-) create mode 120000 deploy-service/common/src/main/resources/sql/schema-update-20.sql create mode 100644 tools/mysql/schema-update-20.sql diff --git a/.gitignore b/.gitignore index 43bf1857bf..283192a180 100644 --- a/.gitignore +++ b/.gitignore @@ -82,9 +82,5 @@ MANIFEST docs/docs_generator/bin/ # Bazel -bazel-bin -bazel-out -bazel-testlogs -bazel-weave -bazel-deploy-agent +bazel-* MODULE.bazel* diff --git a/README.md b/README.md index 54dbab9127..472c13d87c 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,16 @@ Check out [Integrate with Teletraan](https://github.com/pinterest/teletraan/wiki [Quick start guide!](https://github.com/pinterest/teletraan/wiki/Quickstart-Guide) +#### Before you commit new code + +1. Install [pre-commit](https://pre-commit.com/#install) +```bash +cd teletraan +pip install pre-commit +pre-commit install +``` + + ### Documentation [Check out our wiki!](https://github.com/pinterest/teletraan/wiki) diff --git a/deploy-agent/README.md b/deploy-agent/README.md index 75cf171f0a..d99705e957 100644 --- a/deploy-agent/README.md +++ b/deploy-agent/README.md @@ -5,6 +5,7 @@ See https://github.com/pinterest/teletraan/wiki for more details. 1. Install [pre-commit](https://pre-commit.com/#install) ```bash +cd teletraan pip install pre-commit pre-commit install ``` @@ -15,6 +16,7 @@ Ensure that your python version is at least python3.8. ## Building ```bash +cd teletraan/deploy-agent/ sudo bazel build //deployd:deploy-agent ``` diff --git a/deploy-agent/deployd/client/client.py b/deploy-agent/deployd/client/client.py index 549833608c..b58846ba3e 100644 --- a/deploy-agent/deployd/client/client.py +++ b/deploy-agent/deployd/client/client.py @@ -19,6 +19,9 @@ import socket import traceback import json +from pathlib import Path +import re +import subprocess from deployd.client.base_client import BaseClient from deployd.client.restfulclient import RestfulClient @@ -32,6 +35,9 @@ log = logging.getLogger(__name__) +NORMANDIE_CERT_FILEPATH = "/var/lib/normandie/fuse/cert/generic" +SAN_URI_PATTERN = r"URI:(\S+),?" + class Client(BaseClient): def __init__(self, config=None, hostname=None, ip=None, hostgroup=None, @@ -51,6 +57,8 @@ def __init__(self, config=None, hostname=None, ip=None, hostgroup=None, # keep trying to fetch it from facter every time self._stage_type_fetched = False self._account_id = None + self._normandie_status = None + self._knox_status = None def _read_host_info(self) -> bool: if self._use_facter: @@ -196,10 +204,20 @@ def _read_host_info(self) -> bool: info = json.loads(ec2_metadata) self._account_id = info.get('AccountId', None) + # Retrieve Normandie status + spiffId = self.get_spiffe_id_from_normandie() + if spiffId: + self._normandie_status = 'OK' + else: + self._normandie_status = 'ERROR' + + # TODO retrieve knox status + self._knox_status = 'NotImplementedYet' + log.info("Host information is loaded. " "Host name: {}, IP: {}, host id: {}, agent_version={}, autoscaling_group: {}, " - "availability_zone: {}, ec2_tags: {}, stage_type: {}, group: {}, account id: {}".format(self._hostname, self._ip, self._id, - self._agent_version, self._autoscaling_group, self._availability_zone, self._ec2_tags, self._stage_type, self._hostgroup, self._account_id)) + "availability_zone: {}, ec2_tags: {}, stage_type: {}, group: {}, account id: {}, normandie_status: {}, knox_status: {}".format(self._hostname, self._ip, self._id, + self._agent_version, self._autoscaling_group, self._availability_zone, self._ec2_tags, self._stage_type, self._hostgroup, self._account_id, self._normandie_status, self._knox_status)) if not self._availability_zone: log.error("Fail to read host info: availablity zone") @@ -209,6 +227,32 @@ def _read_host_info(self) -> bool: return True + def get_spiffe_id_from_normandie(self) -> Optional[str]: + path = Path(NORMANDIE_CERT_FILEPATH) + cmd = [ + "openssl", + "x509", + "-in", + path.as_posix(), + "-noout", + "-text", + "-certopt", + "no_subject,no_header,no_version,no_serial,no_signame,no_validity,no_issuer,no_pubkey,no_sigdump,no_aux", + ] + matcher = None + try: + cert = subprocess.check_output(cmd).decode("utf-8") + except subprocess.CalledProcessError as e: + log.exception(f"failed to get spiffe id from normandie: {e}") + return None + else: + matcher = re.search(SAN_URI_PATTERN, cert) + + if matcher: + return matcher.group(1) + + return None + def send_reports(self, env_reports=None) -> Optional[PingResponse]: try: if self._read_host_info(): @@ -229,7 +273,9 @@ def send_reports(self, env_reports=None) -> Optional[PingResponse]: availabilityZone=self._availability_zone, ec2Tags=self._ec2_tags, stageType=self._stage_type, - accountId=self._account_id) + accountId=self._account_id, + normandieStatus=self._normandie_status, + knoxStatus=self._knox_status) with create_stats_timer('deploy.agent.request.latency', tags={'host': self._hostname}): diff --git a/deploy-agent/deployd/types/ping_request.py b/deploy-agent/deployd/types/ping_request.py index 88651b1434..76206cdaae 100644 --- a/deploy-agent/deployd/types/ping_request.py +++ b/deploy-agent/deployd/types/ping_request.py @@ -3,9 +3,9 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,7 +20,8 @@ class PingRequest(object): def __init__(self, hostId=None, hostName=None, hostIp=None, groups=None, reports=None, - agentVersion=None, autoscalingGroup=None, availabilityZone=None, ec2Tags=None, stageType=None, accountId=None): + agentVersion=None, autoscalingGroup=None, availabilityZone=None, ec2Tags=None, stageType=None, + accountId=None, normandieStatus=None, knoxStatus=None): self.hostId = hostId self.hostName = hostName self.hostIp = hostIp @@ -32,6 +33,8 @@ def __init__(self, hostId=None, hostName=None, hostIp=None, groups=None, reports self.ec2Tags = ec2Tags self.stageType = stageType self.accountId = accountId + self.normandieStatus = normandieStatus + self.knoxStatus = knoxStatus def to_json(self): ping_requests = {} @@ -52,6 +55,10 @@ def to_json(self): ping_requests["accountId"] = self.accountId if self.ec2Tags: ping_requests["ec2Tags"] = self.ec2Tags + if self.normandieStatus: + ping_requests["normandieStatus"] = self.normandieStatus + if self.knoxStatus: + ping_requests["knoxStatus"] = self.knoxStatus ping_requests["reports"] = [] for report in self.reports: @@ -76,16 +83,16 @@ def to_json(self): ping_report["deployAlias"] = report.deployAlias ping_report["containerHealthStatus"] = report.containerHealthStatus ping_report["agentState"] = report.state - + if report.extraInfo: ping_report["extraInfo"] = \ json.dumps(report.extraInfo, ensure_ascii=False).encode('utf8') - + ping_requests["reports"].append(ping_report) return ping_requests def __str__(self): return "PingRequest(hostId={}, hostName={}, hostIp={}, agentVersion={}, autoscalingGroup={}, " \ - "availabilityZone={}, ec2Tags={}, stageType={}, groups={}, accountId={}, reports={})".format(self.hostId, self.hostName, + "availabilityZone={}, ec2Tags={}, stageType={}, groups={}, accountId={}, normandieStatus={}, knoxStatus={}, reports={})".format(self.hostId, self.hostName, self.hostIp, self.agentVersion, self.autoscalingGroup, self.availabilityZone, self.ec2Tags, self.stageType, - self.groups, self.accountId, ",".join(str(v) for v in self.reports)) + self.groups, self.accountId, self.normandieStatus, self.knoxStatus, ",".join(str(v) for v in self.reports)) diff --git a/deploy-agent/tests/unit/deploy/client/test_client.py b/deploy-agent/tests/unit/deploy/client/test_client.py index 7dd5052723..ff5c0e377f 100644 --- a/deploy-agent/tests/unit/deploy/client/test_client.py +++ b/deploy-agent/tests/unit/deploy/client/test_client.py @@ -23,6 +23,15 @@ def test_read_host_info(self): self.assertIsNotNone(client._ip) self.assertTrue(return_value) + def test_read_host_info_normandie(self): + client = Client(config=Config()) + client._ec2_tags = {} + client._availability_zone = "us-east-1" + return_value: bool = client._read_host_info() + self.assertIsNotNone(client._normandie_status) + self.assertEquals(client._normandie_status, 'OK') + self.assertTrue(return_value) + def test_read_host_info_no_ec2_tags_provided(self): client = Client(config=Config()) with self.assertRaises(AttributeError): diff --git a/deploy-service/common/pom.xml b/deploy-service/common/pom.xml index c7656f7784..cbcf49d224 100644 --- a/deploy-service/common/pom.xml +++ b/deploy-service/common/pom.xml @@ -207,5 +207,11 @@ org.quartz-scheduler quartz + + org.projectlombok + lombok + 1.18.28 + provided + diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/bean/HostAgentBean.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/bean/HostAgentBean.java index f7811e2c10..7b52c5a042 100644 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/bean/HostAgentBean.java +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/bean/HostAgentBean.java @@ -16,8 +16,17 @@ package com.pinterest.deployservice.bean; import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import lombok.Data; + +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Data public class HostAgentBean implements Updatable { @JsonProperty("hostName") private String host_name; @@ -40,61 +49,11 @@ public class HostAgentBean implements Updatable { @JsonProperty("autoScalingGroup") private String auto_scaling_group; - public String getHost_name() { - return host_name; - } - - public void setHost_name(String host_name) { - this.host_name = host_name; - } - - public String getIp() { - return ip; - } - - public void setIp(String ip) { - this.ip = ip; - } - - public String getHost_id() { - return host_id; - } - - public void setHost_id(String host_id) { - this.host_id = host_id; - } - - public Long getCreate_date() { - return create_date; - } + @JsonProperty("normandieStatus") + private String normandie_status; - public void setCreate_date(Long create_date) { - this.create_date = create_date; - } - - public Long getLast_update() { - return last_update; - } - - public void setLast_update(Long last_update) { - this.last_update = last_update; - } - - public String getAgent_Version() { - return agent_version; - } - - public void setAgent_Version(String agent_version) { - this.agent_version = agent_version; - } - - public String getAuto_scaling_group() { - return this.auto_scaling_group; - } - - public void setAuto_scaling_group(String asg) { - this.auto_scaling_group = asg; - } + @JsonProperty("knoxStatus") + private String knox_status; @Override public SetClause genSetClause() { @@ -106,6 +65,8 @@ public SetClause genSetClause() { clause.addColumn("last_update", last_update); clause.addColumn("agent_version", agent_version); clause.addColumn("auto_scaling_group", auto_scaling_group); + clause.addColumn("normandie_status", normandie_status); + clause.addColumn("knox_status", knox_status); return clause; } @@ -115,7 +76,9 @@ public SetClause genSetClause() { + "host_id=VALUES(host_id)," + "last_update=VALUES(last_update)," + "agent_version=VALUES(agent_version)," - + "auto_scaling_group=VALUES(auto_scaling_group)"; + + "auto_scaling_group=VALUES(auto_scaling_group)," + + "normandie_status=VALUES(normandie_status)," + + "knox_status=VALUES(knox_status)"; @Override public String toString() { diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/bean/PingRequestBean.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/bean/PingRequestBean.java index bd7bc84bf1..59e0ce91f6 100644 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/bean/PingRequestBean.java +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/bean/PingRequestBean.java @@ -41,6 +41,10 @@ public class PingRequestBean { private String accountId; + private String normandieStatus; + + private String knoxStatus; + private List reports; public String getHostId() { @@ -127,6 +131,14 @@ public void setAccountId(String accountId) { this.accountId = accountId; } + public String getNormandieStatus() { return normandieStatus; } + + public void setNormandieStatus(String normandieStatus) { this.normandieStatus = normandieStatus; } + + public String getKnoxStatus() { return knoxStatus; } + + public void setKnoxStatus(String knoxStatus) { this.knoxStatus = knoxStatus; } + public void setReports(List reports) { this.reports = reports; } diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/PingHandler.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/PingHandler.java index cc704b5d87..806e7d34e7 100644 --- a/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/PingHandler.java +++ b/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/PingHandler.java @@ -209,7 +209,8 @@ void updateHosts( } void updateHostStatus( - String hostId, String hostName, String hostIp, String agentVersion, String asg) + String hostId, String hostName, String hostIp, String agentVersion, String asg, + String normandieStatus, String knoxStatus) throws Exception { HostAgentBean hostAgentBean = hostAgentDAO.getHostById(hostId); long currentTime = System.currentTimeMillis(); @@ -223,8 +224,10 @@ void updateHostStatus( hostAgentBean.setHost_name(hostName); hostAgentBean.setIp(hostIp); hostAgentBean.setLast_update(currentTime); - hostAgentBean.setAgent_Version(agentVersion); + hostAgentBean.setAgent_version(agentVersion); hostAgentBean.setAuto_scaling_group(asg); + hostAgentBean.setNormandie_status(normandieStatus); + hostAgentBean.setKnox_status(knoxStatus); if (!isExisting) { // First ping @@ -830,7 +833,10 @@ public PingResult ping(PingRequestBean pingRequest, boolean rate_limited) throws String agentVersion = pingRequest.getAgentVersion() != null ? pingRequest.getAgentVersion() : "UNKNOWN"; - this.updateHostStatus(hostId, hostName, hostIp, agentVersion, asg); + String normandieStatus = pingRequest.getNormandieStatus() != null ? pingRequest.getNormandieStatus() : "UNKNOWN"; + String knoxStatus = pingRequest.getKnoxStatus() != null ? pingRequest.getKnoxStatus() : "UNKNOWN"; + + this.updateHostStatus(hostId, hostName, hostIp, agentVersion, asg, normandieStatus, knoxStatus); // update the host <-> groups mapping this.updateHosts(hostName, hostIp, hostId, groups, accountId); diff --git a/deploy-service/common/src/main/resources/sql/schema-update-20.sql b/deploy-service/common/src/main/resources/sql/schema-update-20.sql new file mode 120000 index 0000000000..2d9a9b0d0b --- /dev/null +++ b/deploy-service/common/src/main/resources/sql/schema-update-20.sql @@ -0,0 +1 @@ +../../../../../../tools/mysql/schema-update-20.sql \ No newline at end of file diff --git a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java index 7674e992a0..ddd9d577fb 100644 --- a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java +++ b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java @@ -21,6 +21,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.fasterxml.jackson.annotation.JsonProperty; import com.pinterest.deployservice.bean.AcceptanceStatus; import com.pinterest.deployservice.bean.AgentBean; import com.pinterest.deployservice.bean.AgentErrorBean; @@ -37,6 +38,7 @@ import com.pinterest.deployservice.bean.DeployType; import com.pinterest.deployservice.bean.EnvironBean; import com.pinterest.deployservice.bean.GroupRolesBean; +import com.pinterest.deployservice.bean.HostAgentBean; import com.pinterest.deployservice.bean.HostBean; import com.pinterest.deployservice.bean.HostState; import com.pinterest.deployservice.bean.HostTagBean; @@ -62,6 +64,7 @@ import com.pinterest.deployservice.dao.EnvironDAO; import com.pinterest.deployservice.dao.GroupDAO; import com.pinterest.deployservice.dao.GroupRolesDAO; +import com.pinterest.deployservice.dao.HostAgentDAO; import com.pinterest.deployservice.dao.HostDAO; import com.pinterest.deployservice.dao.HostTagDAO; import com.pinterest.deployservice.dao.PromoteDAO; @@ -81,6 +84,8 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.UUID; + import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.lang3.builder.EqualsBuilder; import org.joda.time.DateTime; @@ -108,6 +113,7 @@ public class DBDAOTest { private static TagDAO tagDAO; private static ScheduleDAO scheduleDAO; private static UtilDAO utilDAO; + private static HostAgentDAO hostAgentDAO; private static BasicDataSource dataSource; @BeforeAll @@ -132,6 +138,7 @@ public static void setUpClass() throws Exception { tagDAO = new DBTagDAOImpl(dataSource); scheduleDAO = new DBScheduleDAOImpl(dataSource); utilDAO = new DBUtilDAOImpl(dataSource); + hostAgentDAO = new DBHostAgentDAOImpl(dataSource); } @AfterEach @@ -1115,6 +1122,66 @@ public void testUtilDAO() throws Exception { } } + @Test + public void testHostAgentDAO() throws Exception { + final String hostId = "host-1"; + + // Test Insert and getById + HostAgentBean hostAgentBean = genDefaultHostAgentBean(hostId); + hostAgentDAO.insert(hostAgentBean); + HostAgentBean getByIdBean = hostAgentDAO.getHostById(hostId); + assertEquals(hostAgentBean, getByIdBean); + + // Test Update and getById + hostAgentBean.setIp("192.168.0.1"); + hostAgentDAO.update(hostId, hostAgentBean); + HostAgentBean getByIdBean2 = hostAgentDAO.getHostById(hostId); + assertEquals(hostAgentBean, getByIdBean2); + + // Test getHostByName + HostAgentBean getByNameBean = hostAgentDAO.getHostByName(hostAgentBean.getHost_name()); + assertEquals(hostAgentBean, getByNameBean); + + // Test getDistinctHostsCount + long hostCount = hostAgentDAO.getDistinctHostsCount(); + assertEquals(1, hostCount); + + // Test getStaleHosts + List staleHosts = hostAgentDAO.getStaleHosts(System.currentTimeMillis() - 100_000); + assertTrue(staleHosts.isEmpty()); + + List staleHosts2 = hostAgentDAO.getStaleHosts(System.currentTimeMillis() + 100_000); + assertEquals(1, staleHosts2.size()); + assertEquals(hostAgentBean, staleHosts2.get(0)); + + List staleHosts3 = hostAgentDAO.getStaleHosts( + System.currentTimeMillis() - 100_000, System.currentTimeMillis() + 100_000); + assertEquals(1, staleHosts3.size()); + assertEquals(hostAgentBean, staleHosts3.get(0)); + + // Test Delete + hostAgentDAO.delete(hostId); + HostAgentBean getByIdBean3 = hostAgentDAO.getHostById(hostId); + assertNull(getByIdBean3); + long hostCount2 = hostAgentDAO.getDistinctHostsCount(); + assertEquals(0, hostCount2); + + } + + private HostAgentBean genDefaultHostAgentBean(String hostId) { + return HostAgentBean.builder() + .ip("127.0.0.1") + .host_id(hostId) + .host_name(UUID.randomUUID().toString()) + .create_date(System.currentTimeMillis()) + .last_update(System.currentTimeMillis()) + .agent_version("1.0") + .auto_scaling_group("auto-scaling-group") + .normandie_status("normandie status normal") + .knox_status("knox status") + .build(); + } + private EnvironBean genDefaultEnvBean( String envId, String envName, String envStage, String deployId) { EnvironBean envBean = EnvironBeanFixture.createRandomEnvironBean(); diff --git a/tools/mysql/schema-update-20.sql b/tools/mysql/schema-update-20.sql new file mode 100644 index 0000000000..a16b3a0f1b --- /dev/null +++ b/tools/mysql/schema-update-20.sql @@ -0,0 +1,12 @@ +-- !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +-- ALWAYS BACKUP YOUR DATA BEFORE EXECUTING THIS SCRIPT +-- !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +-- This script upgrade DB schema from version 19 to version 20 + +ALTER TABLE hosts_and_agents ADD COLUMN normandie_status VARCHAR(64) DEFAULT NULL; +ALTER TABLE hosts_and_agents ADD COLUMN knox_status VARCHAR(64) DEFAULT NULL; + + +-- make sure to update the schema version to 13 +UPDATE schema_versions SET version=20;