diff --git a/proxysql_tools/aws/aws.py b/proxysql_tools/aws/aws.py index f5387b7..aea538a 100644 --- a/proxysql_tools/aws/aws.py +++ b/proxysql_tools/aws/aws.py @@ -1,139 +1,18 @@ # pylint: skip-file import os -import time from ConfigParser import NoOptionError -from subprocess import Popen +from subprocess import CalledProcessError -import netifaces -import requests -import boto3 -from botocore.exceptions import ClientError +import time from proxysql_tools import LOG +from proxysql_tools.aws.notify_master import server_ready, eth1_present, restart_proxy, change_names_to, \ + log_remaining_sessions, stop_proxy, start_proxy -DEVICE_INDEX = 1 - - -def get_my_instance_id(): - r = requests.get('http://169.254.169.254/latest/meta-data/instance-id') - r.raise_for_status() - return r.content - - -def get_network_interface(ip): - client = boto3.client('ec2') - response = client.describe_network_interfaces( - Filters=[ - { - 'Name': 'addresses.private-ip-address', - 'Values': [ - ip, - ] - }, - ] - ) - network_interface_id = \ - response['NetworkInterfaces'][0]['NetworkInterfaceId'] - return network_interface_id - - -def ensure_local_interface_is_gone(local_interface): - while local_interface in netifaces.interfaces(): - pass - - -def get_network_interface_state(network_interface): - client = boto3.client('ec2') - response = client.describe_network_interfaces( - NetworkInterfaceIds=[network_interface] - ) - status = response['NetworkInterfaces'][0]['Attachment']['Status'] - LOG.debug('Interface %s, status = %s', network_interface, status) - return status - - -def network_interface_attached(network_interface): - """ - Check whether network interface is attached - - :param network_interface: network interface id - :return: True or False - """ - try: - status = get_network_interface_state(network_interface) - return status == 'attached' - except KeyError: - return False - - -def detach_network_interface(network_interface): - - def get_attachment_id(boto_client, interface): - response = boto_client.describe_network_interfaces( - NetworkInterfaceIds=[ - interface - ] - ) - return response['NetworkInterfaces'][0]['Attachment']['AttachmentId'] - - client = boto3.client('ec2') - client.detach_network_interface( - AttachmentId=get_attachment_id(client, network_interface) - ) - - -def ensure_network_interface_is_detached(network_interface): - try: - while get_network_interface_state(network_interface) != 'detached': - time.sleep(1) - except KeyError: - pass +def aws_notify_master(cfg, proxy_a, proxy_b, vip, dns, + mysql_user, mysql_password, mysql_port): -def attach_network_interface(network_interface, instance_id): - client = boto3.client('ec2') - for _ in xrange(10): - try: - client.attach_network_interface( - NetworkInterfaceId=network_interface, - InstanceId=instance_id, - DeviceIndex=DEVICE_INDEX - ) - except ClientError: - time.sleep(1) - - -def configure_local_interface(local_interface, ip, netmask): - - cmd = [ - 'ifconfig', - local_interface, - 'inet', - ip, - 'netmask', - netmask - ] - for _ in xrange(10): - os.environ['PATH'] = "%s:/sbin" % os.environ['PATH'] - env = os.environ - proc = Popen(cmd, env=env) - proc.communicate() - if proc.returncode: - time.sleep(1) - else: - return - - -def aws_notify_master(cfg): - """The function moves network interface to local instance and brings it up. - Steps: - - - Detach network interface if attached to anywhere. - - Attach the network interface to the local instance. - - Configure IP address on this instance - - :param cfg: config object - """ try: os.environ['AWS_ACCESS_KEY_ID'] = cfg.get('aws', 'aws_access_key_id') os.environ['AWS_SECRET_ACCESS_KEY'] = cfg.get('aws', @@ -145,25 +24,85 @@ def aws_notify_master(cfg): 'aws section of the config file.') exit(-1) - instance_id = get_my_instance_id() + for host in [proxy_a, proxy_b, vip]: + if not server_ready(host, + user=mysql_user, + password=mysql_password, + port=mysql_port): + LOG.error('Server %s must be up and running to do switchover', + host) + exit(1) + if proxy_a == proxy_b: + LOG.error('Proxy A and Proxy B cannot be same') + exit(1) + + if vip in [proxy_a, proxy_b]: + LOG.error('VIP address cannot be equal to proxy A or proxy B') + exit(1) + + LOG.info("Switching active ProxySQL from %s to %s", proxy_a, proxy_b) + LOG.debug('DNS names: %s', ', '.join(dns)) + + if not eth1_present(proxy_a): + LOG.error('It looks like %s is not active proxy', proxy_a) + exit(1) + + if eth1_present(proxy_b): + LOG.error('Interface eth1 is expected on %s, but found on %s. ' + 'Exiting', proxy_a, proxy_b) + exit(1) + + # Step 1. Restart ProxyB try: - ip = cfg.get('proxysql', 'virtual_ip') - netmask = cfg.get('proxysql', 'virtual_netmask') - - network_interface = get_network_interface(ip) - - if network_interface_attached(network_interface): - detach_network_interface(network_interface) - - local_interface = "eth%d" % DEVICE_INDEX - ensure_local_interface_is_gone(local_interface) - - ensure_network_interface_is_detached(network_interface) - - attach_network_interface(network_interface, instance_id) - - configure_local_interface(local_interface, ip, netmask) - except NoOptionError as err: - LOG.error('virtual_ip and virtual_netmask must be defined in ' - 'proxysql section of the config file.') + restart_proxy(proxy_b) + pass + except CalledProcessError as err: LOG.error(err) + exit(1) + + # Step 2, 3. Change DNS so dns points to Proxy B Private IP + change_names_to(dns, proxy_b) + + # Step 4. Wait TTL * 2 time + wait_time = 30 * 60 + timeout = time.time() + wait_time + n_conn = 0 + while time.time() < timeout: + # Step 5. Check if any MySQL users are connected to Proxy A + # (and log if any) + n_conn = log_remaining_sessions(proxy_a, + mysql_user, + mysql_password, + mysql_port) + if not n_conn: + LOG.info('All sessions disconnected from %s', proxy_a) + break + LOG.info('There are still %d open sessions on %s', n_conn, proxy_a) + time.sleep(3) + if n_conn > 0: + LOG.warning('Will kill existing sessions anyway') + # 6. Stop Proxy A + stop_proxy(proxy_a) + + # 7. Wait until eth1 shows up on Proxy B. If not - log error and stop. + wait_time = 60 + timeout = time.time() + wait_time + while time.time() < timeout: + if eth1_present(proxy_b) and server_ready(vip, + user=mysql_user, + password=mysql_password, + port=mysql_port): + break + time.sleep(3) + + if not eth1_present(proxy_b): + LOG.error('Keepalived failed to move VIP to %s', proxy_b) + exit(1) + + # 8. Start Proxy A + start_proxy(proxy_a) + + # 9. Change DNS so dns points to points to VIP + # 10. Change DNS so dns points to points to VIP + change_names_to(dns, vip) + LOG.info('Successfully done.') diff --git a/proxysql_tools/aws/notify_master.py b/proxysql_tools/aws/notify_master.py new file mode 100644 index 0000000..9e074fd --- /dev/null +++ b/proxysql_tools/aws/notify_master.py @@ -0,0 +1,168 @@ +""" +Module that defines functions for notify_master +""" +from __future__ import print_function +from contextlib import contextmanager +from subprocess import check_call, CalledProcessError + +import boto3 +import pymysql +from pymysql import MySQLError +from pymysql.cursors import DictCursor + +from proxysql_tools import LOG + + +def domainname(name): + """Extracts domain name from a fqdn + :param name: FQDN like www.google.com or www.yahoo.com. + :type name: str + :return: domain name like google.com or yahoo.com. + :rtype: str + """ + result = name.split('.') + result.pop(0) + return '.'.join(result) + + +def start_proxy(proxy): + """Start ProxySQL on a remote server proxy""" + cmd = [ + 'sudo', + 'ssh', + '-t', + proxy, + 'sudo /etc/init.d/proxysql start' + ] + LOG.info('Executing: %s', ' '.join(cmd)) + check_call(cmd) + + +def stop_proxy(proxy): + """Stop ProxySQL on a remote server proxy""" + cmd = [ + 'sudo', + 'ssh', + '-t', + proxy, + 'sudo killall -9 proxysql' + ] + LOG.info('Executing: %s', ' '.join(cmd)) + check_call(cmd) + + +def restart_proxy(proxy): + """Restart ProxySQL on server proxy""" + LOG.info('Restarting %s', proxy) + stop_proxy(proxy) + start_proxy(proxy) + + +def change_names_to(names, ip_addr): + """ + Change DNS name for ip_addr + :param names: DNS names + :param ip_addr: Ip Address + """ + client = boto3.client('route53') + if names: + for name in names: + LOG.info('Updating A record of %s to %s', name, ip_addr) + print(name) + response = client.list_hosted_zones_by_name( + DNSName=domainname(name), + ) + zone_id = response['HostedZones'][0]['Id'] + request = { + 'HostedZoneId': zone_id, + 'ChangeBatch': { + 'Comment': 'Automated switchover DNS update', + 'Changes': [ + { + 'Action': 'UPSERT', + 'ResourceRecordSet': { + 'Name': name, + 'Type': 'A', + 'TTL': 300, + 'ResourceRecords': [ + { + 'Value': ip_addr + }, + ] + } + } + ] + } + } + client.change_resource_record_sets(**request) + + +def log_remaining_sessions(host, user='root', password='', port=3306): + """Connect to host and print existing sessions. + :return: Number of connected sessions + :rtype: int + """ + with _connect(host, user=user, password=password, port=port) as conn: + cursor = conn.cursor() + query = "SHOW PROCESSLIST" + cursor.execute(query) + nrows = cursor.rowcount + while True: + row = cursor.fetchone() + if row: + print(row) + else: + break + + return nrows + + +def server_ready(host, user='root', password='', port=3306): + """Connect to host and execute SELECT 1 to make sure + it's up and running. + :return: True if server is ready for connections + :rtype: bool + """ + try: + with _connect(host, user=user, password=password, port=port) as conn: + cursor = conn.cursor() + query = "SELECT 1" + cursor.execute(query) + return True + except MySQLError as err: + LOG.error(err) + return False + + +def eth1_present(proxy): + """Check if eth1 is up on remote host proxy""" + cmd = [ + 'sudo', + 'ssh', + '-t', + proxy, + '/sbin/ifconfig eth1' + ] + LOG.info('Executing: %s', ' '.join(cmd)) + try: + check_call(cmd) + return True + except CalledProcessError: + return False + + +@contextmanager +def _connect(host, user='root', password='', port=3306): + """Connect to ProxySQL admin interface.""" + connect_args = { + 'host': host, + 'port': port, + 'user': user, + 'passwd': password, + 'connect_timeout': 60, + 'cursorclass': DictCursor + } + + conn = pymysql.connect(**connect_args) + yield conn + conn.close() diff --git a/proxysql_tools/cli.py b/proxysql_tools/cli.py index 5c481e9..1e0cea9 100644 --- a/proxysql_tools/cli.py +++ b/proxysql_tools/cli.py @@ -71,12 +71,26 @@ def aws(): @aws.command() +@click.argument('vip') +@click.argument('proxy_a') +@click.argument('proxy_b') +@click.argument('vip') +@click.option('--dns', multiple=True, + help='Update given DNS names.') +@click.option('--mysql-user', help='MySQL user to connect to proxies.', + default='root', show_default=True) +@click.option('--mysql-password', help='Password for MySQL user.', + default='', show_default=True) +@click.option('--mysql-port', help='TCP port that open for MySQL connections.', + default=3306, show_default=True) @PASS_CFG -def notify_master(cfg): +def notify_master(cfg, proxy_a, proxy_b, vip, dns, # pylint: disable=too-many-arguments + mysql_user, mysql_password, mysql_port): """The notify_master script for keepalived.""" LOG.debug('Switching to master role and executing keepalived ' 'notify_master script.') - aws_notify_master(cfg) + aws_notify_master(cfg, proxy_a, proxy_b, vip, dns, + mysql_user, mysql_password, mysql_port) @main.group() diff --git a/tests/unit/test_aws.py b/tests/unit/test_aws.py deleted file mode 100644 index 2728b0e..0000000 --- a/tests/unit/test_aws.py +++ /dev/null @@ -1,47 +0,0 @@ -from mock import MagicMock - -from proxysql_tools.aws.aws import get_network_interface, \ - get_network_interface_state, network_interface_attached, \ - detach_network_interface - - -def test__get_network_interface(mocker): - mock_boto3 = mocker.patch('proxysql_tools.aws.aws.boto3') - mock_ec2 = MagicMock() - mock_boto3.client.return_value = mock_ec2 - get_network_interface('0.0.0.0') - mock_boto3.client.assert_called_once_with('ec2') - - -def test__get_network_interface_state(mocker): - mock_boto3 = mocker.patch('proxysql_tools.aws.aws.boto3') - mock_ec2 = MagicMock() - mock_boto3.client.return_value = mock_ec2 - get_network_interface_state('some interface') - mock_boto3.client.assert_called_once_with('ec2') - - -def test__network_interface_state_returns_true_if_attached(mocker): - mock_get_interface_state = mocker.patch('proxysql_tools.aws.aws.get_network_interface_state') - mock_get_interface_state.return_value = 'attached' - assert network_interface_attached('1.1.1.1') - - -def test__network_interface_state_returns_false_if_not_attached(mocker): - mock_get_interface_state = mocker.patch('proxysql_tools.aws.aws.get_network_interface_state') - mock_get_interface_state.return_value = 'detached' - assert not network_interface_attached('1.1.1.1') - - -def test__network_interface_attached_if_side_effect_is_key_error(mocker): - mock_get_interface_state = mocker.patch('proxysql_tools.aws.aws.get_network_interface_state') - mock_get_interface_state.side_effect = KeyError - assert not network_interface_attached('1.1.1.1') - - -def test__detach_network_interface(mocker): - mock_boto3 = mocker.patch('proxysql_tools.aws.aws.boto3') - mock_ec2 = MagicMock() - mock_boto3.client.return_value = mock_ec2 - detach_network_interface('stub') - mock_boto3.client.assert_called_once_with('ec2')