From c38ca306b35a0c844698a8f0e3295c4c7d5e54d1 Mon Sep 17 00:00:00 2001 From: darefalola Date: Thu, 14 Nov 2024 16:01:31 +0000 Subject: [PATCH 1/3] added gRPC reflection to Thoas client --- common/db.py | 23 ++++++++++++++++++-- graphql_service/server.py | 3 ++- grpc_service/grpc_model.py | 43 ++++++++++++++++++++++++++++++++------ requirements.txt | 3 ++- 4 files changed, 62 insertions(+), 10 deletions(-) diff --git a/common/db.py b/common/db.py index 3987651..8b4447b 100644 --- a/common/db.py +++ b/common/db.py @@ -16,7 +16,9 @@ import pymongo import mongomock import grpc -from ensembl.production.metadata.grpc import ensembl_metadata_pb2_grpc + +# from ensembl.production.metadata.grpc import ensembl_metadata_pb2_grpc +from yagrc import reflector as yagrc_reflector from common.utils import process_release_version @@ -117,8 +119,25 @@ def __init__(self, config): "{}:{}".format(host, port), options=(("grpc.enable_http_proxy", 0),) ) + # create reflector for querying server using reflection + self.reflector = yagrc_reflector.GrpcReflectionClient() + + # use reflection to load service definitions and message types + self.reflector.load_protocols( + self.channel, symbols=["ensembl_metadata.EnsemblMetadata"] + ) + + # dynamically retrieve the client stub class for service + stub_class = self.reflector.service_stub_class( + "ensembl_metadata.EnsemblMetadata" + ) + # bind the client and the server - self.stub = ensembl_metadata_pb2_grpc.EnsemblMetadataStub(self.channel) + # self.stub = ensembl_metadata_pb2_grpc.EnsemblMetadataStub(self.channel) + self.stub = stub_class(self.channel) def get_grpc_stub(self): return self.stub + + def get_grpc_reflector(self): + return self.reflector diff --git a/graphql_service/server.py b/graphql_service/server.py index cacbf8d..562a6c5 100644 --- a/graphql_service/server.py +++ b/graphql_service/server.py @@ -64,7 +64,8 @@ GRPC_SERVER = db.GRPCServiceClient(os.environ) GRPC_STUB = GRPC_SERVER.get_grpc_stub() -GRPC_MODEL = grpc_model.GRPC_MODEL(GRPC_STUB) +GRPC_REFLECTOR = GRPC_SERVER.get_grpc_reflector() +GRPC_MODEL = grpc_model.GRPC_MODEL(GRPC_STUB, GRPC_REFLECTOR) EXECUTABLE_SCHEMA = prepare_executable_schema() diff --git a/grpc_service/grpc_model.py b/grpc_service/grpc_model.py index 85c56d7..f44b2a2 100644 --- a/grpc_service/grpc_model.py +++ b/grpc_service/grpc_model.py @@ -1,13 +1,14 @@ import logging -from ensembl.production.metadata.grpc import ensembl_metadata_pb2 +# from ensembl.production.metadata.grpc import ensembl_metadata_pb2 logger = logging.getLogger(__name__) class GRPC_MODEL: # pylint: disable=invalid-name - def __init__(self, grpc_stub): + def __init__(self, grpc_stub, grpc_reflector): self.grpc_stub = grpc_stub + self.reflector = grpc_reflector def get_genome_by_genome_uuid(self, genome_uuid, release_version=None): logger.debug( @@ -15,10 +16,17 @@ def get_genome_by_genome_uuid(self, genome_uuid, release_version=None): genome_uuid, release_version, ) - request = ensembl_metadata_pb2.GenomeUUIDRequest( + request_class = self.reflector.message_class( + "ensembl_metadata.GenomeUUIDRequest" + ) + # request = ensembl_metadata_pb2.GenomeUUIDRequest( + # genome_uuid=genome_uuid, release_version=release_version + # ) + request = request_class( genome_uuid=genome_uuid, release_version=release_version ) response = self.grpc_stub.GetGenomeByUUID(request) + return response def get_genome_by_specific_keyword( @@ -47,7 +55,22 @@ def get_genome_by_specific_keyword( species_taxonomy_id, release_version, ) - request = ensembl_metadata_pb2.GenomeBySpecificKeywordRequest( + + request_class = self.reflector.message_class( + "ensembl_metadata.GenomeBySpecificKeywordRequest" + ) + # request = ensembl_metadata_pb2.GenomeBySpecificKeywordRequest( + # tolid=tolid, + # assembly_accession_id=assembly_accession_id, + # assembly_name=assembly_name, + # ensembl_name=ensembl_name, + # common_name=common_name, + # scientific_name=scientific_name, + # scientific_parlance_name=scientific_parlance_name, + # species_taxonomy_id=species_taxonomy_id, + # release_version=release_version, + # ) + request = request_class( tolid=tolid, assembly_accession_id=assembly_accession_id, assembly_name=assembly_name, @@ -67,13 +90,21 @@ def get_datasets_list_by_uuid(self, genome_uuid, release_version=None): genome_uuid, release_version, ) - request = ensembl_metadata_pb2.DatasetsRequest( + request_class = self.reflector.message_class("ensembl_metadata.DatasetsRequest") + # request = ensembl_metadata_pb2.DatasetsRequest( + # genome_uuid=genome_uuid, release_version=release_version + # ) + request = request_class( genome_uuid=genome_uuid, release_version=release_version ) response = self.grpc_stub.GetDatasetsListByUUID(request) return response def get_release_by_genome_uuid(self, genome_uuid): - request = ensembl_metadata_pb2.ReleaseVersionRequest(genome_uuid=genome_uuid) + request_class = self.reflector.message_class( + "ensembl_metadata.ReleaseVersionRequest" + ) + # request = ensembl_metadata_pb2.ReleaseVersionRequest(genome_uuid=genome_uuid) + request = request_class(genome_uuid=genome_uuid) response = self.grpc_stub.GetReleaseVersionByUUID(request) return response diff --git a/requirements.txt b/requirements.txt index f0152fa..3f6a545 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,8 +5,9 @@ aiodataloader==0.4.0 ariadne==0.23 python-dotenv==1.0.1 uvicorn==0.18.1 -ensembl-metadata-api@git+https://github.com/Ensembl/ensembl-metadata-api.git@3.2.1a1 grpcio==1.66.1 grpcio-tools==1.66.1 ensembl-utils==0.4.4 ensembl-py==2.1.3 +yagrc==1.1.2 +ensembl-metadata-api@git+https://github.com/Ensembl/ensembl-metadata-api.git@3.2.1a1 \ No newline at end of file From 1059b01c2c6a73fff0b146e95309bbe2d51ef796 Mon Sep 17 00:00:00 2001 From: darefalola Date: Thu, 14 Nov 2024 16:24:23 +0000 Subject: [PATCH 2/3] fixed mypy issues --- mypy.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy.ini b/mypy.ini index 1fba86e..f13e361 100644 --- a/mypy.ini +++ b/mypy.ini @@ -22,3 +22,5 @@ ignore_missing_imports = True [mypy-ensembl.production.metadata.*] ignore_missing_imports = True +[mypy-yagrc.*] +ignore_missing_imports = True \ No newline at end of file From 14db9684cf04eb67760de63d6eadd9e148353163 Mon Sep 17 00:00:00 2001 From: darefalola Date: Tue, 19 Nov 2024 11:07:29 +0000 Subject: [PATCH 3/3] added PR requested changes --- common/db.py | 2 -- common/extensions.py | 1 - common/utils.py | 15 --------------- graphql_service/resolver/gene_model.py | 3 +-- grpc_service/grpc_model.py | 24 ++++-------------------- requirements.txt | 3 +-- 6 files changed, 6 insertions(+), 42 deletions(-) diff --git a/common/db.py b/common/db.py index 8b4447b..fde15a4 100644 --- a/common/db.py +++ b/common/db.py @@ -17,7 +17,6 @@ import mongomock import grpc -# from ensembl.production.metadata.grpc import ensembl_metadata_pb2_grpc from yagrc import reflector as yagrc_reflector from common.utils import process_release_version @@ -133,7 +132,6 @@ def __init__(self, config): ) # bind the client and the server - # self.stub = ensembl_metadata_pb2_grpc.EnsemblMetadataStub(self.channel) self.stub = stub_class(self.channel) def get_grpc_stub(self): diff --git a/common/extensions.py b/common/extensions.py index bb2daec..960a463 100644 --- a/common/extensions.py +++ b/common/extensions.py @@ -33,6 +33,5 @@ def format(self, context): ) return { "execution_time_in_seconds": exec_time_in_secs, - "metadata_api_version": utils.get_ensembl_metadata_api_version(), } return None diff --git a/common/utils.py b/common/utils.py index 70b2860..2e9dfd5 100644 --- a/common/utils.py +++ b/common/utils.py @@ -59,21 +59,6 @@ def process_release_version(grpc_response): return "release_" + release_version -def get_ensembl_metadata_api_version(): - """ - Get the Metadata API tag from requirement.txt file - """ - version = "unknown" # default version - with open("requirements.txt", "r", encoding="UTF-8") as file: - lines = file.readlines() - for line in lines: - if "ensembl-metadata-api" in line: - # Extract the tag part from the line - version = line.strip().split("@")[-1] - break - return version - - def check_requested_fields(info: GraphQLResolveInfo, fields: List[str]) -> List[bool]: """ Check if specific fields are requested in the GraphQL query. diff --git a/graphql_service/resolver/gene_model.py b/graphql_service/resolver/gene_model.py index c2646df..00a8562 100644 --- a/graphql_service/resolver/gene_model.py +++ b/graphql_service/resolver/gene_model.py @@ -17,7 +17,6 @@ from typing import Dict, Optional, List, Any, Mapping from ariadne import QueryType, ObjectType -from ensembl.production.metadata.api.models import Genome from graphql import GraphQLResolveInfo, GraphQLError from pymongo.database import Database, Collection @@ -868,7 +867,7 @@ def resolve_genome(_, info: GraphQLResolveInfo, by_genome_uuid: Dict[str, str]) def create_genome_response( - genome: Genome, + genome, dataset_data: Optional[List] = None, assembly_data: Optional[Mapping[Any, Any]] = None, ) -> Dict: diff --git a/grpc_service/grpc_model.py b/grpc_service/grpc_model.py index f44b2a2..4a00d56 100644 --- a/grpc_service/grpc_model.py +++ b/grpc_service/grpc_model.py @@ -1,7 +1,5 @@ import logging -# from ensembl.production.metadata.grpc import ensembl_metadata_pb2 - logger = logging.getLogger(__name__) @@ -19,9 +17,7 @@ def get_genome_by_genome_uuid(self, genome_uuid, release_version=None): request_class = self.reflector.message_class( "ensembl_metadata.GenomeUUIDRequest" ) - # request = ensembl_metadata_pb2.GenomeUUIDRequest( - # genome_uuid=genome_uuid, release_version=release_version - # ) + request = request_class( genome_uuid=genome_uuid, release_version=release_version ) @@ -59,17 +55,7 @@ def get_genome_by_specific_keyword( request_class = self.reflector.message_class( "ensembl_metadata.GenomeBySpecificKeywordRequest" ) - # request = ensembl_metadata_pb2.GenomeBySpecificKeywordRequest( - # tolid=tolid, - # assembly_accession_id=assembly_accession_id, - # assembly_name=assembly_name, - # ensembl_name=ensembl_name, - # common_name=common_name, - # scientific_name=scientific_name, - # scientific_parlance_name=scientific_parlance_name, - # species_taxonomy_id=species_taxonomy_id, - # release_version=release_version, - # ) + request = request_class( tolid=tolid, assembly_accession_id=assembly_accession_id, @@ -91,9 +77,7 @@ def get_datasets_list_by_uuid(self, genome_uuid, release_version=None): release_version, ) request_class = self.reflector.message_class("ensembl_metadata.DatasetsRequest") - # request = ensembl_metadata_pb2.DatasetsRequest( - # genome_uuid=genome_uuid, release_version=release_version - # ) + request = request_class( genome_uuid=genome_uuid, release_version=release_version ) @@ -104,7 +88,7 @@ def get_release_by_genome_uuid(self, genome_uuid): request_class = self.reflector.message_class( "ensembl_metadata.ReleaseVersionRequest" ) - # request = ensembl_metadata_pb2.ReleaseVersionRequest(genome_uuid=genome_uuid) + request = request_class(genome_uuid=genome_uuid) response = self.grpc_stub.GetReleaseVersionByUUID(request) return response diff --git a/requirements.txt b/requirements.txt index 3f6a545..c64b8a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,4 @@ grpcio==1.66.1 grpcio-tools==1.66.1 ensembl-utils==0.4.4 ensembl-py==2.1.3 -yagrc==1.1.2 -ensembl-metadata-api@git+https://github.com/Ensembl/ensembl-metadata-api.git@3.2.1a1 \ No newline at end of file +yagrc==1.1.2 \ No newline at end of file