Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add script to compare service-generated SEPIO transformation results #7

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add script to compare service-generated SEPIO transformation results #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions test_data/sepio-transform-test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
import argparse
import json
import os
import requests
import subprocess
import sys

arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('-f', '--source-file', help='Source file to test (takes precedence over source-list)')
arg_parser.add_argument('-l', '--source-list', default='test-source-1.json,test-source-2.json,test-source-3.json', help='Comma-separated list of source files to test')
arg_parser.add_argument('-t', '--test-url', default='https://tych0pbo2f.execute-api.us-west-2.amazonaws.com/default/sepio-transform', help='Service URL to generate test results (defaults to non-production service)')
arg_parser.add_argument('-c', '--control-url', default='https://a61m4glrmf.execute-api.us-west-2.amazonaws.com/default/sepio-transform', help='Service URL to generate control results (defaults to production service)')
args = arg_parser.parse_args()

source_file_list = []

# Determine source file(s) to test
try:
if args.source_file:
source_file_list.append(args.source_file)
elif args.source_list:
source_file_list = args.source_list.split(',')

except Exception as e:
sys.exit('Failed to process source file(s) argument:\n{}'.format(e))

print('\nControl service: {}'.format(args.control_url))
print('Test service: {}'.format(args.test_url))

# Run test on each source file
for source_file in source_file_list:
print('\nComparing SEPIO transformation results for {}:'.format(source_file))

try:
# Get request data from source file
source_file_object = open(source_file, 'r')
source_file_data = source_file_object.read()
source_file_object.close()

# Send request data to control service and if successful, save response in a temporary file
service_result_control = requests.post('{}/vci2cgsepio'.format(args.control_url), headers={'Content-Type': 'application/json'}, data=source_file_data, timeout=10)
service_result_control.raise_for_status()

service_result_control_file = open('temp-sepio-transform-control-result.json', 'w')
json.dump(service_result_control.json(), service_result_control_file, indent=' ', sort_keys=True)
service_result_control_file.close()

# Send request data to test service and if successful, save response in a temporary file
service_result_test = requests.post('{}/vci2cgsepio'.format(args.test_url), headers={'Content-Type': 'application/json'}, data=source_file_data, timeout=10)
service_result_test.raise_for_status()

service_result_test_file = open('temp-sepio-transform-test-result.json', 'w')
json.dump(service_result_test.json(), service_result_test_file, indent=' ', sort_keys=True)
service_result_test_file.close()

# Use diff to compare results
service_result_comparison = subprocess.run('diff temp-sepio-transform-control-result.json temp-sepio-transform-test-result.json', shell=True, text=True)

# Delete temporary files
os.remove('temp-sepio-transform-control-result.json')
os.remove('temp-sepio-transform-test-result.json')

except Exception as e:
print('Test failed:\n{}'.format(e))
pass
1 change: 1 addition & 0 deletions test_data/test-source-1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"snapshots": ["acbae081-2e95-4122-a539-a9b3da0dbab5", "c2fc7b36-2394-4628-a7ce-026d4728a8c1", "0b44a35d-85c8-401b-98bd-d1d5d1410467", "cc5d077c-d81f-4f34-a32d-c1728b72bf62", "6f1f18ce-40af-44da-b423-cb3dab355a3e", "3f332225-1e55-42ca-b1f1-03bf3a38830e", "c8b459e9-5686-48dd-972e-e61223f88ff5"], "status": "Approved", "provisionalVariant": {"reason": "Setting a modified classification...", "approvalDate": "2022-11-21T21:07:02.598Z", "date_created": "2022-11-21T20:32:50.438Z", "provisionedClassification": true, "approvalSubmitter": "John Doe", "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "classificationApprover": "Jane Doe", "evidenceSummary": "Evidence summary, no met criteria...", "alteredClassification": "Uncertain significance - conflicting evidence", "provisionalSubmitter": "John Doe", "affiliation": "10004", "classificationDate": "2022-11-21T21:06:53.575Z", "provisionalDate": "2022-11-21T21:06:58.522Z", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "approvedClassification": true, "classificationStatus": "Approved", "publishClassification": false, "last_modified": "2022-11-21T21:07:02.598Z"}, "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "date_created": "2022-11-21T20:28:28.772464", "item_type": "interpretation", "evaluations": [{"criteriaStatus": "not-met", "affiliation": "10004", "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "variant": "0372c8d4-bec2-4521-b57d-fc626683956b", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "date_created": "2022-11-21T20:50:11.123704", "PK": "bd903380-9ab6-406d-bef2-a117e3c08e44", "last_modified": "2022-11-21T20:50:11.123704", "criteria": "PM1", "item_type": "evaluation"}], "affiliation": "10004", "variant": {"variationType": "single nucleotide variant", "clinvarVariantTitle": "NM_000548.5(TSC2):c.451A>T (p.Arg151Ter)", "molecularConsequenceList": [{"hgvsName": "NM_000548.5:c.451A>T", "term": "nonsense", "soId": "SO:0001587"}, {"hgvsName": "NM_001077183.3:c.451A>T", "term": "nonsense", "soId": "SO:0001587"}, {"hgvsName": "NM_001114382.3:c.451A>T", "term": "nonsense", "soId": "SO:0001587"}, {"hgvsName": "NM_001318827.2:c.340A>T", "term": "nonsense", "soId": "SO:0001587"}, {"hgvsName": "NM_001318829.2:c.304A>T", "term": "nonsense", "soId": "SO:0001587"}, {"hgvsName": "NM_001318832.2:c.484A>T", "term": "nonsense", "soId": "SO:0001587"}, {"hgvsName": "NM_001363528.2:c.451A>T", "term": "nonsense", "soId": "SO:0001587"}, {"hgvsName": "NM_001370404.1:c.451A>T", "term": "nonsense", "soId": "SO:0001587"}, {"hgvsName": "NM_001370405.1:c.451A>T", "term": "nonsense", "soId": "SO:0001587"}, {"hgvsName": "NM_021055.3:c.451A>T", "term": "nonsense", "soId": "SO:0001587"}, {"hgvsName": "NM_001318831.2:c.-1-1786A>T", "term": "intron variant", "soId": "SO:0001627"}], "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "date_created": "2022-11-21T20:28:13.180914", "dbSNPIds": ["137854199"], "preferredTitle": "NM_000548.5(TSC2):c.451A>T (p.Arg151Ter)", "hgvsNames": {"others": ["LRG_487:g.10105A>T", "LRG_487t1:c.451A>T", "NG_005895.1:g.10105A>T", "NM_000548.5:c.451A>T", "NP_000539.2:p.Arg151Ter", "NM_001077183.3:c.451A>T", "NP_001070651.1:p.Arg151Ter", "NM_001114382.3:c.451A>T", "NP_001107854.1:p.Arg151Ter", "NM_001318827.2:c.340A>T", "NP_001305756.1:p.Arg114Ter", "NM_001318829.2:c.304A>T", "NP_001305758.1:p.Arg102Ter", "NM_001318832.2:c.484A>T", "NP_001305761.1:p.Arg162Ter", "NM_001363528.2:c.451A>T", "NP_001350457.1:p.Arg151Ter", "NM_001370404.1:c.451A>T", "NP_001357333.1:p.Arg151Ter", "NM_001370405.1:c.451A>T", "NP_001357334.1:p.Arg151Ter", "NM_021055.3:c.451A>T", "NP_066399.2:p.Arg151Ter", "NM_001318831.2:c.-1-1786A>T"], "GRCh38": "NC_000016.10:g.2054410A>T", "GRCh37": "NC_000016.9:g.2104411A>T"}, "item_type": "variant", "communityStandardTitle": "NM_000548.5(TSC2):c.451A>T (p.Arg151Ter)", "clinvarVariantId": "50000", "carId": "CA020595", "PK": "0372c8d4-bec2-4521-b57d-fc626683956b", "last_modified": "2022-11-21T20:28:13.180914"}, "PK": "7906ba28-6ee1-41f6-81b9-1f2c516900eb", "last_modified": "2022-11-21T21:07:03.345881"}
1 change: 1 addition & 0 deletions test_data/test-source-2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"cspec": {"documentName": "ClinGen Hearing Loss Expert Panel Specifications to the ACMG/AMP Variant Interpretation Guidelines for CDH23, COCH, GJB2,\nKCNQ4, MYO6, MYO7A, SLC26A4, TECTA and USH2A Version 2", "cspecUrl": "https://cspec.genome.network/cspec/SequenceVariantInterpretation/id/135637577", "ruleSetIri": "https://cspec.genome.network/cspec/RuleSet/id/135640633", "version": "2.0.0", "cspecId": "135640633"}, "snapshots": ["3890bc00-3d08-490f-ad25-8ed8735d024f"], "status": "Approved", "provisionalVariant": {"autoClassification": "Likely pathogenic", "approvalDate": "2022-11-21T22:12:46.514Z", "date_created": "2022-11-21T22:12:35.126Z", "provisionedClassification": true, "approvalSubmitter": "John Doe", "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "classificationApprover": "Jane Doe", "evidenceSummary": "Evidence summary, several met/supporting pathogenic criteria...", "provisionalSubmitter": "John Doe", "affiliation": "10007", "classificationDate": "2022-11-21T22:12:35.126Z", "provisionalDate": "2022-11-21T22:12:38.188Z", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "approvedClassification": true, "classificationStatus": "Approved", "last_modified": "2022-11-21T22:12:46.514Z"}, "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "diseaseTerm": "uterine ligament adenosarcoma", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "date_created": "2022-11-21T22:04:41.431864", "item_type": "interpretation", "evaluations": [{"affiliation": "10007", "population": "4e4d6fd7-5d0b-4a01-8d13-232f8fdfcafa", "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "variant": "5ca2daf8-d209-4a59-8ce5-ade4ec5ef05a", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "PK": "2227036d-888d-4340-97c1-2555e3e68663", "date_created": "2022-11-21T22:08:11.536Z", "last_modified": "2022-11-21T22:08:11.738854", "criteria": "BA1", "item_type": "evaluation"}, {"criteriaStatus": "not-evaluated", "affiliation": "10007", "population": "4e4d6fd7-5d0b-4a01-8d13-232f8fdfcafa", "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "variant": "5ca2daf8-d209-4a59-8ce5-ade4ec5ef05a", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "PK": "32ececc1-bf8c-426b-a057-8fecc6c4063d", "date_created": "2022-11-21T22:08:11.537Z", "last_modified": "2022-11-21T22:08:11.798467", "criteria": "BS1", "item_type": "evaluation"}, {"criteriaStatus": "met", "population": "4e4d6fd7-5d0b-4a01-8d13-232f8fdfcafa", "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "date_created": "2022-11-21T22:08:11.537Z", "item_type": "evaluation", "affiliation": "10007", "variant": "5ca2daf8-d209-4a59-8ce5-ade4ec5ef05a", "PK": "8ffab765-d59c-4df2-92e2-3baca069a10c", "last_modified": "2022-11-21T22:08:11.811200", "criteria": "PM2", "explanation": "PM2 criteria met"}, {"affiliation": "10007", "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "variant": "5ca2daf8-d209-4a59-8ce5-ade4ec5ef05a", "computational": "c07cf205-fc9f-4d9c-bfec-505b90395fbe", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "date_created": "2022-11-21T22:09:15.879393", "PK": "cf5abd68-328b-4f5b-b608-f27eb6882d35", "last_modified": "2022-11-21T22:09:15.879393", "criteria": "BP1", "item_type": "evaluation"}, {"affiliation": "10007", "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "variant": "5ca2daf8-d209-4a59-8ce5-ade4ec5ef05a", "computational": "c07cf205-fc9f-4d9c-bfec-505b90395fbe", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "date_created": "2022-11-21T22:09:15.976104", "PK": "87011def-6afc-4cd0-b244-a09e76e99db0", "last_modified": "2022-11-21T22:09:15.976104", "criteria": "BP4", "item_type": "evaluation"}, {"affiliation": "10007", "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "variant": "5ca2daf8-d209-4a59-8ce5-ade4ec5ef05a", "computational": "c07cf205-fc9f-4d9c-bfec-505b90395fbe", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "date_created": "2022-11-21T22:09:15.915897", "PK": "ee56d161-a770-4182-bfd2-5962871a8046", "last_modified": "2022-11-21T22:09:15.915897", "criteria": "PP2", "item_type": "evaluation"}, {"criteriaStatus": "met", "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "date_created": "2022-11-21T22:09:15.952560", "item_type": "evaluation", "criteriaModifier": "moderate", "affiliation": "10007", "variant": "5ca2daf8-d209-4a59-8ce5-ade4ec5ef05a", "computational": "c07cf205-fc9f-4d9c-bfec-505b90395fbe", "PK": "2057ba16-941e-4cd4-adac-2e9119ab6f8f", "last_modified": "2022-11-21T22:09:15.952560", "criteria": "PP3", "explanation": "PP3 criteria moderate"}, {"criteriaStatus": "met", "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "date_created": "2022-11-21T22:09:51.541843", "item_type": "evaluation", "criteriaModifier": "supporting", "affiliation": "10007", "variant": "5ca2daf8-d209-4a59-8ce5-ade4ec5ef05a", "PK": "79c29731-a840-47c2-89d5-9619ae237f5e", "last_modified": "2022-11-21T22:09:51.541843", "criteria": "PM1", "explanation": "PM1 criteria supporting"}, {"criteriaStatus": "met", "affiliation": "10007", "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "variant": "5ca2daf8-d209-4a59-8ce5-ade4ec5ef05a", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "date_created": "2022-11-21T22:10:47.268761", "PK": "808fcc9f-122f-4955-a83b-3f1bef87cfe6", "last_modified": "2022-11-21T22:10:47.268761", "criteria": "PS4", "item_type": "evaluation", "explanation": "PS4 criteria met"}], "pathogenicity": "Likely pathogenic", "affiliation": "10007", "disease": {"term": "uterine ligament adenosarcoma", "PK": "MONDO_0200000", "date_created": "2022-11-21T22:06:25.181941", "last_modified": "2022-11-21T22:06:25.181941", "item_type": "disease"}, "variant": {"variationType": "Deletion", "clinvarVariantTitle": "NM_000249.4(MLH1):c.201del (p.Ile68fs)", "molecularConsequenceList": [{"hgvsName": "NM_000249.4:c.201del", "term": "frameshift variant", "soId": "SO:0001589"}, {"hgvsName": "NM_001167617.3:c.-89del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001167618.3:c.-523del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001167619.3:c.-431del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001258271.2:c.201del", "term": "frameshift variant", "soId": "SO:0001589"}, {"hgvsName": "NM_001258274.3:c.-668del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354615.2:c.-426del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354616.2:c.-431del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354617.2:c.-523del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354618.2:c.-523del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354619.2:c.-523del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354620.2:c.-89del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354621.2:c.-616del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354622.2:c.-729del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354624.2:c.-626del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354625.2:c.-529del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354626.2:c.-626del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354627.2:c.-626del", "term": "5 prime UTR variant", "soId": "SO:0001623"}, {"hgvsName": "NM_001354628.2:c.201del", "term": "frameshift variant", "soId": "SO:0001589"}, {"hgvsName": "NM_001354629.2:c.201del", "term": "frameshift variant", "soId": "SO:0001589"}, {"hgvsName": "NM_001354630.2:c.201del", "term": "frameshift variant", "soId": "SO:0001589"}, {"hgvsName": "NM_001354623.2:c.-723+2813del", "term": "intron variant", "soId": "SO:0001627"}, {"hgvsName": "NM_001258273.2:c.-517+3040del", "term": "intron variant", "soId": "SO:0001627"}], "submitted_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "modified_by": "b1f1a7c4-8987-4177-bcd0-c8a94f3b2df0", "date_created": "2022-11-21T22:04:33.050728", "dbSNPIds": ["587778968"], "preferredTitle": "NM_000249.4(MLH1):c.201del (p.Ile68SerfsTer24)", "hgvsNames": {"others": ["LRG_216:g.8354del", "NM_000249.3:c.201delG", "NG_007109.2:g.8354del", "NG_008418.1:g.1604del", "NM_000249.4:c.201del", "NP_000240.1:p.Ile68fs", "NM_001167617.3:c.-89del", "NM_001167618.3:c.-523del", "NM_001167619.3:c.-431del", "NM_001258271.2:c.201del", "NP_001245200.1:p.Ile68fs", "NM_001258274.3:c.-668del", "NM_001354615.2:c.-426del", "NM_001354616.2:c.-431del", "NM_001354617.2:c.-523del", "NM_001354618.2:c.-523del", "NM_001354619.2:c.-523del", "NM_001354620.2:c.-89del", "NM_001354621.2:c.-616del", "NM_001354622.2:c.-729del", "NM_001354624.2:c.-626del", "NM_001354625.2:c.-529del", "NM_001354626.2:c.-626del", "NM_001354627.2:c.-626del", "NM_001354628.2:c.201del", "NP_001341557.1:p.Ile68fs", "NM_001354629.2:c.201del", "NP_001341558.1:p.Ile68fs", "NM_001354630.2:c.201del", "NP_001341559.1:p.Ile68fs", "NM_001354623.2:c.-723+2813del", "NM_001258273.2:c.-517+3040del"], "GRCh38": "NC_000003.12:g.36996703del", "GRCh37": "NC_000003.11:g.37038194del"}, "item_type": "variant", "communityStandardTitle": "NM_000249.4(MLH1):c.201del (p.Ile68SerfsTer24)", "clinvarVariantId": "90000", "carId": "CA008214", "PK": "5ca2daf8-d209-4a59-8ce5-ade4ec5ef05a", "last_modified": "2022-11-21T22:04:33.050728"}, "PK": "ac7bbd73-138f-4cc7-b47f-93c4d47eac74", "modeInheritance": "Autosomal recessive inheritance (HP:0000007)", "last_modified": "2022-11-21T22:12:46.722016"}
Loading