From 474bdbca320c0a4f39ec428b17016e28cb5b21a0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 10 Oct 2024 16:38:21 -0400 Subject: [PATCH 01/35] note locations to update --- seqr/utils/search/add_data_utils.py | 2 +- seqr/utils/search/elasticsearch/es_search.py | 4 ++-- seqr/utils/search/hail_search_utils.py | 2 +- seqr/views/apis/individual_api.py | 6 ------ seqr/views/utils/anvil_metadata_utils.py | 4 ++-- seqr/views/utils/individual_utils.py | 7 ------- seqr/views/utils/pedigree_info_utils.py | 9 +++++---- ui/pages/Project/selectors.js | 2 +- ui/shared/components/icons/PedigreeIcon.jsx | 2 +- .../components/panel/variants/VariantIndividuals.jsx | 2 +- .../panel/view-pedigree-image/LazyPedigreeImagePanel.jsx | 2 +- ui/shared/utils/constants.js | 2 +- 12 files changed, 16 insertions(+), 28 deletions(-) diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index 5a3e0c221c..63eee40eab 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -140,7 +140,7 @@ def _upload_data_loading_files(projects: list[Project], user: User, file_path: s 'Project_GUID': F('family__project__guid'), 'Family_GUID': F('family__guid'), 'Family_ID': F('family__family_id'), 'Individual_ID': F('individual_id'), - 'Paternal_ID': F('father__individual_id'), 'Maternal_ID': F('mother__individual_id'), 'Sex': F('sex'), + 'Paternal_ID': F('father__individual_id'), 'Maternal_ID': F('mother__individual_id'), 'Sex': F('sex'), # TODO sex update }) annotations = {'project': F('family__project__guid'), **file_annotations} individual_filter = {'id__in': individual_ids} if individual_ids else {'family__project__in': projects} diff --git a/seqr/utils/search/elasticsearch/es_search.py b/seqr/utils/search/elasticsearch/es_search.py index 2c5b74e300..901f4267fc 100644 --- a/seqr/utils/search/elasticsearch/es_search.py +++ b/seqr/utils/search/elasticsearch/es_search.py @@ -878,7 +878,7 @@ def _is_matched_sample(family_guid, sample): {'sample_id': sample_id}, genotype_fields_config) genotypes[sample.individual.guid]['isRef'] = True genotypes[sample.individual.guid]['cn'] = \ - 1 if hit['contig'] == 'X' and sample.individual.sex == Individual.SEX_MALE else 2 + 1 if hit['contig'] == 'X' and sample.individual.sex == Individual.SEX_MALE else 2 # TODO sex update return family_guids, genotypes @@ -1367,7 +1367,7 @@ def _family_genotype_inheritance_filter(inheritance_mode, inheritance_filter, sa samples_q = Q('range', xpos={'gte': get_xpos('X', 1), 'lte': get_xpos('Y', 1)}) for individual in individuals: if individual_affected_status[individual.guid] == Individual.AFFECTED_STATUS_UNAFFECTED \ - and individual.sex == Individual.SEX_MALE: + and individual.sex == Individual.SEX_MALE: # TODO sex update individual_genotype_filter[individual.guid] = REF_REF is_sv_comp_het = inheritance_mode == COMPOUND_HET and 'samples' in index_fields diff --git a/seqr/utils/search/hail_search_utils.py b/seqr/utils/search/hail_search_utils.py index 0c93cf4377..85b43d457a 100644 --- a/seqr/utils/search/hail_search_utils.py +++ b/seqr/utils/search/hail_search_utils.py @@ -141,7 +141,7 @@ def _get_sample_data(samples, inheritance_filter=None, inheritance_mode=None, ** affected=F('individual__affected'), ) if inheritance_mode == X_LINKED_RECESSIVE: - sample_values['sex'] = F('individual__sex') + sample_values['sex'] = F('individual__sex') # TODO sex update sample_data = samples.order_by('guid').values('individual__individual_id', 'dataset_type', 'sample_type', **sample_values) custom_affected = (inheritance_filter or {}).pop('affected', None) diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index b6d82c7b3a..21a2ef166c 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -29,12 +29,6 @@ from seqr.views.utils.individual_utils import delete_individuals, add_or_update_individuals_and_families from seqr.views.utils.variant_utils import bulk_create_tagged_variants -_SEX_TO_EXPORTED_VALUE = dict(Individual.SEX_LOOKUP) -_SEX_TO_EXPORTED_VALUE['U'] = '' - -__AFFECTED_TO_EXPORTED_VALUE = dict(Individual.AFFECTED_STATUS_LOOKUP) -__AFFECTED_TO_EXPORTED_VALUE['U'] = '' - @login_and_policies_required def update_individual_handler(request, individual_guid): diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index d63df5c93c..855b7e3738 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -293,7 +293,7 @@ def _get_genotype_zygosity(genotype, individual=None, variant=None): num_alt = genotype.get('numAlt') cn = genotype.get('cn') if num_alt == 2 or cn == 0 or (cn != None and cn > 3): - return HEMI if (variant or {}).get('chrom') == 'X' and individual.sex == Individual.SEX_MALE else HOM_ALT + return HEMI if (variant or {}).get('chrom') == 'X' and individual.sex == Individual.SEX_MALE else HOM_ALT # TODO sex update if num_alt == 1 or cn == 1 or cn == 3: return HET return None @@ -413,7 +413,7 @@ def _get_subject_row(individual, has_dbgap_submission, airtable_metadata, indivi maternal_ids = individual_ids_map.get(individual.mother_id, ('', '')) subject_row = { 'participant_id': format_id(individual.individual_id), - 'sex': Individual.SEX_LOOKUP[individual.sex], + 'sex': Individual.SEX_LOOKUP[individual.sex], # TODO sex update 'reported_race': ANCESTRY_MAP.get(individual.population, ''), 'ancestry_detail': ANCESTRY_DETAIL_MAP.get(individual.population, ''), 'reported_ethnicity': ETHNICITY_MAP.get(individual.population, ''), diff --git a/seqr/views/utils/individual_utils.py b/seqr/views/utils/individual_utils.py index a9bc940dd4..94d93337e4 100644 --- a/seqr/views/utils/individual_utils.py +++ b/seqr/views/utils/individual_utils.py @@ -13,13 +13,6 @@ from seqr.views.utils.pedigree_info_utils import JsonConstants -_SEX_TO_EXPORTED_VALUE = dict(Individual.SEX_LOOKUP) -_SEX_TO_EXPORTED_VALUE['U'] = '' - -__AFFECTED_TO_EXPORTED_VALUE = dict(Individual.AFFECTED_STATUS_LOOKUP) -__AFFECTED_TO_EXPORTED_VALUE['U'] = '' - - def _get_record_family_id(record): # family id will be in different places in the json depending on whether it comes from a flat uploaded file or from the nested individual object return record.get(JsonConstants.FAMILY_ID_COLUMN) or record.get('family', {})['familyId'] diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py index 91b74f8566..44960c3399 100644 --- a/seqr/views/utils/pedigree_info_utils.py +++ b/seqr/views/utils/pedigree_info_utils.py @@ -128,6 +128,7 @@ def _parse_pedigree_table_json(project, rows, header=None, column_map=None, erro def _parse_sex(sex): + # TODO sex update if sex == '1' or sex.upper().startswith('M'): return 'M' elif sex == '2' or sex.upper().startswith('F'): @@ -290,7 +291,7 @@ def validate_fam_file_records(project, records, fail_on_warnings=False, errors=N errors.append(f'{individual_id} already has loaded data and cannot be moved to a different family') # check proband relationship has valid gender - if r.get(JsonConstants.PROBAND_RELATIONSHIP) and r.get(JsonConstants.SEX_COLUMN): + if r.get(JsonConstants.PROBAND_RELATIONSHIP) and r.get(JsonConstants.SEX_COLUMN): # TODO sex update invalid_choices = {} if r[JsonConstants.SEX_COLUMN] == Individual.SEX_MALE: invalid_choices = Individual.FEMALE_RELATIONSHIP_CHOICES @@ -365,7 +366,7 @@ def _validate_parent(row, parent_id_type, parent_id_field, expected_sex, individ errors.append('{} is recorded as their own {}'.format(parent_id, parent_id_type)) # is father male and mother female? - if JsonConstants.SEX_COLUMN in records_by_id[parent_id]: + if JsonConstants.SEX_COLUMN in records_by_id[parent_id]: # TODO sex update actual_sex = records_by_id[parent_id][JsonConstants.SEX_COLUMN] if actual_sex != expected_sex: actual_sex_label = dict(Individual.SEX_CHOICES)[actual_sex] @@ -1035,7 +1036,7 @@ class DSMConstants: MALE_SEX = 'MALE' FEMALE_SEX = 'FEMALE' - RELATIONSHIP_MAP = { + RELATIONSHIP_MAP = { # TODO sex update 'MYSELF': {MALE_SEX: 'Myself (male)', FEMALE_SEX: 'Myself (female)', PREFER_NOT_ANSWER: 'Myself (unspecified sex)'}, 'CHILD': {MALE_SEX: 'Son', FEMALE_SEX: 'Daughter', PREFER_NOT_ANSWER: 'Child (unspecified sex)'}, 'SIBLING': {MALE_SEX: 'Brother', FEMALE_SEX: 'Sister', PREFER_NOT_ANSWER: 'Sibling (unspecified sex)'}, @@ -1105,7 +1106,7 @@ class DSMConstants: OTHER_RELATIVES: 'RELATIVE', } - RELATIVE_SEX_MAP = { + RELATIVE_SEX_MAP = { # TODO sex update SIBLINGS: {'MALE': 'Brother', 'FEMALE': 'Sister', 'Other': 'Sibling (unspecified sex)'}, CHILDREN: {'MALE': 'Son', 'FEMALE': 'Daughter', 'Other': 'Child (unspecified sex)'}, OTHER_RELATIVES: {'MALE': 'Male', 'FEMALE': 'Female', 'Other': 'unspecified sex'}, diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index c619ef3b3b..f7126f56ab 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -632,7 +632,7 @@ export const getMmeDefaultContactEmail = createSelector( const individualOption = ({ individualGuid, displayName }) => ({ value: individualGuid, text: displayName }) -export const getParentOptionsByIndividual = createSelector( +export const getParentOptionsByIndividual = createSelector( // TODO sex update getSortedIndividualsByFamily, individualsByFamily => Object.values(individualsByFamily).reduce((acc, individuals) => ({ ...acc, diff --git a/ui/shared/components/icons/PedigreeIcon.jsx b/ui/shared/components/icons/PedigreeIcon.jsx index 16becb6eaa..29db239889 100644 --- a/ui/shared/components/icons/PedigreeIcon.jsx +++ b/ui/shared/components/icons/PedigreeIcon.jsx @@ -33,7 +33,7 @@ const ICON_LOOKUP = { } const PedigreeIcon = React.memo((props) => { - const iconProps = ICON_LOOKUP[`${props.sex}${props.affected}`] + const iconProps = ICON_LOOKUP[`${props.sex}${props.affected}`] // TODO sex update return ( individual.sex === 'M' && (variant.chrom === 'X' || variant.chrom === 'Y') && + (variant, individual) => individual.sex === 'M' && (variant.chrom === 'X' || variant.chrom === 'Y') && // TODO sex update PAR_REGIONS[variant.genomeVersion][variant.chrom].every(region => variant.pos < region[0] || variant.pos > region[1]) const missingParentVariant = variant => (parentGuid) => { diff --git a/ui/shared/components/panel/view-pedigree-image/LazyPedigreeImagePanel.jsx b/ui/shared/components/panel/view-pedigree-image/LazyPedigreeImagePanel.jsx index 8e5978e652..0ed970ade5 100644 --- a/ui/shared/components/panel/view-pedigree-image/LazyPedigreeImagePanel.jsx +++ b/ui/shared/components/panel/view-pedigree-image/LazyPedigreeImagePanel.jsx @@ -73,7 +73,7 @@ const EDIT_INDIVIDUAL_FIELDS = [ const INDIVIDUAL_FIELD_MAP = { name: 'individualGuid', label: 'displayName', - sex: 'sex', + sex: 'sex', // TODO sex update affected: 'affected', yob: 'birthYear', mother: 'maternalGuid', diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 3415a1287e..53212a1dc9 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -395,7 +395,7 @@ export const CATEGORY_FAMILY_FILTERS = { // INDIVIDUAL FIELDS -export const SEX_OPTIONS = [ +export const SEX_OPTIONS = [ // TODO sex update { value: 'M', text: 'Male' }, { value: 'F', text: 'Female' }, { value: 'U', text: '?' }, From 4e2ac75ce4d343cd5561176b1132a711e015b70f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 10 Oct 2024 16:45:13 -0400 Subject: [PATCH 02/35] clean up --- seqr/models.py | 1 - seqr/views/utils/anvil_metadata_utils.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/seqr/models.py b/seqr/models.py index e571e184fa..e58e84c8f4 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -583,7 +583,6 @@ class Individual(ModelWithGUID): (UNSOLVED, 'Unsolved'), ] - SEX_LOOKUP = dict(SEX_CHOICES) AFFECTED_STATUS_LOOKUP = dict(AFFECTED_STATUS_CHOICES) CASE_REVIEW_STATUS_LOOKUP = dict(CASE_REVIEW_STATUS_CHOICES) CASE_REVIEW_STATUS_REVERSE_LOOKUP = {name.lower(): key for key, name in CASE_REVIEW_STATUS_CHOICES} diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 855b7e3738..931809e426 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -413,7 +413,7 @@ def _get_subject_row(individual, has_dbgap_submission, airtable_metadata, indivi maternal_ids = individual_ids_map.get(individual.mother_id, ('', '')) subject_row = { 'participant_id': format_id(individual.individual_id), - 'sex': Individual.SEX_LOOKUP[individual.sex], # TODO sex update + 'sex': dict(Individual.SEX_CHOICES)[individual.sex], # TODO sex update 'reported_race': ANCESTRY_MAP.get(individual.population, ''), 'ancestry_detail': ANCESTRY_DETAIL_MAP.get(individual.population, ''), 'reported_ethnicity': ETHNICITY_MAP.get(individual.population, ''), From 79ca3bb9df846f965cbdfec5391231d50d30d3a7 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 10 Oct 2024 16:52:12 -0400 Subject: [PATCH 03/35] update sex options --- seqr/migrations/0076_alter_individual_sex.py | 18 ++++++++++++++++++ seqr/models.py | 6 +++++- 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 seqr/migrations/0076_alter_individual_sex.py diff --git a/seqr/migrations/0076_alter_individual_sex.py b/seqr/migrations/0076_alter_individual_sex.py new file mode 100644 index 0000000000..93f5967eac --- /dev/null +++ b/seqr/migrations/0076_alter_individual_sex.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.13 on 2024-10-10 20:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('seqr', '0075_alter_individual_primary_biosample'), + ] + + operations = [ + migrations.AlterField( + model_name='individual', + name='sex', + field=models.CharField(choices=[('M', 'Male'), ('F', 'Female'), ('U', 'Unknown'), ('XXY', 'XXY'), ('XYY', 'XYY'), ('XXX', 'XXX'), ('X0', 'X0')], default='U', max_length=3), + ), + ] diff --git a/seqr/models.py b/seqr/models.py index e58e84c8f4..33a0b2b303 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -447,10 +447,14 @@ class Individual(ModelWithGUID): SEX_MALE = 'M' SEX_FEMALE = 'F' SEX_UNKNOWN = 'U' + MALE_ANEUPLOIDIES = ['XXY', 'XYY'] + FEMALE_ANEUPLOIDIES = ['XXX', 'X0'] SEX_CHOICES = ( (SEX_MALE, 'Male'), ('F', 'Female'), ('U', 'Unknown'), + *[(sex, sex) for sex in MALE_ANEUPLOIDIES], + *[(sex, sex) for sex in FEMALE_ANEUPLOIDIES], ) AFFECTED_STATUS_AFFECTED = 'A' @@ -602,7 +606,7 @@ class Individual(ModelWithGUID): mother = models.ForeignKey('seqr.Individual', null=True, blank=True, on_delete=models.SET_NULL, related_name='maternal_children') father = models.ForeignKey('seqr.Individual', null=True, blank=True, on_delete=models.SET_NULL, related_name='paternal_children') - sex = models.CharField(max_length=1, choices=SEX_CHOICES, default='U') + sex = models.CharField(max_length=3, choices=SEX_CHOICES, default='U') affected = models.CharField(max_length=1, choices=AFFECTED_STATUS_CHOICES, default=AFFECTED_STATUS_UNKNOWN) # TODO once sample and individual ids are fully decoupled no reason to maintain this field From dfcdece12b326b3c53d7938c18e858a926756d8c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 10 Oct 2024 16:56:10 -0400 Subject: [PATCH 04/35] add test case --- seqr/fixtures/1kg_project.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index d0c79afaed..b5d090221d 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -444,7 +444,7 @@ "individual_id": "NA19675_1", "mother_id": 3, "father_id": 2, - "sex": "M", + "sex": "XXY", "affected": "A", "display_name": "", "notes": "", From c51ea06655677977b03bcaa06316f88be4e0eb5b Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 10 Oct 2024 17:12:38 -0400 Subject: [PATCH 05/35] add aneuploidy data to loading pedigrees --- seqr/utils/search/add_data_utils.py | 2 +- seqr/views/apis/data_manager_api_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index 63eee40eab..5a3e0c221c 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -140,7 +140,7 @@ def _upload_data_loading_files(projects: list[Project], user: User, file_path: s 'Project_GUID': F('family__project__guid'), 'Family_GUID': F('family__guid'), 'Family_ID': F('family__family_id'), 'Individual_ID': F('individual_id'), - 'Paternal_ID': F('father__individual_id'), 'Maternal_ID': F('mother__individual_id'), 'Sex': F('sex'), # TODO sex update + 'Paternal_ID': F('father__individual_id'), 'Maternal_ID': F('mother__individual_id'), 'Sex': F('sex'), }) annotations = {'project': F('family__project__guid'), **file_annotations} individual_filter = {'id__in': individual_ids} if individual_ids else {'family__project__in': projects} diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 3429db7123..9b25cf4500 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -390,7 +390,7 @@ PEDIGREE_HEADER = ['Project_GUID', 'Family_GUID', 'Family_ID', 'Individual_ID', 'Paternal_ID', 'Maternal_ID', 'Sex'] EXPECTED_PEDIGREE_ROWS = [ - ['R0001_1kg', 'F000001_1', '1', 'NA19675_1', 'NA19678', 'NA19679', 'M'], + ['R0001_1kg', 'F000001_1', '1', 'NA19675_1', 'NA19678', 'NA19679', 'XXY'], ['R0001_1kg', 'F000001_1', '1', 'NA19678', '', '', 'M'], ['R0001_1kg', 'F000001_1', '1', 'NA19679', '', '', 'F'], ['R0001_1kg', 'F000002_2', '2', 'HG00731', 'HG00732', 'HG00733', 'F'], From fd8d556d6eaffd67d19ee2124ce690035169ed23 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 10 Oct 2024 17:18:44 -0400 Subject: [PATCH 06/35] report sex aneuploidy --- seqr/views/apis/report_api_tests.py | 4 ++-- seqr/views/utils/anvil_metadata_utils.py | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 7f9fc0b255..37db2aab7b 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -525,7 +525,7 @@ 'missing_variant_case', ], [ 'Broad_NA19675_1', 'Broad_1kg project nme with unide', 'BROAD', 'HMB', 'Yes', 'IKBKAP|CCDC102B|CMA - normal', - '34415322', 'Broad_1', 'Broad_NA19678', 'Broad_NA19679', '', 'Self', '', 'Male', '', + '34415322', 'Broad_1', 'Broad_NA19678', 'Broad_NA19679', '', 'Self', '', 'Male', 'XXY', 'Middle Eastern or North African', '', '', '21', 'Affected', 'myopathy', '18', 'Unsolved', 'No', ], [ 'Broad_HG00731', 'Broad_1kg project nme with unide', 'BROAD', 'HMB', '', '', '', 'Broad_2', 'Broad_HG00732', @@ -874,7 +874,7 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo [c for c in PARTICIPANT_TABLE[0] if c not in {'pmid_id', 'ancestry_detail', 'age_at_last_observation', 'missing_variant_case'}], [ 'Broad_NA19675_1', 'Broad_1kg project nme with unide', 'BROAD', 'HMB', 'Yes', 'IKBKAP|CCDC102B|CMA - normal', - 'Broad_1', 'Broad_NA19678', 'Broad_NA19679', '', 'Self', '', 'Male', '', 'Middle Eastern or North African', + 'Broad_1', 'Broad_NA19678', 'Broad_NA19679', '', 'Self', '', 'Male', 'XXY', 'Middle Eastern or North African', '', 'Affected', 'myopathy', '18', 'Unsolved', ], [ 'Broad_NA19678', 'Broad_1kg project nme with unide', 'BROAD', 'HMB', '', '', 'Broad_1', '0', '0', '', '', diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 931809e426..ba5f4698c2 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -411,9 +411,18 @@ def _get_transcript_field(field, config, transcript): def _get_subject_row(individual, has_dbgap_submission, airtable_metadata, individual_ids_map, get_additional_individual_fields, format_id): paternal_ids = individual_ids_map.get(individual.father_id, ('', '')) maternal_ids = individual_ids_map.get(individual.mother_id, ('', '')) + sex = individual.sex + sex_detail = None + if sex in Individual.MALE_ANEUPLOIDIES: + sex_detail = sex + sex = Individual.SEX_MALE + elif sex in Individual.FEMALE_ANEUPLOIDIES: + sex_detail = sex + sex = Individual.SEX_FEMALE subject_row = { 'participant_id': format_id(individual.individual_id), - 'sex': dict(Individual.SEX_CHOICES)[individual.sex], # TODO sex update + 'sex': dict(Individual.SEX_CHOICES)[sex], + 'sex_detail': sex_detail, 'reported_race': ANCESTRY_MAP.get(individual.population, ''), 'ancestry_detail': ANCESTRY_DETAIL_MAP.get(individual.population, ''), 'reported_ethnicity': ETHNICITY_MAP.get(individual.population, ''), From 3362fc811862388aae4bb058b645554fa43f04ae Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 10 Oct 2024 17:22:54 -0400 Subject: [PATCH 07/35] fix sex search determination --- seqr/models.py | 3 ++- seqr/utils/search/elasticsearch/es_search.py | 4 ++-- seqr/utils/search/hail_search_utils.py | 2 +- seqr/views/utils/anvil_metadata_utils.py | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/seqr/models.py b/seqr/models.py index 33a0b2b303..0cf5fa8dda 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -447,8 +447,9 @@ class Individual(ModelWithGUID): SEX_MALE = 'M' SEX_FEMALE = 'F' SEX_UNKNOWN = 'U' - MALE_ANEUPLOIDIES = ['XXY', 'XYY'] FEMALE_ANEUPLOIDIES = ['XXX', 'X0'] + MALE_ANEUPLOIDIES = ['XXY', 'XYY'] + MALE_SEXES = [SEX_MALE] + MALE_ANEUPLOIDIES SEX_CHOICES = ( (SEX_MALE, 'Male'), ('F', 'Female'), diff --git a/seqr/utils/search/elasticsearch/es_search.py b/seqr/utils/search/elasticsearch/es_search.py index 901f4267fc..2c5b74e300 100644 --- a/seqr/utils/search/elasticsearch/es_search.py +++ b/seqr/utils/search/elasticsearch/es_search.py @@ -878,7 +878,7 @@ def _is_matched_sample(family_guid, sample): {'sample_id': sample_id}, genotype_fields_config) genotypes[sample.individual.guid]['isRef'] = True genotypes[sample.individual.guid]['cn'] = \ - 1 if hit['contig'] == 'X' and sample.individual.sex == Individual.SEX_MALE else 2 # TODO sex update + 1 if hit['contig'] == 'X' and sample.individual.sex == Individual.SEX_MALE else 2 return family_guids, genotypes @@ -1367,7 +1367,7 @@ def _family_genotype_inheritance_filter(inheritance_mode, inheritance_filter, sa samples_q = Q('range', xpos={'gte': get_xpos('X', 1), 'lte': get_xpos('Y', 1)}) for individual in individuals: if individual_affected_status[individual.guid] == Individual.AFFECTED_STATUS_UNAFFECTED \ - and individual.sex == Individual.SEX_MALE: # TODO sex update + and individual.sex == Individual.SEX_MALE: individual_genotype_filter[individual.guid] = REF_REF is_sv_comp_het = inheritance_mode == COMPOUND_HET and 'samples' in index_fields diff --git a/seqr/utils/search/hail_search_utils.py b/seqr/utils/search/hail_search_utils.py index c9d89d4a92..5433dc4650 100644 --- a/seqr/utils/search/hail_search_utils.py +++ b/seqr/utils/search/hail_search_utils.py @@ -141,7 +141,7 @@ def _get_sample_data(samples, inheritance_filter=None, inheritance_mode=None, ** affected=F('individual__affected'), ) if inheritance_mode == X_LINKED_RECESSIVE: - sample_values['is_male'] = Case(When(individual__sex=Individual.SEX_MALE, then=True), default=False) # TODO sex update + sample_values['is_male'] = Case(When(individual__sex__in=Individual.MALE_SEXES, then=True), default=False) sample_data = samples.order_by('guid').values('individual__individual_id', 'dataset_type', 'sample_type', **sample_values) custom_affected = (inheritance_filter or {}).pop('affected', None) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index ba5f4698c2..baa13b89a6 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -293,7 +293,7 @@ def _get_genotype_zygosity(genotype, individual=None, variant=None): num_alt = genotype.get('numAlt') cn = genotype.get('cn') if num_alt == 2 or cn == 0 or (cn != None and cn > 3): - return HEMI if (variant or {}).get('chrom') == 'X' and individual.sex == Individual.SEX_MALE else HOM_ALT # TODO sex update + return HEMI if (variant or {}).get('chrom') == 'X' and individual.sex in Individual.MALE_SEXES else HOM_ALT if num_alt == 1 or cn == 1 or cn == 3: return HET return None From 87afc6974840c651ff6fb6a2f823a77029f054fd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Thu, 10 Oct 2024 17:29:11 -0400 Subject: [PATCH 08/35] better test updates --- seqr/fixtures/1kg_project.json | 2 +- seqr/views/apis/data_manager_api_tests.py | 2 +- seqr/views/apis/report_api_tests.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/seqr/fixtures/1kg_project.json b/seqr/fixtures/1kg_project.json index b5d090221d..3b737b363b 100644 --- a/seqr/fixtures/1kg_project.json +++ b/seqr/fixtures/1kg_project.json @@ -536,7 +536,7 @@ "individual_id": "HG00731", "mother_id": 6, "father_id": 5, - "sex": "F", + "sex": "X0", "affected": "A", "proband_relationship": "S", "display_name": "HG00731_a", diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 9b25cf4500..e79dab6b07 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -393,7 +393,7 @@ ['R0001_1kg', 'F000001_1', '1', 'NA19675_1', 'NA19678', 'NA19679', 'XXY'], ['R0001_1kg', 'F000001_1', '1', 'NA19678', '', '', 'M'], ['R0001_1kg', 'F000001_1', '1', 'NA19679', '', '', 'F'], - ['R0001_1kg', 'F000002_2', '2', 'HG00731', 'HG00732', 'HG00733', 'F'], + ['R0001_1kg', 'F000002_2', '2', 'HG00731', 'HG00732', 'HG00733', 'X0'], ] PROJECT_OPTION = { diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 37db2aab7b..52dbc9808e 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -529,7 +529,7 @@ 'Middle Eastern or North African', '', '', '21', 'Affected', 'myopathy', '18', 'Unsolved', 'No', ], [ 'Broad_HG00731', 'Broad_1kg project nme with unide', 'BROAD', 'HMB', '', '', '', 'Broad_2', 'Broad_HG00732', - 'Broad_HG00733', '', 'Self', '', 'Female', '', '', 'Hispanic or Latino', 'Other', '', 'Affected', + 'Broad_HG00733', '', 'Self', '', 'Female', 'X0', '', 'Hispanic or Latino', 'Other', '', 'Affected', 'microcephaly; seizures', '', 'Unsolved', 'No', ], [ 'Broad_HG00732', 'Broad_1kg project nme with unide', 'BROAD', 'HMB', '', '', '', 'Broad_2', '0', '0', '', @@ -881,7 +881,7 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo '', 'Male', '', '', '', 'Unaffected', 'myopathy', '', 'Unaffected', ], [ 'Broad_HG00731', 'Broad_1kg project nme with unide', 'BROAD', 'HMB', '', '', 'Broad_2', 'Broad_HG00732', - 'Broad_HG00733', '', 'Self', '', 'Female', '', '', 'Hispanic or Latino', 'Affected', + 'Broad_HG00733', '', 'Self', '', 'Female', 'X0', '', 'Hispanic or Latino', 'Affected', 'microcephaly; seizures', '', 'Unsolved', ]], additional_calls=10) self._assert_expected_file(read_file, [READ_TABLE_HEADER, [ From 0b2aa5c5026e7633a21ab71cc019a2bc08bb7085 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 11 Oct 2024 10:52:01 -0400 Subject: [PATCH 09/35] fix test fixture --- seqr/views/apis/anvil_workspace_api_tests.py | 2 +- seqr/views/apis/individual_api_tests.py | 2 +- seqr/views/apis/variant_search_api_tests.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/seqr/views/apis/anvil_workspace_api_tests.py b/seqr/views/apis/anvil_workspace_api_tests.py index 36490f516a..f5db4e7859 100644 --- a/seqr/views/apis/anvil_workspace_api_tests.py +++ b/seqr/views/apis/anvil_workspace_api_tests.py @@ -730,7 +730,7 @@ def _assert_valid_operation(self, project, test_add_data=True): ['R0001_1kg', 'F000001_1', '1', 'NA19675_1', 'NA19678', '', 'F'], ['R0001_1kg', 'F000001_1', '1', 'NA19678', '', '', 'M'], ['R0001_1kg', 'F000001_1', '1', 'NA19679', '', '', 'F'], - ['R0001_1kg', 'F000002_2', '2', 'HG00731', 'HG00732', 'HG00733', 'F'], + ['R0001_1kg', 'F000002_2', '2', 'HG00731', 'HG00732', 'HG00733', 'X0'], ['R0001_1kg', 'F000002_2', '2', 'HG00732', '', '', 'M'], ['R0001_1kg', 'F000002_2', '2', 'HG00733', '', '', 'F'], ['R0001_1kg', 'F000003_3', '3', 'NA20870', '', '', 'M'], diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index cae1f5f062..66f7451c2c 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -899,7 +899,7 @@ def _is_expected_individuals_metadata_upload(self, response, expected_families=F response_json['individualsByGuid']['I000001_na19675']['absentFeatures'], [{'id': 'HP:0012469', 'category': 'HP:0025031', 'label': 'Infantile spasms'}] ) - self.assertEqual(response_json['individualsByGuid']['I000001_na19675']['sex'], 'M') + self.assertEqual(response_json['individualsByGuid']['I000001_na19675']['sex'], 'XXY') self.assertEqual(response_json['individualsByGuid']['I000001_na19675']['birthYear'], 2000) self.assertTrue(response_json['individualsByGuid']['I000001_na19675']['affectedRelatives']) self.assertEqual(response_json['individualsByGuid']['I000001_na19675']['onsetAge'], 'J') diff --git a/seqr/views/apis/variant_search_api_tests.py b/seqr/views/apis/variant_search_api_tests.py index cf9576bf6e..eef51399c7 100644 --- a/seqr/views/apis/variant_search_api_tests.py +++ b/seqr/views/apis/variant_search_api_tests.py @@ -836,7 +836,7 @@ def test_variant_lookup(self, mock_variant_lookup): 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }, 'I2_F0_1-10439-AC-A': { - 'affected': 'A', 'familyGuid': 'F0_1-10439-AC-A', 'individualGuid': 'I2_F0_1-10439-AC-A', 'sex': 'F', + 'affected': 'A', 'familyGuid': 'F0_1-10439-AC-A', 'individualGuid': 'I2_F0_1-10439-AC-A', 'sex': 'X0', 'features': [{'category': 'HP:0000707', 'label': '1 terms'}, {'category': 'HP:0001626', 'label': '1 terms'}], 'vlmContactEmail': 'test@broadinstitute.org,vlm@broadinstitute.org', }, From 27234f8f69de8453fac0fe549d9bfbd3758c7ef4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 11 Oct 2024 11:08:27 -0400 Subject: [PATCH 10/35] handle sex detail in summary data --- seqr/views/apis/summary_data_api_tests.py | 2 ++ ui/pages/SummaryData/components/IndividualMetadata.jsx | 2 +- ui/pages/SummaryData/components/IndividualMetadata.test.js | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/seqr/views/apis/summary_data_api_tests.py b/seqr/views/apis/summary_data_api_tests.py index dde9e17a00..15df14d46c 100644 --- a/seqr/views/apis/summary_data_api_tests.py +++ b/seqr/views/apis/summary_data_api_tests.py @@ -73,6 +73,7 @@ 'seqr_chosen_consequence-1': 'intron_variant', "ancestry": "Ashkenazi Jewish", "sex": "Female", + 'sex_detail': None, "chrom-1": "1", "alt-1": "T", "gene_of_interest-1": "OR4G11P", @@ -154,6 +155,7 @@ 'pmid_id': None, 'proband_relationship': 'Self', 'sex': 'Female', + 'sex_detail': None, 'solve_status': 'Unsolved', 'alt-1': 'T', 'chrom-1': '1', diff --git a/ui/pages/SummaryData/components/IndividualMetadata.jsx b/ui/pages/SummaryData/components/IndividualMetadata.jsx index 431c8c6d9b..7460a2a278 100644 --- a/ui/pages/SummaryData/components/IndividualMetadata.jsx +++ b/ui/pages/SummaryData/components/IndividualMetadata.jsx @@ -34,7 +34,7 @@ const CORE_COLUMNS = [ { name: 'paternal_id', secondaryExportColumn: 'paternal_guid' }, { name: 'maternal_id', secondaryExportColumn: 'maternal_guid' }, { name: 'proband_relationship' }, - { name: 'sex' }, + { name: 'sex', format: ({ sex, sex_detail: sexDetail }) => (sexDetail ? `${sex} (${sexDetail})` : sex) }, { name: 'ancestry' }, { name: 'affected_status' }, { name: 'hpo_present', style: { minWidth: '400px' } }, diff --git a/ui/pages/SummaryData/components/IndividualMetadata.test.js b/ui/pages/SummaryData/components/IndividualMetadata.test.js index f6abaaf0ac..1166facee6 100644 --- a/ui/pages/SummaryData/components/IndividualMetadata.test.js +++ b/ui/pages/SummaryData/components/IndividualMetadata.test.js @@ -52,6 +52,7 @@ const DATA = [ 'seqr_chosen_consequence-1': 'intron_variant', ancestry: 'Ashkenazi Jewish', sex: 'Female', + sex_detail: 'XXX', 'chrom-1': '1', 'alt-1': 'T', 'gene_of_interest-1': 'OR4G11P', @@ -99,7 +100,7 @@ test('IndividualMetadata render and export', () => { 'phenotype_contribution-2', 'partial_contribution_explained-2', 'notes-2', 'ClinGen_allele_ID-2']) expect(exportConfig.processRow(DATA[0])).toEqual([ 'Test Reprocessed Project', 'R0003_test', '12', 'F000012_12', 'NA20889', 'I000017_na20889', '', '', '', '', - 'Self', 'Female', 'Ashkenazi Jewish', 'Affected', 'HP:0011675 (Arrhythmia)|HP:0001509 ()', '', 'Yes', null, + 'Self', 'Female (XXX)', 'Ashkenazi Jewish', 'Affected', 'HP:0011675 (Arrhythmia)|HP:0001509 ()', '', 'Yes', null, 'OMIM:616126', 'Immunodeficiency 38', 'Autosomal recessive', null, null, undefined, 'Waiting for data', 'Tier 1', 'WES', '2017-02-05', '', undefined, 'Yes', 'NA20889_1_248367227', undefined, '1', 248367227, null, null, 'TC', 'T', 'OR4G11P', 'ENSG00000240361', 'intron_variant', 'ENST00000505820', 'c.3955G>A', 'c.1586-17C>G', 'Heterozygous', null, undefined, undefined, undefined, From cceadfa3d879212395930bbc7162a7b518e72af4 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 11 Oct 2024 11:15:58 -0400 Subject: [PATCH 11/35] clean up --- seqr/models.py | 1 + seqr/views/utils/anvil_metadata_utils.py | 2 +- seqr/views/utils/pedigree_info_utils.py | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/seqr/models.py b/seqr/models.py index 0cf5fa8dda..77f10893eb 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -588,6 +588,7 @@ class Individual(ModelWithGUID): (UNSOLVED, 'Unsolved'), ] + SEX_LOOKUP = dict(SEX_CHOICES) AFFECTED_STATUS_LOOKUP = dict(AFFECTED_STATUS_CHOICES) CASE_REVIEW_STATUS_LOOKUP = dict(CASE_REVIEW_STATUS_CHOICES) CASE_REVIEW_STATUS_REVERSE_LOOKUP = {name.lower(): key for key, name in CASE_REVIEW_STATUS_CHOICES} diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index baa13b89a6..43f19e54a5 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -421,7 +421,7 @@ def _get_subject_row(individual, has_dbgap_submission, airtable_metadata, indivi sex = Individual.SEX_FEMALE subject_row = { 'participant_id': format_id(individual.individual_id), - 'sex': dict(Individual.SEX_CHOICES)[sex], + 'sex': Individual.SEX_LOOKUP[sex], 'sex_detail': sex_detail, 'reported_race': ANCESTRY_MAP.get(individual.population, ''), 'ancestry_detail': ANCESTRY_DETAIL_MAP.get(individual.population, ''), diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py index 44960c3399..4e88ac5fdc 100644 --- a/seqr/views/utils/pedigree_info_utils.py +++ b/seqr/views/utils/pedigree_info_utils.py @@ -301,7 +301,7 @@ def validate_fam_file_records(project, records, fail_on_warnings=False, errors=N message = 'Invalid proband relationship "{relationship}" for {individual_id} with given gender {sex}'.format( relationship=Individual.RELATIONSHIP_LOOKUP[r[JsonConstants.PROBAND_RELATIONSHIP]], individual_id=individual_id, - sex=dict(Individual.SEX_CHOICES)[r[JsonConstants.SEX_COLUMN]] + sex=Individual.SEX_LOOKUP[r[JsonConstants.SEX_COLUMN]] ) if clear_invalid_values: r[JsonConstants.PROBAND_RELATIONSHIP] = None @@ -369,7 +369,7 @@ def _validate_parent(row, parent_id_type, parent_id_field, expected_sex, individ if JsonConstants.SEX_COLUMN in records_by_id[parent_id]: # TODO sex update actual_sex = records_by_id[parent_id][JsonConstants.SEX_COLUMN] if actual_sex != expected_sex: - actual_sex_label = dict(Individual.SEX_CHOICES)[actual_sex] + actual_sex_label = Individual.SEX_LOOKUP[actual_sex] errors.append( "%(parent_id)s is recorded as %(actual_sex_label)s sex and also as the %(parent_id_type)s of %(individual_id)s" % locals()) @@ -1036,7 +1036,7 @@ class DSMConstants: MALE_SEX = 'MALE' FEMALE_SEX = 'FEMALE' - RELATIONSHIP_MAP = { # TODO sex update + RELATIONSHIP_MAP = { 'MYSELF': {MALE_SEX: 'Myself (male)', FEMALE_SEX: 'Myself (female)', PREFER_NOT_ANSWER: 'Myself (unspecified sex)'}, 'CHILD': {MALE_SEX: 'Son', FEMALE_SEX: 'Daughter', PREFER_NOT_ANSWER: 'Child (unspecified sex)'}, 'SIBLING': {MALE_SEX: 'Brother', FEMALE_SEX: 'Sister', PREFER_NOT_ANSWER: 'Sibling (unspecified sex)'}, @@ -1106,7 +1106,7 @@ class DSMConstants: OTHER_RELATIVES: 'RELATIVE', } - RELATIVE_SEX_MAP = { # TODO sex update + RELATIVE_SEX_MAP = { SIBLINGS: {'MALE': 'Brother', 'FEMALE': 'Sister', 'Other': 'Sibling (unspecified sex)'}, CHILDREN: {'MALE': 'Son', 'FEMALE': 'Daughter', 'Other': 'Child (unspecified sex)'}, OTHER_RELATIVES: {'MALE': 'Male', 'FEMALE': 'Female', 'Other': 'unspecified sex'}, From 2167e7b27fe399efb14392f8ce9fba8d1e7469d3 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 11 Oct 2024 11:36:15 -0400 Subject: [PATCH 12/35] support aneuploidy in pedigree upload --- seqr/models.py | 1 + seqr/views/apis/individual_api_tests.py | 8 ++++++-- seqr/views/utils/pedigree_info_utils.py | 19 +++++++++---------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/seqr/models.py b/seqr/models.py index 77f10893eb..267ec0d247 100644 --- a/seqr/models.py +++ b/seqr/models.py @@ -449,6 +449,7 @@ class Individual(ModelWithGUID): SEX_UNKNOWN = 'U' FEMALE_ANEUPLOIDIES = ['XXX', 'X0'] MALE_ANEUPLOIDIES = ['XXY', 'XYY'] + FEMALE_SEXES = [SEX_FEMALE] + FEMALE_ANEUPLOIDIES MALE_SEXES = [SEX_MALE] + MALE_ANEUPLOIDIES SEX_CHOICES = ( (SEX_MALE, 'Male'), diff --git a/seqr/views/apis/individual_api_tests.py b/seqr/views/apis/individual_api_tests.py index 66f7451c2c..5c10820ee8 100644 --- a/seqr/views/apis/individual_api_tests.py +++ b/seqr/views/apis/individual_api_tests.py @@ -453,7 +453,8 @@ def test_individuals_table_handler_errors(self): rows += [ '"1" "NA19675_1" "NA19675_1" "F" "Father"', - '"2" "NA19675_2" "NA19675_1" "M" ""', + '"2" "NA19675_2" "NA19675_1" "XXX" "Nephew"', + '"2" "NA19677" "NA19675_2" "M" ""', ] response = self.client.post(individuals_url, { 'f': SimpleUploadedFile('test.tsv', '\n'.join(rows).encode('utf-8'))}) @@ -463,8 +464,10 @@ def test_individuals_table_handler_errors(self): 'Invalid proband relationship "Father" for NA19675_1 with given gender Female', 'NA19675_1 is recorded as their own father', 'NA19675_1 is recorded as Female sex and also as the father of NA19675_1', + 'Invalid proband relationship "Nephew" for NA19675_2 with given gender XXX', 'NA19675_1 is recorded as Female sex and also as the father of NA19675_2', 'NA19675_1 is recorded as the father of NA19675_2 but they have different family ids: 1 and 2', + 'NA19675_2 is recorded as XXX sex and also as the father of NA19677', 'NA19675_1 is included as 2 separate records, but must be unique within the project', ], 'warnings': [missing_entry_warning], @@ -477,7 +480,7 @@ def test_individuals_table_handler(self): data = 'Family ID Individual ID Previous Individual ID Paternal ID Maternal ID Sex Affected Status Notes familyNotes\n\ "1" " NA19675_1 " "" "NA19678 " "NA19679" "Female" "Affected" "A affected individual, test1-zsf" ""\n\ -"1" "NA19678" "" "" "" "Male" "Unaffected" "a individual note" ""\n\ +"1" "NA19678" "" "" "" "XXY" "Unaffected" "a individual note" ""\n\ "4" "NA20872_update" "NA20872" "" "" "Male" "Affected" "" ""\n\ "21" " HG00735" "" "" "" "Female" "Unaffected" "" "a new family""' @@ -525,6 +528,7 @@ def test_individuals_table_handler(self): self.assertEqual(response_json['individualsByGuid']['I000001_na19675']['sex'], 'F') self.assertEqual( response_json['individualsByGuid']['I000001_na19675']['notes'], 'A affected individual, test1-zsf') + self.assertEqual(response_json['individualsByGuid']['I000002_na19678']['sex'], 'XXY') self.assertEqual(response_json['individualsByGuid'][new_indiv_guid]['individualId'], 'HG00735') self.assertEqual(response_json['individualsByGuid'][new_indiv_guid]['sex'], 'F') self.assertEqual(response_json['individualsByGuid']['I000008_na20872']['individualId'], 'NA20872_update') diff --git a/seqr/views/utils/pedigree_info_utils.py b/seqr/views/utils/pedigree_info_utils.py index 4e88ac5fdc..87867602b7 100644 --- a/seqr/views/utils/pedigree_info_utils.py +++ b/seqr/views/utils/pedigree_info_utils.py @@ -128,14 +128,13 @@ def _parse_pedigree_table_json(project, rows, header=None, column_map=None, erro def _parse_sex(sex): - # TODO sex update if sex == '1' or sex.upper().startswith('M'): return 'M' elif sex == '2' or sex.upper().startswith('F'): return 'F' elif sex == '0' or not sex or sex.lower() in {'unknown', 'prefer_not_answer'}: return 'U' - return None + return Individual.SEX_LOOKUP.get(sex) def _parse_affected(affected): @@ -291,11 +290,11 @@ def validate_fam_file_records(project, records, fail_on_warnings=False, errors=N errors.append(f'{individual_id} already has loaded data and cannot be moved to a different family') # check proband relationship has valid gender - if r.get(JsonConstants.PROBAND_RELATIONSHIP) and r.get(JsonConstants.SEX_COLUMN): # TODO sex update + if r.get(JsonConstants.PROBAND_RELATIONSHIP) and r.get(JsonConstants.SEX_COLUMN): invalid_choices = {} - if r[JsonConstants.SEX_COLUMN] == Individual.SEX_MALE: + if r[JsonConstants.SEX_COLUMN] in Individual.MALE_SEXES: invalid_choices = Individual.FEMALE_RELATIONSHIP_CHOICES - elif r[JsonConstants.SEX_COLUMN] == Individual.SEX_FEMALE: + elif r[JsonConstants.SEX_COLUMN] in Individual.FEMALE_SEXES: invalid_choices = Individual.MALE_RELATIONSHIP_CHOICES if invalid_choices and r[JsonConstants.PROBAND_RELATIONSHIP] in invalid_choices: message = 'Invalid proband relationship "{relationship}" for {individual_id} with given gender {sex}'.format( @@ -311,8 +310,8 @@ def validate_fam_file_records(project, records, fail_on_warnings=False, errors=N # check maternal and paternal ids for consistency for parent in [ - ('father', JsonConstants.PATERNAL_ID_COLUMN, 'M'), - ('mother', JsonConstants.MATERNAL_ID_COLUMN, 'F') + ('father', JsonConstants.PATERNAL_ID_COLUMN, Individual.MALE_SEXES), + ('mother', JsonConstants.MATERNAL_ID_COLUMN, Individual.FEMALE_SEXES) ]: _validate_parent(r, *parent, individual_id, family_id, records_by_id, warnings, errors, clear_invalid_values) @@ -346,7 +345,7 @@ def get_valid_hpo_terms(records, additional_feature_columns=None): return set(HumanPhenotypeOntology.objects.filter(hpo_id__in=all_hpo_terms).values_list('hpo_id', flat=True)) -def _validate_parent(row, parent_id_type, parent_id_field, expected_sex, individual_id, family_id, records_by_id, warnings, errors, clear_invalid_values): +def _validate_parent(row, parent_id_type, parent_id_field, expected_sexes, individual_id, family_id, records_by_id, warnings, errors, clear_invalid_values): parent_id = row.get(parent_id_field) if not parent_id: return @@ -366,9 +365,9 @@ def _validate_parent(row, parent_id_type, parent_id_field, expected_sex, individ errors.append('{} is recorded as their own {}'.format(parent_id, parent_id_type)) # is father male and mother female? - if JsonConstants.SEX_COLUMN in records_by_id[parent_id]: # TODO sex update + if JsonConstants.SEX_COLUMN in records_by_id[parent_id]: actual_sex = records_by_id[parent_id][JsonConstants.SEX_COLUMN] - if actual_sex != expected_sex: + if actual_sex not in expected_sexes: actual_sex_label = Individual.SEX_LOOKUP[actual_sex] errors.append( "%(parent_id)s is recorded as %(actual_sex_label)s sex and also as the %(parent_id_type)s of %(individual_id)s" % locals()) From eaf500e7cd7e6dfba017b75e4a060bcea5563416 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 11 Oct 2024 12:56:03 -0400 Subject: [PATCH 13/35] fix sex options ui display --- .../Project/components/FamilyTable/IndividualRow.jsx | 9 +++++++++ ui/shared/utils/constants.js | 8 ++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx index c650625e85..0e312f9913 100644 --- a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx +++ b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx @@ -64,6 +64,15 @@ const IndividualContainer = styled.div` const PaddedRadioButtonGroup = styled(RadioButtonGroup)` padding: 10px; + + .button { + padding-left: 1em !important; + padding-right: 1em !important; + + &.labeled .label { + margin-left: 0px !important; + } + } ` const POPULATION_MAP = { diff --git a/ui/shared/utils/constants.js b/ui/shared/utils/constants.js index 53212a1dc9..cdfb22a4ae 100644 --- a/ui/shared/utils/constants.js +++ b/ui/shared/utils/constants.js @@ -395,10 +395,14 @@ export const CATEGORY_FAMILY_FILTERS = { // INDIVIDUAL FIELDS -export const SEX_OPTIONS = [ // TODO sex update +export const SEX_OPTIONS = [ { value: 'M', text: 'Male' }, { value: 'F', text: 'Female' }, { value: 'U', text: '?' }, + { value: 'XXY', text: 'Male (XXY)' }, + { value: 'XYY', text: 'Male (XYY)' }, + { value: 'XXX', text: 'Female (XXX)' }, + { value: 'X0', text: 'Female (X0)' }, ] export const SEX_LOOKUP = SEX_OPTIONS.reduce( @@ -477,7 +481,7 @@ export const INDIVIDUAL_FIELD_CONFIGS = { format: sex => SEX_LOOKUP[sex], width: 3, description: 'Male, Female, or Unknown', - formFieldProps: { component: RadioGroup, options: SEX_OPTIONS }, + formFieldProps: { component: Select, options: SEX_OPTIONS }, }, [INDIVIDUAL_FIELD_AFFECTED]: { label: 'Affected Status', From e47a0f4b1caf61da0532a02de4fa4c326b4f296e Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 11 Oct 2024 13:07:13 -0400 Subject: [PATCH 14/35] show sex onn individual row --- .../Project/components/FamilyTable/IndividualRow.jsx | 9 +++++++++ .../components/panel/variants/VariantIndividuals.jsx | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx index 0e312f9913..4a0688b18d 100644 --- a/ui/pages/Project/components/FamilyTable/IndividualRow.jsx +++ b/ui/pages/Project/components/FamilyTable/IndividualRow.jsx @@ -422,6 +422,8 @@ const CASE_REVIEW_FIELDS = [ ...INDIVIDUAL_FIELDS, ] +const INDIVIDUAL_FIELD_CONFIG_SEX = INDIVIDUAL_FIELD_CONFIGS[INDIVIDUAL_FIELD_SEX] + const NON_CASE_REVIEW_FIELDS = [ { component: OptionFieldView, @@ -438,6 +440,13 @@ const NON_CASE_REVIEW_FIELDS = [ isVisible: caseReviewStatus === CASE_REVIEW_STATUS_MORE_INFO_NEEDED, }), }, + { + field: INDIVIDUAL_FIELD_SEX, + fieldName: INDIVIDUAL_FIELD_CONFIG_SEX.label, + isEditable: false, + component: OptionFieldView, + tagOptions: INDIVIDUAL_FIELD_CONFIG_SEX.formFieldProps.options, + }, { field: 'analyteType', fieldName: 'Analyte Type', diff --git a/ui/shared/components/panel/variants/VariantIndividuals.jsx b/ui/shared/components/panel/variants/VariantIndividuals.jsx index 19efb51b85..d5e156124b 100644 --- a/ui/shared/components/panel/variants/VariantIndividuals.jsx +++ b/ui/shared/components/panel/variants/VariantIndividuals.jsx @@ -84,7 +84,7 @@ const PAR_REGIONS = { } const isHemiXVariant = - (variant, individual) => individual.sex === 'M' && (variant.chrom === 'X' || variant.chrom === 'Y') && // TODO sex update + (variant, individual) => individual.sex === 'M' && (variant.chrom === 'X' || variant.chrom === 'Y') && // TODO sex update, show warning for all X variants PAR_REGIONS[variant.genomeVersion][variant.chrom].every(region => variant.pos < region[0] || variant.pos > region[1]) const missingParentVariant = variant => (parentGuid) => { From 6a127f49bd94ee1c2736e07a419c02451dd70469 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 11 Oct 2024 13:25:13 -0400 Subject: [PATCH 15/35] fix aneuploidy pedigree display --- ui/shared/components/icons/PedigreeIcon.jsx | 4 ++-- .../LazyPedigreeImagePanel.jsx | 12 +++++++++--- ui/shared/utils/constants.js | 16 +++++++++++----- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/ui/shared/components/icons/PedigreeIcon.jsx b/ui/shared/components/icons/PedigreeIcon.jsx index 29db239889..04a35d8ec5 100644 --- a/ui/shared/components/icons/PedigreeIcon.jsx +++ b/ui/shared/components/icons/PedigreeIcon.jsx @@ -4,7 +4,7 @@ import PropTypes from 'prop-types' import { Icon, Popup } from 'semantic-ui-react' import styled from 'styled-components' -import { SEX_LOOKUP, AFFECTED_LOOKUP } from 'shared/utils/constants' +import { SEX_LOOKUP, SIMPLIFIED_SEX_LOOKUP, AFFECTED_LOOKUP } from 'shared/utils/constants' const RotatedIcon = styled(Icon)` transform: rotate(45deg); @@ -33,7 +33,7 @@ const ICON_LOOKUP = { } const PedigreeIcon = React.memo((props) => { - const iconProps = ICON_LOOKUP[`${props.sex}${props.affected}`] // TODO sex update + const iconProps = ICON_LOOKUP[`${SIMPLIFIED_SEX_LOOKUP[props.sex]}${props.affected}`] return ( ({ value, text: `Male (${value})` })), + ...FEMALE_ANEUPLOIDIES.map(value => ({ value, text: `Female (${value})` })), ] +export const SIMPLIFIED_SEX_LOOKUP = { + ...[SEX_MALE, ...MALE_ANEUPLOIDIES].reduce((acc, val) => ({ ...acc, [val]: SEX_MALE }), {}), + ...[SEX_FEMALE, ...FEMALE_ANEUPLOIDIES].reduce((acc, val) => ({ ...acc, [val]: SEX_FEMALE }), {}), +} + export const SEX_LOOKUP = SEX_OPTIONS.reduce( (acc, opt) => ({ ...acc, From c036abf560291e3361d60d460a58c9027fea1ebd Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 11 Oct 2024 13:27:07 -0400 Subject: [PATCH 16/35] fix parent lookup --- ui/pages/Project/selectors.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index f7126f56ab..1c14417e9a 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -12,6 +12,7 @@ import { INDIVIDUAL_HAS_DATA_FIELD, MME_TAG_NAME, TISSUE_DISPLAY, + SIMPLIFIED_SEX_LOOKUP, } from 'shared/utils/constants' import { toCamelcase, toSnakecase, snakecaseToTitlecase } from 'shared/utils/stringUtils' @@ -639,8 +640,8 @@ export const getParentOptionsByIndividual = createSelector( // TODO sex update ...individuals.reduce((indAcc, { individualGuid }) => ({ ...indAcc, [individualGuid]: { - M: individuals.filter(i => i.sex === 'M' && i.individualGuid !== individualGuid).map(individualOption), - F: individuals.filter(i => i.sex === 'F' && i.individualGuid !== individualGuid).map(individualOption), + M: individuals.filter(i => SIMPLIFIED_SEX_LOOKUP[i.sex] === 'M' && i.individualGuid !== individualGuid).map(individualOption), + F: individuals.filter(i => SIMPLIFIED_SEX_LOOKUP[i.sex] === 'F' && i.individualGuid !== individualGuid).map(individualOption), }, }), {}), }), {}), From aedd0581c0691ab411aa3b5f2cd611c57b80b447 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 11 Oct 2024 13:27:18 -0400 Subject: [PATCH 17/35] fix parent lookup --- ui/pages/Project/selectors.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/pages/Project/selectors.js b/ui/pages/Project/selectors.js index 1c14417e9a..703423596a 100644 --- a/ui/pages/Project/selectors.js +++ b/ui/pages/Project/selectors.js @@ -633,7 +633,7 @@ export const getMmeDefaultContactEmail = createSelector( const individualOption = ({ individualGuid, displayName }) => ({ value: individualGuid, text: displayName }) -export const getParentOptionsByIndividual = createSelector( // TODO sex update +export const getParentOptionsByIndividual = createSelector( getSortedIndividualsByFamily, individualsByFamily => Object.values(individualsByFamily).reduce((acc, individuals) => ({ ...acc, From 90c8497f06b59a82fd338c70f003c92d490bff5f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 11 Oct 2024 13:32:09 -0400 Subject: [PATCH 18/35] show warning for sex chromosome variant aneuploidy --- ui/shared/components/panel/variants/VariantIndividuals.jsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ui/shared/components/panel/variants/VariantIndividuals.jsx b/ui/shared/components/panel/variants/VariantIndividuals.jsx index d5e156124b..eada714968 100644 --- a/ui/shared/components/panel/variants/VariantIndividuals.jsx +++ b/ui/shared/components/panel/variants/VariantIndividuals.jsx @@ -11,6 +11,7 @@ import { INDIVIDUAL_FIELD_POP_FILTERS, INDIVIDUAL_FIELD_SV_FLAGS, INDIVIDUAL_FIELD_LOOKUP, + SIMPLIFIED_SEX_LOOKUP, } from 'shared/utils/constants' import BaseFieldView from '../view-fields/BaseFieldView' import PedigreeIcon from '../../icons/PedigreeIcon' @@ -84,7 +85,7 @@ const PAR_REGIONS = { } const isHemiXVariant = - (variant, individual) => individual.sex === 'M' && (variant.chrom === 'X' || variant.chrom === 'Y') && // TODO sex update, show warning for all X variants + (variant, individual) => SIMPLIFIED_SEX_LOOKUP[individual.sex] === 'M' && (variant.chrom === 'X' || variant.chrom === 'Y') && PAR_REGIONS[variant.genomeVersion][variant.chrom].every(region => variant.pos < region[0] || variant.pos > region[1]) const missingParentVariant = variant => (parentGuid) => { @@ -298,6 +299,10 @@ const Genotype = React.memo(({ variant, individual, isCompoundHet, genesById }) warnings.push('Common low heteroplasmy') } + if ((variant.chrom === 'X' || variant.chrom === 'Y') && SIMPLIFIED_SEX_LOOKUP[individual.sex] !== individual.sex) { + warnings.push(`Sex Aneuploidy - ${individual.sex}`) + } + const warning = warnings.join('. ') let previousCall From 423c95ac5ac15ee59cdba4761b1cbedb67e2c2a2 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Fri, 11 Oct 2024 14:26:20 -0400 Subject: [PATCH 19/35] fix matchmaersex --- matchmaker/models.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/matchmaker/models.py b/matchmaker/models.py index 36c676069f..3797c8a242 100644 --- a/matchmaker/models.py +++ b/matchmaker/models.py @@ -8,7 +8,10 @@ class MatchmakerSubmission(ModelWithGUID): - SEX_LOOKUP = {Individual.SEX_MALE: 'MALE', Individual.SEX_FEMALE: 'FEMALE'} + SEX_LOOKUP = { + **{sex: 'MALE' for sex in Individual.MALE_SEXES}, + **{sex: 'FEMALE' for sex in Individual.FEMALE_SEXES}, + } individual = models.OneToOneField(Individual, on_delete=models.PROTECT) From b6d9f42ca156f0cf1a682c8f36bf10797fafd642 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 15 Oct 2024 11:26:03 -0400 Subject: [PATCH 20/35] remove unneeded comp het filter --- hail_search/queries/base.py | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/hail_search/queries/base.py b/hail_search/queries/base.py index a04a2708a4..c132e954d8 100644 --- a/hail_search/queries/base.py +++ b/hail_search/queries/base.py @@ -624,14 +624,11 @@ def _annotate_families_inheritance( if genotype: entry_indices_by_gt[genotype][family_index].append(sample_index) - min_unaffected = None if inheritance_mode == COMPOUND_HET: family_unaffected_counts = [ len(i) for i in entry_indices_by_gt[INHERITANCE_FILTERS[COMPOUND_HET][UNAFFECTED_ID]].values() ] self.max_unaffected_samples = max(family_unaffected_counts) if family_unaffected_counts else 0 - if self.max_unaffected_samples > 1: - min_unaffected = min(family_unaffected_counts) for genotype, entry_indices in entry_indices_by_gt.items(): if not entry_indices: @@ -639,28 +636,16 @@ def _annotate_families_inheritance( entry_indices = hl.dict(entry_indices) ht = ht.annotate(**{ annotation: hl.enumerate(ht[entries_ht_field]).starmap( - lambda i, family_samples: self._valid_genotype_family_entries( - family_samples, entry_indices.get(i), genotype, min_unaffected - ) + lambda family_i, family_samples: hl.or_missing( + ~entry_indices.get(family_i).any( + lambda sample_i: ~self.GENOTYPE_QUERY_MAP[genotype](family_samples[sample_i].GT) + ), family_samples, + ), ) }) return ht - @classmethod - def _valid_genotype_family_entries(cls, entries: list, genotype_entry_indices, genotype: str, min_unaffected: int): - is_valid = hl.is_missing(genotype_entry_indices) | genotype_entry_indices.all( - lambda i: cls.GENOTYPE_QUERY_MAP[genotype](entries[i].GT) - ) - if min_unaffected is not None and genotype == HAS_REF: - unaffected_filter = genotype_entry_indices.any( - lambda i: cls.GENOTYPE_QUERY_MAP[REF_REF](entries[i].GT) - ) - if min_unaffected < 2: - unaffected_filter |= genotype_entry_indices.size() < 2 - is_valid &= unaffected_filter - return hl.or_missing(is_valid, entries) - def _get_family_passes_quality_filter(self, quality_filter, ht, **kwargs): quality_filter = quality_filter or {} From 5b732c7155668fec1791f7b19deaeb6e00ccf3ae Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Tue, 15 Oct 2024 11:41:34 -0400 Subject: [PATCH 21/35] undo failed logic negation --- hail_search/queries/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hail_search/queries/base.py b/hail_search/queries/base.py index c132e954d8..c16779a576 100644 --- a/hail_search/queries/base.py +++ b/hail_search/queries/base.py @@ -637,8 +637,8 @@ def _annotate_families_inheritance( ht = ht.annotate(**{ annotation: hl.enumerate(ht[entries_ht_field]).starmap( lambda family_i, family_samples: hl.or_missing( - ~entry_indices.get(family_i).any( - lambda sample_i: ~self.GENOTYPE_QUERY_MAP[genotype](family_samples[sample_i].GT) + ~entry_indices.contains(family_i) | entry_indices[family_i].all( + lambda sample_i: self.GENOTYPE_QUERY_MAP[genotype](family_samples[sample_i].GT) ), family_samples, ), ) From 2d8beff14cdb7cdfe19b2e728fbe5b71098c4c0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 20:00:42 +0000 Subject: [PATCH 22/35] Bump cookie and express in /ui Bumps [cookie](https://github.com/jshttp/cookie) and [express](https://github.com/expressjs/express). These dependencies needed to be updated together. Updates `cookie` from 0.6.0 to 0.7.1 - [Release notes](https://github.com/jshttp/cookie/releases) - [Commits](https://github.com/jshttp/cookie/compare/v0.6.0...v0.7.1) Updates `express` from 4.20.0 to 4.21.1 - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/4.21.1/History.md) - [Commits](https://github.com/expressjs/express/compare/4.20.0...4.21.1) --- updated-dependencies: - dependency-name: cookie dependency-type: indirect - dependency-name: express dependency-type: indirect ... Signed-off-by: dependabot[bot] --- ui/package-lock.json | 215 ++++++++++++++----------------------------- 1 file changed, 67 insertions(+), 148 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index a96ea05d9a..b400c69b00 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -4796,21 +4796,6 @@ "node": ">= 0.8" } }, - "node_modules/body-parser/node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", - "dev": true, - "dependencies": { - "side-channel": "^1.0.6" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/bonjour": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/bonjour/-/bonjour-3.5.0.tgz", @@ -5459,9 +5444,9 @@ } }, "node_modules/cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", "dev": true, "engines": { "node": ">= 0.6" @@ -7988,9 +7973,9 @@ } }, "node_modules/express": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/express/-/express-4.20.0.tgz", - "integrity": "sha512-pLdae7I6QqShF5PnNTCVn4hI91Dx0Grkn2+IAsMTgMIKuQVte2dN9PeGSSAME2FR8anOhVA62QDIUaWVfEXVLw==", + "version": "4.21.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", + "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", "dev": true, "dependencies": { "accepts": "~1.3.8", @@ -7998,14 +7983,14 @@ "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.6.0", + "cookie": "0.7.1", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.2.0", + "finalhandler": "1.3.1", "fresh": "0.5.2", "http-errors": "2.0.0", "merge-descriptors": "1.0.3", @@ -8014,11 +7999,11 @@ "parseurl": "~1.3.3", "path-to-regexp": "0.1.10", "proxy-addr": "~2.0.7", - "qs": "6.11.0", + "qs": "6.13.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "0.19.0", - "serve-static": "1.16.0", + "serve-static": "1.16.2", "setprototypeof": "1.2.0", "statuses": "2.0.1", "type-is": "~1.6.18", @@ -8248,13 +8233,13 @@ } }, "node_modules/finalhandler": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", - "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", "dev": true, "dependencies": { "debug": "2.6.9", - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "on-finished": "2.4.1", "parseurl": "~1.3.3", @@ -8265,6 +8250,15 @@ "node": ">= 0.8" } }, + "node_modules/finalhandler/node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/finalhandler/node_modules/statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -14558,12 +14552,12 @@ } }, "node_modules/qs": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", - "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", "dev": true, "dependencies": { - "side-channel": "^1.0.4" + "side-channel": "^1.0.6" }, "engines": { "node": ">=0.6" @@ -15983,63 +15977,24 @@ "dev": true }, "node_modules/serve-static": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.0.tgz", - "integrity": "sha512-pDLK8zwl2eKaYrs8mrPZBJua4hMplRWJ1tIFksVC3FtBEBnl8dxgeHtsaMS8DhS9i4fLObaon6ABoc4/hQGdPA==", + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", "dev": true, "dependencies": { - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.18.0" + "send": "0.19.0" }, "engines": { "node": ">= 0.8.0" } }, - "node_modules/serve-static/node_modules/depd": { + "node_modules/serve-static/node_modules/encodeurl": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/serve-static/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - }, - "node_modules/serve-static/node_modules/send": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", - "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", - "dev": true, - "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/serve-static/node_modules/statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", "dev": true, "engines": { "node": ">= 0.8" @@ -22863,15 +22818,6 @@ "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "dev": true - }, - "qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", - "dev": true, - "requires": { - "side-channel": "^1.0.6" - } } } }, @@ -23433,9 +23379,9 @@ } }, "cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", "dev": true }, "cookie-signature": { @@ -25537,9 +25483,9 @@ } }, "express": { - "version": "4.20.0", - "resolved": "https://registry.npmjs.org/express/-/express-4.20.0.tgz", - "integrity": "sha512-pLdae7I6QqShF5PnNTCVn4hI91Dx0Grkn2+IAsMTgMIKuQVte2dN9PeGSSAME2FR8anOhVA62QDIUaWVfEXVLw==", + "version": "4.21.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", + "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", "dev": true, "requires": { "accepts": "~1.3.8", @@ -25547,14 +25493,14 @@ "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.6.0", + "cookie": "0.7.1", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", - "finalhandler": "1.2.0", + "finalhandler": "1.3.1", "fresh": "0.5.2", "http-errors": "2.0.0", "merge-descriptors": "1.0.3", @@ -25563,11 +25509,11 @@ "parseurl": "~1.3.3", "path-to-regexp": "0.1.10", "proxy-addr": "~2.0.7", - "qs": "6.11.0", + "qs": "6.13.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "0.19.0", - "serve-static": "1.16.0", + "serve-static": "1.16.2", "setprototypeof": "1.2.0", "statuses": "2.0.1", "type-is": "~1.6.18", @@ -25766,13 +25712,13 @@ "requires": {} }, "finalhandler": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", - "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", "dev": true, "requires": { "debug": "2.6.9", - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "on-finished": "2.4.1", "parseurl": "~1.3.3", @@ -25780,6 +25726,12 @@ "unpipe": "~1.0.0" }, "dependencies": { + "encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "dev": true + }, "statuses": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", @@ -30850,12 +30802,12 @@ } }, "qs": { - "version": "6.11.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", - "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", "dev": true, "requires": { - "side-channel": "^1.0.4" + "side-channel": "^1.0.6" } }, "query-string": { @@ -32005,54 +31957,21 @@ } }, "serve-static": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.0.tgz", - "integrity": "sha512-pDLK8zwl2eKaYrs8mrPZBJua4hMplRWJ1tIFksVC3FtBEBnl8dxgeHtsaMS8DhS9i4fLObaon6ABoc4/hQGdPA==", + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", "dev": true, "requires": { - "encodeurl": "~1.0.2", + "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "parseurl": "~1.3.3", - "send": "0.18.0" + "send": "0.19.0" }, "dependencies": { - "depd": { + "encodeurl": { "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true - }, - "send": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", - "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", - "dev": true, - "requires": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" - } - }, - "statuses": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", - "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", "dev": true } } From 491cc63608a2bc9e2d6e882f1a0d1af51f43e8fe Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Wed, 16 Oct 2024 15:07:12 +0000 Subject: [PATCH 23/35] fix: requirements-dev.txt to reduce vulnerabilities The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-DJANGO-7435780 - https://snyk.io/vuln/SNYK-PYTHON-DJANGO-7436273 - https://snyk.io/vuln/SNYK-PYTHON-DJANGO-7436514 - https://snyk.io/vuln/SNYK-PYTHON-DJANGO-7436646 - https://snyk.io/vuln/SNYK-PYTHON-DJANGO-7642790 - https://snyk.io/vuln/SNYK-PYTHON-DJANGO-7642791 - https://snyk.io/vuln/SNYK-PYTHON-DJANGO-7642813 - https://snyk.io/vuln/SNYK-PYTHON-DJANGO-7642814 - https://snyk.io/vuln/SNYK-PYTHON-DJANGO-7886958 - https://snyk.io/vuln/SNYK-PYTHON-DJANGO-7886959 - https://snyk.io/vuln/SNYK-PYTHON-REQUESTS-6928867 - https://snyk.io/vuln/SNYK-PYTHON-SQLPARSE-6615674 - https://snyk.io/vuln/SNYK-PYTHON-ZIPP-7430899 From b58445a4eac382e143e504e86721095300cf9b08 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 16 Oct 2024 11:28:41 -0400 Subject: [PATCH 24/35] conditionally show dataset type selector for local installs --- .../DataManagement/components/LoadData.jsx | 98 ++++++++++++------- 1 file changed, 60 insertions(+), 38 deletions(-) diff --git a/ui/pages/DataManagement/components/LoadData.jsx b/ui/pages/DataManagement/components/LoadData.jsx index c9c24f24e8..4d88ec93b8 100644 --- a/ui/pages/DataManagement/components/LoadData.jsx +++ b/ui/pages/DataManagement/components/LoadData.jsx @@ -1,6 +1,9 @@ import React from 'react' +import PropTypes from 'prop-types' +import { connect } from 'react-redux' import { FormSpy } from 'react-final-form' +import { getUser } from 'redux/selectors' import { validators } from 'shared/components/form/FormHelpers' import FormWizard from 'shared/components/form/FormWizard' import { ButtonRadioGroup } from 'shared/components/form/Inputs' @@ -44,41 +47,49 @@ const LoadedProjectOptions = props => ( ) -const LOAD_DATA_PAGES = [ - { - fields: [ - { - name: 'filePath', - label: 'Callset File Path', - placeholder: 'gs://', - validate: validators.required, - }, - { - name: 'sampleType', - label: 'Sample Type', - component: ButtonRadioGroup, - options: [SAMPLE_TYPE_EXOME, SAMPLE_TYPE_GENOME].map(value => ({ value, text: value })), - validate: validators.required, - }, - { - name: 'datasetType', - label: 'Dataset Type', - component: ButtonRadioGroup, - options: [ - DATASET_TYPE_SNV_INDEL_CALLS, - DATASET_TYPE_SV_CALLS, - DATASET_TYPE_MITO_CALLS, - ].map(value => ({ value, text: value.replace('_', '/') })), - validate: validators.required, - }, - { - ...GENOME_VERSION_FIELD, - component: ButtonRadioGroup, - validate: validators.required, - }, - ], - submitUrl: '/api/data_management/validate_callset', - }, +const CALLSET_PAGE = { + fields: [ + { + name: 'filePath', + label: 'Callset File Path', + placeholder: 'gs://', + validate: validators.required, + }, + { + ...GENOME_VERSION_FIELD, + component: ButtonRadioGroup, + validate: validators.required, + }, + { + name: 'sampleType', + label: 'Sample Type', + component: ButtonRadioGroup, + options: [SAMPLE_TYPE_EXOME, SAMPLE_TYPE_GENOME].map(value => ({ value, text: value })), + validate: validators.required, + }, + ], + submitUrl: '/api/data_management/validate_callset', +} + +const MULTI_DATA_TYPE_CALLSET_PAGE = { + ...CALLSET_PAGE, + fields: [ + ...CALLSET_PAGE.fields, + { + name: 'datasetType', + label: 'Dataset Type', + component: ButtonRadioGroup, + options: [ + DATASET_TYPE_SNV_INDEL_CALLS, + DATASET_TYPE_SV_CALLS, + DATASET_TYPE_MITO_CALLS, + ].map(value => ({ value, text: value.replace('_', '/') })), + validate: validators.required, + }, + ], +} + +const ADDITIONAL_LOAD_DATA_PAGES = [ { fields: [ { @@ -91,15 +102,26 @@ const LOAD_DATA_PAGES = [ }, ] +const LOAD_DATA_PAGES = [CALLSET_PAGE, ...ADDITIONAL_LOAD_DATA_PAGES] +const MULTI_DATA_TYPE_LOAD_DATA_PAGES = [MULTI_DATA_TYPE_CALLSET_PAGE, ...ADDITIONAL_LOAD_DATA_PAGES] + const formatSubmitUrl = () => '/api/data_management/load_data' -const LoadData = () => ( +const LoadData = ({ user }) => ( ) -export default LoadData +LoadData.propTypes = { + user: PropTypes.object, +} + +const mapStateToProps = state => ({ + user: getUser(state), +}) + +export default connect(mapStateToProps)(LoadData) From e2e5250b869503824cd4eff43a3e0759412d71f0 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 16 Oct 2024 11:46:25 -0400 Subject: [PATCH 25/35] add skip validatioon option --- ui/pages/DataManagement/components/LoadData.jsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ui/pages/DataManagement/components/LoadData.jsx b/ui/pages/DataManagement/components/LoadData.jsx index 4d88ec93b8..23a8a57f88 100644 --- a/ui/pages/DataManagement/components/LoadData.jsx +++ b/ui/pages/DataManagement/components/LoadData.jsx @@ -6,7 +6,7 @@ import { FormSpy } from 'react-final-form' import { getUser } from 'redux/selectors' import { validators } from 'shared/components/form/FormHelpers' import FormWizard from 'shared/components/form/FormWizard' -import { ButtonRadioGroup } from 'shared/components/form/Inputs' +import { ButtonRadioGroup, InlineToggle } from 'shared/components/form/Inputs' import LoadOptionsSelect from 'shared/components/form/LoadOptionsSelect' import { SAMPLE_TYPE_EXOME, @@ -55,6 +55,12 @@ const CALLSET_PAGE = { placeholder: 'gs://', validate: validators.required, }, + { + name: 'skipValidation', + label: 'Skip Callset Validation', + component: InlineToggle, + asFormInput: true, + }, { ...GENOME_VERSION_FIELD, component: ButtonRadioGroup, From 6854c648be257015c1507bad1c7784b1e60a8026 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 16 Oct 2024 11:56:58 -0400 Subject: [PATCH 26/35] support updated load data fields --- seqr/utils/search/add_data_utils.py | 4 +++- seqr/views/apis/data_manager_api.py | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/seqr/utils/search/add_data_utils.py b/seqr/utils/search/add_data_utils.py index 5a3e0c221c..88a52a13aa 100644 --- a/seqr/utils/search/add_data_utils.py +++ b/seqr/utils/search/add_data_utils.py @@ -116,7 +116,7 @@ def format_email(sample_summary, project_link, num_new_samples): def prepare_data_loading_request(projects: list[Project], sample_type: str, dataset_type: str, genome_version: str, data_path: str, user: User, pedigree_dir: str, raise_pedigree_error: bool = False, - individual_ids: list[str] = None): + individual_ids: list[str] = None, skip_validation: bool = False): project_guids = sorted([p.guid for p in projects]) variables = { 'projects_to_run': project_guids, @@ -125,6 +125,8 @@ def prepare_data_loading_request(projects: list[Project], sample_type: str, data 'dataset_type': _dag_dataset_type(sample_type, dataset_type), 'reference_genome': GENOME_VERSION_LOOKUP[genome_version], } + if skip_validation: + variables['skip_validation'] = True file_path = _get_pedigree_path(pedigree_dir, genome_version, sample_type, dataset_type) _upload_data_loading_files(projects, user, file_path, individual_ids, raise_pedigree_error) return variables, file_path diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index f5ccabc62a..f54ee2af7f 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -447,8 +447,9 @@ def load_phenotype_prioritization_data(request): @pm_or_data_manager_required def validate_callset(request): request_json = json.loads(request.body) + dataset_type = request_json.get('datasetType', Sample.DATASET_TYPE_VARIANT_CALLS) validate_vcf_exists( - _callset_path(request_json), request.user, allowed_exts=DATA_TYPE_FILE_EXTS.get(request_json['datasetType']), + _callset_path(request_json), request.user, allowed_exts=DATA_TYPE_FILE_EXTS.get(dataset_type), path_name=request_json['filePath'], ) return create_json_response({'success': True}) @@ -508,7 +509,8 @@ def _fetch_airtable_loadable_project_samples(user): def load_data(request): request_json = json.loads(request.body) sample_type = request_json['sampleType'] - dataset_type = request_json['datasetType'] + dataset_type = request_json.get('datasetType', Sample.DATASET_TYPE_VARIANT_CALLS) + skip_validation = request_json.get('skipValidation', False) projects = [json.loads(project) for project in request_json['projects']] project_samples = {p['projectGuid']: p.get('sampleIds') for p in projects} @@ -525,16 +527,17 @@ def load_data(request): loading_args = ( project_models, sample_type, dataset_type, request_json['genomeVersion'], _callset_path(request_json), ) + loading_kwargs = {'user': request.user, 'skip_validation': skip_validation} if has_airtable: success_message = f'*{request.user.email}* triggered loading internal {sample_type} {dataset_type} data for {len(projects)} projects' error_message = f'ERROR triggering internal {sample_type} {dataset_type} loading' trigger_airflow_data_loading( - *loading_args, user=request.user, success_message=success_message, error_message=error_message, + *loading_args, **loading_kwargs, success_message=success_message, error_message=error_message, success_slack_channel=SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, is_internal=True, individual_ids=individual_ids, ) else: request_json, _ = prepare_data_loading_request( - *loading_args, user=request.user, pedigree_dir=LOADING_DATASETS_DIR, raise_pedigree_error=True, + *loading_args, **loading_kwargs, pedigree_dir=LOADING_DATASETS_DIR, raise_pedigree_error=True, ) response = requests.post(f'{PIPELINE_RUNNER_SERVER}/loading_pipeline_enqueue', json=request_json, timeout=60) if response.status_code == 409: From ebc3ab25bbacb7dc433441f8af0b05ea4ed218ed Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 16 Oct 2024 11:59:24 -0400 Subject: [PATCH 27/35] do not allow triggering non-snv indel pipeline --- seqr/views/apis/data_manager_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index f54ee2af7f..8270790f89 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -536,6 +536,8 @@ def load_data(request): success_slack_channel=SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, is_internal=True, individual_ids=individual_ids, ) else: + if dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS: + return create_json_response({'error': f'Invalid dataset type {dataset_type}'}, status=400) request_json, _ = prepare_data_loading_request( *loading_args, **loading_kwargs, pedigree_dir=LOADING_DATASETS_DIR, raise_pedigree_error=True, ) From de7f5d2f703471a90b45e8880499dfd2ec9fe700 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 16 Oct 2024 12:00:55 -0400 Subject: [PATCH 28/35] Revert "do not allow triggering non-snv indel pipeline" This reverts commit ebc3ab25bbacb7dc433441f8af0b05ea4ed218ed. --- seqr/views/apis/data_manager_api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/seqr/views/apis/data_manager_api.py b/seqr/views/apis/data_manager_api.py index 8270790f89..f54ee2af7f 100644 --- a/seqr/views/apis/data_manager_api.py +++ b/seqr/views/apis/data_manager_api.py @@ -536,8 +536,6 @@ def load_data(request): success_slack_channel=SEQR_SLACK_LOADING_NOTIFICATION_CHANNEL, is_internal=True, individual_ids=individual_ids, ) else: - if dataset_type != Sample.DATASET_TYPE_VARIANT_CALLS: - return create_json_response({'error': f'Invalid dataset type {dataset_type}'}, status=400) request_json, _ = prepare_data_loading_request( *loading_args, **loading_kwargs, pedigree_dir=LOADING_DATASETS_DIR, raise_pedigree_error=True, ) From 7e6f24943b725c11dad2762d984889e497d408c1 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 16 Oct 2024 12:14:53 -0400 Subject: [PATCH 29/35] update unit tests --- seqr/views/apis/data_manager_api_tests.py | 22 +++++++++++++++------- seqr/views/utils/test_utils.py | 4 +++- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/seqr/views/apis/data_manager_api_tests.py b/seqr/views/apis/data_manager_api_tests.py index 3429db7123..5d1e43c38f 100644 --- a/seqr/views/apis/data_manager_api_tests.py +++ b/seqr/views/apis/data_manager_api_tests.py @@ -1399,7 +1399,7 @@ def test_validate_callset(self, mock_subprocess, mock_glob, mock_os_isfile): mock_subprocess.return_value.communicate.return_value = ( b'', b'CommandException: One or more URLs matched no objects.', ) - body = {'filePath': f'{self.CALLSET_DIR}/sharded_vcf/part0*.vcf', 'datasetType': 'SNV_INDEL'} + body = {'filePath': f'{self.CALLSET_DIR}/sharded_vcf/part0*.vcf'} response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 400) self.assertListEqual( @@ -1467,7 +1467,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_mkdir): mock_temp_dir.return_value.__enter__.return_value = '/mock/tmp' body = {'filePath': f'{self.CALLSET_DIR}/mito_callset.mt', 'datasetType': 'MITO', 'sampleType': 'WGS', 'genomeVersion': '38', 'projects': [ json.dumps({'projectGuid': 'R0001_1kg'}), json.dumps(PROJECT_OPTION), json.dumps({'projectGuid': 'R0005_not_project'}), - ]} + ], 'skipValidation': True} response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self.assertEqual(response.status_code, 400) self.assertDictEqual(response.json(), {'error': 'The following projects are invalid: R0005_not_project'}) @@ -1478,7 +1478,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_mkdir): self.assertEqual(response.status_code, 200) self.assertDictEqual(response.json(), {'success': True}) - self._assert_expected_load_data_requests() + self._assert_expected_load_data_requests(skip_validation=True) self._has_expected_ped_files(mock_open, mock_mkdir, 'MITO') dag_json = { @@ -1490,6 +1490,7 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_mkdir): 'sample_type': 'WGS', 'dataset_type': 'MITO', 'reference_genome': 'GRCh38', + 'skip_validation': True, } self._assert_success_notification(dag_json) @@ -1500,6 +1501,8 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_mkdir): responses.calls.reset() self.reset_logs() + del body['skipValidation'] + del dag_json['skip_validation'] body.update({'datasetType': 'SV', 'filePath': f'{self.CALLSET_DIR}/sv_callset.vcf', 'sampleType': 'WES'}) self._trigger_error(url, body, dag_json, mock_open, mock_mkdir) @@ -1508,7 +1511,8 @@ def test_load_data(self, mock_temp_dir, mock_open, mock_mkdir): responses.calls.reset() mock_open.reset_mock() mock_mkdir.reset_mock() - body.update({'datasetType': 'SNV_INDEL', 'sampleType': 'WGS', 'projects': [json.dumps(PROJECT_SAMPLES_OPTION)]}) + body.update({'sampleType': 'WGS', 'projects': [json.dumps(PROJECT_SAMPLES_OPTION)]}) + del body['datasetType'] response = self.client.post(url, content_type='application/json', data=json.dumps(body)) self._test_load_sample_subset(mock_open, mock_mkdir, response, url, body) @@ -1593,18 +1597,21 @@ def _add_file_iter(self, stdout): def _assert_expected_get_projects_requests(self): self.assertEqual(len(responses.calls), 0) - def _assert_expected_load_data_requests(self, dataset_type='MITO', sample_type='WGS', trigger_error=False, skip_project=False): + def _assert_expected_load_data_requests(self, dataset_type='MITO', sample_type='WGS', trigger_error=False, skip_project=False, skip_validation=False): self.assertEqual(len(responses.calls), 1) projects = [PROJECT_GUID, NON_ANALYST_PROJECT_GUID] if skip_project: projects = projects[1:] - self.assertDictEqual(json.loads(responses.calls[0].request.body), { + body = { 'projects_to_run': projects, 'callset_path': '/local_datasets/sv_callset.vcf' if trigger_error else '/local_datasets/mito_callset.mt', 'sample_type': sample_type, 'dataset_type': dataset_type, 'reference_genome': 'GRCh38', - }) + } + if skip_validation: + body['skip_validation'] = True + self.assertDictEqual(json.loads(responses.calls[0].request.body), body) @staticmethod def _local_pedigree_path(dataset_type, sample_type): @@ -1721,6 +1728,7 @@ def _get_dag_variable_overrides(*args, **kwargs): 'sample_source': 'Broad_Internal', 'sample_type': 'WGS', 'dataset_type': 'MITO', + 'skip_validation': True, } def _assert_expected_load_data_requests(self, **kwargs): diff --git a/seqr/views/utils/test_utils.py b/seqr/views/utils/test_utils.py index e9d6aa3598..5bf621d676 100644 --- a/seqr/views/utils/test_utils.py +++ b/seqr/views/utils/test_utils.py @@ -659,8 +659,10 @@ def assert_airflow_calls(self, trigger_error=False, additional_tasks_check=False 'sample_type': dag_variable_overrides['sample_type'], 'dataset_type': dataset_type or dag_variable_overrides['dataset_type'], 'reference_genome': dag_variable_overrides.get('reference_genome', 'GRCh38'), - 'sample_source': dag_variable_overrides['sample_source'], } + if dag_variable_overrides.get('skip_validation'): + dag_variables['skip_validation'] = True + dag_variables['sample_source'] = dag_variable_overrides['sample_source'] self._assert_airflow_calls(dag_variables, call_count) def _assert_airflow_calls(self, dag_variables, call_count, offset=0): From 3626293db4fad1c531ddf5b61182eac513254bba Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 16 Oct 2024 12:47:40 -0400 Subject: [PATCH 30/35] include vairant type in metadata report --- seqr/views/apis/report_api.py | 1 - seqr/views/apis/report_api_tests.py | 2 ++ seqr/views/apis/summary_data_api.py | 2 +- seqr/views/utils/anvil_metadata_utils.py | 2 ++ ui/pages/Report/components/VariantMetadata.jsx | 1 + 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/seqr/views/apis/report_api.py b/seqr/views/apis/report_api.py index ed42ad2909..a7cba7fcc6 100644 --- a/seqr/views/apis/report_api.py +++ b/seqr/views/apis/report_api.py @@ -588,7 +588,6 @@ def _post_process_gregor_variant(row, gene_variants): 'linked_variant': next( v['genetic_findings_id'] for v in gene_variants if v['genetic_findings_id'] != row['genetic_findings_id'] ) if len(gene_variants) > 1 else None, - 'variant_type': 'SNV/INDEL' if row['alt'] else 'SV', } diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 7f9fc0b255..f28a225ebc 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -514,6 +514,7 @@ 'sv_name': None, 'transcript': None, 'validated_name': None, + 'variant_type': 'SNV/INDEL', } PARTICIPANT_TABLE = [ @@ -1403,6 +1404,7 @@ def test_variant_metadata(self): 'tags': ['Tier 1 - Novel gene and phenotype'], 'variant_inheritance': 'unknown', 'variant_reference_assembly': 'GRCh37', + 'variant_type': 'SV', 'zygosity': 'Heterozygous', }) diff --git a/seqr/views/apis/summary_data_api.py b/seqr/views/apis/summary_data_api.py index 96cb2b6547..51193cd389 100644 --- a/seqr/views/apis/summary_data_api.py +++ b/seqr/views/apis/summary_data_api.py @@ -302,7 +302,7 @@ def _add_row(row, family_id, row_type): elif row_type == DISCOVERY_ROW_TYPE: for i, discovery_row in enumerate(row): participant_id = discovery_row.pop('participant_id') - parsed_row = {'{}-{}'.format(k, i + 1): v for k, v in discovery_row.items() if k != 'allele_balance_or_heteroplasmy_percentage'} + parsed_row = {'{}-{}'.format(k, i + 1): v for k, v in discovery_row.items() if k not in {'allele_balance_or_heteroplasmy_percentage', 'variant_type'}} parsed_row['num_saved_variants'] = len(row) rows_by_subject_family_id[(participant_id, family_id)].update(parsed_row) elif row_type == SUBJECT_ROW_TYPE: diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index d63df5c93c..1f90f7d69e 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -340,6 +340,7 @@ def _get_parsed_saved_discovery_variants_by_family( gene_id = main_transcript.get('geneId') gene_ids.add(gene_id) sv_type = variant_json.get('svType') + variant_type = 'SV' if sv_type else 'SNV/INDEL' partial_hpo_terms = variant.partial_hpo_terms[0] if variant.partial_hpo_terms else '' phenotype_contribution = 'Partial' if partial_hpo_terms else 'Full' @@ -358,6 +359,7 @@ def _get_parsed_saved_discovery_variants_by_family( 'partial_contribution_explained': partial_hpo_terms.replace(', ', '|'), 'sv_type': sv_type, 'sv_name': (variant_json.get('svName') or '{svType}:chr{chrom}:{pos}-{end}'.format(**variant_json)) if sv_type else None, + 'variant_type': variant_type, 'validated_name': variant.validated_name[0] if variant.validated_name else None, **{k: _get_transcript_field(k, config, main_transcript) for k, config in TRANSCRIPT_FIELDS.items()}, **{k: variant_json.get(k) for k in ['genotypes'] + (variant_json_fields or [])}, diff --git a/ui/pages/Report/components/VariantMetadata.jsx b/ui/pages/Report/components/VariantMetadata.jsx index ee7fe71e10..828c65f416 100644 --- a/ui/pages/Report/components/VariantMetadata.jsx +++ b/ui/pages/Report/components/VariantMetadata.jsx @@ -11,6 +11,7 @@ const VIEW_ALL_PAGES = [ const COLUMNS = [ { name: 'participant_id' }, ...VARIANT_METADATA_COLUMNS.slice(0, -1), + { name: 'variant_type' }, { name: 'allele_balance_or_heteroplasmy_percentage' }, { name: 'Clinvar allele ID', format: ({ clinvar }) => clinvar?.alleleId }, { name: 'ClinVar Clinical Significance', format: ({ clinvar }) => clinvarSignificance(clinvar).pathogenicity }, From 1829ea2944de4ab64479d7f36ae5a72f2ab2222a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 16 Oct 2024 13:03:40 -0400 Subject: [PATCH 31/35] update varian type logic --- seqr/views/utils/anvil_metadata_utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/seqr/views/utils/anvil_metadata_utils.py b/seqr/views/utils/anvil_metadata_utils.py index 1f90f7d69e..6ad49d87b1 100644 --- a/seqr/views/utils/anvil_metadata_utils.py +++ b/seqr/views/utils/anvil_metadata_utils.py @@ -313,6 +313,11 @@ def _get_discovery_notes(variant, gene_variants, omit_parent_mnvs): nested_mnvs = sorted([v for v in mnv_names if v != parent_name]) return f'The following variants are part of the {variant_type} variant {parent}: {", ".join(nested_mnvs)}' +VARIANT_TYPES = [ + ('SV', lambda ref, alt: not alt), + ('SNV', lambda ref, alt: len(ref) == 1 and len(alt) == 1), + ('RE', lambda ref, alt: all(['[' in allele and ']' in allele for allele in [ref, alt]])), +] def _get_parsed_saved_discovery_variants_by_family( families: Iterable[Family], include_metadata: bool, variant_json_fields: list[str], @@ -340,7 +345,9 @@ def _get_parsed_saved_discovery_variants_by_family( gene_id = main_transcript.get('geneId') gene_ids.add(gene_id) sv_type = variant_json.get('svType') - variant_type = 'SV' if sv_type else 'SNV/INDEL' + variant_type = next( + (variant_type for variant_type, has_type in VARIANT_TYPES if has_type(variant.ref, variant.alt)), + 'INDEL') partial_hpo_terms = variant.partial_hpo_terms[0] if variant.partial_hpo_terms else '' phenotype_contribution = 'Partial' if partial_hpo_terms else 'Full' From 50d80d27b4bb12b36789eef96057ab34622afd85 Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 16 Oct 2024 13:08:35 -0400 Subject: [PATCH 32/35] update tests --- seqr/views/apis/report_api_tests.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index f28a225ebc..0ec0bd88e3 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -410,12 +410,12 @@ {'column': 'genetic_findings_id', 'required': True}, {'column': 'participant_id', 'required': True}, {'column': 'experiment_id'}, - {'column': 'variant_type', 'required': True, 'data_type': 'enumeration', 'enumerations': ['SNV/INDEL', 'SV', 'CNV', 'RE', 'MEI']}, + {'column': 'variant_type', 'required': True, 'data_type': 'enumeration', 'enumerations': ['SNV', 'INDEL', 'SV', 'CNV', 'RE', 'MEI']}, {'column': 'variant_reference_assembly', 'required': True, 'data_type': 'enumeration', 'enumerations': ['GRCh37', 'GRCh38']}, {'column': 'chrom', 'required': True}, {'column': 'pos', 'required': True, 'data_type': 'integer'}, - {'column': 'ref','required': 'CONDITIONAL (variant_type = SNV/INDEL, variant_type = RE)'}, - {'column': 'alt', 'required': 'CONDITIONAL (variant_type = SNV/INDEL, variant_type = RE)'}, + {'column': 'ref','required': 'CONDITIONAL (variant_type = SNV, variant_type = INDEL, variant_type = RE)'}, + {'column': 'alt', 'required': 'CONDITIONAL (variant_type = SNV, variant_type = INDEL, variant_type = RE)'}, {'column': 'ClinGen_allele_ID'}, {'column': 'gene_of_interest', 'required': True}, {'column': 'transcript'}, @@ -514,7 +514,7 @@ 'sv_name': None, 'transcript': None, 'validated_name': None, - 'variant_type': 'SNV/INDEL', + 'variant_type': 'INDEL', } PARTICIPANT_TABLE = [ @@ -604,22 +604,22 @@ 'phenotype_contribution', 'partial_contribution_explained', 'additional_family_members_with_variant', 'method_of_discovery', 'notes', 'sv_type', 'chrom_end', 'pos_end', 'copy_number', 'hgvs', 'gene_disease_validity', ], [ - 'Broad_NA19675_1_21_3343353', 'Broad_NA19675_1', '', 'SNV/INDEL', 'GRCh37', '21', '3343353', 'GAGA', 'G', '', + 'Broad_NA19675_1_21_3343353', 'Broad_NA19675_1', '', 'INDEL', 'GRCh37', '21', '3343353', 'GAGA', 'G', '', 'RP11', 'ENST00000258436.5', 'c.375_377delTCT', 'p.Leu126del', 'Heterozygous', '', 'de novo', '', '', 'Candidate', 'Myasthenic syndrome, congenital, 8, with pre- and postsynaptic defects', 'OMIM:615120', 'Autosomal recessive|X-linked', 'Full', '', '', 'SR-ES', 'This individual is published in PMID34415322', '', '', '', '', '', '', ], [ - 'Broad_HG00731_1_248367227', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV/INDEL', 'GRCh37', '1', + 'Broad_HG00731_1_248367227', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'INDEL', 'GRCh37', '1', '248367227', 'TC', 'T', 'CA1501729', 'RP11', '', '', '', 'Homozygous', '', 'paternal', '', '', 'Known', '', 'MONDO:0044970', '', 'Uncertain', '', 'Broad_HG00732', 'SR-ES', '', '', '', '', '', '', '', ], [ - 'Broad_HG00731_19_1912632', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV/INDEL', 'GRCh38', '19', + 'Broad_HG00731_19_1912632', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'INDEL', 'GRCh38', '19', '1912632', 'GC', 'TT', '', 'OR4G11P', 'ENST00000371839', 'c.586_587delinsTT', 'p.Ala196Leu', 'Heterozygous', '', 'unknown', 'Broad_HG00731_19_1912634', '', 'Known', '', 'MONDO:0044970', '', 'Full', '', '', 'SR-ES', 'The following variants are part of the multinucleotide variant 19-1912632-GC-TT (c.586_587delinsTT, p.Ala196Leu): 19-1912633-G-T, 19-1912634-C-T', '', '', '', '', '', '', ], [ - 'Broad_NA20889_1_248367227', 'Broad_NA20889', '', 'SNV/INDEL', 'GRCh37', '1', '248367227', 'TC', 'T', + 'Broad_NA20889_1_248367227', 'Broad_NA20889', '', 'INDEL', 'GRCh37', '1', '248367227', 'TC', 'T', 'CA1501729', 'OR4G11P', 'ENST00000505820', 'c.3955G>A', 'c.1586-17C>G', 'Heterozygous', '', 'unknown', 'Broad_NA20889_1_249045487_DEL', '', 'Candidate', 'Immunodeficiency 38', 'OMIM:616126', 'Autosomal recessive', 'Partial', 'HP:0000501|HP:0000365', '', 'SR-ES', '', '', '', '', '', '', '', @@ -901,13 +901,13 @@ def _test_gregor_export(self, url, mock_subprocess, mock_temp_dir, mock_open, mo '80.2', '1.05', '', '', '', '', '', ]]) self._assert_expected_file(genetic_findings_file, [GENETIC_FINDINGS_TABLE[0], [ - 'Broad_NA19675_1_21_3343353', 'Broad_NA19675_1', '', 'SNV/INDEL', 'GRCh37', '21', '3343353', 'GAGA', 'G', '', + 'Broad_NA19675_1_21_3343353', 'Broad_NA19675_1', '', 'INDEL', 'GRCh37', '21', '3343353', 'GAGA', 'G', '', 'RP11', 'ENST00000258436.5', 'c.375_377delTCT', 'p.Leu126del', 'Heterozygous', '', 'de novo', '', '', 'Candidate', 'Myasthenic syndrome, congenital, 8, with pre- and postsynaptic defects', 'OMIM:615120', 'Autosomal recessive|X-linked', 'Full', '', '', 'SR-ES', 'This individual is published in PMID34415322', '', '', '', '', '', '', ], [ - 'Broad_HG00731_1_248367227', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV/INDEL', 'GRCh37', '1', + 'Broad_HG00731_1_248367227', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'INDEL', 'GRCh37', '1', '248367227', 'TC', 'T', 'CA1501729', 'RP11', '', '', '', 'Homozygous', '', 'paternal', '', '', 'Known', '', 'MONDO:0044970', '', 'Uncertain', '', 'Broad_HG00732', 'SR-ES', '', '', '', '', '', '', '', ]], additional_calls=1) From 06af3082e0b76aead93c00e5bd8f9f356380a53a Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 16 Oct 2024 13:17:03 -0400 Subject: [PATCH 33/35] test SNV --- seqr/fixtures/report_variants.json | 8 ++++---- seqr/views/apis/report_api_tests.py | 17 +++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/seqr/fixtures/report_variants.json b/seqr/fixtures/report_variants.json index e0722385b4..06ed7b2d11 100644 --- a/seqr/fixtures/report_variants.json +++ b/seqr/fixtures/report_variants.json @@ -105,9 +105,9 @@ "last_modified_date": "2018-05-31T16:36:02.805Z", "xpos": 19001912632, "xpos_end": 19001912632, - "ref": "GC", - "alt": "TT", - "variant_id": "19-1912632-GC-TT", + "ref": "G", + "alt": "C", + "variant_id": "19-1912632-G-C", "saved_variant_json": { "pos": 1912632, "end": 1912632, @@ -116,7 +116,7 @@ "genotypes": { "I000004_hg00731": {"numAlt": 1} }, - "variantId": "19-1912632-GC-TT", + "variantId": "19-1912632-G-C", "chrom": "19", "mainTranscriptId": "ENST00000371839", "transcripts": { diff --git a/seqr/views/apis/report_api_tests.py b/seqr/views/apis/report_api_tests.py index 0ec0bd88e3..767af58fe3 100644 --- a/seqr/views/apis/report_api_tests.py +++ b/seqr/views/apis/report_api_tests.py @@ -613,10 +613,10 @@ '248367227', 'TC', 'T', 'CA1501729', 'RP11', '', '', '', 'Homozygous', '', 'paternal', '', '', 'Known', '', 'MONDO:0044970', '', 'Uncertain', '', 'Broad_HG00732', 'SR-ES', '', '', '', '', '', '', '', ], [ - 'Broad_HG00731_19_1912632', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'INDEL', 'GRCh38', '19', - '1912632', 'GC', 'TT', '', 'OR4G11P', 'ENST00000371839', 'c.586_587delinsTT', 'p.Ala196Leu', 'Heterozygous', '', 'unknown', + 'Broad_HG00731_19_1912632', 'Broad_HG00731', 'Broad_exome_VCGS_FAM203_621_D2', 'SNV', 'GRCh38', '19', + '1912632', 'G', 'C', '', 'OR4G11P', 'ENST00000371839', 'c.586_587delinsTT', 'p.Ala196Leu', 'Heterozygous', '', 'unknown', 'Broad_HG00731_19_1912634', '', 'Known', '', 'MONDO:0044970', '', 'Full', '', '', 'SR-ES', - 'The following variants are part of the multinucleotide variant 19-1912632-GC-TT (c.586_587delinsTT, p.Ala196Leu): 19-1912633-G-T, 19-1912634-C-T', + 'The following variants are part of the multinucleotide variant 19-1912632-G-C (c.586_587delinsTT, p.Ala196Leu): 19-1912633-G-T, 19-1912634-C-T', '', '', '', '', '', '', ], [ 'Broad_NA20889_1_248367227', 'Broad_NA20889', '', 'INDEL', 'GRCh37', '1', '248367227', 'TC', 'T', @@ -759,13 +759,13 @@ def _check_anvil_export_response(self, response, mock_zip, no_analyst_project_ur self.assertIn([ '19_1912633_HG00731', 'HG00731', 'HG00731', 'OR4G11P', 'Known', 'unknown', 'Heterozygous', 'GRCh38', '19', '1912633', 'G', 'T', '-', '-', 'ENST00000371839', '-', '-', '-', - 'The following variants are part of the multinucleotide variant 19-1912632-GC-TT ' + 'The following variants are part of the multinucleotide variant 19-1912632-G-C ' '(c.586_587delinsTT, p.Ala196Leu): 19-1912633-G-T, 19-1912634-C-T'], discovery_file) self.assertIn([ '19_1912634_HG00731', 'HG00731', 'HG00731', 'OR4G11P', 'Known', 'unknown', 'Heterozygous', 'GRCh38', '19', '1912634', 'C', 'T', '-', '-', 'ENST00000371839', '-', '-', '-', - 'The following variants are part of the multinucleotide variant 19-1912632-GC-TT (c.586_587delinsTT, ' + 'The following variants are part of the multinucleotide variant 19-1912632-G-C (c.586_587delinsTT, ' 'p.Ala196Leu): 19-1912633-G-T, 19-1912634-C-T'], discovery_file) @@ -1307,7 +1307,7 @@ def test_variant_metadata(self): self.assertDictEqual(response_json['rows'][1], expected_row) expected_mnv = { **BASE_VARIANT_METADATA_ROW, - 'alt': 'TT', + 'alt': 'C', 'chrom': '19', 'condition_id': 'MONDO:0044970', 'condition_inheritance': 'Unknown', @@ -1321,15 +1321,16 @@ def test_variant_metadata(self): 'hgvsc': 'c.586_587delinsTT', 'hgvsp': 'p.Ala196Leu', 'known_condition_name': 'mitochondrial disease', - 'notes': 'The following variants are part of the multinucleotide variant 19-1912632-GC-TT (c.586_587delinsTT, p.Ala196Leu): 19-1912633-G-T, 19-1912634-C-T', + 'notes': 'The following variants are part of the multinucleotide variant 19-1912632-G-C (c.586_587delinsTT, p.Ala196Leu): 19-1912633-G-T, 19-1912634-C-T', 'participant_id': 'HG00731', 'pos': 1912632, 'projectGuid': 'R0001_1kg', - 'ref': 'GC', + 'ref': 'G', 'tags': ['Known gene for phenotype'], 'transcript': 'ENST00000371839', 'variant_inheritance': 'unknown', 'variant_reference_assembly': 'GRCh38', + 'variant_type': 'SNV', 'zygosity': 'Heterozygous', } self.assertDictEqual(response_json['rows'][2], expected_mnv) From 1ccd6cbdf50e527a7e172902309dd518e7c4014f Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Wed, 16 Oct 2024 13:17:48 -0400 Subject: [PATCH 34/35] update gregor import --- seqr/views/apis/individual_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seqr/views/apis/individual_api.py b/seqr/views/apis/individual_api.py index b6d82c7b3a..9aad4dc754 100644 --- a/seqr/views/apis/individual_api.py +++ b/seqr/views/apis/individual_api.py @@ -875,7 +875,7 @@ def import_gregor_metadata(request, project_guid): genes = set() for row in _iter_metadata_table( metadata_files_path, FINDINGS_TABLE, request.user, - lambda r: r['participant_id'] in participant_individual_map and r['variant_type'] == 'SNV/INDEL', + lambda r: r['participant_id'] in participant_individual_map and r['variant_type'] in {'SNV/INDEL', 'SNV', 'INDEL'}, ): individual = participant_individual_map[row['participant_id']] variant_id = '-'.join([row[col] for col in ['chrom', 'pos', 'ref', 'alt']]) From b86c70f21c67eab703732a9f1746beabe8c5027c Mon Sep 17 00:00:00 2001 From: Hana Snow Date: Mon, 21 Oct 2024 14:14:01 -0400 Subject: [PATCH 35/35] get cached gene counts for comp hets --- seqr/utils/search/search_utils_tests.py | 2 +- seqr/utils/search/utils.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/seqr/utils/search/search_utils_tests.py b/seqr/utils/search/search_utils_tests.py index 3bd88e0903..f60cd785df 100644 --- a/seqr/utils/search/search_utils_tests.py +++ b/seqr/utils/search/search_utils_tests.py @@ -550,7 +550,7 @@ def test_get_variant_query_gene_counts(self, mock_call): def test_cached_get_variant_query_gene_counts(self): super(HailSearchUtilsTests, self).test_cached_get_variant_query_gene_counts() - self.set_cache({'all_results': PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT + [SV_VARIANT1], 'total_results': 3}) + self.set_cache({'all_results': [PARSED_COMPOUND_HET_VARIANTS_MULTI_PROJECT] + [SV_VARIANT1], 'total_results': 2}) gene_counts = get_variant_query_gene_counts(self.results_model, self.user) self.assertDictEqual(gene_counts, { 'ENSG00000135953': {'total': 2, 'families': {'F000003_3': 2, 'F000011_11': 2}}, diff --git a/seqr/utils/search/utils.py b/seqr/utils/search/utils.py index 5fdfbb4d45..b4081bf69a 100644 --- a/seqr/utils/search/utils.py +++ b/seqr/utils/search/utils.py @@ -314,7 +314,12 @@ def get_variant_query_gene_counts(search_model, user): def _get_gene_aggs_for_cached_variants(previous_search_results): gene_aggs = defaultdict(lambda: {'total': 0, 'families': defaultdict(int)}) - for var in previous_search_results['all_results']: + # ES caches compound hets separately from main results, hail search caches everything together + flattened_variants = backend_specific_call( + lambda results: results, + lambda results: [v for variants in results for v in (variants if isinstance(variants, list) else [variants])], + )(previous_search_results['all_results']) + for var in flattened_variants: # ES only reports breakdown for main transcript gene only, hail backend reports for all genes gene_ids = backend_specific_call( lambda variant_transcripts: next((