Skip to content

Commit

Permalink
For Issue dmwm#94: Removed unused code, added integration tests, fixe…
Browse files Browse the repository at this point in the history 
…d issue with parentage check

Kept validation check
Partial Parentage is indicated by providing -1 to ChildParentIDList
Only insert valid parentage
  • Loading branch information
@d-ylee
d-ylee committed Sep 27, 2023
1 parent 35bb188 commit 33ea8b3
Show file tree
Hide file tree
Showing 4 changed files with 153 additions and 70 deletions.
50 changes: 17 additions & 33 deletions dbs/fileparents.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"fmt"
"io"
"log"
"strings"

"github.com/dmwm/dbs2go/utils"
)
Expand Down Expand Up @@ -363,31 +362,12 @@ func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error {
log.Println("block fids != file ids")
log.Println("block ids", bfids)
log.Println("file ids", fids)

// return list of children with missing parents
missingFids := make([]int64, 0)
if rec.MissingFiles > 0 {
for _, v := range rec.ChildParentIDList {
parentFileID := v[1]
if parentFileID == 0 {
missingFids = append(missingFids, v[0])
}
}
}

// check if number missing files matches the given missingFiles. returns error if len does not match
if len(missingFids) != int(rec.MissingFiles) {
msg := fmt.Sprintf("not all files present in block")
log.Println("number of child fileIDs without parentage does not match the provided missingFiles")
return Error(RecordErr, ParametersErrorCode, msg, "dbs.fileparents.InsertFileParentsBlockTxt")
// return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt")
} else {
log.Println("number of child fileIDs without parentage matches given missingFiles")
}
msg := fmt.Sprintf("not all files present in block")
return Error(RecordErr, ParametersErrorCode, msg, "dbs.fileparents.InsertFileParentsBlockTxt")
}

// now we can loop over provided list and insert file parents
var missingFiles int64
var missingFiles []FileParents
var validatedChildParentIDList []FileParents
for _, v := range rec.ChildParentIDList {
var r FileParents
Expand All @@ -398,20 +378,24 @@ func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error {
}
err = r.Validate()
if err != nil {
if strings.Contains(err.(*DBSError).Message, "missing parent_file_id") {
log.Printf("record has missing parent_file_id: %v", r)
missingFiles = missingFiles + 1
continue
} else {
log.Println("unable to validate the record", r, "error", err)
return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt")
}
log.Println("unable to validate the record", r, "error", err)
return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt")
}
// only provide list of child parent that have parentage
// otherwise, skip partial parentage and add to the missingFiles count
if r.PARENT_FILE_ID > 0 {
validatedChildParentIDList = append(validatedChildParentIDList, r)
} else {
missingFiles = append(missingFiles, r)
}
validatedChildParentIDList = append(validatedChildParentIDList, r)
}

if len(missingFiles) > 0 {
log.Printf("files without parentage: %v", missingFiles)
}

// check if provided MissingFiles matches the amount of mising files found
if missingFiles != rec.MissingFiles {
if len(missingFiles) != int(rec.MissingFiles) {
log.Println("provided missingFiles does not match number of pairs with no parentage")
return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt")
}
Expand Down
64 changes: 32 additions & 32 deletions test/data/integration/bulkblocks_data.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/0.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/0.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand All @@ -28,7 +28,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/1.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/1.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand All @@ -39,7 +39,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/2.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/2.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand All @@ -50,7 +50,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/3.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/3.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand All @@ -61,7 +61,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/4.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/4.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand Down Expand Up @@ -95,7 +95,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/0.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/0.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -125,7 +125,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/1.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/1.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -155,7 +155,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/2.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/2.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -185,7 +185,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/3.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/3.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -215,7 +215,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/4.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/4.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand All @@ -232,19 +232,19 @@
"create_by": "WMAgent",
"primary_ds_type": "test",
"primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain",
"creation_date": 1652798955
"creation_date": 1695842582
},
"dataset": {
"dataset_id": 0,
"create_by": "WMAgent",
"creation_date": 1652798955,
"creation_date": 1695842582,
"physics_group_name": "Tracker",
"dataset_access_type": "PRODUCTION",
"data_tier_name": "GEN-SIM-RAW",
"last_modified_by": "WMAgent",
"processed_ds_name": "acq_era_8268-ptsr-v8268",
"xtcrosssection": 0,
"last_modification_date": 1652798955,
"last_modification_date": 1695842582,
"dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-ptsr-v8268/GEN-SIM-RAW",
"prep_id": "TestPrepID"
},
Expand Down Expand Up @@ -507,19 +507,19 @@
"create_by": "WMAgent",
"primary_ds_type": "test",
"primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain",
"creation_date": 1652798955
"creation_date": 1695842582
},
"dataset": {
"dataset_id": 0,
"create_by": "WMAgent",
"creation_date": 1652798955,
"creation_date": 1695842582,
"physics_group_name": "Tracker",
"dataset_access_type": "PRODUCTION",
"data_tier_name": "GEN-SIM-RAW",
"last_modified_by": "WMAgent",
"processed_ds_name": "acq_era_8268-v8268",
"xtcrosssection": 0,
"last_modification_date": 1652798955,
"last_modification_date": 1695842582,
"dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-v8268/GEN-SIM-RAW",
"prep_id": "TestPrepID"
},
Expand Down Expand Up @@ -591,7 +591,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/5.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/5.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -621,7 +621,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/6.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/6.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -651,7 +651,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/7.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/7.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -681,7 +681,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/8.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/8.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -711,7 +711,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/9.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/9.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand All @@ -728,19 +728,19 @@
"create_by": "WMAgent",
"primary_ds_type": "test",
"primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain2",
"creation_date": 1652798955
"creation_date": 1695842582
},
"dataset": {
"dataset_id": 0,
"create_by": "WMAgent",
"creation_date": 1652798955,
"creation_date": 1695842582,
"physics_group_name": "Tracker",
"dataset_access_type": "PRODUCTION",
"data_tier_name": "GEN-SIM-RAW",
"last_modified_by": "WMAgent",
"processed_ds_name": "acq_era_8268-ptsr-v82682",
"xtcrosssection": 0,
"last_modification_date": 1652798955,
"last_modification_date": 1695842582,
"dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-ptsr-v8268/GEN-SIM-RAW2",
"prep_id": "TestPrepID"
},
Expand Down Expand Up @@ -810,7 +810,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/5.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/5.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -840,7 +840,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/6.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/6.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -870,7 +870,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/7.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/7.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -900,7 +900,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/8.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/8.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -930,7 +930,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/9.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/9.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand All @@ -947,19 +947,19 @@
"create_by": "WMAgent",
"primary_ds_type": "test",
"primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain2",
"creation_date": 1652798955
"creation_date": 1695842582
},
"dataset": {
"dataset_id": 0,
"create_by": "WMAgent",
"creation_date": 1652798955,
"creation_date": 1695842582,
"physics_group_name": "Tracker",
"dataset_access_type": "PRODUCTION",
"data_tier_name": "GEN-SIM-RAW",
"last_modified_by": "WMAgent",
"processed_ds_name": "acq_era_8268-v82682",
"xtcrosssection": 0,
"last_modification_date": 1652798955,
"last_modification_date": 1695842582,
"dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-v8268/GEN-SIM-RAW2",
"prep_id": "TestPrepID"
},
Expand Down
Loading

0 comments on commit 33ea8b3

Please sign in to comment.