From 35bb1888360183df4b8adf72b16fc886fb3c22f2 Mon Sep 17 00:00:00 2001 From: Dennis Lee Date: Tue, 26 Sep 2023 21:35:50 +0000 Subject: [PATCH 1/2] Initial changes for partial parentage. Start of unit tests. --- .github/workflows/build.yml | 1 + dbs/fileparents.go | 58 +++++++++++++++++++++++++++++++++---- static/parameters.json | 2 +- test/int_fileparents.go | 24 ++++++++++++++- utils/utils.go | 29 ++++++++++++++++--- 5 files changed, 103 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a48868db..ee6dfa10 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -119,6 +119,7 @@ jobs: # - run: docker push cmssw/dbs2go:${{steps.get-ref.outputs.tag}} - name: Push new image to k8s + if: "!contains(${{steps.get-ref.outputs.tag}}, 'dev')" run: | curl -ksLO https://raw.githubusercontent.com/vkuznet/imagebot/main/imagebot.sh sed -i -e "s,COMMIT,${{github.sha}},g" -e "s,REPOSITORY,${{github.repository}},g" -e "s,NAMESPACE,dbs,g" -e "s,TAG,${{steps.get-ref.outputs.tag}},g" -e "s,IMAGE,registry.cern.ch/cmsweb/dbs2go,g" -e "s,SERVICE,dbs2go,g" -e "s,HOST,${{secrets.IMAGEBOT_URL}},g" imagebot.sh diff --git a/dbs/fileparents.go b/dbs/fileparents.go index 1ee873f0..d71b4f1d 100644 --- a/dbs/fileparents.go +++ b/dbs/fileparents.go @@ -3,9 +3,11 @@ package dbs import ( "database/sql" "encoding/json" + "errors" "fmt" "io" "log" + "strings" "github.com/dmwm/dbs2go/utils" ) @@ -60,10 +62,11 @@ func (a *API) FileParents() error { // FileParents represents file parents DBS DB table type FileParents struct { THIS_FILE_ID int64 `json:"this_file_id" validate:"required,number,gt=0"` - PARENT_FILE_ID int64 `json:"parent_file_id" validate:"required,number,gt=0"` + PARENT_FILE_ID int64 `json:"parent_file_id" validate:"required,number"` // TODO: may need to modify to handle `none` from python } // Insert implementation of FileParents +// //gocyclo:ignore func (r *FileParents) Insert(tx *sql.Tx) error { var tid int64 @@ -206,6 +209,7 @@ func (r *FileParents) Insert(tx *sql.Tx) error { } // Validate implementation of FileParents +// TODO: handle this for partial parentage func (r *FileParents) Validate() error { if err := RecordValidator.Struct(*r); err != nil { return DecodeValidatorError(r, err) @@ -239,6 +243,7 @@ func (r *FileParents) Decode(reader io.Reader) error { // err := decoder.Decode(&rec) if err != nil { log.Println("fail to decode data", err) + errors.Unwrap(err) return Error(err, UnmarshalErrorCode, "", "dbs.fileparents.Decode") } return nil @@ -274,12 +279,17 @@ func (a *API) InsertFileParents() error { } // FileParentBlockRecord represents file parent DBS record +// BlockName: name of the block +// ChildParentIDList: list of child and parent file ids +// MissingFiles: Number of missing files in the child and parent file list type FileParentBlockRecord struct { BlockName string `json:"block_name"` ChildParentIDList [][]int64 `json:"child_parent_id_list"` + MissingFiles int64 `json:"missing_files"` } // InsertFileParentsBlockTxt DBS API +// //gocyclo:ignore func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error { // read given input @@ -353,11 +363,32 @@ func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error { log.Println("block fids != file ids") log.Println("block ids", bfids) log.Println("file ids", fids) - msg := fmt.Sprintf("not all files present in block") - return Error(RecordErr, ParametersErrorCode, msg, "dbs.fileparents.InsertFileParentsBlockTxt") + + // return list of children with missing parents + missingFids := make([]int64, 0) + if rec.MissingFiles > 0 { + for _, v := range rec.ChildParentIDList { + parentFileID := v[1] + if parentFileID == 0 { + missingFids = append(missingFids, v[0]) + } + } + } + + // check if number missing files matches the given missingFiles. returns error if len does not match + if len(missingFids) != int(rec.MissingFiles) { + msg := fmt.Sprintf("not all files present in block") + log.Println("number of child fileIDs without parentage does not match the provided missingFiles") + return Error(RecordErr, ParametersErrorCode, msg, "dbs.fileparents.InsertFileParentsBlockTxt") + // return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt") + } else { + log.Println("number of child fileIDs without parentage matches given missingFiles") + } } // now we can loop over provided list and insert file parents + var missingFiles int64 + var validatedChildParentIDList []FileParents for _, v := range rec.ChildParentIDList { var r FileParents r.THIS_FILE_ID = v[0] @@ -367,9 +398,26 @@ func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error { } err = r.Validate() if err != nil { - log.Println("unable to validate the record", r, "error", err) - return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt") + if strings.Contains(err.(*DBSError).Message, "missing parent_file_id") { + log.Printf("record has missing parent_file_id: %v", r) + missingFiles = missingFiles + 1 + continue + } else { + log.Println("unable to validate the record", r, "error", err) + return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt") + } } + validatedChildParentIDList = append(validatedChildParentIDList, r) + } + + // check if provided MissingFiles matches the amount of mising files found + if missingFiles != rec.MissingFiles { + log.Println("provided missingFiles does not match number of pairs with no parentage") + return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt") + } + + // only insert valid child parent id lists + for _, r := range validatedChildParentIDList { err = r.Insert(tx) if err != nil { if utils.VERBOSE > 1 { diff --git a/static/parameters.json b/static/parameters.json index ee2ca801..15468178 100644 --- a/static/parameters.json +++ b/static/parameters.json @@ -156,7 +156,7 @@ { "api": "fileparents", "parameters": [ - "logical_file_name", "block_name", "block_id" + "logical_file_name", "block_name", "block_id", "missing_files" ] }, { diff --git a/test/int_fileparents.go b/test/int_fileparents.go index 7a50288f..28425bfe 100644 --- a/test/int_fileparents.go +++ b/test/int_fileparents.go @@ -9,7 +9,19 @@ import ( "github.com/dmwm/dbs2go/web" ) +// response for fileparent +type fileParentResponse struct { + LOGICAL_FILE_NAME string `json:"logical_file_name"` + PARENT_FILE_ID int `json:"parent_file_id"` + PARENT_LOGICAL_FILE_NAME string `json:"parent_logical_file_name"` +} + func getFileParentsTestTable(t *testing.T) EndpointTestCase { + fpResp := fileParentResponse{ + LOGICAL_FILE_NAME: TestData.Files[0], + PARENT_FILE_ID: 1, + PARENT_LOGICAL_FILE_NAME: TestData.ParentFiles[0], + } dbsError := dbs.DBSError{ Reason: dbs.InvalidParamErr.Error(), Message: "logical_file_name, block_id or block_name is required for fileparents api", @@ -30,7 +42,7 @@ func getFileParentsTestTable(t *testing.T) EndpointTestCase { defaultEndpoint: "/dbs/fileparents", testCases: []testCase{ { // DBSClientReader_t.test041 - description: "Test fileparents with lfn", + description: "Test fileparents with parent lfn", method: "GET", serverType: "DBSReader", params: url.Values{ @@ -39,6 +51,16 @@ func getFileParentsTestTable(t *testing.T) EndpointTestCase { output: []Response{}, respCode: http.StatusOK, }, + { // DBSClientReader_t.test041 + description: "Test fileparents with file lfn", + method: "GET", + serverType: "DBSReader", + params: url.Values{ + "logical_file_name": []string{TestData.Files[0]}, + }, + output: []Response{fpResp}, + respCode: http.StatusOK, + }, { // DBSClientReader_t.test042 description: "Test fileparents with no params", method: "GET", diff --git a/utils/utils.go b/utils/utils.go index 16e3a0cb..f682a5ab 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -73,10 +73,11 @@ func ErrPropagate(api string) { // ErrPropagate2Channel helper function which can be used in goroutines as // ch := make(chan interface{}) -// go func() { -// defer ErrPropagate2Channel(api, ch) -// someFunction() -// }() +// +// go func() { +// defer ErrPropagate2Channel(api, ch) +// someFunction() +// }() func ErrPropagate2Channel(api string, ch chan interface{}) { if err := recover(); err != nil { log.Println("ERROR", api, "error", err, Stack()) @@ -157,6 +158,26 @@ func Equal[T ListEntry](a, b []T) bool { return true } +// Diff provides a slice of the different elements from a and b +func Diff[T ListEntry](a, b []T) []T { + diff := make([]T, 0) + m := map[T]int{} + for _, aVal := range a { + m[aVal] = 1 + } + for _, bVal := range b { + m[bVal] = m[bVal] + 1 + } + + for mKey, mVal := range m { + if mVal == 1 { + diff = append(diff, mKey) + } + } + + return diff +} + // MapKeys returns string keys from a map func MapKeys(rec map[string]interface{}) []string { keys := make([]string, 0, len(rec)) From 33ea8b3794fd1d6b36d3651db62ace5e06e11c00 Mon Sep 17 00:00:00 2001 From: Dennis Lee Date: Wed, 27 Sep 2023 20:36:09 +0000 Subject: [PATCH 2/2] For Issue #94: Removed unused code, added integration tests, fixed issue with parentage check Kept validation check Partial Parentage is indicated by providing -1 to ChildParentIDList Only insert valid parentage --- dbs/fileparents.go | 50 ++++------- test/data/integration/bulkblocks_data.json | 64 +++++++------- test/int_fileparents.go | 99 ++++++++++++++++++++++ test/integration_cases.go | 10 +-- 4 files changed, 153 insertions(+), 70 deletions(-) diff --git a/dbs/fileparents.go b/dbs/fileparents.go index d71b4f1d..3980c2fa 100644 --- a/dbs/fileparents.go +++ b/dbs/fileparents.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "log" - "strings" "github.com/dmwm/dbs2go/utils" ) @@ -363,31 +362,12 @@ func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error { log.Println("block fids != file ids") log.Println("block ids", bfids) log.Println("file ids", fids) - - // return list of children with missing parents - missingFids := make([]int64, 0) - if rec.MissingFiles > 0 { - for _, v := range rec.ChildParentIDList { - parentFileID := v[1] - if parentFileID == 0 { - missingFids = append(missingFids, v[0]) - } - } - } - - // check if number missing files matches the given missingFiles. returns error if len does not match - if len(missingFids) != int(rec.MissingFiles) { - msg := fmt.Sprintf("not all files present in block") - log.Println("number of child fileIDs without parentage does not match the provided missingFiles") - return Error(RecordErr, ParametersErrorCode, msg, "dbs.fileparents.InsertFileParentsBlockTxt") - // return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt") - } else { - log.Println("number of child fileIDs without parentage matches given missingFiles") - } + msg := fmt.Sprintf("not all files present in block") + return Error(RecordErr, ParametersErrorCode, msg, "dbs.fileparents.InsertFileParentsBlockTxt") } // now we can loop over provided list and insert file parents - var missingFiles int64 + var missingFiles []FileParents var validatedChildParentIDList []FileParents for _, v := range rec.ChildParentIDList { var r FileParents @@ -398,20 +378,24 @@ func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error { } err = r.Validate() if err != nil { - if strings.Contains(err.(*DBSError).Message, "missing parent_file_id") { - log.Printf("record has missing parent_file_id: %v", r) - missingFiles = missingFiles + 1 - continue - } else { - log.Println("unable to validate the record", r, "error", err) - return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt") - } + log.Println("unable to validate the record", r, "error", err) + return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt") + } + // only provide list of child parent that have parentage + // otherwise, skip partial parentage and add to the missingFiles count + if r.PARENT_FILE_ID > 0 { + validatedChildParentIDList = append(validatedChildParentIDList, r) + } else { + missingFiles = append(missingFiles, r) } - validatedChildParentIDList = append(validatedChildParentIDList, r) + } + + if len(missingFiles) > 0 { + log.Printf("files without parentage: %v", missingFiles) } // check if provided MissingFiles matches the amount of mising files found - if missingFiles != rec.MissingFiles { + if len(missingFiles) != int(rec.MissingFiles) { log.Println("provided missingFiles does not match number of pairs with no parentage") return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt") } diff --git a/test/data/integration/bulkblocks_data.json b/test/data/integration/bulkblocks_data.json index 1a657ed4..fe366575 100755 --- a/test/data/integration/bulkblocks_data.json +++ b/test/data/integration/bulkblocks_data.json @@ -17,7 +17,7 @@ "release_version": "CMSSW_1_2_3", "pset_hash": "76e303993a1c2f842159dbfeeed9a0dd", "pset_name": "", - "lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/0.root", + "lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/0.root", "app_name": "cmsRun", "output_module_label": "merged", "global_tag": "my-cms-gtag_8268", @@ -28,7 +28,7 @@ "release_version": "CMSSW_1_2_3", "pset_hash": "76e303993a1c2f842159dbfeeed9a0dd", "pset_name": "", - "lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/1.root", + "lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/1.root", "app_name": "cmsRun", "output_module_label": "merged", "global_tag": "my-cms-gtag_8268", @@ -39,7 +39,7 @@ "release_version": "CMSSW_1_2_3", "pset_hash": "76e303993a1c2f842159dbfeeed9a0dd", "pset_name": "", - "lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/2.root", + "lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/2.root", "app_name": "cmsRun", "output_module_label": "merged", "global_tag": "my-cms-gtag_8268", @@ -50,7 +50,7 @@ "release_version": "CMSSW_1_2_3", "pset_hash": "76e303993a1c2f842159dbfeeed9a0dd", "pset_name": "", - "lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/3.root", + "lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/3.root", "app_name": "cmsRun", "output_module_label": "merged", "global_tag": "my-cms-gtag_8268", @@ -61,7 +61,7 @@ "release_version": "CMSSW_1_2_3", "pset_hash": "76e303993a1c2f842159dbfeeed9a0dd", "pset_name": "", - "lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/4.root", + "lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/4.root", "app_name": "cmsRun", "output_module_label": "merged", "global_tag": "my-cms-gtag_8268", @@ -95,7 +95,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/0.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/0.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -125,7 +125,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/1.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/1.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -155,7 +155,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/2.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/2.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -185,7 +185,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/3.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/3.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -215,7 +215,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/4.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/4.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -232,19 +232,19 @@ "create_by": "WMAgent", "primary_ds_type": "test", "primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain", - "creation_date": 1652798955 + "creation_date": 1695842582 }, "dataset": { "dataset_id": 0, "create_by": "WMAgent", - "creation_date": 1652798955, + "creation_date": 1695842582, "physics_group_name": "Tracker", "dataset_access_type": "PRODUCTION", "data_tier_name": "GEN-SIM-RAW", "last_modified_by": "WMAgent", "processed_ds_name": "acq_era_8268-ptsr-v8268", "xtcrosssection": 0, - "last_modification_date": 1652798955, + "last_modification_date": 1695842582, "dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-ptsr-v8268/GEN-SIM-RAW", "prep_id": "TestPrepID" }, @@ -507,19 +507,19 @@ "create_by": "WMAgent", "primary_ds_type": "test", "primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain", - "creation_date": 1652798955 + "creation_date": 1695842582 }, "dataset": { "dataset_id": 0, "create_by": "WMAgent", - "creation_date": 1652798955, + "creation_date": 1695842582, "physics_group_name": "Tracker", "dataset_access_type": "PRODUCTION", "data_tier_name": "GEN-SIM-RAW", "last_modified_by": "WMAgent", "processed_ds_name": "acq_era_8268-v8268", "xtcrosssection": 0, - "last_modification_date": 1652798955, + "last_modification_date": 1695842582, "dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-v8268/GEN-SIM-RAW", "prep_id": "TestPrepID" }, @@ -591,7 +591,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/5.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/5.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -621,7 +621,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/6.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/6.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -651,7 +651,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/7.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/7.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -681,7 +681,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/8.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/8.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -711,7 +711,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/9.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/9.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -728,19 +728,19 @@ "create_by": "WMAgent", "primary_ds_type": "test", "primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain2", - "creation_date": 1652798955 + "creation_date": 1695842582 }, "dataset": { "dataset_id": 0, "create_by": "WMAgent", - "creation_date": 1652798955, + "creation_date": 1695842582, "physics_group_name": "Tracker", "dataset_access_type": "PRODUCTION", "data_tier_name": "GEN-SIM-RAW", "last_modified_by": "WMAgent", "processed_ds_name": "acq_era_8268-ptsr-v82682", "xtcrosssection": 0, - "last_modification_date": 1652798955, + "last_modification_date": 1695842582, "dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-ptsr-v8268/GEN-SIM-RAW2", "prep_id": "TestPrepID" }, @@ -810,7 +810,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/5.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/5.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -840,7 +840,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/6.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/6.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -870,7 +870,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/7.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/7.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -900,7 +900,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/8.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/8.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -930,7 +930,7 @@ "file_type": "EDM", "last_modified_by": "", "last_modification_date": 0, - "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/9.root", + "logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/9.root", "md5": "", "auto_cross_section": 0, "is_file_valid": 1 @@ -947,19 +947,19 @@ "create_by": "WMAgent", "primary_ds_type": "test", "primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain2", - "creation_date": 1652798955 + "creation_date": 1695842582 }, "dataset": { "dataset_id": 0, "create_by": "WMAgent", - "creation_date": 1652798955, + "creation_date": 1695842582, "physics_group_name": "Tracker", "dataset_access_type": "PRODUCTION", "data_tier_name": "GEN-SIM-RAW", "last_modified_by": "WMAgent", "processed_ds_name": "acq_era_8268-v82682", "xtcrosssection": 0, - "last_modification_date": 1652798955, + "last_modification_date": 1695842582, "dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-v8268/GEN-SIM-RAW2", "prep_id": "TestPrepID" }, diff --git a/test/int_fileparents.go b/test/int_fileparents.go index 28425bfe..68e686fe 100644 --- a/test/int_fileparents.go +++ b/test/int_fileparents.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "net/http" "net/url" "testing" @@ -36,6 +37,61 @@ func getFileParentsTestTable(t *testing.T) EndpointTestCase { Type: "HTTPError", Message: dbsError.Error(), } + + // insert fileparent record requests + partialChildParentIDList := [][]int64{ + {21, 25}, + {22, -1}, + {23, -1}, + {24, -1}, + {25, -1}, + } + fpBlockPartialRecordGood := dbs.FileParentBlockRecord{ + BlockName: BulkBlocksData.SequentialParentData.Block.BlockName, + ChildParentIDList: partialChildParentIDList, + MissingFiles: 4, + } + fpBlockPartialRecordBad := dbs.FileParentBlockRecord{ + BlockName: BulkBlocksData.SequentialParentData.Block.BlockName, + ChildParentIDList: partialChildParentIDList, + MissingFiles: 3, + } + + completeChildParentIDList := [][]int64{ + {21, 1}, + {22, 2}, + {23, 3}, + {24, 4}, + {25, 5}, + } + fpBlockCompleteRecordGood := dbs.FileParentBlockRecord{ + BlockName: BulkBlocksData.SequentialParentData.Block.BlockName, + ChildParentIDList: completeChildParentIDList, + MissingFiles: 0, + } + + // responses for fileparents + // logical_file_name:/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/5.root + // parent_file_id:1 + // parent_logical_file_name:/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/IDEAL_/p8268/1.root + var fpRespList []Response + for i := 1; i <= 5; i++ { + lfn := fmt.Sprintf("/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/IDEAL_/p%v/%v.root", TestData.UID, i) + fpR := fileParentResponse{ + LOGICAL_FILE_NAME: BulkBlocksData.SequentialChildData.Files[i-1].LogicalFileName, + PARENT_FILE_ID: i, + PARENT_LOGICAL_FILE_NAME: lfn, + } + fpRespList = append(fpRespList, fpR) + } + fpR := fileParentResponse{ + LOGICAL_FILE_NAME: BulkBlocksData.SequentialChildData.Files[0].LogicalFileName, + PARENT_FILE_ID: 25, + PARENT_LOGICAL_FILE_NAME: BulkBlocksData.SequentialParentData.Files[4].LogicalFileName, + } + fpRespList = append(fpRespList[:2], fpRespList[1:]...) + fpRespList[1] = fpR + return EndpointTestCase{ description: "Test fileparents", defaultHandler: web.FileParentsHandler, @@ -69,6 +125,49 @@ func getFileParentsTestTable(t *testing.T) EndpointTestCase { output: []Response{errorResp}, respCode: http.StatusBadRequest, }, + { + description: "Test fileparents insert with partial parentage and bad missingFiles", + method: "POST", + serverType: "DBSWriter", + params: url.Values{ + "block_name": []string{BulkBlocksData.SequentialChildData.Block.BlockName}, + }, + input: fpBlockPartialRecordBad, + output: []Response{}, + respCode: http.StatusBadRequest, + }, + { + description: "Test fileparents insert with partial parentage and good missingFiles", + method: "POST", + serverType: "DBSWriter", + params: url.Values{ + "block_name": []string{BulkBlocksData.SequentialChildData.Block.BlockName}, + }, + input: fpBlockPartialRecordGood, + output: []Response{}, + respCode: http.StatusOK, + }, + { + description: "Test fileparents insert with complete parentage and good missingFiles", + method: "POST", + serverType: "DBSWriter", + params: url.Values{ + "block_name": []string{BulkBlocksData.SequentialChildData.Block.BlockName}, + }, + input: fpBlockCompleteRecordGood, + output: []Response{}, + respCode: http.StatusOK, + }, + { + description: "Test GET fileparents with parent block name", + method: "GET", + serverType: "DBSReader", + params: url.Values{ + "block_name": []string{BulkBlocksData.SequentialParentData.Block.BlockName}, + }, + output: fpRespList, + respCode: http.StatusOK, + }, }, } } diff --git a/test/integration_cases.go b/test/integration_cases.go index 6c87ea30..dd18725b 100644 --- a/test/integration_cases.go +++ b/test/integration_cases.go @@ -209,7 +209,7 @@ func generateBaseData(t *testing.T, filepath string) { } // creates a file for bulkblocks -func createFile(t *testing.T, i int) dbs.File { +func createFile(t *testing.T, parent string, i int) dbs.File { return dbs.File{ Adler32: "NOTSET", FileType: "EDM", @@ -234,7 +234,7 @@ func createFile(t *testing.T, i int) dbs.File { }, }, EventCount: 201, - LogicalFileName: fmt.Sprintf("/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p%v/%v.root", TestData.UID, i), + LogicalFileName: fmt.Sprintf("/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_%v/p%v/%v.root", parent, TestData.UID, i), IsFileValid: 1, } } @@ -330,7 +330,7 @@ func generateBulkBlocksData(t *testing.T, filepath string) { var parentFileList2 []dbs.File var childFileList2 []dbs.File for i := 0; i < fileCount; i++ { - f := createFile(t, i) + f := createFile(t, "ptr", i) parentFileList = append(parentFileList, f) var parentAlgo dbs.FileConfig @@ -353,7 +353,7 @@ func generateBulkBlocksData(t *testing.T, filepath string) { childAlgo.LFN = childf.LogicalFileName bulk.FileConfigList = append(bulk.FileConfigList, childAlgo) - f2 := createFile(t, i+fileCount) + f2 := createFile(t, "ptr", i+fileCount) parentFileList2 = append(parentFileList2, f2) childf2 := f2 childFileList2 = append(childFileList2, childf2) @@ -456,7 +456,7 @@ func generateLargeBulkBlocksData(t *testing.T, fileLumiChunkSize int, filepath s parentBulk.Dataset.ProcessedDSName = TestData.ParentProcDataset + "3" var parentFileList []dbs.File - f := createFile(t, 1) + f := createFile(t, "ptr", 1) var fileLumiList []dbs.FileLumi for i := 0; i < fileLumiChunkSize; i++ {