Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow Partial Parentage #101

Merged
merged 2 commits into from
Oct 23, 2023
Merged

Allow Partial Parentage #101

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
35bb188
Initial changes for partial parentage. Start of unit tests.
d-ylee Sep 26, 2023
33ea8b3
For Issue #94: Removed unused code, added integration tests, fixed is…
d-ylee Sep 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ jobs:
# - run: docker push cmssw/dbs2go:${{steps.get-ref.outputs.tag}}

- name: Push new image to k8s
if: "!contains(${{steps.get-ref.outputs.tag}}, 'dev')"
run: |
curl -ksLO https://raw.githubusercontent.com/vkuznet/imagebot/main/imagebot.sh
sed -i -e "s,COMMIT,${{github.sha}},g" -e "s,REPOSITORY,${{github.repository}},g" -e "s,NAMESPACE,dbs,g" -e "s,TAG,${{steps.get-ref.outputs.tag}},g" -e "s,IMAGE,registry.cern.ch/cmsweb/dbs2go,g" -e "s,SERVICE,dbs2go,g" -e "s,HOST,${{secrets.IMAGEBOT_URL}},g" imagebot.sh
Expand Down
34 changes: 33 additions & 1 deletion dbs/fileparents.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package dbs
import (
"database/sql"
"encoding/json"
"errors"
"fmt"
"io"
"log"
Expand Down Expand Up @@ -60,10 +61,11 @@ func (a *API) FileParents() error {
// FileParents represents file parents DBS DB table
type FileParents struct {
THIS_FILE_ID int64 `json:"this_file_id" validate:"required,number,gt=0"`
PARENT_FILE_ID int64 `json:"parent_file_id" validate:"required,number,gt=0"`
PARENT_FILE_ID int64 `json:"parent_file_id" validate:"required,number"` // TODO: may need to modify to handle `none` from python
}

// Insert implementation of FileParents
//
//gocyclo:ignore
func (r *FileParents) Insert(tx *sql.Tx) error {
var tid int64
Expand Down Expand Up @@ -206,6 +208,7 @@ func (r *FileParents) Insert(tx *sql.Tx) error {
}

// Validate implementation of FileParents
// TODO: handle this for partial parentage
func (r *FileParents) Validate() error {
if err := RecordValidator.Struct(*r); err != nil {
return DecodeValidatorError(r, err)
Expand Down Expand Up @@ -239,6 +242,7 @@ func (r *FileParents) Decode(reader io.Reader) error {
// err := decoder.Decode(&rec)
if err != nil {
log.Println("fail to decode data", err)
errors.Unwrap(err)
return Error(err, UnmarshalErrorCode, "", "dbs.fileparents.Decode")
}
return nil
Expand Down Expand Up @@ -274,12 +278,17 @@ func (a *API) InsertFileParents() error {
}

// FileParentBlockRecord represents file parent DBS record
// BlockName: name of the block
// ChildParentIDList: list of child and parent file ids
// MissingFiles: Number of missing files in the child and parent file list
type FileParentBlockRecord struct {
BlockName string `json:"block_name"`
ChildParentIDList [][]int64 `json:"child_parent_id_list"`
MissingFiles int64 `json:"missing_files"`
}

// InsertFileParentsBlockTxt DBS API
//
//gocyclo:ignore
func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error {
// read given input
Expand Down Expand Up @@ -358,6 +367,8 @@ func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error {
}

// now we can loop over provided list and insert file parents
var missingFiles []FileParents
var validatedChildParentIDList []FileParents
for _, v := range rec.ChildParentIDList {
var r FileParents
r.THIS_FILE_ID = v[0]
Expand All @@ -370,6 +381,27 @@ func (a *API) InsertFileParentsBlockTxt(tx *sql.Tx) error {
log.Println("unable to validate the record", r, "error", err)
return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt")
}
// only provide list of child parent that have parentage
// otherwise, skip partial parentage and add to the missingFiles count
if r.PARENT_FILE_ID > 0 {
validatedChildParentIDList = append(validatedChildParentIDList, r)
} else {
missingFiles = append(missingFiles, r)
}
}

if len(missingFiles) > 0 {
log.Printf("files without parentage: %v", missingFiles)
}

// check if provided MissingFiles matches the amount of mising files found
if len(missingFiles) != int(rec.MissingFiles) {
log.Println("provided missingFiles does not match number of pairs with no parentage")
return Error(err, ValidateErrorCode, "", "dbs.fileparents.InsertFileParentsBlockTxt")
}

// only insert valid child parent id lists
for _, r := range validatedChildParentIDList {
err = r.Insert(tx)
if err != nil {
if utils.VERBOSE > 1 {
Expand Down
2 changes: 1 addition & 1 deletion static/parameters.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@
{
"api": "fileparents",
"parameters": [
"logical_file_name", "block_name", "block_id"
"logical_file_name", "block_name", "block_id", "missing_files"
]
},
{
Expand Down
64 changes: 32 additions & 32 deletions test/data/integration/bulkblocks_data.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/0.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/0.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand All @@ -28,7 +28,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/1.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/1.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand All @@ -39,7 +39,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/2.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/2.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand All @@ -50,7 +50,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/3.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/3.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand All @@ -61,7 +61,7 @@
"release_version": "CMSSW_1_2_3",
"pset_hash": "76e303993a1c2f842159dbfeeed9a0dd",
"pset_name": "",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/4.root",
"lfn": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/4.root",
"app_name": "cmsRun",
"output_module_label": "merged",
"global_tag": "my-cms-gtag_8268",
Expand Down Expand Up @@ -95,7 +95,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/0.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/0.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -125,7 +125,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/1.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/1.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -155,7 +155,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/2.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/2.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -185,7 +185,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/3.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/3.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -215,7 +215,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/4.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/4.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand All @@ -232,19 +232,19 @@
"create_by": "WMAgent",
"primary_ds_type": "test",
"primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain",
"creation_date": 1652798955
"creation_date": 1695842582
},
"dataset": {
"dataset_id": 0,
"create_by": "WMAgent",
"creation_date": 1652798955,
"creation_date": 1695842582,
"physics_group_name": "Tracker",
"dataset_access_type": "PRODUCTION",
"data_tier_name": "GEN-SIM-RAW",
"last_modified_by": "WMAgent",
"processed_ds_name": "acq_era_8268-ptsr-v8268",
"xtcrosssection": 0,
"last_modification_date": 1652798955,
"last_modification_date": 1695842582,
"dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-ptsr-v8268/GEN-SIM-RAW",
"prep_id": "TestPrepID"
},
Expand Down Expand Up @@ -507,19 +507,19 @@
"create_by": "WMAgent",
"primary_ds_type": "test",
"primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain",
"creation_date": 1652798955
"creation_date": 1695842582
},
"dataset": {
"dataset_id": 0,
"create_by": "WMAgent",
"creation_date": 1652798955,
"creation_date": 1695842582,
"physics_group_name": "Tracker",
"dataset_access_type": "PRODUCTION",
"data_tier_name": "GEN-SIM-RAW",
"last_modified_by": "WMAgent",
"processed_ds_name": "acq_era_8268-v8268",
"xtcrosssection": 0,
"last_modification_date": 1652798955,
"last_modification_date": 1695842582,
"dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-v8268/GEN-SIM-RAW",
"prep_id": "TestPrepID"
},
Expand Down Expand Up @@ -591,7 +591,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/5.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/5.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -621,7 +621,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/6.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/6.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -651,7 +651,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/7.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/7.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -681,7 +681,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/8.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/8.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -711,7 +711,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/9.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/9.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand All @@ -728,19 +728,19 @@
"create_by": "WMAgent",
"primary_ds_type": "test",
"primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain2",
"creation_date": 1652798955
"creation_date": 1695842582
},
"dataset": {
"dataset_id": 0,
"create_by": "WMAgent",
"creation_date": 1652798955,
"creation_date": 1695842582,
"physics_group_name": "Tracker",
"dataset_access_type": "PRODUCTION",
"data_tier_name": "GEN-SIM-RAW",
"last_modified_by": "WMAgent",
"processed_ds_name": "acq_era_8268-ptsr-v82682",
"xtcrosssection": 0,
"last_modification_date": 1652798955,
"last_modification_date": 1695842582,
"dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-ptsr-v8268/GEN-SIM-RAW2",
"prep_id": "TestPrepID"
},
Expand Down Expand Up @@ -810,7 +810,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/5.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/5.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -840,7 +840,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/6.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/6.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -870,7 +870,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/7.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/7.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -900,7 +900,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/8.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/8.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand Down Expand Up @@ -930,7 +930,7 @@
"file_type": "EDM",
"last_modified_by": "",
"last_modification_date": 0,
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_/p8268/9.root",
"logical_file_name": "/store/mc/Fall08/BBJets250to500-madgraph/GEN-SIM-RAW/StepChain_ptr/p8268/9.root",
"md5": "",
"auto_cross_section": 0,
"is_file_valid": 1
Expand All @@ -947,19 +947,19 @@
"create_by": "WMAgent",
"primary_ds_type": "test",
"primary_ds_name": "unittest_web_primary_ds_name_8268_stepchain2",
"creation_date": 1652798955
"creation_date": 1695842582
},
"dataset": {
"dataset_id": 0,
"create_by": "WMAgent",
"creation_date": 1652798955,
"creation_date": 1695842582,
"physics_group_name": "Tracker",
"dataset_access_type": "PRODUCTION",
"data_tier_name": "GEN-SIM-RAW",
"last_modified_by": "WMAgent",
"processed_ds_name": "acq_era_8268-v82682",
"xtcrosssection": 0,
"last_modification_date": 1652798955,
"last_modification_date": 1695842582,
"dataset": "/unittest_web_primary_ds_name_8268_stepchain/acq_era_8268-v8268/GEN-SIM-RAW2",
"prep_id": "TestPrepID"
},
Expand Down
Loading
Loading