Skip to content

Commit

Permalink
add Keysets to to table dto / info (#452)
Browse files Browse the repository at this point in the history
* wip

* wip2

* wip3

* extend serializable

* implement serializable
  • Loading branch information
zcxcasd authored Aug 24, 2021
1 parent 04eac5b commit 6bb1a68
Show file tree
Hide file tree
Showing 13 changed files with 409 additions and 63 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package com.netflix.metacat.common.server.connectors.model;

import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;

import java.io.Serializable;
import java.util.List;

/**
* Key Info.
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = false)
public class KeyInfo implements Serializable {
private static final long serialVersionUID = 7254898853779135216L;

private String name;
private List<String> fields;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package com.netflix.metacat.common.server.connectors.model;

import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;

import java.io.Serializable;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;

/**
* KeySet Info.
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = false)
public class KeySetInfo implements Serializable {
private static final long serialVersionUID = 3659843901964058788L;

private static final String PARTITION_KEY_DEFAULT_NAME = "partition";
private static final String PRIMARY_KEY_DEFAULT_NAME = "primary";
private static final String SORT_KEY_DEFAULT_NAME = "sort";
private static final String INDEX_KEY_DEFAULT_NAME = "index";

private List<KeyInfo> partition;
private List<KeyInfo> primary;
private List<KeyInfo> sort;
private List<KeyInfo> index;

/**
* builds a keyset from fieldInfo list.
*
* @param fields list of fieldInfo
* @return keyset
*/
public static KeySetInfo buildKeySet(final List<FieldInfo> fields) {
return buildKeySet(fields, null);
}

/**
* builds a keyset from fieldInfo list and primary key list.
*
* @param fields list of fieldInfo
* @param primary list of primary keys
* @return keyset
*/
public static KeySetInfo buildKeySet(final List<FieldInfo> fields, final List<KeyInfo> primary) {
if (fields == null) {
return null;
} else if (fields.isEmpty()) {
return new KeySetInfo();
}

final List<String> partitionKeys = new LinkedList<>();
final List<String> sortKeys = new LinkedList<>();
final List<String> indexKeys = new LinkedList<>();
for (FieldInfo field : fields) {
if (field.isPartitionKey()) {
partitionKeys.add(field.getName());
}
if (field.getIsSortKey() != null && field.getIsSortKey()) {
sortKeys.add(field.getName());
}
if (field.getIsIndexKey() != null && field.getIsIndexKey()) {
indexKeys.add(field.getName());
}
}

final KeySetInfo keySetInfo = new KeySetInfo();
keySetInfo.partition = partitionKeys.isEmpty() ? Collections.emptyList()
: Arrays.asList(
KeyInfo.builder().name(PARTITION_KEY_DEFAULT_NAME).fields(partitionKeys).build());
keySetInfo.sort = sortKeys.isEmpty() ? Collections.emptyList()
: Arrays.asList(KeyInfo.builder().name(SORT_KEY_DEFAULT_NAME).fields(sortKeys).build());
keySetInfo.index = indexKeys.isEmpty() ? Collections.emptyList()
: Arrays.asList(KeyInfo.builder().name(INDEX_KEY_DEFAULT_NAME).fields(indexKeys).build());
keySetInfo.primary = (primary == null || primary.isEmpty()) ? Collections.emptyList()
: primary;

return keySetInfo;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ public class TableInfo extends BaseInfo {
private List<FieldInfo> fields;
private StorageInfo serde;
private ViewInfo view;
private KeySetInfo keys;

/**
* Constructor.
Expand All @@ -57,11 +58,35 @@ private TableInfo(
final Map<String, String> metadata,
final List<FieldInfo> fields,
final StorageInfo serde,
final ViewInfo view
final ViewInfo view,
final KeySetInfo keys
) {
super(name, auditInfo, metadata);
this.fields = fields;
this.serde = serde;
this.view = view;
this.keys = keys;
}

/**
* builds key set info from the fields.
*
* @return key set
*/
public KeySetInfo getKeys() {
if (this.keys != null) {
return keys;
}
keys = KeySetInfo.buildKeySet(this.fields);
return keys;
}

/**
* sets the keyset.
*
* @param keys keyset to be set
*/
public void setKeys(final KeySetInfo keys) {
this.keys = keys;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import com.netflix.metacat.common.dto.DatabaseDto;
import com.netflix.metacat.common.dto.FieldDto;
import com.netflix.metacat.common.dto.GetPartitionsRequestDto;
import com.netflix.metacat.common.dto.KeyDto;
import com.netflix.metacat.common.dto.KeySetDto;
import com.netflix.metacat.common.dto.ViewDto;
import com.netflix.metacat.common.dto.Pageable;
import com.netflix.metacat.common.dto.PartitionDto;
Expand All @@ -39,6 +41,8 @@
import com.netflix.metacat.common.server.connectors.model.ClusterInfo;
import com.netflix.metacat.common.server.connectors.model.DatabaseInfo;
import com.netflix.metacat.common.server.connectors.model.FieldInfo;
import com.netflix.metacat.common.server.connectors.model.KeyInfo;
import com.netflix.metacat.common.server.connectors.model.KeySetInfo;
import com.netflix.metacat.common.server.connectors.model.ViewInfo;
import com.netflix.metacat.common.server.connectors.model.PartitionInfo;
import com.netflix.metacat.common.server.connectors.model.PartitionListRequest;
Expand Down Expand Up @@ -101,6 +105,8 @@ protected void configure() {
mapping(AuditDto.class, AuditInfo.class);
mapping(ViewDto.class, ViewInfo.class);
mapping(StorageDto.class, StorageInfo.class);
mapping(KeySetDto.class, KeySetInfo.class);
mapping(KeyDto.class, KeyInfo.class);
}
};
dozerBeanMapper.addMapping(builder);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,13 @@ class ConverterUtilSpec extends Specification {
given:
def dto = new TableDto(name: QualifiedName.ofTable('prodhive', 'amajumdar', 'part'),
audit: new AuditDto('test', new Date(), 'test', new Date()),
fields: [FieldDto.builder().name('esn').type('string').source_type('string').jsonType(new TextNode('string')).pos(0).build()] ,
serde: new StorageDto(owner: 'test'))
fields: [FieldDto.builder().name('esn').type('string').source_type('string')
.jsonType(new TextNode('string')).pos(0).build()] ,
serde: new StorageDto(owner: 'test'),
keys: KeySetDto.builder()
.partition(Arrays.asList(KeyDto.builder().name('partition').fields(Arrays.asList('esn')).build()))
.build()
)
when:
def info = converter.fromTableDto(dto)
def resultDto = converter.toTableDto(info)
Expand All @@ -65,7 +70,11 @@ class ConverterUtilSpec extends Specification {
audit: new AuditDto('test', new Date(), 'test', new Date()),
fields: [FieldDto.builder().name('esn').type('string').source_type('string').jsonType(new TextNode('string')).pos(0).build()],
serde: new StorageDto(owner: 'test'),
view: new ViewDto("select test", "select test2"))
view: new ViewDto("select test", "select test2"),
keys: KeySetDto.builder()
.partition(Arrays.asList(KeyDto.builder().name('partition').fields(Arrays.asList('esn')).build()))
.build()
)
when:
def info = converter.fromTableDto(dto)
def resultDto = converter.toTableDto(info)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,20 @@
/*
*
* Copyright 2016 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.netflix.metacat.common.dto;

import io.swagger.annotations.ApiModel;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
*
* Copyright 2016 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.netflix.metacat.common.dto;

import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;

import java.util.List;

/**
* Key DTO.
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = false)
public class KeyDto extends BaseDto {
private static final long serialVersionUID = 5511551575484406779L;

private String name;
private List<String> fields;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
*
* Copyright 2016 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.netflix.metacat.common.dto;

import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;

import java.util.List;

/**
* KeySet DTO.
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@EqualsAndHashCode(callSuper = false)
public class KeySetDto extends BaseDto {
private static final long serialVersionUID = 6250093794701822334L;

private List<KeyDto> partition;
private List<KeyDto> primary;
private List<KeyDto> sort;
private List<KeyDto> index;
}
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ public class TableDto extends BaseDto implements HasDataMetadata, HasDefinitionM
//Naming as view required by dozer mapping
private ViewDto view;

@ApiModelProperty(value = "keys defined in the table")
@JsonProperty
private KeySetDto keys;

@Nonnull
@Override
@JsonIgnore
Expand All @@ -97,19 +101,25 @@ public QualifiedName getDefinitionName() {
@JsonProperty
@SuppressWarnings("checkstyle:methodname")
public List<String> getPartition_keys() {
if (fields == null) {
return null;
} else if (fields.isEmpty()) {
return Collections.emptyList();
}
if (this.keys == null) {
if (fields == null) {
return null;
} else if (fields.isEmpty()) {
return Collections.emptyList();
}

final List<String> keys = new LinkedList<>();
for (FieldDto field : fields) {
if (field.isPartition_key()) {
keys.add(field.getName());
final List<String> partitionKeys = new LinkedList<>();
for (FieldDto field : fields) {
if (field.isPartition_key()) {
partitionKeys.add(field.getName());
}
}
return partitionKeys;
}
return keys;

return this.keys.getPartition().isEmpty()
? Collections.EMPTY_LIST
: this.keys.getPartition().get(0).getFields();
}

/**
Expand All @@ -120,6 +130,19 @@ public List<String> getPartition_keys() {
public void setPartition_keys(final List<String> ignored) {
}

@JsonIgnore
public KeySetDto getKeys() {
return keys;
}

/**
* Sets the keyset.
* @param keys keyset
*/
public void setKeys(final KeySetDto keys) {
this.keys = keys;
}

@Override
@JsonProperty
public boolean isDataExternal() {
Expand Down
Loading

0 comments on commit 6bb1a68

Please sign in to comment.