diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index dd9fb09b572c0..c434da9ecfd7d 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -71,6 +71,7 @@ void OlapTableIndexSchema::to_protobuf(POlapTableIndexSchema* pindex) const { pindex->set_id(index_id); pindex->set_schema_hash(schema_hash); pindex->set_schema_id(schema_id); + pindex->set_is_shadow(is_shadow); for (auto slot : slots) { pindex->add_columns(slot->col_name()); } @@ -128,6 +129,13 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { index->column_to_expr_value.insert({entry.first, entry.second}); } + if (p_index.has_is_shadow()) { + index->is_shadow = p_index.is_shadow(); + if (index->is_shadow) { + _shadow_indexes++; + } + } + _indexes.emplace_back(index); } @@ -187,6 +195,12 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema, RuntimeS index->column_to_expr_value.insert({entry.first, entry.second}); } } + if (t_index.__isset.is_shadow) { + index->is_shadow = t_index.is_shadow; + if (index->is_shadow) { + _shadow_indexes++; + } + } _indexes.emplace_back(index); } @@ -477,10 +491,11 @@ Status OlapTablePartitionParam::add_partitions(const std::vectornum_buckets = t_part.num_buckets; auto num_indexes = _schema->indexes().size(); - if (t_part.indexes.size() != num_indexes) { + if (t_part.indexes.size() != num_indexes - _schema->shadow_index_size()) { std::stringstream ss; ss << "number of partition's index is not equal with schema's" - << ", num_part_indexes=" << t_part.indexes.size() << ", num_schema_indexes=" << num_indexes; + << ", num_part_indexes=" << t_part.indexes.size() << ", num_schema_indexes=" << num_indexes + << ", num_shadow_indexes=" << _schema->shadow_index_size(); LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } @@ -490,16 +505,25 @@ Status OlapTablePartitionParam::add_partitions(const std::vectorindexes[j].index_id != _schema->indexes()[j]->index_id) { + // If an add_partition operation is executed during the ALTER process, the ALTER operation will be canceled first. + // Therefore, the latest indexes will not include shadow indexes. + // However, the schema's index may still contain shadow indexes, so these shadow indexes need to be ignored. + int j = 0; + for (int i = 0; i < num_indexes; ++i) { + if (_schema->indexes()[i]->is_shadow) { + continue; + } + if (part->indexes[j].index_id != _schema->indexes()[i]->index_id) { std::stringstream ss; ss << "partition's index is not equal with schema's" << ", part_index=" << part->indexes[j].index_id - << ", schema_index=" << _schema->indexes()[j]->index_id; + << ", schema_index=" << _schema->indexes()[i]->index_id; LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } + j++; } + _partitions.emplace(part->id, part); if (t_part.__isset.in_keys) { for (auto& in_key : part->in_keys) { diff --git a/be/src/exec/tablet_info.h b/be/src/exec/tablet_info.h index a86963b0c4098..9f2ac9f2d1184 100644 --- a/be/src/exec/tablet_info.h +++ b/be/src/exec/tablet_info.h @@ -51,6 +51,7 @@ struct OlapTableIndexSchema { OlapTableColumnParam* column_param; ExprContext* where_clause = nullptr; std::map column_to_expr_value; + bool is_shadow = false; void to_protobuf(POlapTableIndexSchema* pindex) const; }; @@ -81,6 +82,7 @@ class OlapTableSchemaParam { return _proto_schema; } + int64_t shadow_index_size() const { return _shadow_indexes; } std::string debug_string() const; private: @@ -92,6 +94,8 @@ class OlapTableSchemaParam { mutable POlapTableSchemaParam* _proto_schema = nullptr; std::vector _indexes; mutable ObjectPool _obj_pool; + + int64_t _shadow_indexes = 0; }; using OlapTableIndexTablets = TOlapTableIndexTablets; diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/OlapTableSink.java b/fe/fe-core/src/main/java/com/starrocks/planner/OlapTableSink.java index 835a70998b724..c7eff7fc88d4a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/com/starrocks/planner/OlapTableSink.java @@ -376,6 +376,7 @@ public static TOlapTableSchemaParam createSchema(long dbId, OlapTable table, Tup .stream() .map(column -> column.isShadowColumn() ? column.getName() : column.getColumnId().getId()) .collect(Collectors.toList())); + boolean isShadow = indexMeta.getSchema().stream().anyMatch(column -> column.isShadowColumn()); for (Column column : indexMeta.getSchema()) { TColumn tColumn = column.toThrift(); tColumn.setColumn_name(column.getColumnId().getId()); @@ -400,6 +401,7 @@ public static TOlapTableSchemaParam createSchema(long dbId, OlapTable table, Tup indexSchema.setColumn_param(columnParam); indexSchema.setSchema_id(indexMeta.getSchemaId()); indexSchema.setColumn_to_expr_value(columnToExprValue); + indexSchema.setIs_shadow(isShadow); schemaParam.addToIndexes(indexSchema); if (indexMeta.getWhereClause() != null) { Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(dbId); diff --git a/gensrc/proto/descriptors.proto b/gensrc/proto/descriptors.proto index c81728df7fbb8..4215e63639b24 100644 --- a/gensrc/proto/descriptors.proto +++ b/gensrc/proto/descriptors.proto @@ -76,6 +76,7 @@ message POlapTableIndexSchema { optional POlapTableColumnParam column_param = 4; optional int64 schema_id = 5; map column_to_expr_value = 6; + optional bool is_shadow = 7; }; message POlapTableSchemaParam { diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 6a2d34bde1478..34b1bf4b6b63a 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -288,6 +288,7 @@ struct TOlapTableIndexSchema { 5: optional Exprs.TExpr where_clause 6: optional i64 schema_id // schema id 7: optional map column_to_expr_value + 8: optional bool is_shadow } struct TOlapTableSchemaParam {