Merge pull request #24138 from vespa-engine/toregge/only-save-config-…

…needed-for-replay-of-transaction-log

Only save config needed for replay of transaction log.

searchcore/src/apps/tests/persistenceconformance_test.cpp

@@ -16,7 +16,6 @@
 #include <vespa/document/test/make_bucket_space.h>
 #include <vespa/fastos/file.h>
 #include <vespa/persistence/conformancetest/conformancetest.h>
-#include <vespa/searchcommon/common/schemaconfigurer.h>
 #include <vespa/searchcore/proton/common/alloc_config.h>
 #include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcore/proton/matching/querylimiter.h>
@@ -61,7 +60,6 @@ using document::test::makeBucketSpace;
 using search::TuneFileDocumentDB;
 using search::index::DummyFileHeaderContext;
 using search::index::Schema;
-using search::index::SchemaBuilder;
 using search::transactionlog::TransLogServer;
 using storage::spi::ConformanceTest;
 using storage::spi::PersistenceProvider;
@@ -126,10 +124,7 @@ class ConfigFactory {
         CS::IndexschemaConfigSP indexschema = _schemaFactory->createIndexSchema(*docType);
         CS::AttributesConfigSP attributes = _schemaFactory->createAttributes(*docType);
         CS::SummaryConfigSP summary = _schemaFactory->createSummary(*docType);
-        Schema::SP schema(new Schema());
-        SchemaBuilder::build(*indexschema, *schema);
-        SchemaBuilder::build(*attributes, *schema);
-        SchemaBuilder::build(*summary, *schema);
+        auto schema = DocumentDBConfig::build_schema(*attributes, *summary, *indexschema);
         return std::make_shared<DocumentDBConfig>(
                         1,
                         std::make_shared<RankProfilesConfig>(),

searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp

@@ -123,22 +123,27 @@ class MyConfigBuilder {
 };
 
 struct Fixture {
-    Schema::SP schema;
+    std::shared_ptr<Schema> basic_schema;
+    std::shared_ptr<Schema> full_schema;
     std::shared_ptr<const DocumentTypeRepo> repo;
     ConfigSP basicCfg;
     ConfigSP fullCfg;
     ConfigSP replayCfg;
     ConfigSP nullCfg;
     Fixture()
-        : schema(make_shared<Schema>()),
+        : basic_schema(make_shared<Schema>()),
+          full_schema(make_shared<Schema>()),
           repo(make_shared<DocumentTypeRepo>()),
           basicCfg(),
           fullCfg(),
           replayCfg(),
           nullCfg()
     {
-        basicCfg = MyConfigBuilder(4, schema, repo).addAttribute().addSummary(false, false).build();
-        fullCfg = MyConfigBuilder(4, schema, repo).addAttribute().
+        basic_schema->addAttributeField(Schema::AttributeField("my_attribute", schema::DataType::INT32));
+        full_schema->addAttributeField(Schema::AttributeField("my_attribute", schema::DataType::INT32));
+        full_schema->addSummaryField(Schema::SummaryField("my_attribute", schema::DataType::INT32));
+        basicCfg = MyConfigBuilder(4, basic_schema, repo).addAttribute().addSummary(false, false).build();
+        fullCfg = MyConfigBuilder(4, full_schema, repo).addAttribute().
                                                    addRankProfile().
                                                    addRankingConstant().
                                                    addRankingExpression().

searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp

@@ -20,7 +20,6 @@
 #include <vespa/searchcore/proton/server/i_proton_disk_layout.h>
 #include <vespa/searchcore/proton/server/threading_service_config.h>
 #include <vespa/searchsummary/config/config-juniperrc.h>
-#include <vespa/searchcommon/common/schemaconfigurer.h>
 #include <vespa/vespalib/util/size_literals.h>
 #include <vespa/vespalib/util/threadstackexecutor.h>
 #include <vespa/vespalib/test/insertion_operators.h>
@@ -42,7 +41,6 @@ using document::DocumentTypeRepo;
 using search::TuneFileDocumentDB;
 using std::map;
 using search::index::Schema;
-using search::index::SchemaBuilder;
 using proton::matching::RankingConstants;
 using proton::matching::RankingExpressions;
 using proton::matching::OnnxModels;
@@ -58,11 +56,7 @@ struct DBConfigFixture {
 
     Schema::SP buildSchema()
     {
-        Schema::SP schema(std::make_shared<Schema>());
-        SchemaBuilder::build(_attributesBuilder, *schema);
-        SchemaBuilder::build(_summaryBuilder, *schema);
-        SchemaBuilder::build(_indexschemaBuilder, *schema);
-        return schema;
+        return DocumentDBConfig::build_schema(_attributesBuilder, _summaryBuilder, _indexschemaBuilder);
     }
 
     static RankingConstants::SP buildRankingConstants()

searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp

@@ -31,7 +31,6 @@
 #include <vespa/messagebus/config-messagebus.h>
 #include <vespa/messagebus/testlib/slobrok.h>
 #include <vespa/metrics/config-metricsmanager.h>
-#include <vespa/searchcommon/common/schemaconfigurer.h>
 #include <vespa/searchcore/proton/common/alloc_config.h>
 #include <vespa/searchcore/proton/matching/querylimiter.h>
 #include <vespa/searchcore/proton/metrics/metricswireservice.h>
@@ -91,7 +90,6 @@ using proton::DocumentDB;
 using proton::DocumentDBConfig;
 using proton::HwInfo;
 using search::index::Schema;
-using search::index::SchemaBuilder;
 using search::transactionlog::TransLogServer;
 using storage::MergeThrottler;
 using storage::distributor::BucketSpacesStatsProvider;
@@ -178,10 +176,7 @@ std::shared_ptr<DocumentDBConfig> make_document_db_config(std::shared_ptr<Docume
     auto indexschema = std::make_shared<IndexschemaConfig>();
     auto attributes = make_attributes_config();
     auto summary = std::make_shared<SummaryConfig>();
-    std::shared_ptr<Schema> schema(new Schema());
-    SchemaBuilder::build(*indexschema, *schema);
-    SchemaBuilder::build(*attributes, *schema);
-    SchemaBuilder::build(*summary, *schema);
+    auto schema = DocumentDBConfig::build_schema(*attributes, *summary, *indexschema);
     return std::make_shared<DocumentDBConfig>(
             1,
             std::make_shared<RankProfilesConfig>(),

searchcore/src/vespa/searchcore/proton/server/documentdb.cpp

@@ -233,7 +233,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir,
 
     _feedHandler->init(_config_store->getOldestSerialNum());
     _feedHandler->setBucketDBHandler(&_subDBs.getBucketDBHandler());
-    saveInitialConfig(*configSnapshot);
+    saveInitialConfig(configSnapshot);
     resumeSaveConfig();
     SerialNum configSerial = _config_store->getPrevValidSerial(_feedHandler->getPrunedSerialNum() + 1);
     assert(configSerial > 0);
@@ -473,15 +473,17 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum
     const ReconfigParams params(cmpres);
 
     // Save config via config manager if replay is done.
+    auto replay_config = DocumentDBConfig::makeReplayConfig(configSnapshot);
     bool equalReplayConfig =
-        *DocumentDBConfig::makeReplayConfig(configSnapshot) ==
+        *replay_config ==
         *DocumentDBConfig::makeReplayConfig(_activeConfigSnapshot);
     bool tlsReplayDone = _feedHandler->getTransactionLogReplayDone();
     FeedHandler::CommitResult commit_result;
     if (!equalReplayConfig && tlsReplayDone) {
         sync(_feedHandler->getSerialNum());
         serialNum = _feedHandler->inc_serial_num();
-        _config_store->saveConfig(*configSnapshot, serialNum);
+        _config_store->saveConfig(*replay_config, serialNum);
+        replay_config.reset();
         // save entry in transaction log
         NewConfigOperation op(serialNum, *_config_store);
         commit_result = _feedHandler->storeOperationSync(op);
@@ -632,7 +634,7 @@ DocumentDB::getNumActiveDocs() const
 }
 
 void
-DocumentDB::saveInitialConfig(const DocumentDBConfig &configSnapshot)
+DocumentDB::saveInitialConfig(std::shared_ptr<DocumentDBConfig> configSnapshot)
 {
     // Only called from ctor
 
@@ -657,7 +659,8 @@ DocumentDB::saveInitialConfig(const DocumentDBConfig &configSnapshot)
         LOG(warning, "DocumentDB(%s): saveInitialConfig() failed pruning due to '%s'",
             _docTypeName.toString().c_str(), e.what());
     }
-    _config_store->saveConfig(configSnapshot, confSerial);
+    auto replay_config = DocumentDBConfig::makeReplayConfig(configSnapshot);
+    _config_store->saveConfig(*replay_config, confSerial);
 }
 
 void

searchcore/src/vespa/searchcore/proton/server/documentdb.h

@@ -155,7 +155,7 @@ class DocumentDB : public DocumentDBConfigOwner,
      *
      * @param configSnapshot initial config snapshot.
      */
-    void saveInitialConfig(const DocumentDBConfig &configSnapshot);
+    void saveInitialConfig(std::shared_ptr<DocumentDBConfig> configSnapshot);
 
     /**
      * Resume interrupted config save if needed.

searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp

@@ -10,6 +10,7 @@
 #include <vespa/document/config/documenttypes_config_fwd.h>
 #include <vespa/document/config/config-documenttypes.h>
 #include <vespa/document/repo/documenttyperepo.h>
+#include <vespa/searchcommon/common/schemaconfigurer.h>
 #include <vespa/searchcore/config/config-ranking-constants.h>
 #include <vespa/searchcore/config/config-onnx-models.h>
 #include <vespa/searchcore/proton/attribute/attribute_aspect_delayer.h>
@@ -24,6 +25,7 @@ using namespace vespa::config::search;
 using document::DocumentTypeRepo;
 using search::TuneFileDocumentDB;
 using search::index::Schema;
+using search::index::SchemaBuilder;
 using vespa::config::search::core::RankingConstantsConfig;
 using vespa::config::search::core::OnnxModelsConfig;
 
@@ -233,7 +235,12 @@ DocumentDBConfig::SP
 DocumentDBConfig::makeReplayConfig(const SP & orig)
 {
     const DocumentDBConfig &o = *orig;
-
+
+    auto replay_summary_config = emptyConfig(o._summary);
+    auto replay_schema = build_schema(*o._attributes, *replay_summary_config, *o._indexschema);
+    if (*replay_schema == *o._schema) {
+        replay_schema = o._schema;
+    }
     SP ret = std::make_shared<DocumentDBConfig>(
                 o._generation,
                 emptyConfig(o._rankProfiles),
@@ -242,13 +249,13 @@ DocumentDBConfig::makeReplayConfig(const SP & orig)
                 std::make_shared<OnnxModels>(),
                 o._indexschema,
                 o._attributes,
-                emptyConfig(o._summary),
+                replay_summary_config,
                 o._juniperrc,
                 o._documenttypes,
                 o._repo,
                 std::make_shared<ImportedFieldsConfig>(),
                 o._tuneFileDocumentDB,
-                o._schema,
+                replay_schema,
                 o._maintenance,
                 o._storeConfig,
                 o._threading_service_config,
@@ -347,5 +354,16 @@ DocumentDBConfig::getDocumentType() const
     return _repo->getDocumentType(getDocTypeName());
 }
 
+std::shared_ptr<Schema>
+DocumentDBConfig::build_schema(const AttributesConfig& attributes_config,
+                               const SummaryConfig& summary_config,
+                               const IndexschemaConfig &indexschema_config)
+{
+    auto schema = std::make_shared<Schema>();
+    SchemaBuilder::build(attributes_config, *schema);
+    SchemaBuilder::build(summary_config, *schema);
+    SchemaBuilder::build(indexschema_config, *schema);
+    return schema;
+}
 
 } // namespace proton

searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h

@@ -250,6 +250,11 @@ class DocumentDBConfig
      * reprocessing.
      */
     static SP makeDelayedAttributeAspectConfig(const SP &newCfg, const DocumentDBConfig &oldCfg);
+
+    static std::shared_ptr<search::index::Schema>
+    build_schema(const AttributesConfig& attributes_config,
+                 const SummaryConfig& summary_config,
+                 const IndexschemaConfig &indexschema_config);
 };
 
 } // namespace proton

searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp

@@ -72,22 +72,6 @@ DocumentDBConfigManager::createConfigKeySet() const
     return set;
 }
 
-namespace {
-
-Schema::SP
-buildNewSchema(const AttributesConfig &newAttributesConfig,
-               const SummaryConfig &newSummaryConfig,
-               const IndexschemaConfig &newIndexschemaConfig)
-{
-    Schema::SP schema = std::make_shared<Schema>();
-    SchemaBuilder::build(newAttributesConfig, *schema);
-    SchemaBuilder::build(newSummaryConfig, *schema);
-    SchemaBuilder::build(newIndexschemaConfig, *schema);
-    return schema;
-}
-
-}
-
 Schema::SP
 DocumentDBConfigManager::buildSchema(const AttributesConfig &newAttributesConfig,
                                      const SummaryConfig &newSummaryConfig,
@@ -99,14 +83,14 @@ DocumentDBConfigManager::buildSchema(const AttributesConfig &newAttributesConfig
         oldSchema = _pendingConfigSnapshot->getSchemaSP();
     }
     if (!oldSchema) {
-        return buildNewSchema(newAttributesConfig, newSummaryConfig, newIndexschemaConfig);
+        return DocumentDBConfig::build_schema(newAttributesConfig, newSummaryConfig, newIndexschemaConfig);
     }
     const DocumentDBConfig &old = *_pendingConfigSnapshot;
     if (old.getAttributesConfig() != newAttributesConfig ||
         old.getSummaryConfig() != newSummaryConfig ||
         old.getIndexschemaConfig() != newIndexschemaConfig)
     {
-        Schema::SP schema(buildNewSchema(newAttributesConfig, newSummaryConfig, newIndexschemaConfig));
+        auto schema = DocumentDBConfig::build_schema(newAttributesConfig, newSummaryConfig, newIndexschemaConfig);
         return (*oldSchema == *schema) ? oldSchema : schema;
     }
     return oldSchema;