diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp index dca567417a8..8eb7b4340c6 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.cpp @@ -1507,7 +1507,7 @@ void TIndexTabletActor::HandleForcedOperationCompletedImpl( TOverloaded{ [&](const TForcedRangeOperationState& state) { - LOG_DEBUG( + LOG_INFO( ctx, TFileStoreComponents::TABLET, "%s ForcedRangeOperation mode=%u completed (%s)", @@ -1517,7 +1517,7 @@ void TIndexTabletActor::HandleForcedOperationCompletedImpl( }, [&](const TForcedTabletOperationState& state) { - LOG_DEBUG( + LOG_INFO( ctx, TFileStoreComponents::TABLET, "%s ForcedTabletOperation mode=%u completed (%s)", diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_forced_range_operation.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_forced_range_operation.cpp index 795b0298a2f..8e564c0846f 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_forced_range_operation.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_forced_range_operation.cpp @@ -271,7 +271,7 @@ void TIndexTabletActor::HandleForcedRangeOperation( { auto* msg = ev->Get(); - LOG_DEBUG( + LOG_INFO( ctx, TFileStoreComponents::TABLET, "%s ForcedRangeOperation request mode=%u for %lu ranges", diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_forced_tablet_operation.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_forced_tablet_operation.cpp index dd3a3ba7553..c32fc3a3fc5 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_forced_tablet_operation.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_forced_tablet_operation.cpp @@ -13,46 +13,18 @@ namespace { //////////////////////////////////////////////////////////////////////////////// -struct TDefaultCompletionPolicy -{ - static bool IsCompleted(const NProto::TError& error) - { - Y_UNUSED(error); - return true; - } -}; - -struct TFlushBytesCompletionPolicy -{ - static bool IsCompleted(const NProto::TError& error) - { - // A successful flush processes only one chunk and may reply before - // trimming it. Only an empty flush confirms that all chunks are - // drained. - return error.GetCode() == S_FALSE; - } -}; - -//////////////////////////////////////////////////////////////////////////////// - /** * @brief An actor that performs forced flush, flush_bytes, collect_garbage ops. * It is implemented as a template class to avoid code duplication. - * - * @tparam TCompletionPolicy Determines whether a successful response completes - * the forced operation. Errors use the same retry policy for all operations. */ -template < - typename TResponseType, - typename TRequestType, - typename TCompletionPolicy = TDefaultCompletionPolicy> +template class TForcedOperationActor final : public TActorBootstrapped< - TForcedOperationActor> + TForcedOperationActor> { private: using TBase = NActors::TActorBootstrapped< - TForcedOperationActor>; + TForcedOperationActor>; const TActorId Tablet; const TString LogTag; @@ -91,28 +63,21 @@ class TForcedOperationActor final //////////////////////////////////////////////////////////////////////////////// -template < - typename TResponseType, - typename TRequestType, - typename TCompletionPolicy> -TForcedOperationActor:: - TForcedOperationActor( - TActorId tablet, - TString logTag, - TDuration retry, - TRequestInfoPtr requestInfo) +template +TForcedOperationActor::TForcedOperationActor( + TActorId tablet, + TString logTag, + TDuration retry, + TRequestInfoPtr requestInfo) : Tablet(tablet) , LogTag(std::move(logTag)) , RetryTimeout(retry) , RequestInfo(std::move(requestInfo)) {} -template < - typename TResponseType, - typename TRequestType, - typename TCompletionPolicy> -void TForcedOperationActor:: - Bootstrap(const TActorContext& ctx) +template +void TForcedOperationActor::Bootstrap( + const TActorContext& ctx) { TBase::Become(&TBase::TThis::StateWork); @@ -124,23 +89,16 @@ void TForcedOperationActor:: SendOperationRequest(ctx); } -template < - typename TResponseType, - typename TRequestType, - typename TCompletionPolicy> -void TForcedOperationActor:: - SendOperationRequest(const TActorContext& ctx) +template +void TForcedOperationActor::SendOperationRequest( + const TActorContext& ctx) { auto request = std::make_unique(); ctx.Send(Tablet, request.release()); } -template < - typename TResponseType, - typename TRequestType, - typename TCompletionPolicy> -STFUNC((TForcedOperationActor:: - StateWork)) +template +STFUNC((TForcedOperationActor::StateWork)) { switch (ev->GetTypeRewrite()) { HFunc(TEvents::TEvWakeup, HandleWakeUp); @@ -157,11 +115,8 @@ STFUNC((TForcedOperationActor:: } } -template < - typename TResponseType, - typename TRequestType, - typename TCompletionPolicy> -void TForcedOperationActor:: +template +void TForcedOperationActor:: HandleOperationResponse( const TResponseType::TPtr& ev, const TActorContext& ctx) @@ -177,43 +132,31 @@ void TForcedOperationActor:: return ReplyAndDie(ctx, msg->Error); } - if (TCompletionPolicy::IsCompleted(msg->Error)) { - return ReplyAndDie(ctx, {}); - } - - SendOperationRequest(ctx); + return ReplyAndDie(ctx, {}); } -template < - typename TResponseType, - typename TRequestType, - typename TCompletionPolicy> -void TForcedOperationActor:: - HandleWakeUp(const TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx) +template +void TForcedOperationActor::HandleWakeUp( + const TEvents::TEvWakeup::TPtr& ev, + const TActorContext& ctx) { Y_UNUSED(ev); SendOperationRequest(ctx); } -template < - typename TResponseType, - typename TRequestType, - typename TCompletionPolicy> -void TForcedOperationActor:: - HandlePoisonPill( - const TEvents::TEvPoison::TPtr& ev, - const TActorContext& ctx) +template +void TForcedOperationActor::HandlePoisonPill( + const TEvents::TEvPoison::TPtr& ev, + const TActorContext& ctx) { Y_UNUSED(ev); ReplyAndDie(ctx, MakeError(E_FAIL, "actor killed")); } -template < - typename TResponseType, - typename TRequestType, - typename TCompletionPolicy> -void TForcedOperationActor:: - ReplyAndDie(const TActorContext& ctx, const NProto::TError& error) +template +void TForcedOperationActor::ReplyAndDie( + const TActorContext& ctx, + const NProto::TError& error) { { // notify tablet @@ -243,12 +186,9 @@ using TForcedFlushActor = TForcedOperationActor< TEvIndexTabletPrivate::TEvFlushResponse, TEvIndexTabletPrivate::TEvFlushRequest>; -// Note: only use with no concurrent writes to the fs. Otherwise the -// actor may continue issuing unbounded number of flush bytes requests. using TForcedFlushBytesActor = TForcedOperationActor< TEvIndexTabletPrivate::TEvFlushBytesResponse, - TEvIndexTabletPrivate::TEvFlushBytesRequest, - TFlushBytesCompletionPolicy>; + TEvIndexTabletPrivate::TEvFlushBytesRequest>; using TForcedCollectGarbageActor = TForcedOperationActor< TEvIndexTabletPrivate::TEvCollectGarbageResponse, @@ -264,7 +204,7 @@ void TIndexTabletActor::HandleForcedTabletOperation( { auto* msg = ev->Get(); - LOG_DEBUG( + LOG_INFO( ctx, TFileStoreComponents::TABLET, "%s ForcedTabletOperation mode=%u request", diff --git a/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp b/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp index 8f110aaf49a..9753a031fd1 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp @@ -4959,68 +4959,6 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data) testEnvConfig); } - TABLET_TEST(ShouldDrainAllFreshBytesWithForcedFlushBytes) - { - const auto block = tabletConfig.BlockSize; - NProto::TStorageConfig storageConfig; - storageConfig.SetWriteBlobThreshold(1_GB); - storageConfig.SetFlushThreshold(1_GB); - storageConfig.SetFlushBytesThreshold(1_GB); - storageConfig.SetTrimBytesItemCount(1); - - TTestEnv env(testEnvConfig, storageConfig); - auto& runtime = env.GetRuntime(); - const auto nodeIdx = env.AddDynamicNode(); - const auto tabletId = env.BootIndexTablet(nodeIdx); - TIndexTabletClient tablet(runtime, nodeIdx, tabletId, tabletConfig); - tablet.InitSession("client", "session"); - - const auto id = - CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test")); - const auto handle = CreateHandle(tablet, id); - for (ui32 i = 0; i < 2; ++i) { - tablet.WriteData(handle, i * block, block, 'a'); - tablet.WriteData(handle, i * block + 100, 10, 'b'); - } - tablet.CreateCheckpoint("first"); - - // Seal a second chunk containing only deletion markers. - const auto sparseId = - CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "sparse")); - tablet.SetNodeAttr(TSetNodeAttrArgs(sparseId).SetSize(3 * block)); - tablet.SetNodeAttr(TSetNodeAttrArgs(sparseId).SetSize(2 * block)); - tablet.SetNodeAttr(TSetNodeAttrArgs(sparseId).SetSize(block)); - tablet.CreateCheckpoint("second"); - - // Leave a third chunk active, below the background flush threshold. - for (ui32 i = 0; i < 2; ++i) { - tablet.WriteData(handle, i * block + 100, 10, 'c'); - } - - { - const auto response = tablet.GetStorageStats(); - const auto& stats = response->Record.GetStats(); - UNIT_ASSERT_VALUES_EQUAL(40, stats.GetFreshBytesCount()); - UNIT_ASSERT_VALUES_EQUAL(4, stats.GetFreshBytesItemCount()); - UNIT_ASSERT_VALUES_EQUAL( - 2 * block, - stats.GetDeletedFreshBytesCount()); - } - - auto request = std::make_unique< - TEvIndexTabletPrivate::TEvForcedTabletOperationRequest>( - TEvIndexTabletPrivate::EForcedTabletOperationMode::FlushBytes, - CreateGuidAsString()); - tablet.SendRequest(std::move(request)); - tablet.AssertForcedTabletOperationResponse(S_OK); - - const auto response = tablet.GetStorageStats(); - const auto& stats = response->Record.GetStats(); - UNIT_ASSERT_VALUES_EQUAL(0, stats.GetFreshBytesCount()); - UNIT_ASSERT_VALUES_EQUAL(0, stats.GetFreshBytesItemCount()); - UNIT_ASSERT_VALUES_EQUAL(0, stats.GetDeletedFreshBytesCount()); - } - TABLET_TEST(ShouldCompleteForcedFlushBytesWhenEmpty) { TTestEnv env(testEnvConfig); diff --git a/cloud/filestore/private/api/protos/tablet.proto b/cloud/filestore/private/api/protos/tablet.proto index 599db154f97..225a12887a0 100644 --- a/cloud/filestore/private/api/protos/tablet.proto +++ b/cloud/filestore/private/api/protos/tablet.proto @@ -702,7 +702,7 @@ message TForcedOperationRequest // Tablet operations E_FLUSH = 3; - E_FLUSH_BYTES = 4; // Only use when no concurrent writes are possible + E_FLUSH_BYTES = 4; E_COLLECT_GARBAGE = 5; }; diff --git a/cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt b/cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt index 009092fa46d..12a39a138f3 100644 --- a/cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt +++ b/cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt @@ -22,5 +22,8 @@ Tests { WriteBytes: 4096 MinFileCount: 3 } + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-auth-test/index.txt b/cloud/filestore/tests/loadtest/service-kikimr-auth-test/index.txt index 600542b27f5..7df9f883c9a 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-auth-test/index.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-auth-test/index.txt @@ -21,5 +21,8 @@ Tests { IODepth: 64 RequestsCount: 1024 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } -} \ No newline at end of file +} diff --git a/cloud/filestore/tests/loadtest/service-kikimr-auth-test/read-write.txt b/cloud/filestore/tests/loadtest/service-kikimr-auth-test/read-write.txt index 4816f1af8e3..02bfbfbd565 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-auth-test/read-write.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-auth-test/read-write.txt @@ -25,5 +25,8 @@ Tests { } IODepth: 64 TestDuration: 240 + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-datashard-like-test/datashard-like-read-write-with-page-size.txt b/cloud/filestore/tests/loadtest/service-kikimr-datashard-like-test/datashard-like-read-write-with-page-size.txt index a36cc3376ab..9dc6a96b7a6 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-datashard-like-test/datashard-like-read-write-with-page-size.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-datashard-like-test/datashard-like-read-write-with-page-size.txt @@ -27,5 +27,8 @@ Tests { } IODepth: 4 TestDuration: 30 + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-datashard-like-test/datashard-like-read-write.txt b/cloud/filestore/tests/loadtest/service-kikimr-datashard-like-test/datashard-like-read-write.txt index 44ef213671c..d813f53a7c0 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-datashard-like-test/datashard-like-read-write.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-datashard-like-test/datashard-like-read-write.txt @@ -27,5 +27,8 @@ Tests { } IODepth: 4 TestDuration: 30 + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-emergency-test/read-write-validation.txt b/cloud/filestore/tests/loadtest/service-kikimr-emergency-test/read-write-validation.txt index b1cd8fd0cbd..2e6dc60331f 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-emergency-test/read-write-validation.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-emergency-test/read-write-validation.txt @@ -22,5 +22,8 @@ Tests { IODepth: 64 RequestsCount: 4096 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/create-remove.txt b/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/create-remove.txt index 1212041dd57..b6bd17df777 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/create-remove.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/create-remove.txt @@ -20,5 +20,9 @@ Tests { } IODepth: 32 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + TimeoutSeconds: 300 + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/intrahost-migration.txt b/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/intrahost-migration.txt index 8689fd0889e..a3f10aae89f 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/intrahost-migration.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/intrahost-migration.txt @@ -24,5 +24,9 @@ Tests { MigrationPeriod: 20 StateTransferDelay: 10 } + IndexConsistencyValidation: { + Enabled: true + TimeoutSeconds: 300 + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/read-write-unaligned.txt b/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/read-write-unaligned.txt index a04525c5809..c50a34c4987 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/read-write-unaligned.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/read-write-unaligned.txt @@ -23,5 +23,9 @@ Tests { } IODepth: 4 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + TimeoutSeconds: 300 + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/read-write-validation.txt b/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/read-write-validation.txt index a534bc7de81..c10c2d99d46 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/read-write-validation.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/read-write-validation.txt @@ -25,6 +25,10 @@ Tests { } IODepth: 1 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + TimeoutSeconds: 300 + } } } Tests { @@ -54,5 +58,9 @@ Tests { } IODepth: 1 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + TimeoutSeconds: 300 + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/read-write.txt b/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/read-write.txt index 6e0f9a3146a..48fb9ec15e7 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/read-write.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/read-write.txt @@ -20,6 +20,10 @@ Tests { } IODepth: 64 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + TimeoutSeconds: 300 + } } } Tests { @@ -49,5 +53,9 @@ Tests { } IODepth: 64 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + TimeoutSeconds: 300 + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/get-node-attr.txt b/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/get-node-attr.txt index fdf3bc2467a..ac8a940f3b8 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/get-node-attr.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/get-node-attr.txt @@ -18,5 +18,8 @@ Tests { IODepth: 64 RequestsCount: 16384 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/read-write-sequential-validation.txt b/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/read-write-sequential-validation.txt index 9ea0a629f33..6863c4dcee6 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/read-write-sequential-validation.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/read-write-sequential-validation.txt @@ -27,6 +27,9 @@ Tests { } IODepth: 4 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } Tests { @@ -58,5 +61,8 @@ Tests { } IODepth: 4 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/read-write-validation.txt b/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/read-write-validation.txt index 5cb1b91625f..e61a61b5906 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/read-write-validation.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-newfeatures-test/read-write-validation.txt @@ -25,6 +25,9 @@ Tests { } IODepth: 4 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } Tests { @@ -54,6 +57,9 @@ Tests { } IODepth: 4 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } Tests { @@ -83,5 +89,8 @@ Tests { } IODepth: 2 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-test/create-rename.txt b/cloud/filestore/tests/loadtest/service-kikimr-test/create-rename.txt index 764eef9dbc2..7df9f883c9a 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-test/create-rename.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-test/create-rename.txt @@ -21,5 +21,8 @@ Tests { IODepth: 64 RequestsCount: 1024 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-test/get-node-attr.txt b/cloud/filestore/tests/loadtest/service-kikimr-test/get-node-attr.txt index fdf3bc2467a..ac8a940f3b8 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-test/get-node-attr.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-test/get-node-attr.txt @@ -18,5 +18,8 @@ Tests { IODepth: 64 RequestsCount: 16384 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-test/intrahost-migration.txt b/cloud/filestore/tests/loadtest/service-kikimr-test/intrahost-migration.txt index 8689fd0889e..03b14b12d56 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-test/intrahost-migration.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-test/intrahost-migration.txt @@ -24,5 +24,8 @@ Tests { MigrationPeriod: 20 StateTransferDelay: 10 } + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-test/read-write-unaligned.txt b/cloud/filestore/tests/loadtest/service-kikimr-test/read-write-unaligned.txt index e996367e485..60401c265f5 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-test/read-write-unaligned.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-test/read-write-unaligned.txt @@ -24,5 +24,8 @@ Tests { IODepth: 4 RequestsCount: 200000 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-test/read-write-validation.txt b/cloud/filestore/tests/loadtest/service-kikimr-test/read-write-validation.txt index a534bc7de81..a1ebf7130c8 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-test/read-write-validation.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-test/read-write-validation.txt @@ -25,6 +25,9 @@ Tests { } IODepth: 1 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } Tests { @@ -54,5 +57,8 @@ Tests { } IODepth: 1 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tests/loadtest/service-kikimr-test/read-write.txt b/cloud/filestore/tests/loadtest/service-kikimr-test/read-write.txt index 6e0f9a3146a..4dc60bf434a 100644 --- a/cloud/filestore/tests/loadtest/service-kikimr-test/read-write.txt +++ b/cloud/filestore/tests/loadtest/service-kikimr-test/read-write.txt @@ -20,6 +20,9 @@ Tests { } IODepth: 64 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } Tests { @@ -49,5 +52,8 @@ Tests { } IODepth: 64 TestDuration: 60 + IndexConsistencyValidation: { + Enabled: true + } } } diff --git a/cloud/filestore/tools/testing/loadtest/lib/filestore_index_operations.cpp b/cloud/filestore/tools/testing/loadtest/lib/filestore_index_operations.cpp new file mode 100644 index 00000000000..845a4f8d65b --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/lib/filestore_index_operations.cpp @@ -0,0 +1,286 @@ +#include "filestore_index_operations.h" + +#include +#include + +#include + +#include +#include +#include + +#include + +namespace NCloud::NFileStore::NLoadTest { + +TFileStoreIndexOperations::TFileStoreIndexOperations( + TLog& log, + const IFileStoreServicePtr& client, + const TString& filesystemId, + TInstant deadlineTs) + : Log(log) + , Client(client) + , FilesystemId(filesystemId) + , DeadlineTs(deadlineTs) + , FilesystemIds{filesystemId} +{ + const auto shardIds = GetShardFileSystemIds(); + FilesystemIds.insert(FilesystemIds.end(), shardIds.begin(), shardIds.end()); +} + +NProtoPrivate::TStorageStats TFileStoreIndexOperations::GetStorageStats() +{ + NProtoPrivate::TGetStorageStatsRequest request; + request.SetFileSystemId(FilesystemId); + request.SetCacheTTL(0); + NProtoPrivate::TGetStorageStatsResponse response; + ExecuteAction("getstoragestats", request, &response); + if (HasError(response)) { + throw yexception() << FilesystemId << ": " + << "failed to execute getstoragestats: " + << response.GetError(); + } + return response.GetStats(); +} + +TVector TFileStoreIndexOperations::GetShardFileSystemIds() +{ + NProtoPrivate::TGetFileSystemTopologyRequest request; + request.SetFileSystemId(FilesystemId); + NProtoPrivate::TGetFileSystemTopologyResponse response; + ExecuteAction("getfilesystemtopology", request, &response); + if (HasError(response)) { + throw yexception() << FilesystemId << ": " + << "failed to execute getfilesystemtopology: " + << response.GetError(); + } + + const auto& shardIds = response.GetShardFileSystemIds(); + return {shardIds.begin(), shardIds.end()}; +} + +void TFileStoreIndexOperations::FlushBytes() +{ + // Each request processes one chunk and may reply before trimming finishes. + ForcedOperationUntil( + NProtoPrivate::TForcedOperationRequest::E_FLUSH_BYTES, + [](const NProtoPrivate::TStorageStats& stats) + { + return stats.GetFreshBytesCount() == 0 && + stats.GetDeletedFreshBytesCount() == 0; + }); +} + +void TFileStoreIndexOperations::Flush() +{ + ForcedOperation(NProtoPrivate::TForcedOperationRequest::E_FLUSH); +} + +void TFileStoreIndexOperations::Compaction() +{ + ForcedOperation(NProtoPrivate::TForcedOperationRequest::E_COMPACTION); +} + +void TFileStoreIndexOperations::Cleanup() +{ + ForcedOperation(NProtoPrivate::TForcedOperationRequest::E_CLEANUP); +} + +void TFileStoreIndexOperations::CollectGarbage() +{ + // Pending barriers may prevent a request from draining the garbage queue. + ForcedOperationUntil( + NProtoPrivate::TForcedOperationRequest::E_COLLECT_GARBAGE, + [](const NProtoPrivate::TStorageStats& stats) + { return stats.GetGarbageQueueSize() == 0; }); +} + +template +void TFileStoreIndexOperations::ForcedOperationUntil( + NProtoPrivate::TForcedOperationRequest::EForcedOperationType type, + TPredicate isCompleted) +{ + while (true) { + ForcedOperation(type); + if (isCompleted(GetStorageStats())) { + return; + } + + if (TInstant::Now() >= DeadlineTs) { + throw yexception() << "fs=" << FilesystemId << " " + << NProtoPrivate::TForcedOperationRequest:: + EForcedOperationType_Name(type) + << ": deadline exceeded."; + } + Sleep(TDuration::Seconds(1)); + } +} + +void TFileStoreIndexOperations::ForcedOperation( + NProtoPrivate::TForcedOperationRequest::EForcedOperationType type) +{ + for (const auto& filesystemId: FilesystemIds) { + ForcedOperation(filesystemId, type); + } +} + +void TFileStoreIndexOperations::ForcedOperation( + const TString& filesystemId, + NProtoPrivate::TForcedOperationRequest::EForcedOperationType type) +{ + const auto& typeName = + NProtoPrivate::TForcedOperationRequest::EForcedOperationType_Name(type); + + auto operationId = RunForcedOperation(filesystemId, type); + + while (true) { + NProtoPrivate::TForcedOperationStatusRequest statusRequest; + statusRequest.SetFileSystemId(filesystemId); + statusRequest.SetOperationId(operationId); + NProtoPrivate::TForcedOperationStatusResponse statusResponse; + ExecuteAction("forcedoperationstatus", statusRequest, &statusResponse); + + if (statusResponse.GetError().GetCode() == E_NOT_FOUND) { + STORAGE_WARN( + TStringBuilder() << "fs=" << filesystemId << " " << typeName + << ": operation status not found, restarting"); + operationId = RunForcedOperation(filesystemId, type); + continue; + } + + if (HasError(statusResponse)) { + throw yexception() + << "fs=" << filesystemId << " " << typeName + << ": failed to read op status: " << statusResponse.GetError(); + } + + switch (statusResponse.GetOperationStatus()) { + case NProtoPrivate::TForcedOperationStatusResponse::E_UNKNOWN: + case NProtoPrivate::TForcedOperationStatusResponse::E_PENDING: + case NProtoPrivate::TForcedOperationStatusResponse::E_RUNNING: + STORAGE_INFO( + TStringBuilder() << "fs=" << filesystemId << " " << typeName + << ": waiting..."); + break; + case NProtoPrivate::TForcedOperationStatusResponse::E_COMPLETED: + STORAGE_INFO( + TStringBuilder() << "fs=" << filesystemId << " " << typeName + << ": completed"); + return; + case NProtoPrivate::TForcedOperationStatusResponse::E_FAILED: { + const auto& error = statusResponse.GetOperationError(); + if (GetErrorKind(error) == EErrorKind::ErrorRetriable) { + STORAGE_WARN( + TStringBuilder() + << "fs=" << filesystemId << " " << typeName + << ": operation failed, restarting: " + << FormatError(error)); + Sleep(TDuration::Seconds(1)); + operationId = RunForcedOperation(filesystemId, type); + continue; + } + throw yexception() + << "fs=" << filesystemId << " " << typeName << ": failed: " + << (HasError(error) ? FormatError(error) : "without error"); + } + default: + STORAGE_WARN( + TStringBuilder() << "fs=" << filesystemId << " " << typeName + << ": unknown status"); + break; + } + + Sleep(TDuration::Seconds(1)); + } +} + +TString TFileStoreIndexOperations::RunForcedOperation( + const TString& filesystemId, + NProtoPrivate::TForcedOperationRequest::EForcedOperationType type) +{ + while (true) { + NProtoPrivate::TForcedOperationRequest request; + request.SetFileSystemId(filesystemId); + request.SetOpType(type); + + NProtoPrivate::TForcedOperationResponse response; + ExecuteAction("forcedoperation", request, &response); + if (HasError(response)) { + if (response.GetError().GetCode() == E_TRY_AGAIN) { + Sleep(TDuration::Seconds(1)); + continue; + } + const auto& typeName = NProtoPrivate::TForcedOperationRequest:: + EForcedOperationType_Name(type); + throw yexception() << "fs=" << filesystemId << " " + << "failed to start forced operation " + << typeName << ": " << response.GetError(); + } + + // The same type of operation is already running. + // In essence, this is a form of E_TRY_AGAIN. + // TODO: return current operationId in response. + if (response.GetError().GetCode() == S_ALREADY) { + Sleep(TDuration::Seconds(1)); + continue; + } + return response.GetOperationId(); + } +} + +template +void TFileStoreIndexOperations::ExecuteAction( + const TString& action, + const TRequest& requestProto, + TResponse* responseProto) +{ + const auto& filesystemId = requestProto.GetFileSystemId(); + if (TInstant::Now() >= DeadlineTs) { + throw yexception() << "fs=" << filesystemId << " " << action + << ": deadline exceeded"; + } + + TString input; + auto status = + google::protobuf::util::MessageToJsonString(requestProto, &input); + if (!status.ok()) { + responseProto->MutableError()->CopyFrom(MakeError( + E_BADMSG, + TStringBuilder() << "failed to serialize request in json: " + << status.ToString())); + return; + } + + auto request = std::make_shared(); + request->SetAction(action); + request->SetInput(input); + + const auto requestId = GetRequestId(*request); + auto future = Client->ExecuteAction( + MakeIntrusive(filesystemId, requestId), + std::move(request)); + if (!future.Wait(DeadlineTs)) { + throw yexception() + << "fs=" << filesystemId + << ": deadline exceeded waiting for response; request=" << input; + } + const auto& result = future.GetValue(); + + if (HasError(result)) { + responseProto->MutableError()->CopyFrom(result.GetError()); + return; + } + + if (!google::protobuf::util::JsonStringToMessage( + result.GetOutput(), + responseProto) + .ok()) + { + responseProto->MutableError()->CopyFrom(MakeError( + E_BADMSG, + TStringBuilder() + << "failed to parse response json: " << result.GetOutput())); + } +} + +} // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/filestore_index_operations.h b/cloud/filestore/tools/testing/loadtest/lib/filestore_index_operations.h new file mode 100644 index 00000000000..260ade91a0e --- /dev/null +++ b/cloud/filestore/tools/testing/loadtest/lib/filestore_index_operations.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace NCloud::NFileStore::NLoadTest { + +class TFileStoreIndexOperations +{ + TLog& Log; + IFileStoreServicePtr Client; + TString FilesystemId; + TInstant DeadlineTs; + TVector FilesystemIds; + +public: + TFileStoreIndexOperations( + TLog& log, + const IFileStoreServicePtr& client, + const TString& filesystemId, + TInstant deadlineTs); + + NProtoPrivate::TStorageStats GetStorageStats(); + + void FlushBytes(); + void Flush(); + void Compaction(); + void Cleanup(); + void CollectGarbage(); + +private: + TVector GetShardFileSystemIds(); + + void ForcedOperation( + NProtoPrivate::TForcedOperationRequest::EForcedOperationType type); + template + void ForcedOperationUntil( + NProtoPrivate::TForcedOperationRequest::EForcedOperationType type, + TPredicate isCompleted); + void ForcedOperation( + const TString& filesystemId, + NProtoPrivate::TForcedOperationRequest::EForcedOperationType type); + TString RunForcedOperation( + const TString& filesystemId, + NProtoPrivate::TForcedOperationRequest::EForcedOperationType type); + + template + void ExecuteAction( + const TString& action, + const TRequest& requestProto, + TResponse* responseProto); +}; + +} // namespace NCloud::NFileStore::NLoadTest diff --git a/cloud/filestore/tools/testing/loadtest/lib/test.cpp b/cloud/filestore/tools/testing/loadtest/lib/test.cpp index 81d4f834da9..ab0c766f4d2 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/test.cpp +++ b/cloud/filestore/tools/testing/loadtest/lib/test.cpp @@ -2,6 +2,7 @@ #include "client.h" #include "context.h" +#include "filestore_index_operations.h" #include "request.h" #include "shm_client.h" @@ -73,14 +74,19 @@ struct TTestStats template T WaitForCompletion( const TString& request, - const TFuture& future) + const TFuture& future, + TInstant deadlineTs = TInstant::Max()) { - const auto& response = future.GetValue(TDuration::Max()); + if (!future.Wait(deadlineTs)) { + throw yexception() << "Failed to execute " << request + << " request: deadline exceeded"; + } + + const auto& response = future.GetValue(); if (HasError(response)) { const auto& error = response.GetError(); - throw yexception() - << "Failed to execute " << request << " request: " - << FormatError(error); + throw yexception() << "Failed to execute " << request + << " request: " << FormatError(error); } return response; @@ -1132,6 +1138,21 @@ class TLoadTestController final stopTest(SourceTest, SourceFuture); stopTest(TargetTest, TargetFuture); + const auto indexConsistencyValidation = + Config.GetIndexConsistencyValidation(); + if (indexConsistencyValidation.GetEnabled()) { + TString filesystemId; + if (Config.HasFileSystemId()) { + filesystemId = Config.GetFileSystemId(); + } else if (Config.HasCreateFileStoreRequest()) { + filesystemId = + Config.GetCreateFileStoreRequest().GetFileSystemId(); + } + ValidateIndexConsistency( + indexConsistencyValidation, + filesystemId); + } + if (Config.HasCreateFileStoreRequest() && !Config.GetKeepFileStore()) { auto filesystemId = Config.GetCreateFileStoreRequest().GetFileSystemId(); @@ -1217,11 +1238,100 @@ class TLoadTestController final Exception; } - const TString& MakeTestTag() const { return TestTag; } + + void ValidateIndexConsistency( + const NProto::TIndexConsistencyValidation& config, + const TString& filesystemId) + { + STORAGE_INFO( + "%s Starting index consistency validation", + MakeTestTag().c_str()); + + const auto timeout = + config.GetTimeoutSeconds() ? config.GetTimeoutSeconds() : 60; + const auto deadlineTs = TInstant::Now() + TDuration::Seconds(timeout); + + try { + TFileStoreIndexOperations filestoreCleanup( + Log, + Client, + filesystemId, + deadlineTs); + filestoreCleanup.Flush(); + filestoreCleanup.FlushBytes(); + filestoreCleanup.Cleanup(); + filestoreCleanup.Compaction(); + // We need to create a new commit to make collect_garbage handle + // new blobs + CreateCommit(filesystemId, deadlineTs); + filestoreCleanup.CollectGarbage(); + ValidateStorageStats(filestoreCleanup.GetStorageStats()); + } catch (...) { + SaveException(); + } + } + + void ValidateStorageStats(const NProtoPrivate::TStorageStats& stats) + { + TStringBuilder builder; + auto check = [&](const char* name, ui64 value) + { + if (value != 0) { + if (!builder.empty()) { + builder << ", "; + } + builder << name << "=" << value; + } + }; + + check("FreshBlocksCount", stats.GetFreshBlocksCount()); + check("FreshBytesItemCount", stats.GetFreshBytesItemCount()); + check("DeletedFreshBytesCount", stats.GetDeletedFreshBytesCount()); + check("DeletionMarkersCount", stats.GetDeletionMarkersCount()); + // TODO: there is no forced op to trigger processing of large deletions + // check(LargeDeletionMarkersCount); + check("GarbageQueueSize", stats.GetGarbageQueueSize()); + check("GarbageBlocksCount", stats.GetGarbageBlocksCount()); + check("UnconfirmedDataCount", stats.GetUnconfirmedDataCount()); + + if (!builder.empty()) { + throw yexception() << "ValidateStorageStats failed: " << builder; + } + + STORAGE_INFO("Storage GC stats validation passed"); + } + + // Creating and destroying a session should be sufficient to increment + // commit id across all shards. + void CreateCommit(const TString& filesystemId, TInstant deadlineTs) + { + NProto::TSessionConfig proto; + proto.SetFileSystemId(filesystemId); + proto.SetClientId( + Config.GetClientId() ? Config.GetClientId() : "test-client"); + proto.SetSessionPingTimeout(Config.GetSessionPingTimeout()); + + auto session = NClient::CreateSession( + Logging, + Timer, + Scheduler, + Client, + std::make_shared(proto)); + + WaitForCompletion( + "create session", + session->CreateSession(false, SessionSeqNo), + deadlineTs); + + WaitForCompletion( + "destroy session", + session->DestroySession(), + deadlineTs); + } }; } // namespace diff --git a/cloud/filestore/tools/testing/loadtest/lib/ya.make b/cloud/filestore/tools/testing/loadtest/lib/ya.make index 28f49781557..2550d2e95a1 100644 --- a/cloud/filestore/tools/testing/loadtest/lib/ya.make +++ b/cloud/filestore/tools/testing/loadtest/lib/ya.make @@ -17,6 +17,7 @@ ENDIF() SRCS( client.cpp context.h + filestore_index_operations.cpp executor.cpp request_data.cpp request_index.cpp diff --git a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto index 31a3b3a33f7..387bfc9408c 100644 --- a/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto +++ b/cloud/filestore/tools/testing/loadtest/protos/loadtest.proto @@ -176,6 +176,15 @@ message TMigrationSpec uint32 StateTransferDelay = 2; } +message TIndexConsistencyValidation +{ + // Enables the forced cleanup and checks (false by default) + bool Enabled = 1; + + // Maximum duration of final garbage collection (60s by default) + uint32 TimeoutSeconds = 2; +} + message TLoadTest { string Name = 1; @@ -213,6 +222,10 @@ message TLoadTest // Maximum number of nodes created by this synthetic load test. // Zero means unlimited. Ignored by replay specs. uint64 MaxNodes = 22; + + // Enables final cleanup and validation of filesystem metrics, assuring + // no garbage is left + TIndexConsistencyValidation IndexConsistencyValidation = 23; } ////////////////////////////////////////////////////////////////////////////////