[Filestore] issue-4115: validate garbage collection in load tests

cloud/filestore/libs/storage/tablet/tablet_actor.cpp

@@ -1507,7 +1507,7 @@ void TIndexTabletActor::HandleForcedOperationCompletedImpl(
         TOverloaded{
             [&](const TForcedRangeOperationState& state)
             {
-                LOG_DEBUG(
+                LOG_INFO(
                     ctx,
                     TFileStoreComponents::TABLET,
                     "%s ForcedRangeOperation mode=%u completed (%s)",
@@ -1517,7 +1517,7 @@ void TIndexTabletActor::HandleForcedOperationCompletedImpl(
             },
             [&](const TForcedTabletOperationState& state)
             {
-                LOG_DEBUG(
+                LOG_INFO(
                     ctx,
                     TFileStoreComponents::TABLET,
                     "%s ForcedTabletOperation mode=%u completed (%s)",

cloud/filestore/libs/storage/tablet/tablet_actor_forced_range_operation.cpp

@@ -271,7 +271,7 @@ void TIndexTabletActor::HandleForcedRangeOperation(
 {
     auto* msg = ev->Get();
 
-    LOG_DEBUG(
+    LOG_INFO(
         ctx,
         TFileStoreComponents::TABLET,
         "%s ForcedRangeOperation request mode=%u for %lu ranges",

cloud/filestore/libs/storage/tablet/tablet_actor_forced_tablet_operation.cpp

@@ -13,46 +13,18 @@ namespace {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-struct TDefaultCompletionPolicy
-{
-    static bool IsCompleted(const NProto::TError& error)
-    {
-        Y_UNUSED(error);
-        return true;
-    }
-};
-
-struct TFlushBytesCompletionPolicy
-{
-    static bool IsCompleted(const NProto::TError& error)
-    {
-        // A successful flush processes only one chunk and may reply before
-        // trimming it. Only an empty flush confirms that all chunks are
-        // drained.
-        return error.GetCode() == S_FALSE;
-    }
-};
-
-////////////////////////////////////////////////////////////////////////////////
-
 /**
  * @brief An actor that performs forced flush, flush_bytes, collect_garbage ops.
  * It is implemented as a template class to avoid code duplication.
- *
- * @tparam TCompletionPolicy Determines whether a successful response completes
- * the forced operation. Errors use the same retry policy for all operations.
  */
-template <
-    typename TResponseType,
-    typename TRequestType,
-    typename TCompletionPolicy = TDefaultCompletionPolicy>
+template <typename TResponseType, typename TRequestType>
 class TForcedOperationActor final
     : public TActorBootstrapped<
-          TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>>
+          TForcedOperationActor<TResponseType, TRequestType>>
 {
 private:
     using TBase = NActors::TActorBootstrapped<
-        TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>>;
+        TForcedOperationActor<TResponseType, TRequestType>>;
 
     const TActorId Tablet;
     const TString LogTag;
@@ -91,28 +63,21 @@ class TForcedOperationActor final
 
 ////////////////////////////////////////////////////////////////////////////////
 
-template <
-    typename TResponseType,
-    typename TRequestType,
-    typename TCompletionPolicy>
-TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>::
-    TForcedOperationActor(
-        TActorId tablet,
-        TString logTag,
-        TDuration retry,
-        TRequestInfoPtr requestInfo)
+template <typename TResponseType, typename TRequestType>
+TForcedOperationActor<TResponseType, TRequestType>::TForcedOperationActor(
+    TActorId tablet,
+    TString logTag,
+    TDuration retry,
+    TRequestInfoPtr requestInfo)
     : Tablet(tablet)
     , LogTag(std::move(logTag))
     , RetryTimeout(retry)
     , RequestInfo(std::move(requestInfo))
 {}
 
-template <
-    typename TResponseType,
-    typename TRequestType,
-    typename TCompletionPolicy>
-void TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>::
-    Bootstrap(const TActorContext& ctx)
+template <typename TResponseType, typename TRequestType>
+void TForcedOperationActor<TResponseType, TRequestType>::Bootstrap(
+    const TActorContext& ctx)
 {
     TBase::Become(&TBase::TThis::StateWork);
 
@@ -124,23 +89,16 @@ void TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>::
     SendOperationRequest(ctx);
 }
 
-template <
-    typename TResponseType,
-    typename TRequestType,
-    typename TCompletionPolicy>
-void TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>::
-    SendOperationRequest(const TActorContext& ctx)
+template <typename TResponseType, typename TRequestType>
+void TForcedOperationActor<TResponseType, TRequestType>::SendOperationRequest(
+    const TActorContext& ctx)
 {
     auto request = std::make_unique<TRequestType>();
     ctx.Send(Tablet, request.release());
 }
 
-template <
-    typename TResponseType,
-    typename TRequestType,
-    typename TCompletionPolicy>
-STFUNC((TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>::
-            StateWork))
+template <typename TResponseType, typename TRequestType>
+STFUNC((TForcedOperationActor<TResponseType, TRequestType>::StateWork))
 {
     switch (ev->GetTypeRewrite()) {
         HFunc(TEvents::TEvWakeup, HandleWakeUp);
@@ -157,11 +115,8 @@ STFUNC((TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>::
     }
 }
 
-template <
-    typename TResponseType,
-    typename TRequestType,
-    typename TCompletionPolicy>
-void TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>::
+template <typename TResponseType, typename TRequestType>
+void TForcedOperationActor<TResponseType, TRequestType>::
     HandleOperationResponse(
         const TResponseType::TPtr& ev,
         const TActorContext& ctx)
@@ -177,43 +132,31 @@ void TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>::
         return ReplyAndDie(ctx, msg->Error);
     }
 
-    if (TCompletionPolicy::IsCompleted(msg->Error)) {
-        return ReplyAndDie(ctx, {});
-    }
-
-    SendOperationRequest(ctx);
+    return ReplyAndDie(ctx, {});
 }
 
-template <
-    typename TResponseType,
-    typename TRequestType,
-    typename TCompletionPolicy>
-void TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>::
-    HandleWakeUp(const TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx)
+template <typename TResponseType, typename TRequestType>
+void TForcedOperationActor<TResponseType, TRequestType>::HandleWakeUp(
+    const TEvents::TEvWakeup::TPtr& ev,
+    const TActorContext& ctx)
 {
     Y_UNUSED(ev);
     SendOperationRequest(ctx);
 }
 
-template <
-    typename TResponseType,
-    typename TRequestType,
-    typename TCompletionPolicy>
-void TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>::
-    HandlePoisonPill(
-        const TEvents::TEvPoison::TPtr& ev,
-        const TActorContext& ctx)
+template <typename TResponseType, typename TRequestType>
+void TForcedOperationActor<TResponseType, TRequestType>::HandlePoisonPill(
+    const TEvents::TEvPoison::TPtr& ev,
+    const TActorContext& ctx)
 {
     Y_UNUSED(ev);
     ReplyAndDie(ctx, MakeError(E_FAIL, "actor killed"));
 }
 
-template <
-    typename TResponseType,
-    typename TRequestType,
-    typename TCompletionPolicy>
-void TForcedOperationActor<TResponseType, TRequestType, TCompletionPolicy>::
-    ReplyAndDie(const TActorContext& ctx, const NProto::TError& error)
+template <typename TResponseType, typename TRequestType>
+void TForcedOperationActor<TResponseType, TRequestType>::ReplyAndDie(
+    const TActorContext& ctx,
+    const NProto::TError& error)
 {
     {
         // notify tablet
@@ -243,12 +186,9 @@ using TForcedFlushActor = TForcedOperationActor<
     TEvIndexTabletPrivate::TEvFlushResponse,
     TEvIndexTabletPrivate::TEvFlushRequest>;
 
-// Note: only use with no concurrent writes to the fs. Otherwise the
-//       actor may continue issuing unbounded number of flush bytes requests.
 using TForcedFlushBytesActor = TForcedOperationActor<
     TEvIndexTabletPrivate::TEvFlushBytesResponse,
-    TEvIndexTabletPrivate::TEvFlushBytesRequest,
-    TFlushBytesCompletionPolicy>;
+    TEvIndexTabletPrivate::TEvFlushBytesRequest>;
 
 using TForcedCollectGarbageActor = TForcedOperationActor<
     TEvIndexTabletPrivate::TEvCollectGarbageResponse,
@@ -264,7 +204,7 @@ void TIndexTabletActor::HandleForcedTabletOperation(
 {
     auto* msg = ev->Get();
 
-    LOG_DEBUG(
+    LOG_INFO(
         ctx,
         TFileStoreComponents::TABLET,
         "%s ForcedTabletOperation mode=%u request",

cloud/filestore/libs/storage/tablet/tablet_ut_data.cpp

@@ -4959,68 +4959,6 @@ Y_UNIT_TEST_SUITE(TIndexTabletTest_Data)
             testEnvConfig);
     }
 
-    TABLET_TEST(ShouldDrainAllFreshBytesWithForcedFlushBytes)
-    {
-        const auto block = tabletConfig.BlockSize;
-        NProto::TStorageConfig storageConfig;
-        storageConfig.SetWriteBlobThreshold(1_GB);
-        storageConfig.SetFlushThreshold(1_GB);
-        storageConfig.SetFlushBytesThreshold(1_GB);
-        storageConfig.SetTrimBytesItemCount(1);
-
-        TTestEnv env(testEnvConfig, storageConfig);
-        auto& runtime = env.GetRuntime();
-        const auto nodeIdx = env.AddDynamicNode();
-        const auto tabletId = env.BootIndexTablet(nodeIdx);
-        TIndexTabletClient tablet(runtime, nodeIdx, tabletId, tabletConfig);
-        tablet.InitSession("client", "session");
-
-        const auto id =
-            CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "test"));
-        const auto handle = CreateHandle(tablet, id);
-        for (ui32 i = 0; i < 2; ++i) {
-            tablet.WriteData(handle, i * block, block, 'a');
-            tablet.WriteData(handle, i * block + 100, 10, 'b');
-        }
-        tablet.CreateCheckpoint("first");
-
-        // Seal a second chunk containing only deletion markers.
-        const auto sparseId =
-            CreateNode(tablet, TCreateNodeArgs::File(RootNodeId, "sparse"));
-        tablet.SetNodeAttr(TSetNodeAttrArgs(sparseId).SetSize(3 * block));
-        tablet.SetNodeAttr(TSetNodeAttrArgs(sparseId).SetSize(2 * block));
-        tablet.SetNodeAttr(TSetNodeAttrArgs(sparseId).SetSize(block));
-        tablet.CreateCheckpoint("second");
-
-        // Leave a third chunk active, below the background flush threshold.
-        for (ui32 i = 0; i < 2; ++i) {
-            tablet.WriteData(handle, i * block + 100, 10, 'c');
-        }
-
-        {
-            const auto response = tablet.GetStorageStats();
-            const auto& stats = response->Record.GetStats();
-            UNIT_ASSERT_VALUES_EQUAL(40, stats.GetFreshBytesCount());
-            UNIT_ASSERT_VALUES_EQUAL(4, stats.GetFreshBytesItemCount());
-            UNIT_ASSERT_VALUES_EQUAL(
-                2 * block,
-                stats.GetDeletedFreshBytesCount());
-        }
-
-        auto request = std::make_unique<
-            TEvIndexTabletPrivate::TEvForcedTabletOperationRequest>(
-            TEvIndexTabletPrivate::EForcedTabletOperationMode::FlushBytes,
-            CreateGuidAsString());
-        tablet.SendRequest(std::move(request));
-        tablet.AssertForcedTabletOperationResponse(S_OK);
-
-        const auto response = tablet.GetStorageStats();
-        const auto& stats = response->Record.GetStats();
-        UNIT_ASSERT_VALUES_EQUAL(0, stats.GetFreshBytesCount());
-        UNIT_ASSERT_VALUES_EQUAL(0, stats.GetFreshBytesItemCount());
-        UNIT_ASSERT_VALUES_EQUAL(0, stats.GetDeletedFreshBytesCount());
-    }
-
     TABLET_TEST(ShouldCompleteForcedFlushBytesWhenEmpty)
     {
         TTestEnv env(testEnvConfig);

cloud/filestore/private/api/protos/tablet.proto

@@ -702,7 +702,7 @@ message TForcedOperationRequest
 
         // Tablet operations
         E_FLUSH = 3;
-        E_FLUSH_BYTES = 4; // Only use when no concurrent writes are possible
+        E_FLUSH_BYTES = 4;
         E_COLLECT_GARBAGE = 5;
     };
 

cloud/filestore/tests/loadtest/max-file-count-test/max-file-count.txt

@@ -22,5 +22,8 @@ Tests {
             WriteBytes: 4096
             MinFileCount: 3
         }
+        IndexConsistencyValidation: {
+            Enabled: true
+        }
     }
 }

cloud/filestore/tests/loadtest/service-kikimr-auth-test/index.txt

@@ -21,5 +21,8 @@ Tests {
         IODepth: 64
         RequestsCount: 1024
         TestDuration: 60
+        IndexConsistencyValidation: {
+            Enabled: true
+        }
     }
-}
+}

cloud/filestore/tests/loadtest/service-kikimr-auth-test/read-write.txt

@@ -25,5 +25,8 @@ Tests {
         }
         IODepth: 64
         TestDuration: 240
+        IndexConsistencyValidation: {
+            Enabled: true
+        }
     }
 }

cloud/filestore/tests/loadtest/service-kikimr-datashard-like-test/datashard-like-read-write-with-page-size.txt

@@ -27,5 +27,8 @@ Tests {
         }
         IODepth: 4
         TestDuration: 30
+        IndexConsistencyValidation: {
+            Enabled: true
+        }
     }
 }

cloud/filestore/tests/loadtest/service-kikimr-datashard-like-test/datashard-like-read-write.txt

@@ -27,5 +27,8 @@ Tests {
         }
         IODepth: 4
         TestDuration: 30
+        IndexConsistencyValidation: {
+            Enabled: true
+        }
     }
 }

cloud/filestore/tests/loadtest/service-kikimr-emergency-test/read-write-validation.txt

@@ -22,5 +22,8 @@ Tests {
         IODepth: 64
         RequestsCount: 4096
         TestDuration: 60
+        IndexConsistencyValidation: {
+            Enabled: true
+        }
     }
 }

cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/create-remove.txt

@@ -20,5 +20,9 @@ Tests {
         }
         IODepth: 32
         TestDuration: 60
+        IndexConsistencyValidation: {
+            Enabled: true
+            TimeoutSeconds: 300
+        }
     }
 }

cloud/filestore/tests/loadtest/service-kikimr-nemesis-test/intrahost-migration.txt

@@ -24,5 +24,9 @@ Tests {
             MigrationPeriod: 20
             StateTransferDelay: 10
         }
+        IndexConsistencyValidation: {
+            Enabled: true
+            TimeoutSeconds: 300
+        }
     }
 }