Skip to content

Commit

Permalink
Make fake io error tests in PDisk more robust (ydb-platform#10091)
Browse files Browse the repository at this point in the history
  • Loading branch information
@va-kuznecov
va-kuznecov authored Oct 4, 2024
1 parent 2c63185 commit 6536467
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 23 deletions.
21 changes: 12 additions & 9 deletions ydb/core/blobstorage/pdisk/blobstorage_pdisk_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {

Y_UNIT_TEST(TestFakeErrorPDiskManyLogWrite) {
TActorTestContext testCtx({ false });
testCtx.TestCtx.SectorMap->ImitateIoErrorProbability = 1e-4;
testCtx.TestCtx.SectorMap->IoErrorEveryNthRequests = 1000;

const TVDiskID vDiskID(0, 1, 0, 0, 0);
const auto evInitRes = testCtx.TestResponse<NPDisk::TEvYardInitResult>(
Expand All @@ -355,7 +355,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {

ui32 errors = 0;
ui32 lsn = 2;
for (ui32 i = 0; i < 100'000; ++i) {
for (ui32 i = 0; i < 10'000; ++i) {
testCtx.Send(new NPDisk::TEvLog(evInitRes->PDiskParams->Owner, evInitRes->PDiskParams->OwnerRound, 0,
TRcBuf(TString("abc")), TLsnSeg(lsn, lsn), nullptr));
++lsn;
Expand Down Expand Up @@ -384,7 +384,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
vdisk.Init();

// Make sure there will be read error.
testCtx.TestCtx.SectorMap->ImitateReadIoErrorProbability = 1;
testCtx.TestCtx.SectorMap->ReadIoErrorEveryNthRequests = 1;

auto res = vdisk.ReadLog(true);

Expand All @@ -404,7 +404,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
vdisk.InitFull();

// Make sure there will be syslog read error.
testCtx.TestCtx.SectorMap->ImitateReadIoErrorProbability = 1;
testCtx.TestCtx.SectorMap->ReadIoErrorEveryNthRequests = 1;

testCtx.TestResponse<NPDisk::TEvYardControlResult>(
new NPDisk::TEvYardControl(NPDisk::TEvYardControl::PDiskStop, nullptr),
Expand All @@ -417,7 +417,7 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {

Y_UNIT_TEST(TestFakeErrorPDiskManyChunkRead) {
TActorTestContext testCtx({ false });
testCtx.TestCtx.SectorMap->ImitateReadIoErrorProbability = 1e-4;
testCtx.TestCtx.SectorMap->ReadIoErrorEveryNthRequests = 100;

TVDiskMock vdisk(&testCtx);
vdisk.InitFull();
Expand All @@ -434,7 +434,8 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
NKikimrProto::OK);

bool printed = false;
for (ui32 i = 0; i < 100'000; ++i) {
ui32 errors = 0;
for (ui32 i = 0; i < 10'000; ++i) {
testCtx.Send(new NPDisk::TEvChunkRead(vdisk.PDiskParams->Owner, vdisk.PDiskParams->OwnerRound,
reservedChunk, 0, 1024, 0, nullptr));

Expand All @@ -445,30 +446,32 @@ Y_UNIT_TEST_SUITE(TPDiskTest) {
printed = true;
Ctest << res->ToString() << Endl;
}
++errors;
}
}
UNIT_ASSERT(errors > 0);
// Check that PDisk is in working state now
vdisk.InitFull();
}

Y_UNIT_TEST(TestFakeErrorPDiskManyChunkWrite) {
TActorTestContext testCtx({ false });
testCtx.TestCtx.SectorMap->ImitateIoErrorProbability = 1e-4;
testCtx.TestCtx.SectorMap->IoErrorEveryNthRequests = 1000;

const TVDiskID vDiskID(0, 1, 0, 0, 0);
const auto evInitRes = testCtx.TestResponse<NPDisk::TEvYardInitResult>(
new NPDisk::TEvYardInit(2, vDiskID, testCtx.TestCtx.PDiskGuid),
NKikimrProto::OK);

ui32 errors = 0;
const auto evReserveRes = testCtx.TestResponse<NPDisk::TEvChunkReserveResult>(
new NPDisk::TEvChunkReserve(evInitRes->PDiskParams->Owner, evInitRes->PDiskParams->OwnerRound, 1),
NKikimrProto::OK);
UNIT_ASSERT(evReserveRes->ChunkIds.size() == 1);
const ui32 reservedChunk = evReserveRes->ChunkIds.front();

ui32 errors = 0;
bool printed = false;
for (ui32 i = 0; i < 100'000; ++i) {
for (ui32 i = 0; i < 10'000; ++i) {
TString data = PrepareData(1024);
testCtx.Send(new NPDisk::TEvChunkWrite(evInitRes->PDiskParams->Owner, evInitRes->PDiskParams->OwnerRound,
reservedChunk, 0, new NPDisk::TEvChunkWrite::TStrokaBackedUpParts(data), nullptr, false, 0));
Expand Down
31 changes: 23 additions & 8 deletions ydb/library/pdisk_io/aio_map.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,8 @@ class TAsyncIoContextMap : public IAsyncIoContext {

TSpinLock SpinLock;
IAsyncIoOperation* LastOngoingAsyncIoOperation = nullptr;
std::atomic<ui64> GlobalCounter = 0;

public:

TAsyncIoContextMap(const TString &path, ui32 pDiskId, TIntrusivePtr<TSectorMap> sectorMap)
Expand Down Expand Up @@ -248,6 +250,25 @@ class TAsyncIoContextMap : public IAsyncIoContext {
return EIoResult::Ok;
}

EIoResult GenerateResultForOperaion(IAsyncIoOperation::EType type) {
auto result = EIoResult::Ok;

auto globalIdx = GlobalCounter++;
if (auto everyNth = SectorMap->IoErrorEveryNthRequests.load()) {
if (globalIdx % everyNth == everyNth - 1) {
result = EIoResult::FakeError;
}
}
if (type == IAsyncIoOperation::EType::PRead) {
if (auto everyNth = SectorMap->ReadIoErrorEveryNthRequests.load()) {
if (globalIdx % everyNth == everyNth - 1) {
result = EIoResult::FakeError;
}
}
}
return result;
}

i64 GetEvents(ui64 minEvents, ui64 maxEvents, TAsyncIoOperationResult *events, TDuration timeout) override {
ui64 outputIdx = 0;
TInstant startTime = TInstant::Now();
Expand All @@ -258,14 +279,8 @@ class TAsyncIoContextMap : public IAsyncIoContext {
for (TAtomicBase idx = 0; idx < size; ++idx) {
TAsyncIoOperationMap *op = static_cast<TAsyncIoOperationMap*>(CompleteQueue.Pop());
events[outputIdx].Operation = op;
events[outputIdx].Result = (RandomNumber<double>() <
SectorMap->ImitateIoErrorProbability.load())
? EIoResult::FakeError
: EIoResult::Ok;
if (op->GetType() == IAsyncIoOperation::EType::PRead &&
RandomNumber<double>() < SectorMap->ImitateReadIoErrorProbability.load()) {
events[outputIdx].Result = EIoResult::FakeError;
}

events[outputIdx].Result = GenerateResultForOperaion(op->GetType());
events[outputIdx].Operation->ExecCallback(&events[outputIdx]);
++outputIdx;
if (outputIdx == maxEvents) {
Expand Down
12 changes: 6 additions & 6 deletions ydb/library/pdisk_io/sector_map.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,16 +191,16 @@ class TSectorMap : public TThrRefBase {
TTicketLock MapLock;
std::atomic<bool> IsLocked;
std::optional<std::pair<TDuration, TDuration>> ImitateRandomWait;
std::atomic<double> ImitateIoErrorProbability;
std::atomic<double> ImitateReadIoErrorProbability;
std::atomic<ui64> IoErrorEveryNthRequests;
std::atomic<ui64> ReadIoErrorEveryNthRequests;

std::atomic<ui64> AllocatedBytes;

TSectorMap(ui64 deviceSize = 0, NSectorMap::EDiskMode diskMode = NSectorMap::DM_NONE)
: DeviceSize(deviceSize)
, IsLocked(false)
, ImitateIoErrorProbability(0.0)
, ImitateReadIoErrorProbability(0.0)
, IoErrorEveryNthRequests(0)
, ReadIoErrorEveryNthRequests(0)
, AllocatedBytes(0)
, DiskMode(diskMode)
{
Expand Down Expand Up @@ -327,8 +327,8 @@ class TSectorMap : public TThrRefBase {
str << "ImitateRandomWait# [" << ImitateRandomWait->first << ", "
<< ImitateRandomWait->first + ImitateRandomWait->second << ")" << "\n";
}
str << "ImitateReadIoErrorProbability# " << ImitateReadIoErrorProbability.load() << "\n";
str << "ImitateIoErrorProbability# " << ImitateIoErrorProbability.load() << "\n";
str << "ReadIoErrorEveryNthRequests# " << ReadIoErrorEveryNthRequests.load() << "\n";
str << "IoErrorEveryNthRequests# " << IoErrorEveryNthRequests.load() << "\n";
str << "AllocatedBytes (approx.)# " << HumanReadableSize(AllocatedBytes.load(), SF_QUANTITY) << "\n";
str << "DataBytes# " << HumanReadableSize(DataBytes(), SF_QUANTITY) << "\n";
str << "DiskMode# " << DiskModeToString(DiskMode) << "\n";
Expand Down

0 comments on commit 6536467

Please sign in to comment.