From d22ef5c00ef5c65b4564dc3460900ca03144d57c Mon Sep 17 00:00:00 2001 From: Vipin Yadav Date: Thu, 10 Oct 2024 06:01:01 +0000 Subject: [PATCH 1/2] Added a loop to read 5 times from kernel cache and take minimum listing time --- ...ist_dir_with_twelve_thousand_files_test.go | 87 +++++++++++++------ 1 file changed, 60 insertions(+), 27 deletions(-) diff --git a/tools/integration_tests/list_large_dir/list_dir_with_twelve_thousand_files_test.go b/tools/integration_tests/list_large_dir/list_dir_with_twelve_thousand_files_test.go index 56ed17a569..6bbb400445 100644 --- a/tools/integration_tests/list_large_dir/list_dir_with_twelve_thousand_files_test.go +++ b/tools/integration_tests/list_large_dir/list_dir_with_twelve_thousand_files_test.go @@ -15,6 +15,7 @@ package list_large_dir_test import ( + "math" "os" "path" "strconv" @@ -164,21 +165,31 @@ func TestListDirectoryWithTwelveThousandFiles(t *testing.T) { endTime := time.Now() validateDirectoryWithTwelveThousandFiles(objs, t) firstListTime := endTime.Sub(startTime) + // Listing the directory a second time should retrieve the response from the kernel cache. - startTime = time.Now() - objs, err = os.ReadDir(dirPath) - if err != nil { - t.Errorf("Error in listing directory: %v", err) + minSecondListTime := time.Duration(math.MaxInt64) + for i := 0; i < 5; i++ { + startTime = time.Now() + objs, err = os.ReadDir(dirPath) + if err != nil { + t.Errorf("Error in listing directory: %v", err) + } + endTime = time.Now() + validateDirectoryWithTwelveThousandFiles(objs, t) + secondListTime := endTime.Sub(startTime) + + // Update the minimum listing time for the second listing + if secondListTime < minSecondListTime { + minSecondListTime = secondListTime + } } - endTime = time.Now() - validateDirectoryWithTwelveThousandFiles(objs, t) - secondListTime := endTime.Sub(startTime) // Fetching data from the kernel for the second list will be faster. - assert.Less(t, secondListTime, firstListTime) + assert.Less(t, minSecondListTime, firstListTime) // The second directory listing should be 2 times better performant since it // will be retrieved from the kernel cache. - assert.Less(t, 2*secondListTime, firstListTime) + assert.Less(t, 2*minSecondListTime, firstListTime) + // Clear the data after testing. setup.RunScriptForTestData("testdata/delete_objects.sh", testDirPathOnBucket) } @@ -189,6 +200,7 @@ func TestListDirectoryWithTwelveThousandFilesAndHundredExplicitDir(t *testing.T) testDirPath := path.Join(setup.MntDir(), DirectoryForListLargeFileTests) testDirPathOnBucket := path.Join(setup.TestBucket(), DirectoryForListLargeFileTests) dirPath := path.Join(testDirPath, DirectoryWithTwelveThousandFiles) + // Create hundred explicit directories. createHundredExplicitDir(dirPath, t) @@ -201,21 +213,31 @@ func TestListDirectoryWithTwelveThousandFilesAndHundredExplicitDir(t *testing.T) endTime := time.Now() validateDirectoryWithTwelveThousandFilesAndHundredExplicitDirectory(objs, t) firstListTime := endTime.Sub(startTime) + // Listing the directory a second time should retrieve the response from the kernel cache. - startTime = time.Now() - objs, err = os.ReadDir(dirPath) - if err != nil { - t.Errorf("Error in listing directory: %v", err) + minSecondListTime := time.Duration(math.MaxInt64) + for i := 0; i < 5; i++ { + startTime = time.Now() + objs, err = os.ReadDir(dirPath) + if err != nil { + t.Errorf("Error in listing directory: %v", err) + } + endTime = time.Now() + validateDirectoryWithTwelveThousandFilesAndHundredExplicitDirectory(objs, t) + secondListTime := endTime.Sub(startTime) + + // Update the minimum listing time for the second listing + if secondListTime < minSecondListTime { + minSecondListTime = secondListTime + } } - endTime = time.Now() - validateDirectoryWithTwelveThousandFilesAndHundredExplicitDirectory(objs, t) - secondListTime := endTime.Sub(startTime) // Fetching data from the kernel for the second list will be faster. - assert.Less(t, secondListTime, firstListTime) + assert.Less(t, minSecondListTime, firstListTime) // The second directory listing should be 2 times better performant since it // will be retrieved from the kernel cache. - assert.Less(t, 2*secondListTime, firstListTime) + assert.Less(t, 2*minSecondListTime, firstListTime) + // Clear the bucket after testing. setup.RunScriptForTestData("testdata/delete_objects.sh", testDirPathOnBucket) } @@ -226,6 +248,7 @@ func TestListDirectoryWithTwelveThousandFilesAndHundredExplicitDirAndHundredImpl testDirPath := path.Join(setup.MntDir(), DirectoryForListLargeFileTests) testDirPathOnBucket := path.Join(setup.TestBucket(), DirectoryForListLargeFileTests) dirPath := path.Join(testDirPath, DirectoryWithTwelveThousandFiles) + // Create hundred explicit directories. createHundredExplicitDir(dirPath, t) subDirPath := path.Join(testDirPathOnBucket, DirectoryWithTwelveThousandFiles) @@ -240,21 +263,31 @@ func TestListDirectoryWithTwelveThousandFilesAndHundredExplicitDirAndHundredImpl endTime := time.Now() validateDirectoryWithTwelveThousandFilesHundredExplicitDirAndHundredImplicitDir(objs, t) firstListTime := endTime.Sub(startTime) + // Listing the directory a second time should retrieve the response from the kernel cache. - startTime = time.Now() - objs, err = os.ReadDir(dirPath) - if err != nil { - t.Errorf("Error in listing directory: %v", err) + minSecondListTime := time.Duration(math.MaxInt64) + for i := 0; i < 5; i++ { + startTime = time.Now() + objs, err = os.ReadDir(dirPath) + if err != nil { + t.Errorf("Error in listing directory: %v", err) + } + endTime = time.Now() + validateDirectoryWithTwelveThousandFilesHundredExplicitDirAndHundredImplicitDir(objs, t) + secondListTime := endTime.Sub(startTime) + + // Update the minimum listing time for the second listing + if secondListTime < minSecondListTime { + minSecondListTime = secondListTime + } } - endTime = time.Now() - validateDirectoryWithTwelveThousandFilesHundredExplicitDirAndHundredImplicitDir(objs, t) - secondListTime := endTime.Sub(startTime) // Fetching data from the kernel for the second list will be faster. - assert.Less(t, secondListTime, firstListTime) + assert.Less(t, minSecondListTime, firstListTime) // The second directory listing should be 2 times better performant since it // will be retrieved from the kernel cache. - assert.Less(t, 2*secondListTime, firstListTime) + assert.Less(t, 2*minSecondListTime, firstListTime) + // Clear the bucket after testing. setup.RunScriptForTestData("testdata/delete_objects.sh", testDirPathOnBucket) } From 1535e8249f99ee407d9bb8b06b49874918cf44e6 Mon Sep 17 00:00:00 2001 From: Vipin Yadav Date: Thu, 10 Oct 2024 08:33:13 +0000 Subject: [PATCH 2/2] refactored code --- ...ist_dir_with_twelve_thousand_files_test.go | 108 +++++------------- 1 file changed, 28 insertions(+), 80 deletions(-) diff --git a/tools/integration_tests/list_large_dir/list_dir_with_twelve_thousand_files_test.go b/tools/integration_tests/list_large_dir/list_dir_with_twelve_thousand_files_test.go index 6bbb400445..1a3fe1625e 100644 --- a/tools/integration_tests/list_large_dir/list_dir_with_twelve_thousand_files_test.go +++ b/tools/integration_tests/list_large_dir/list_dir_with_twelve_thousand_files_test.go @@ -145,25 +145,15 @@ func createHundredExplicitDir(dirPath string, t *testing.T) { } } -//////////////////////////////////////////////////////////////////////// -// Tests -//////////////////////////////////////////////////////////////////////// - -// Test with a bucket with twelve thousand files. -func TestListDirectoryWithTwelveThousandFiles(t *testing.T) { - createTwelveThousandFilesAndUploadOnTestBucket(t) - testDirPath := path.Join(setup.MntDir(), DirectoryForListLargeFileTests) - testDirPathOnBucket := path.Join(setup.TestBucket(), DirectoryForListLargeFileTests) - dirPath := path.Join(testDirPath, DirectoryWithTwelveThousandFiles) - +func listDirectoryTime(dirPath string, validateDirectory func([]os.DirEntry, *testing.T), t *testing.T) (time.Duration, time.Duration) { // List Directory first time startTime := time.Now() objs, err := os.ReadDir(dirPath) if err != nil { - t.Errorf("Error in listing directory: %v", err) + t.Fatalf("Error in listing directory: %v", err) } endTime := time.Now() - validateDirectoryWithTwelveThousandFiles(objs, t) + validateDirectory(objs, t) firstListTime := endTime.Sub(startTime) // Listing the directory a second time should retrieve the response from the kernel cache. @@ -172,10 +162,10 @@ func TestListDirectoryWithTwelveThousandFiles(t *testing.T) { startTime = time.Now() objs, err = os.ReadDir(dirPath) if err != nil { - t.Errorf("Error in listing directory: %v", err) + t.Fatalf("Error in listing directory: %v", err) } endTime = time.Now() - validateDirectoryWithTwelveThousandFiles(objs, t) + validateDirectory(objs, t) secondListTime := endTime.Sub(startTime) // Update the minimum listing time for the second listing @@ -183,12 +173,27 @@ func TestListDirectoryWithTwelveThousandFiles(t *testing.T) { minSecondListTime = secondListTime } } + return firstListTime, minSecondListTime +} + +//////////////////////////////////////////////////////////////////////// +// Tests +//////////////////////////////////////////////////////////////////////// + +// Test with a bucket with twelve thousand files. +func TestListDirectoryWithTwelveThousandFiles(t *testing.T) { + createTwelveThousandFilesAndUploadOnTestBucket(t) + testDirPath := path.Join(setup.MntDir(), DirectoryForListLargeFileTests) + testDirPathOnBucket := path.Join(setup.TestBucket(), DirectoryForListLargeFileTests) + dirPath := path.Join(testDirPath, DirectoryWithTwelveThousandFiles) + + firstListTime, secondListTime := listDirectoryTime(dirPath, validateDirectoryWithTwelveThousandFiles, t) // Fetching data from the kernel for the second list will be faster. - assert.Less(t, minSecondListTime, firstListTime) + assert.Less(t, secondListTime, firstListTime) // The second directory listing should be 2 times better performant since it // will be retrieved from the kernel cache. - assert.Less(t, 2*minSecondListTime, firstListTime) + assert.Less(t, 2*secondListTime, firstListTime) // Clear the data after testing. setup.RunScriptForTestData("testdata/delete_objects.sh", testDirPathOnBucket) @@ -200,43 +205,15 @@ func TestListDirectoryWithTwelveThousandFilesAndHundredExplicitDir(t *testing.T) testDirPath := path.Join(setup.MntDir(), DirectoryForListLargeFileTests) testDirPathOnBucket := path.Join(setup.TestBucket(), DirectoryForListLargeFileTests) dirPath := path.Join(testDirPath, DirectoryWithTwelveThousandFiles) - - // Create hundred explicit directories. createHundredExplicitDir(dirPath, t) - // List Directory first time - startTime := time.Now() - objs, err := os.ReadDir(dirPath) - if err != nil { - t.Errorf("Error in listing directory: %v", err) - } - endTime := time.Now() - validateDirectoryWithTwelveThousandFilesAndHundredExplicitDirectory(objs, t) - firstListTime := endTime.Sub(startTime) - - // Listing the directory a second time should retrieve the response from the kernel cache. - minSecondListTime := time.Duration(math.MaxInt64) - for i := 0; i < 5; i++ { - startTime = time.Now() - objs, err = os.ReadDir(dirPath) - if err != nil { - t.Errorf("Error in listing directory: %v", err) - } - endTime = time.Now() - validateDirectoryWithTwelveThousandFilesAndHundredExplicitDirectory(objs, t) - secondListTime := endTime.Sub(startTime) - - // Update the minimum listing time for the second listing - if secondListTime < minSecondListTime { - minSecondListTime = secondListTime - } - } + firstListTime, secondListTime := listDirectoryTime(dirPath, validateDirectoryWithTwelveThousandFilesAndHundredExplicitDirectory, t) // Fetching data from the kernel for the second list will be faster. - assert.Less(t, minSecondListTime, firstListTime) + assert.Less(t, secondListTime, firstListTime) // The second directory listing should be 2 times better performant since it // will be retrieved from the kernel cache. - assert.Less(t, 2*minSecondListTime, firstListTime) + assert.Less(t, 2*secondListTime, firstListTime) // Clear the bucket after testing. setup.RunScriptForTestData("testdata/delete_objects.sh", testDirPathOnBucket) @@ -248,46 +225,17 @@ func TestListDirectoryWithTwelveThousandFilesAndHundredExplicitDirAndHundredImpl testDirPath := path.Join(setup.MntDir(), DirectoryForListLargeFileTests) testDirPathOnBucket := path.Join(setup.TestBucket(), DirectoryForListLargeFileTests) dirPath := path.Join(testDirPath, DirectoryWithTwelveThousandFiles) - - // Create hundred explicit directories. createHundredExplicitDir(dirPath, t) subDirPath := path.Join(testDirPathOnBucket, DirectoryWithTwelveThousandFiles) setup.RunScriptForTestData("testdata/create_implicit_dir.sh", subDirPath, PrefixImplicitDirInLargeDirListTest, strconv.Itoa(NumberOfImplicitDirsInDirectoryWithTwelveThousandFiles)) - // List Directory first time - startTime := time.Now() - objs, err := os.ReadDir(dirPath) - if err != nil { - t.Errorf("Error in listing directory: %v", err) - } - endTime := time.Now() - validateDirectoryWithTwelveThousandFilesHundredExplicitDirAndHundredImplicitDir(objs, t) - firstListTime := endTime.Sub(startTime) - - // Listing the directory a second time should retrieve the response from the kernel cache. - minSecondListTime := time.Duration(math.MaxInt64) - for i := 0; i < 5; i++ { - startTime = time.Now() - objs, err = os.ReadDir(dirPath) - if err != nil { - t.Errorf("Error in listing directory: %v", err) - } - endTime = time.Now() - validateDirectoryWithTwelveThousandFilesHundredExplicitDirAndHundredImplicitDir(objs, t) - secondListTime := endTime.Sub(startTime) - - // Update the minimum listing time for the second listing - if secondListTime < minSecondListTime { - minSecondListTime = secondListTime - } - } + firstListTime, secondListTime := listDirectoryTime(dirPath, validateDirectoryWithTwelveThousandFilesHundredExplicitDirAndHundredImplicitDir, t) // Fetching data from the kernel for the second list will be faster. - assert.Less(t, minSecondListTime, firstListTime) + assert.Less(t, secondListTime, firstListTime) // The second directory listing should be 2 times better performant since it // will be retrieved from the kernel cache. - assert.Less(t, 2*minSecondListTime, firstListTime) - + assert.Less(t, 2*secondListTime, firstListTime) // Clear the bucket after testing. setup.RunScriptForTestData("testdata/delete_objects.sh", testDirPathOnBucket) }