diff --git a/mmseg/datasets/builder.py b/mmseg/datasets/builder.py
index e6284e5c9..5994ab233 100644
--- a/mmseg/datasets/builder.py
+++ b/mmseg/datasets/builder.py
@@ -14,8 +14,9 @@ if platform.system() != 'Windows':
     # https://github.com/pytorch/pytorch/issues/973
     import resource
     rlimit = resource.getrlimit(resource.RLIMIT_NOFILE)
+    base_soft_limit = rlimit[0]
     hard_limit = rlimit[1]
-    soft_limit = min(4096, hard_limit)
+    soft_limit = min(max(4096, base_soft_limit), hard_limit)
     resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit))
 
 DATASETS = Registry('dataset')