diff --git a/mmcv/utils/config.py b/mmcv/utils/config.py
index 47af9af26..e9fed8d49 100644
--- a/mmcv/utils/config.py
+++ b/mmcv/utils/config.py
@@ -77,11 +77,13 @@ class Config(object):
         filename = osp.abspath(osp.expanduser(filename))
         check_file_exist(filename)
         if filename.endswith('.py'):
-            sys.path.append(osp.dirname(filename))
             module_name = osp.basename(filename)[:-3]
             if '.' in module_name:
                 raise ValueError('Dots are not allowed in config file path.')
+            config_dir = osp.dirname(filename)
+            sys.path.insert(0, config_dir)
             mod = import_module(module_name)
+            sys.path.pop(0)
             cfg_dict = {
                 name: value
                 for name, value in mod.__dict__.items()