add HTTP backend (#860)

* add HTTP backend

* add HTTPbackend

* fix linting

* fix linting

* add docs attributes http

* fix format

* add real http link test for http backend

mmcv/fileio/file_client.py

@@ -1,5 +1,6 @@
 import inspect
 from abc import ABCMeta, abstractmethod
+from urllib.request import urlopen
 
 
 class BaseStorageBackend(metaclass=ABCMeta):
@@ -191,6 +192,18 @@ class HardDiskBackend(BaseStorageBackend):
         return value_buf
 
 
+class HTTPBackend(BaseStorageBackend):
+    """HTTP and HTTPS storage bachend."""
+
+    def get(self, filepath):
+        value_buf = urlopen(filepath).read()
+        return value_buf
+
+    def get_text(self, filepath):
+        value_buf = urlopen(filepath).read()
+        return value_buf.decode('utf-8')
+
+
 class FileClient:
     """A general file client to access files in different backend.
 
@@ -200,7 +213,7 @@ class FileClient:
 
     Attributes:
         backend (str): The storage backend type. Options are "disk", "ceph",
-            "memcached" and "lmdb".
+            "memcached", "lmdb" and "http".
         client (:obj:`BaseStorageBackend`): The backend object.
     """
 
@@ -210,6 +223,7 @@ class FileClient:
         'memcached': MemcachedBackend,
         'lmdb': LmdbBackend,
         'petrel': PetrelBackend,
+        'http': HTTPBackend,
     }
 
     def __init__(self, backend='disk', **kwargs):

tests/test_fileclient.py

@@ -182,6 +182,32 @@ class TestFileClient:
         img = mmcv.imfrombytes(img_bytes)
         assert img.shape == (120, 125, 3)
 
+    def test_http_backend(self):
+        http_backend = FileClient('http')
+        img_url = 'https://raw.githubusercontent.com/open-mmlab/mmcv/' \
+            'master/tests/data/color.jpg'
+        text_url = 'https://raw.githubusercontent.com/open-mmlab/mmcv/' \
+            'master/tests/data/filelist.txt'
+
+        # input is path or Path object
+        with pytest.raises(Exception):
+            http_backend.get(self.img_path)
+        with pytest.raises(Exception):
+            http_backend.get(str(self.img_path))
+        with pytest.raises(Exception):
+            http_backend.get_text(self.text_path)
+        with pytest.raises(Exception):
+            http_backend.get_text(str(self.text_path))
+
+        # input url is http image
+        img_bytes = http_backend.get(img_url)
+        img = mmcv.imfrombytes(img_bytes)
+        assert img.shape == self.img_shape
+
+        # input url is http text
+        value_buf = http_backend.get_text(text_url)
+        assert self.text_path.open('r').read() == value_buf
+
     def test_register_backend(self):
 
         # name must be a string