有效地读取图像，对比opencv、PIL、turbojpeg、lmdb、tfrecords_python

概述opencv和PIL都是很常见的图像处理库了，就不介绍了，主要介绍后面三个： turbojpeg：libjpeg-turbo的python包装器，用于jpeg图像的解码和编码。基本用法： impor

opencv和PIL都是很常见的图像处理库了，就不介绍了，主要介绍后面三个：

turbojpeg：libjpeg-turbo的python包装器，用于jpeg图像的解码和编码。

基本用法：

import cv2from turbojpeg  TurboJPEG,TJPF_GRAY,TJsAMP_GRAY,TJFLAG_PROGRESSIVE# using default library installationjpeg = TurboJPEG() deCoding input.jpg to BGR arrayin_file = open('input.jpg',rb')bgr_array = jpeg.decode(in_file.read())in_file.close()cv2.imshow(bgr_array,bgr_array)cv2.waitKey(0)

更多信息参考：https://www.cnpython.com/pypi/pyturbojpeg

lmdb：LMDB的全称是lightning Memory-Mapped Database(快如闪电的内存映射数据库)。LMDB文件可以同时由多个进程打开，具有极高的数据存取速度，访问简单，不需要运行单独的数据库管理进程，只要在访问数据的代码里引用LMDB库，访问时给文件路径即可。让系统访问大量小文件的开销很大，而LMDB使用内存映射的方式访问文件，使得文件内寻址的开销非常小，使用指针运算就能实现。数据库单文件还能减少数据集复制/传输过程的开销。

基本用法：

 -*- Coding: utf-8 -*- lmdb   如果train文件夹下没有data.mbd或lock.mdb文件，则会生成一个空的，如果有，不会覆盖# map_size定义最大储存容量，单位是kb，以下定义1TB容量env = lmdb.open("./train"，map_size=1099511627776)env.close()

更多信息参考：https://blog.csdn.net/weixin_41874599/article/details/86631186

tfrecords：frecords是一种二进制编码的文件格式，tensorflow专用。能将任意数据转换为tfrecords。更好的利用内存，更方便复制和移动，并且不需要单独的标签文件。

将图像转换为lmdb格式的数据：

 osfrom argparse  ArgumentParser lmdb numpy as npfrom tools  get_images_pathsdef store_many_lmdb(images_List,save_path):    num_images = len(images_List)   number of images in our folder    file_sizes = [os.path.getsize(item) for item in images_List]   all file sizes    max_size_index = np.argmax(file_sizes)   the maximum file size index     maximum database size in bytes    map_size = num_images * cv2.imread(images_List[max_size_index]).nbytes * 10    env = lmdb.open(save_path,map_size=map_size)   create lmdb environment    with env.begin(write=True) as txn:   start writing to environment        for i,image in enumerate(images_List):            with open(image,") as file:                data = file.read()   read image as bytes                key = f{i:08}"   get image key                txn.put(key.encode(ascii"),data)   put the key-value into database    env.close()   close the environmentif __name__ == __main__:    parser = ArgumentParser()    parser.add_argument(        --path-pstr,required=True,help=path to the images folder to collect--output-opath to the output environment directory file i.e. "path/to/folder/env/" parser.parse_args()    if not os.path.exists(args.output):        os.makedirs(args.output)    images = get_images_paths(args.path)    store_many_lmdb(images,args.output)

将图像转换为tfrecords格式的数据：

 tensorflow as tf _byte_feature(value):    """Convert string / byte into bytes_List."""    if isinstance(value,type(tf.constant(0))):        value = value.numpy()   BytesList can't unpack string from EagerTensor.    return tf.train.Feature(bytes_List=tf.train.BytesList(value=[value])) _int64_feature(value):    Convert bool / enum / int / uint into int64_List.return tf.train.Feature(int64_List=tf.train.Int64List(value= image_example(image_string,label):    feature = {        label: _int64_feature(label),1)">image_raw: _byte_feature(image_string),}    return tf.train.Example(features=tf.train.Features(feature=feature)) store_many_tfrecords(images_List,save_file):    assert save_file.endswith(        .tfrecordsfile path is wrong,it should contain "*myname*.tfrecords"    directory = os.path.dirname(save_file)     os.path.exists(directory):        os.makedirs(directory)    with tf.io.TFRecorDWriter(save_file) as writer:   start writer        for label,filename in enumerate(images_List):   cycle by each image path            image_string = open(filename,1)">").read()   read the image as bytes string            tf_example = image_example(                image_string,label,)   save the data as tf.Example object            writer.write(tf_example.SerializetoString())   and write it into databasepath to the output tfrecords file i.e. "path/to/folder/myname.tfrecords" parser.parse_args()    image_paths = get_images_paths(args.path)    store_many_tfrecords(image_paths,args.output)

使用不同的方式读取图像，同时默认是以BGR的格式读取：

from abc  abstractmethodfrom timeit  default_timer as timer numpy as np tensorflow as tffrom PIL  Image TurboJPEGos.environ[TF_CPP_MIN_LOG_LEVEL"] = 3"class ImageLoader:    extensions: tuple = (.png",1)">.jpg.jpeg.tiff.bmp.gif)    def __init__(self,path: str,mode: str = BGR):        self.path = path        self.mode = mode        self.dataset = self.parse_input(self.path)        self.sample_IDx = 0     parse_input(self,path):         single image or tfrecords file         os.path.isfile(path):             path.lower().endswith(                self.extensions,fUnsupportable extension,please,use one of {self.extensions}"            return [path]         os.path.isdir(path):             lmdb environment            if any([file.endswith(.mdb") for file  os.Listdir(path)]):                 path            else:                 folder with images                paths = [os.path.join(path,image) for image  os.Listdir(path)]                 paths    __iter__(self):        self.sample_IDx = 0         self    __len__(self):         len(self.dataset)    @abstractmethod    __next__pass CV2Loader(ImageLoader):    (self):        start = timer()        path = self.dataset[self.sample_IDx]   get image path by index from the dataset        image = cv2.imread(path)   read the image        full_time = timer() - start        if self.mode == RGB:            start = timer()            image = cv2.cvtcolor(image,cv2.color_BGR2RGB)   change color mode            full_time += timer() - start        self.sample_IDx += 1         image,full_time PILLoader(ImageLoader):     get image path by index from the dataset        image = np.asarray(Image.open(path))   read the image as numpy array        full_time = timer() - TurboJpegLoader(ImageLoader):    kwargs):        super(TurboJpegLoader,self).__init__(path,1)">kwargs)        self.jpeg_reader = TurboJPEG()   create TurboJPEG object for image reading     timer()        file = open(self.dataset[self.sample_IDx],1)">")   open the input file as bytes        full_time = timer() -:            mode =elif self.mode == :            mode = 1        start = timer()        image = self.jpeg_reader.decode(file.read(),mode)   decode raw image        full_time += timer() - LmdbLoader(ImageLoader):    kwargs):        super(LmdbLoader,1)">kwargs)        self.path = path        self._dataset_size = 0        self.dataset = self.open_database()     we need to open the database to read images from it     open_database(self):        lmdb_env = lmdb.open(self.path)   open the environment by path        lmdb_txn = lmdb_env.begin()   start reading        lmdb_cursor = lmdb_txn.cursor()   create cursor to iterate through the database        self._dataset_size = lmdb_env.stat()[            entrIEs        ]   get number of items in full dataset         lmdb_cursor    (self):        self.dataset.first()   return the cursor to the first database element         timer()        raw_image = self.dataset.value()   get raw image        image = np.frombuffer(raw_image,dtype=np.uint8)   convert it to numpy        image = cv2.imdecode(image,cv2.IMREAD_color)   decode image        full_time = timer() - timer()            image = cv2.cvtcolor(image,cv2.color_BGR2RGB)            full_time += timer() - start        start = timer()        self.dataset.next()   step to the next element in database        full_time += timer() -return self._dataset_size   get dataset length TFRecordsLoader(ImageLoader):    kwargs):        super(TFRecordsLoader,1)">kwargs)        self._dataset = open_database(self):         _parse_image_function(example_proto):             tf.io.parse_single_example(example_proto,image_feature_description)         dataset structure description        image_feature_description = {            : tf.io.FixedLenFeature([],tf.int64),tf.string),}        raw_image_dataset = tf.data.TFRecordDataset(self.path)   open dataset by path        parsed_image_dataset = raw_image_dataset.map(            _parse_image_function,1)"> parse dataset using structure description         parsed_image_dataset    (self):        self.dataset = self._dataset.as_numpy_iterator()         timer()        value = next(self.dataset)[             step to the next element in database and get new image        image = tf.image.decode_jpeg(value).numpy()   decode raw image        full_time = timer() - self._dataset.reduce(            np.int64(0),lambda x,_: x + 1 get dataset lengthmethods = {    cv2: CV2Loader,1)">pil: PILLoader,1)">turbojpeg: TurboJpegLoader,1)">lmdb: LmdbLoader,1)">tfrecords: TFRecordsLoader,}

显示图像：

 cv2from loader  (    CV2Loader,LmdbLoader,PILLoader,TFRecordsLoader,TurboJpegLoader,methods,) show_image(method,image):    cv2.imshow(f{method} imageif k == 27:   check ESC pressing         True    :         False show_images(loader):    num_images = len(loader)    loader = iter(loader)    for IDx  range(num_images):        image,time = next(loader)        print_info(image,time)        stop = show_image(type(loader).__name__ stop:            cv2.destroyAllwindows()            return print_info(image,time):    print(        fImage with {image.shape[0]}x{image.shape[1]} size has been loading for {time} seconds demo(method,path):    loader = methods[method](path)   get the image loader    show_images(loader) ArgumentParser()    parser.add_argument(        path to image,folder of images,lmdb environment path or tfrecords database path--method],1)">Image loading methods to use in benchmark parser.parse_args()    demo(args.method,args.path)