![\"@chenghuige\"](\"https://avatars3.githubusercontent.com/u/6323467?s=40&v=4\")
\n \n \n
\n\n\n
\n \n\n\n - \n\n\n\n\n
- \n \n \n \n\n
- \n \n \n\n\n \n\n
- \n \n \n Fork\n\n \n 155\n \n \n
\n \n /wenzheng\n \n\n
\n\n\n \n
\n\n\n \n
\n\n \n\n \n \n\n\n \n Permalink\n\n \n \n\n \n \n\n\n\n\n
\n \n\n \n \n\n\n
\n\n\n\n \n \n Find file\n \n \n Copy path\n \n
\n \n \n wenzheng/projects/kaggle/blindness/keras2/dataset.py\n
\n\n\n \n Find file\n \n \n Copy path\n \n
\n \n Fetching contributors…\n
\n\n \n ![\"\"](\"https://github.githubassets.com/images/spinners/octocat-spinner-32-EAF2F5.gif\")
\n Cannot retrieve contributors at this time\n
\n\n Cannot retrieve contributors at this time\n \n \n
\n\n \n\n \n\n
\n\n\n\n\n \n\n \n executable file\n \n 153 lines (126 sloc)\n \n 5.15 KB\n
\n\n \n\n \n\n
\n\n
\n\n | \n | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. | \n
| \n | # | \n
| \n | # Licensed under the Apache License, Version 2.0 (the "License"); | \n
| \n | # you may not use this file except in compliance with the License. | \n
| \n | # You may obtain a copy of the License at | \n
| \n | # | \n
| \n | # http://www.apache.org/licenses/LICENSE-2.0 | \n
| \n | # | \n
| \n | # Unless required by applicable law or agreed to in writing, software | \n
| \n | # distributed under the License is distributed on an "AS IS" BASIS, | \n
| \n | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | \n
| \n | # See the License for the specific language governing permissions and | \n
| \n | # limitations under the License. | \n
| \n | # ============================================================================== | \n
| \n | """CIFAR-10 data set. | \n
| \n | \n |
| \n | See http://www.cs.toronto.edu/~kriz/cifar.html. | \n
| \n | """ | \n
| \n | import os | \n
| \n | \n | \n
| \n | import tensorflow as tf | \n
| \n | flags = tf.app.flags | \n
| \n | FLAGS = flags.FLAGS | \n
| \n | \n | \n
| \n | flags.DEFINE_bool('random_brightness', False, '') | \n
| \n | flags.DEFINE_bool('random_contrast', False, '') | \n
| \n | flags.DEFINE_bool('return_dict', False, '') | \n
| \n | \n | \n
| \n | HEIGHT = 224 | \n
| \n | WIDTH = 224 | \n
| \n | DEPTH = 3 | \n
| \n | \n | \n
| \n | \n | \n
| \n | class Dataset(object): | \n
| \n | \n | \n
| \n | def __init__(self, data_dir, subset='train', use_distortion=True): | \n
| \n | self.data_dir = data_dir | \n
| \n | self.subset = subset | \n
| \n | self.use_distortion = use_distortion | \n
| \n | \n | \n
| \n | def get_filenames(self): | \n
| \n | if self.subset in ['train', 'valid', 'test']: | \n
| \n | return [os.path.join(self.data_dir, self.subset + '.tfrecords')] | \n
| \n | else: | \n
| \n | raise ValueError('Invalid data subset "%s"' % self.subset) | \n
| \n | \n | \n
| \n | def parser(self, serialized_example): | \n
| \n | """Parses a single tf.Example into image and label tensors.""" | \n
| \n | # Dimensions of the images in the CIFAR-10 dataset. | \n
| \n | # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the | \n
| \n | # input format. | \n
| \n | features = tf.parse_single_example( | \n
| \n | serialized_example, | \n
| \n | features={ | \n
| \n | 'id': tf.FixedLenFeature([], tf.string), | \n
| \n | 'image': tf.FixedLenFeature([], tf.string), | \n
| \n | 'label': tf.FixedLenFeature([], tf.int64), | \n
| \n | }) | \n
| \n | \n | \n
| \n | image = features['image'] | \n
| \n | #image = tf.expand_dims(image, axis=-1) | \n
| \n | print('--------------1', image) | \n
| \n | #image = tf.decode_raw(image, tf.uint8) | \n
| \n | image = tf.image.decode_png(image) | \n
| \n | #image.set_shape([HEIGHT * WIDTH * DEPTH]) | \n
| \n | print('--------------2', image, image.shape) | \n
| \n | \n | \n
| \n | #image = tf.cast( | \n
| \n | # tf.reshape(image, [HEIGHT, WIDTH, DEPTH]), | \n
| \n | # tf.float32) | \n
| \n | image = tf.cast(image, tf.float32) | \n
| \n | \n | \n
| \n | label = tf.cast(features['label'], tf.int32) | \n
| \n | id = features['id'] | \n
| \n | \n | \n
| \n | # Custom preprocessing. | \n
| \n | image = self.preprocess(image) | \n
| \n | \n | \n
| \n | # if not FLAGS.return_dict: | \n
| \n | # return id, image, label | \n
| \n | # else: | \n
| \n | # return {'id': id, 'image': image}, label | \n
| \n | \n | \n
| \n | return image, label | \n
| \n | \n | \n
| \n | \n | \n
| \n | def make_batch(self, batch_size, filenames=None, repeat=None, initializable=None): | \n
| \n | """Read the images and labels from 'filenames'.""" | \n
| \n | filenames = filenames or self.get_filenames() | \n
| \n | \n |
| \n | if initializable is None: | \n
| \n | initializable = self.subset != 'train' | \n
| \n | \n |
| \n | # Repeat infinitely. | \n
| \n | dataset = tf.data.TFRecordDataset(filenames).repeat() | \n
| \n | \n | \n
| \n | # Parse records. | \n
| \n | dataset = dataset.map( | \n
| \n | self.parser, num_parallel_calls=batch_size) | \n
| \n | \n | \n
| \n | # Potentially shuffle records. | \n
| \n | if self.subset == 'train': | \n
| \n | min_queue_examples = int( | \n
| \n | Dataset.num_examples_per_epoch(self.subset) * 0.4) | \n
| \n | # Ensure that the capacity is sufficiently large to provide good random | \n
| \n | # shuffling. | \n
| \n | dataset = dataset.shuffle(buffer_size=min_queue_examples + 3 * batch_size) | \n
| \n | \n | \n
| \n | # Batch it up. | \n
| \n | dataset = dataset.batch(batch_size) | \n
| \n | assert tf.executing_eagerly() | \n
| \n | return dataset | \n
| \n | \n | \n
| \n | def preprocess(self, image): | \n
| \n | """Preprocess a single image in [height, width, depth] layout.""" | \n
| \n | if self.subset == 'train' and self.use_distortion: | \n
| \n | # Pad 4 pixels on each dimension of feature map, done in mini-batch | \n
| \n | #... yes should do something like below.. but you will see with dataset.map.. not ok as summary without scope and finally graph has no these summaries | \n
| \n | # refer to https://stackoverflow.com/questions/47345394/image-summaries-with-tensorflows-dataset-api TODO FIXME | \n
| \n | tf.summary.image('image', image) | \n
| \n | #print('--------', image) | \n
| \n | #image = tf.image.resize_image_with_crop_or_pad(image, 256, 256) | \n
| \n | #image = tf.image.resize_images(image, [256, 256], method=0) | \n
| \n | #image = tf.random_crop(image, [HEIGHT, WIDTH, DEPTH]) | \n
| \n | image = tf.image.random_flip_left_right(image) | \n
| \n | if FLAGS.random_brightness: | \n
| \n | image = tf.image.random_brightness(image, | \n
| \n | max_delta=63) | \n
| \n | if FLAGS.random_contrast: | \n
| \n | distorted_image = tf.image.random_contrast(image, | \n
| \n | lower=0.2, | \n
| \n | upper=1.8) | \n
| \n | tf.summary.image('image/distort', image) | \n
| \n | else: | \n
| \n | image = tf.image.resize_images(image, [HEIGHT, WIDTH], method=0) | \n
| \n | \n | \n
| \n | image = tf.cast( | \n
| \n | tf.reshape(image, [HEIGHT, WIDTH, DEPTH]), | \n
| \n | tf.float32) | \n
| \n | \n | \n
| \n | return image | \n
| \n | \n | \n
| \n | @staticmethod | \n
| \n | def num_examples_per_epoch(subset='train'): | \n
| \n | if subset == 'train': | \n
| \n | return 3295 | \n
| \n | elif subset == 'valid': | \n
| \n | return 367 | \n
| \n | elif subset == 'test': | \n
| \n | return 1928 | \n
| \n | else: | \n
| \n | raise ValueError('Invalid data subset "%s"' % subset) | \n
\n \n \n \n
\n\n \n \n
\n\n \n \n
\n\n \n
\n \n
\n\n\n\n