Commit 7e017801 authored by EC2 Default User's avatar EC2 Default User

Initial version

parent 431554a3
This diff is collapsed.
import argparse, os, json
import numpy as np
import keras
from keras import backend as K
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization, Conv2D, MaxPooling2D
from keras.optimizers import SGD
from keras.utils import multi_gpu_model
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--epochs', type=int, default=10)
parser.add_argument('--learning-rate', type=float, default=0.01)
parser.add_argument('--batch-size', type=int, default=128)
parser.add_argument('--gpu-count', type=int, default=os.environ['SM_NUM_GPUS'])
parser.add_argument('--model-dir', type=str, default=os.environ['SM_MODEL_DIR'])
parser.add_argument('--training', type=str, default=os.environ['SM_CHANNEL_TRAINING'])
parser.add_argument('--validation', type=str, default=os.environ['SM_CHANNEL_VALIDATION'])
args, _ = parser.parse_known_args()
epochs = args.epochs
lr = args.learning_rate
batch_size = args.batch_size
gpu_count = args.gpu_count
model_dir = args.model_dir
training_dir = args.training
validation_dir = args.validation
x_train = np.load(os.path.join(training_dir, 'training.npz'))['image']
y_train = np.load(os.path.join(training_dir, 'training.npz'))['label']
x_val = np.load(os.path.join(validation_dir, 'validation.npz'))['image']
y_val = np.load(os.path.join(validation_dir, 'validation.npz'))['label']
# input image dimensions
img_rows, img_cols = 28, 28
# MXNet needs image channels first, e.g. (batch size, channels, width, height)
print(K.image_data_format())
if K.image_data_format() == 'channels_first':
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_val = x_val.reshape(x_val.shape[0], 1, img_rows, img_cols)
input_shape = (1, img_rows, img_cols)
batch_norm_axis=1
else:
print("Incorrect configuration: MXNet needs channels_first")
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_val.shape[0], 'test samples')
# Normalize pixel values
x_train = x_train.astype('float32')
x_val = x_val.astype('float32')
x_train /= 255
x_val /= 255
# Convert class vectors to binary class matrices
num_classes = 10
y_train = keras.utils.to_categorical(y_train, num_classes)
y_val = keras.utils.to_categorical(y_val, num_classes)
model = Sequential()
# 1st convolution block
model.add(Conv2D(64, kernel_size=(3,3), padding='same', input_shape=input_shape))
model.add(BatchNormalization(axis=batch_norm_axis))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=2))
# 2nd convolution block
model.add(Conv2D(128, kernel_size=(3,3), padding='valid'))
model.add(BatchNormalization(axis=batch_norm_axis))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=2))
# Fully connected block
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.3))
# Output layer
model.add(Dense(num_classes, activation='softmax'))
print(model.summary())
if gpu_count > 1:
model = multi_gpu_model(model, gpus=gpu_count)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True),
metrics=['accuracy'])
model.fit(x_train, y_train, batch_size=batch_size,
validation_data=(x_val, y_val),
epochs=epochs,
verbose=1)
score = model.evaluate(x_val, y_val, verbose=0)
print('Validation loss :', score[0])
print('Validation accuracy:', score[1])
print('Saving model in MXNet format')
model_prefix = os.path.join(model_dir, 'model')
data_name, data_shapes = keras.models.save_mxnet_model(model=model, prefix=model_prefix,
epoch=0)
signature = [{'name': data_name[0], 'shape': [dim for dim in data_desc.shape]}
for data_desc in data_shapes]
with open(os.path.join(model_dir, 'model-shapes.json'), 'w') as f:
json.dump(signature, f)
\ No newline at end of file
import argparse, os
import numpy as np
import tensorflow as tf
import keras
from keras import backend as K
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization, Conv2D, MaxPooling2D
from keras.optimizers import SGD
from keras.utils import multi_gpu_model
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--epochs', type=int, default=10)
parser.add_argument('--learning-rate', type=float, default=0.01)
parser.add_argument('--batch-size', type=int, default=128)
parser.add_argument('--gpu-count', type=int, default=os.environ['SM_NUM_GPUS'])
parser.add_argument('--model-dir', type=str, default=os.environ['SM_MODEL_DIR'])
parser.add_argument('--training', type=str, default=os.environ['SM_CHANNEL_TRAINING'])
parser.add_argument('--validation', type=str, default=os.environ['SM_CHANNEL_VALIDATION'])
args, _ = parser.parse_known_args()
epochs = args.epochs
lr = args.learning_rate
batch_size = args.batch_size
gpu_count = args.gpu_count
model_dir = args.model_dir
training_dir = args.training
validation_dir = args.validation
x_train = np.load(os.path.join(training_dir, 'training.npz'))['image']
y_train = np.load(os.path.join(training_dir, 'training.npz'))['label']
x_val = np.load(os.path.join(validation_dir, 'validation.npz'))['image']
y_val = np.load(os.path.join(validation_dir, 'validation.npz'))['label']
# input image dimensions
img_rows, img_cols = 28, 28
# Tensorflow needs image channels last, e.g. (batch size, width, height, channels)
K.set_image_data_format('channels_last')
print(K.image_data_format())
if K.image_data_format() == 'channels_first':
print("Incorrect configuration: Tensorflow needs channels_last")
else:
# channels last
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_val = x_val.reshape(x_val.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
batch_norm_axis=-1
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_val.shape[0], 'test samples')
# Normalize pixel values
x_train = x_train.astype('float32')
x_val = x_val.astype('float32')
x_train /= 255
x_val /= 255
# Convert class vectors to binary class matrices
num_classes = 10
y_train = keras.utils.to_categorical(y_train, num_classes)
y_val = keras.utils.to_categorical(y_val, num_classes)
model = Sequential()
# 1st convolution block
model.add(Conv2D(64, kernel_size=(3,3), padding='same', input_shape=input_shape))
model.add(BatchNormalization(axis=batch_norm_axis))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=2))
# 2nd convolution block
model.add(Conv2D(128, kernel_size=(3,3), padding='valid'))
model.add(BatchNormalization(axis=batch_norm_axis))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=2))
# Fully connected block
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.3))
# Output layer
model.add(Dense(num_classes, activation='softmax'))
print(model.summary())
if gpu_count > 1:
model = multi_gpu_model(model, gpus=gpu_count)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True),
metrics=['accuracy'])
model.fit(x_train, y_train, batch_size=batch_size,
validation_data=(x_val, y_val),
epochs=epochs,
verbose=1)
score = model.evaluate(x_val, y_val, verbose=0)
print('Validation loss :', score[0])
print('Validation accuracy:', score[1])
# save Keras model for Tensorflow Serving
sess = K.get_session()
tf.saved_model.simple_save(
sess,
os.path.join(model_dir, 'model/1'),
inputs={'inputs': model.input},
outputs={t.name: t for t in model.outputs})
\ No newline at end of file
import argparse, os
import numpy as np
import tensorflow as tf
import keras
from keras import backend as K
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, BatchNormalization, Conv2D, MaxPooling2D
from keras.optimizers import SGD
from keras.utils import multi_gpu_model
from keras.datasets import fashion_mnist
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--epochs', type=int, default=10)
parser.add_argument('--learning-rate', type=float, default=0.01)
parser.add_argument('--batch-size', type=int, default=128)
parser.add_argument('--gpu-count', type=int, default=0)
parser.add_argument('--model-dir', type=str, default='/tmp')
parser.add_argument('--training', type=str, default='data')
parser.add_argument('--validation', type=str, default='data')
args, _ = parser.parse_known_args()
epochs = args.epochs
lr = args.learning_rate
batch_size = args.batch_size
gpu_count = args.gpu_count
model_dir = args.model_dir
training_dir = args.training
validation_dir = args.validation
# Download the data set
os.makedirs("./data", exist_ok = True)
(x_train, y_train), (x_val, y_val) = fashion_mnist.load_data()
np.savez('./data/training', image=x_train, label=y_train)
np.savez('./data/validation', image=x_val, label=y_val)
x_train = np.load(os.path.join(training_dir, 'training.npz'))['image']
y_train = np.load(os.path.join(training_dir, 'training.npz'))['label']
x_val = np.load(os.path.join(validation_dir, 'validation.npz'))['image']
y_val = np.load(os.path.join(validation_dir, 'validation.npz'))['label']
# input image dimensions
img_rows, img_cols = 28, 28
# Tensorflow needs image channels last, e.g. (batch size, width, height, channels)
print(K.image_data_format())
if K.image_data_format() == 'channels_first':
print("Incorrect configuration: Tensorflow needs channels_last")
else:
# channels last
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_val = x_val.reshape(x_val.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
batch_norm_axis=-1
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_val.shape[0], 'test samples')
# Normalize pixel values
x_train = x_train.astype('float32')
x_val = x_val.astype('float32')
x_train /= 255
x_val /= 255
# Convert class vectors to binary class matrices
num_classes = 10
y_train = keras.utils.to_categorical(y_train, num_classes)
y_val = keras.utils.to_categorical(y_val, num_classes)
model = Sequential()
# 1st convolution block
model.add(Conv2D(64, kernel_size=(3,3), padding='same', input_shape=input_shape))
model.add(BatchNormalization(axis=batch_norm_axis))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=2))
# 2nd convolution block
model.add(Conv2D(128, kernel_size=(3,3), padding='valid'))
model.add(BatchNormalization(axis=batch_norm_axis))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2), strides=2))
# Fully connected block
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.3))
# Output layer
model.add(Dense(num_classes, activation='softmax'))
print(model.summary())
if gpu_count > 1:
model = multi_gpu_model(model, gpus=gpu_count)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True),
metrics=['accuracy'])
model.fit(x_train, y_train,
batch_size=batch_size,
validation_data=(x_val, y_val),
epochs=epochs,
verbose=1)
score = model.evaluate(x_val, y_val, verbose=0)
print('Validation loss :', score[0])
print('Validation accuracy:', score[1])
# save Keras model for Tensorflow Serving
sess = K.get_session()
tf.saved_model.simple_save(
sess,
os.path.join(model_dir, 'model/1'),
inputs={'inputs': model.input},
outputs={t.name: t for t in model.outputs})
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment