# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: TensorFlow-1.8
#     language: python
#     name: tensorflow-1.8
# ---

# # 使用MoXing实现手写数字图像识别应用
#
#   &#160;&#160;本内容主要介绍，如何使用MoXing实现手写数字图像的训练、测试应用。  
#   
# ### [1. 准备数据](#data_prepare)
# ### [2. 训练模型](#train)
# ### [3. 预测](#predict)

#
#
# ## <a name="data_prepare">1. 准备数据</a>  
#   &#160;&#160;从obs的mnist桶的mnist_data对象中下载MNIST数据集，并上传至私有的OBS桶中。
#   
# 1.1 &#160; &#160; 下载MNIST数据集， 数据集文件说明如下：
# - t10k-images-idx3-ubyte.gz：验证集，共包含10000个样本。
# - t10k-labels-idx1-ubyte.gz：验证集标签，共包含10000个样本的类别标签。
# - train-images-idx3-ubyte.gz：训练集，共包含60000个样本。
# - train-labels-idx1-ubyte.gz：训练集标签，共包含60000个样本的类别标签。
#
# 1.2 &#160; &#160; .gz数据无需解压，分别上传至华为云OBS桶 ,该数据路径将设置为data_url。

# # <a name="train">2. 训练模型</a>  
#
#   &#160;&#160;通过import加载moxing的tensorflow模块 moxing.tensorflow 

import moxing.tensorflow as mox
import os

# 根据数据存储和数据输出设置data_url和train_url

# +
####### your coding place： begin  ###########
# 此处必须修改为用户数据桶位置

#数据在OBS的存储位置。
# eg. s3:// ：统一路径输入
#     /uBucket ：桶名，用户的私有桶的名称 eg. bucket
#     /notebook/data/： 文件路径

data_url = 's3://bucket/notebook/data/' 

####### your coding place： end  ###########
# -

train_url = './cache/log/'          #训练输出位置。
if not mox.file.exists(data_url):
    raise ValueError('Plese verify your data url!')
if mox.file.exists(train_url):
    mox.file.remove(train_url,recursive=True)
mox.file.make_dirs(train_url)

#  通过mox 能够将数据拷贝到本地，这样能够加快训练。操作如下：

# +
# 本地创建数据存储文件夹
local_url = './cache/local_data/'
if mox.file.exists(local_url):
    mox.file.remove(local_url,recursive=True)
os.makedirs(local_url)

#将私有桶中的数据拷贝到本地mox.file.copy_parallel（）
"""
  Copy all files in src_url to dst_url. Same usage as `shutil.copytree`.
  Note that this method can only copy a directory. If you want to copy a single file,
  please use `mox.file.copy`

  Example::

    copy_parallel(src_url='/tmp', dst_url='s3://bucket_name/my_data')

  Assuming files in `/tmp` are:

  * /tmp:
      * |- train
          * |- 1.jpg
          * |- 2.jpg
      * |- eval
          * |- 3.jpg
          * |- 4.jpg

  Then files after copy in `s3://bucket_name/my_data` are:

  * s3://bucket_name/my_data:
      * |- train
          * |- 1.jpg
          * |- 2.jpg
      * |- eval
          * |- 3.jpg
          * |- 4.jpg

  Directory `tmp` will not be copied. If `file_list` is `['train/1.jpg', 'eval/4.jpg']`,
  then files after copy in `s3://bucket_name/my_data` are:

  * s3://bucket_name/my_data
      * |- train
          * |- 1.jpg
      * |- eval
          * |- 4.jpg

  :param src_url: Source path or s3 url
  :param dst_url: Destination path or s3 url
  :param file_list: A list of relative path to `src_url` of files need to be copied.
  :param threads: Number of threads or processings in Pool.
  :param is_processing: If True, multiprocessing is used. If False, multithreading is used.
  :param use_queue: Whether use queue to manage downloading list.
  :return: None
"""
mox.file.copy_parallel(data_url, local_url)
data_url = local_url
os.listdir(data_url)
# -

from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf
from __future__ import print_function
from __future__ import unicode_literals

# **说明 1**  &#160; &#160; 函数 tf.flags.DEFINE_string('data_url', None, 'Dir of dataset')  数据路径。
#                   函数tf.flags.DEFINE_string('train_url', None, 'Train Url') 日志以及生产模型的存储路径。 当脚本运行的时候可以利用tf.flags传入参数。

# +
tf.flags.DEFINE_string('data_url', None, 'Dir of dataset')
tf.flags.DEFINE_string('train_url', None, 'Train Url')

flags = tf.flags.FLAGS

filenames = ['train-images-idx3-ubyte.gz','train-labels-idx1-ubyte.gz','t10k-images-idx3-ubyte.gz',
             't10k-labels-idx1-ubyte.gz']

for filename in filenames:
  filepath = os.path.join(data_url, filename)
  if not mox.file.exists(filepath):
    raise ValueError('MNIST dataset file %s not found in %s' % (filepath, local_url))


# -

# ####  &#160;&#160;训练的main函数包含三个部分，输入定义、模型定义和运行。
#
# 1） 输入函数：input_fn(run_mode, **kwargs) 用户可以根据自己的输入编写。本例中通过迭代的方式从数据集中取数据。
#
#
# 2） 模型定义：def model_fn(inputs, run_mode, **kwargs): 模型结构定义函数，返回 mox.ModelSpec(），用户作业模式定义返回值。
# 但需要满足如下条件：
#
#  &#160;&#160; For run_mode == ModeKeys.TRAIN: `loss` is required.
#   
#   &#160;&#160;  For run_mode == ModeKeys.EVAL: `log_info` is required.
#   
#   &#160;&#160;  For run_mode == ModeKeys.PREDICT: `output_info` is required.
#   
#   &#160;&#160;  For run_mode == ModeKeys.EXPORT: `export_spec` is required.
#   
#
# 3） 执行训练： mox.run(），训练的过程中可指定optimizer的一些设置，训练batch的大小等，设置内容如下：
#
#
#  &#160;&#160; 输入函数， input_fn: An input_fn defined by user. Allows tfrecord or python data. Returns  input tensor list.
#  
#  &#160;&#160;  模型函数， model_fn: A model_fn defined by user. Returns `mox.ModelSpec`.
#   
#   &#160;&#160; optimizer定义， optimizer_fn: An optimizer_fn defined by user. Returns an optimizer.
#   
#   &#160;&#160; 运行模式选择， run_mode: Only takes mox.ModeKeys.TRAIN or mox.ModeKeys.EVAL or mox.ModeKeys.PREDICT
#   
#   &#160;&#160; batch大小设置， batch_size: Mini-batch size.
#   
#  &#160;&#160;  是否自动化batch， auto_batch: If True, an extra dimension of batch_size will be expanded to the first
#                      dimension of the return value from `get_split`. Default to True.
#                      
#   &#160;&#160; 日志以及checkpoint保存位置， log_dir: The directory to save summaries and checkpoints.
#   
#   &#160;&#160; 最大数量，  max_number_of_steps: Maximum steps for each worker.
#                           
#   &#160;&#160; 日志打印， log_every_n_steps: Step period to print logs to std I/O.
#      
#   &#160;&#160; 是否输出模型， export_model: True or False. Where to export model after running the job.
#
#

# +
def main(*args):
  flags.data_url = data_url
  flags.train_url = train_url
  mnist = input_data.read_data_sets(flags.data_url, one_hot=True)
        

  # define the input dataset, return image and label
  def input_fn(run_mode, **kwargs):
    def gen():
      while True:
        yield mnist.train.next_batch(50)
    ds = tf.data.Dataset.from_generator(
        gen, output_types=(tf.float32, tf.int64),
        output_shapes=(tf.TensorShape([None, 784]), tf.TensorShape([None, 10])))
    return ds.make_one_shot_iterator().get_next()


  # define the model for training or evaling.
  def model_fn(inputs, run_mode, **kwargs):
    x, y_ = inputs
    W = tf.get_variable(name='W', initializer=tf.zeros([784, 10]))
    b = tf.get_variable(name='b', initializer=tf.zeros([10]))
    y = tf.matmul(x, W) + b
    cross_entropy = tf.reduce_mean(
      tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
    predictions = tf.argmax(y, 1)
    correct_predictions = tf.equal(predictions, tf.argmax(y_, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_predictions, tf.float32))
    export_spec = mox.ExportSpec(inputs_dict={'images': x}, outputs_dict={'predictions': predictions}, version='model')
    return mox.ModelSpec(loss=cross_entropy, log_info={'loss': cross_entropy, 'accuracy': accuracy},
                         export_spec=export_spec)


  mox.run(input_fn=input_fn,
          model_fn=model_fn,
          optimizer_fn=mox.get_optimizer_fn('sgd', learning_rate=0.01),
          run_mode=mox.ModeKeys.TRAIN,
          batch_size=50,
          auto_batch=False,
          log_dir=flags.train_url,
          max_number_of_steps=1000,
          log_every_n_steps=10,
          export_model=mox.ExportKeys.TF_SERVING)

if __name__ == '__main__':
  try:
      tf.app.run(main=main)
  except SystemExit:
      pass
# -

# ## <a name="predict">2. 预测</a>  
#
#

# &#8195;&#8195; 在上面训练的基础上，我们可以直接用训练的模型进行预测作业。如读取OBS桶中的数字图片进行识别。input_fn 对输入图片进行简单处理，得到网络允许的输入tensor；model_fn定义一个预测内容，同时，还需定义一个对输出处理的函数output_fn，我们在改函数里对输出进行一个打印输出。
#  
#   还需在 mox.run()函数中加入如下参数：
#   
#  &#8195;&#8195; 输出函数 output_fn: A callback with args of results from sess.run.
#    
# &#8195;&#8195; 模型加载位置 checkpoint_path: Directory or file path of ckpt to restore when `run_mode` is 'evaluation'.
#                           Useless when `run_mode` is 'train'.

# +
####### your coding place： begin###########

#此处必须修改为用户数据存储的OBS位置

# 预测图片在OBS的存储位置。
# eg. 图片名称：  image_number.jpg
#     存储位置为：bucket/test/
src_path = 's3://bucket/test/image_number.jpg'

####### your coding place： end  ###########
# -

# 可以利用moxing 将需要预测的图片从OBS拷贝到本地
if not mox.file.exists(src_path):
    raise ValueError('Plese verify your src_path!')
dst_path =  './cache/test.jpg'
mox.file.copy(src_path,dst_path)

image_path = './cache/test.jpg'            # 指定图片位置
checkpoint_url = './cache/log/'         # 指定checkpoint位置，即上一步训练指定的路径的位置。
print(mox.file.exists(image_path))

import moxing.tensorflow as mox
import os
import tensorflow as tf
from __future__ import print_function
from __future__ import unicode_literals


def predict(*args):
  def input_fn(run_mode, **kwargs):
    image = tf.read_file(image_path)
    img = tf.image.decode_jpeg(image, channels=1)
    img = tf.image.resize_images(img, [28, 28], 0)
    img = tf.reshape(img, [784])
    return img

  def model_fn(inputs, run_mode, **kwargs):
    x = inputs
    W1 = tf.get_variable(name='W', initializer=tf.zeros([784, 10]))
    b1 = tf.get_variable(name='b', initializer=tf.zeros([10]))
    y = tf.matmul(x, W1) + b1
    predictions = tf.argmax(y, 1)
    return mox.ModelSpec(output_info={'predict': predictions})

  def output_fn(outputs):
    for output in outputs:
      result = output['predict']
      print("The result：",result)

  mox.run(input_fn=input_fn,
          model_fn=model_fn,
          output_fn=output_fn,
          run_mode=mox.ModeKeys.PREDICT,
          batch_size=1,
          auto_batch=False,
          max_number_of_steps=1,
          output_every_n_steps=1,
          checkpoint_path=checkpoint_url)
if __name__ == '__main__':
  try:
      tf.app.run(main=predict)
  except SystemExit:
      pass

# 通过预测，我们能够看到结果输出。
#

StarcoderdataJupyterScriptsDedupFiltered