Искусственный интеллект против лжи и коварства

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import NoNorm
%matplotlib inline

import math
from tqdm import tqdm
#from joblib import Parallel, delayed

from skimage.draw import ellipse, polygon

from keras import Model
from keras.optimizers import Adam
from keras.layers import Input,Conv2D,Conv2DTranspose,MaxPooling2D,concatenate
from keras.layers import BatchNormalization,Activation,Add,Dropout
from keras.losses import binary_crossentropy
from keras import backend as K
from keras.models import load_model

import tensorflow as tf
import keras as keras

w_size = 128
train_num = 10000


radius_min = 10
radius_max = 30

def dice_coef(y_true, y_pred):
    y_true_f = K.flatten(y_true)
    y_pred = K.cast(y_pred, 'float32')
    y_pred_f = K.cast(K.greater(K.flatten(y_pred), 0.5), 'float32')
    intersection = y_true_f * y_pred_f
    score = 2. * K.sum(intersection) / (K.sum(y_true_f) + K.sum(y_pred_f))
    return score

def dice_loss(y_true, y_pred):
    smooth = 1.
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = y_true_f * y_pred_f
    score = (2. * K.sum(intersection) + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)
    return 1. - score

def bce_dice_loss(y_true, y_pred):
    return binary_crossentropy(y_true, y_pred) + dice_loss(y_true, y_pred)

def get_iou_vector(A, B):
    # Numpy version
    
    batch_size = A.shape[0]
    metric = 0.0
    for batch in range(batch_size):
        t, p = A[batch], B[batch]
        true = np.sum(t)
        pred = np.sum(p)
        
        # deal with empty mask first
        if true == 0:
            metric += (pred == 0)
            continue
        
        # non empty mask case.  Union is never empty 
        # hence it is safe to divide by its number of pixels
        intersection = np.sum(t * p)
        union = true + pred - intersection
        iou = intersection / union
        
        # iou metrric is a stepwise approximation of the real iou over 0.5
        iou = np.floor(max(0, (iou - 0.45)*20)) / 10
        
        metric += iou
        
    # teake the average over all images in batch
    metric /= batch_size
    return metric


def my_iou_metric(label, pred):
    # Tensorflow version
    return tf.py_func(get_iou_vector, [label, pred > 0.5], tf.float64)



from keras.utils.generic_utils import get_custom_objects

get_custom_objects().update({'bce_dice_loss': bce_dice_loss })
get_custom_objects().update({'dice_loss': dice_loss })
get_custom_objects().update({'dice_coef': dice_coef })
get_custom_objects().update({'my_iou_metric': my_iou_metric })

def build_model(input_layer, start_neurons):
    # 128 -> 64
    conv1 = Conv2D(start_neurons * 1, (3, 3),
                   activation="relu", padding="same")(input_layer)
    conv1 = Conv2D(start_neurons * 1, (3, 3),
                   activation="relu", padding="same")(conv1)
    pool1 = Conv2D(start_neurons * 1, (2, 2),
                   strides=(2, 2), activation="relu", padding="same")(conv1)
#    pool1 = Dropout(0.25)(pool1)

    # 64 -> 32
    conv2 = Conv2D(start_neurons * 2, (3, 3),
                   activation="relu", padding="same")(pool1)
    conv2 = Conv2D(start_neurons * 2, (3, 3),
                   activation="relu", padding="same")(conv2)
    pool2 = Conv2D(start_neurons * 1, (2, 2),
                   strides=(2, 2), activation="relu", padding="same")(conv2)
#    pool2 = Dropout(0.5)(pool2)

    # 32 -> 16
    conv3 = Conv2D(start_neurons * 4, (3, 3),
                   activation="relu", padding="same")(pool2)
    conv3 = Conv2D(start_neurons * 4, (3, 3),
                   activation="relu", padding="same")(conv3)
    pool3 = Conv2D(start_neurons * 1, (2, 2),
                   strides=(2, 2), activation="relu", padding="same")(conv3)
#    pool3 = Dropout(0.5)(pool3)

    # 16 -> 8
    conv4 = Conv2D(start_neurons * 8, (3, 3),
                   activation="relu", padding="same")(pool3)
    conv4 = Conv2D(start_neurons * 8, (3, 3),
                   activation="relu", padding="same")(conv4)
    pool4 = Conv2D(start_neurons * 1, (2, 2),
                   strides=(2, 2), activation="relu", padding="same")(conv4)
#    pool4 = Dropout(0.5)(pool4)

    # Middle
    convm = Conv2D(start_neurons * 16, (3, 3),
                   activation="relu", padding="same")(pool4)
    convm = Conv2D(start_neurons * 16, (3, 3)
                   , activation="relu", padding="same")(convm)

    
    # 8 -> 16
    deconv4 = Conv2DTranspose(start_neurons * 8,
                              (3, 3), strides=(2, 2), padding="same")(convm)
    uconv4 = concatenate([deconv4, conv4])
#    uconv4 = Dropout(0.5)(uconv4)
    uconv4 = Conv2D(start_neurons * 8, (3, 3)
                    , activation="relu", padding="same")(uconv4)
    uconv4 = Conv2D(start_neurons * 8, (3, 3)
                    , activation="relu", padding="same")(uconv4)

    
    # 16 -> 32
    deconv3 = Conv2DTranspose(start_neurons * 4,
                              (3, 3), strides=(2, 2), padding="same")(uconv4)
    uconv3 = concatenate([deconv3, conv3])
#    uconv3 = Dropout(0.5)(uconv3)
    uconv3 = Conv2D(start_neurons * 4, (3, 3)
                    , activation="relu", padding="same")(uconv3)
    uconv3 = Conv2D(start_neurons * 4, (3, 3)
                    , activation="relu", padding="same")(uconv3)

    
    # 32 -> 64
    deconv2 = Conv2DTranspose(start_neurons * 2,
                              (3, 3), strides=(2, 2), padding="same")(uconv3)
    uconv2 = concatenate([deconv2, conv2])
#    uconv2 = Dropout(0.5)(uconv2)
    uconv2 = Conv2D(start_neurons * 2, (3, 3)
                    , activation="relu", padding="same")(uconv2)
    uconv2 = Conv2D(start_neurons * 2, (3, 3)
                    , activation="relu", padding="same")(uconv2)

    # 64 -> 128
    deconv1 = Conv2DTranspose(start_neurons * 1,
                              (3, 3), strides=(2, 2), padding="same")(uconv2)
    uconv1 = concatenate([deconv1, conv1])
#    uconv1 = Dropout(0.5)(uconv1)
    uconv1 = Conv2D(start_neurons * 1, (3, 3)
                    , activation="relu", padding="same")(uconv1)
    uconv1 = Conv2D(start_neurons * 1, (3, 3)
                    , activation="relu", padding="same")(uconv1)
   
#    uncov1 = Dropout(0.5)(uconv1)
    output_layer = Conv2D(1, (1,1), padding="same", activation="sigmoid")(uconv1)
    
    return output_layer

input_layer = Input((w_size, w_size, 1))
output_layer = build_model(input_layer, 27)
model = Model(input_layer, output_layer)
model.compile(loss=bce_dice_loss, optimizer=Adam(lr=1e-4), metrics=[my_iou_metric])

model.summary()

def make_sh(f_imgs, f_msks, val_len):
    precision = 0.85
    batch_size = 50
    
    t = tqdm()
    t_batch_size = 50
    raw_len = val_len
    
    id_train = 1
    #id_select = 1
    v_false = np.zeros((train_num), dtype='float')
    
    while True:
        if id_train == 1:
            fit = model.fit(f_imgs[m2_select>0], f_msks[m2_select>0],
                            batch_size=batch_size,
                            epochs=1,
                            verbose=0
                           )
        
            current_accu = fit.history['my_iou_metric'][0]
            current_loss = fit.history['loss'][0]
            if current_accu > precision:
                id_train = 0
                
        else:
            t_pred = model.predict(
                f_imgs[raw_len: min(raw_len+t_batch_size,f_imgs.shape[0])],
                batch_size=batch_size
                                  )
            for kk in range(t_pred.shape[0]):
                val_iou = get_iou_vector(
                    f_msks[raw_len+kk].reshape(1,w_size,w_size,1),
                    t_pred[kk].reshape(1,w_size,w_size,1) > 0.5)
                v_false[raw_len+kk] = val_iou
                if val_iou < precision*0.95:
                    new_img_test = 1
                    m2_select[raw_len+kk] = 1                
                    val_len += 1
                    break
            raw_len += (kk+1)
            id_train = 1

        t.set_description("Accuracy {0:6.4f} loss {1:6.4f} selected img {2:5d} tested img {3:5d} ".
                          format(current_accu, current_loss, val_len, raw_len))
        t.update(1)
        
        if raw_len >= train_num:
            break
    
    t.close()
    return v_false