!pip install theano
%matplotlib inline
from keras.models import Sequential
from keras.layers.core import Flatten, Dense, Dropout
from keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D
from keras.optimizers import SGD
import cv2, numpy as np
import os
import h5py
from matplotlib import pyplot as plt
from keras.applications import vgg16
from keras.applications import Xception
from keras.preprocessing.image import load_img,img_to_array
from keras.models import Model
from keras.applications.imagenet_utils import preprocess_input
from PIL import Image
import os
import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
import pandas as pd
import theano
theano.config.openmp = True

import re
def sorted_alphanumeric(data):
    convert = lambda text: int(text) if text.isdigit() else text.lower()
    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]
    return sorted(data, key=alphanum_key)
dirlist = sorted_alphanumeric(os.listdir('images'))
r1 = []
r2 = []
for i,x in enumerate(dirlist):
    if x.endswith(".jpg"):
        r1.append((int(x[:-4]),i))
        r2.append((i,int(x[:-4])))
extid_to_intid_dict = dict(r1)
intid_to_extid_dict = dict(r2)

imgs_path = "images/"
imgs_model_width, imgs_model_height = 224, 224
nb_closest_images = 3 # количество ближайших изображений (для тестирования вывода)

vgg_model = vgg16.VGG16(weights='imagenet')

feat_extractor = Model(inputs=vgg_model.input, outputs=vgg_model.get_layer("fc2").output)

feat_extractor.summary()

files = [imgs_path + x for x in os.listdir(imgs_path) if "jpg" in x]
print("number of images:",len(files))

import re
def atof(text):
    try:
        retval = float(text)
    except ValueError:
        retval = text
    return retval
def natural_keys(text):
    '''
    alist.sort(key=natural_keys) sorts in human order
    http://nedbatchelder.com/blog/200712/human_sorting.html
    (See Toothy's implementation in the comments)
    float regex comes from https://stackoverflow.com/a/12643073/190597
    '''
    return [ atof(c) for c in re.split(r'[+-]?([0-9]+(?:[.][0-9]*)?|[.][0-9]+)', text) ]
files.sort(key=natural_keys)

original = load_img(files[1], target_size=(imgs_model_width, imgs_model_height))
plt.imshow(original)
plt.show()
print("image loaded successfully!")

numpy_image = img_to_array(original) # сырое изображения в вектор

image_batch = np.expand_dims(numpy_image, axis=0) # превращаем в вектор-строку (2-dims)
print('image batch size', image_batch.shape)

processed_image = preprocess_input(image_batch.copy()) #  библиотечная подготовка изображения

img_features = feat_extractor.predict(processed_image)

print("features successfully extracted!")
print("number of image features:",img_features.size)
img_features

importedImages = []
for f in files:
    filename = f
    original = load_img(filename, target_size=(224, 224))
    numpy_image = img_to_array(original)
    image_batch = np.expand_dims(numpy_image, axis=0)
    importedImages.append(image_batch)
images = np.vstack(importedImages)
processed_imgs = preprocess_input(images.copy())

imgs_features = feat_extractor.predict(processed_imgs)
print("features successfully extracted!")
imgs_features.shape

cosSimilarities = cosine_similarity(imgs_features)

columns_name = re.findall(r'[0-9]+', str(files))
cos_similarities_df = pd.DataFrame(cosSimilarities, columns=files, index=files)
cos_similarities_df.head()

cos_similarities_df_2.round(2) # cos_similarities_df_2 - название датафрейма с косинусными метриками, которые сохранил выше

cos_similarities_df_2.apply(lambda x: x * 100)
cos_similarities_df_2.apply(lambda x: x.astype(np.uint8))

cos_similarities_df_2.to_hdf('storage/cos_similarities.h5', 'data')

import re
# function to retrieve the most similar products for a given one
def retrieve_most_similar_products(given_img):
    print("-----------------------------------------------------------------------")
    print("original product:")
    original = load_img(given_img, target_size=(imgs_model_width, imgs_model_height))
    original_img = int(re.findall(r'[0-9]+', given_img)[0])
    print((df_items_2.iloc[[original_img]]['name'].iat[0], df_items_2.iloc[[original_img]]['pricer_uah'].iat[0], df_items_2.iloc[[original_img]]['url'].iat[0]))
    plt.imshow(original)
    plt.show()
    print("-----------------------------------------------------------------------")
    print("most similar products:")
    closest_imgs = cos_similarities_df[given_img].sort_values(ascending=False)[1:nb_closest_images+1].index
    closest_imgs_scores = cos_similarities_df[given_img].sort_values(ascending=False)[1:nb_closest_images+1]
    for i in range(0,len(closest_imgs)):
        original = load_img(closest_imgs[i], target_size=(imgs_model_width, imgs_model_height))
        item = int(re.findall(r'[0-9]+', closest_imgs[i])[0])
        print(item)
        print((df_items_2.iloc[[item]]['name'].iat[0], df_items_2.iloc[[item]]['pricer_uah'].iat[0], df_items_2.iloc[[item]]['url'].iat[0]))
        plt.imshow(original)
        plt.show()
        print("similarity score : ",closest_imgs_scores[i])
kbr = '' # напишите сюда название товара
find_rec = int(df_items_2.index[df_items_2['name'] == kbr].tolist()[0]) # df_items_2 название моего базового датафрейма, куда я скачал каталог товаров
print(find_rec)
retrieve_most_similar_products(files[find_rec])

import os
if not os.path.exists('storage'):
    os.makedirs('storage')
if not os.path.exists('images'):
    os.makedirs('images')

# importing required modules
import urllib.request
image_counter = 0
error_list = []
# Функция для загрузки изображения в нужную мне директорию
def image_from_df(row):
    global image_counter
    item_id = image_counter
    filename = f'images/{item_id}.jpg'
    image_url = f'{row.image}'
    try:
      conn = urllib.request.urlopen(image_url)
    except urllib.error.HTTPError as e:
      # Return code error (e.g. 404, 501, ...)
      error_list.append(item_id)
    except urllib.error.URLError as e:
      # Not an HTTP-specific error (e.g. connection refused)
      print('URLError: {}'.format(e.reason))
    else:
      # 200
      urllib.request.urlretrieve(image_url, filename)
      image_counter += 1

df_items_2.apply(lambda row: image_from_df(row), axis=1)

for i in error_list:
  df_items_2.drop(df_items_2.index[i], inplace = True)
  df_items_2.reset_index(drop=True, inplace = True)
print(f'Удалил строки без изображений: {error_list}')
print(len(error_list))