Upload 2 files

movie.py

@@ -0,0 +1,82 @@
+import gradio as gr
+from datasets import load_dataset
+dataset = load_dataset("SandipPalit/Movie_Dataset")
+
+from InstructorEmbedding import INSTRUCTOR
+model = INSTRUCTOR('hkunlp/instructor-xl')
+
+from sklearn.metrics.pairwise import cosine_similarity
+
+def getSimilarity(sentences_a,sentences_b):
+  embeddings_a = model.encode(sentences_a)
+  embeddings_b = model.encode(sentences_b)
+  similarities = cosine_similarity(embeddings_a,embeddings_b)
+  return similarities
+  
+#get the indices of the np_array that has maximum score
+import heapq                           
+def get_top_k(h,k):
+  output=[]
+  for i in range(k):
+     output.append(heapq.heappop(h)[1])
+  return output
+
+def heapsort(np_array,k):
+  h=[] 
+  for idx,score in enumerate(np_array):
+    heapq.heappush(h,(-score,idx))                    #max_heap
+  return get_top_k(h,k)
+
+
+import nltk
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize, sent_tokenize
+from nltk.stem import WordNetLemmatizer
+
+nltk.download('punkt')
+nltk.download('stopwords')
+nltk.download('wordnet')
+
+def preprocess(idx,text,total_length):
+    sentences = sent_tokenize(text)
+    stop_words = set(stopwords.words('english'))
+    lemmatizer = WordNetLemmatizer()
+
+    padding=''+'0'*(len(str(total_length))-len(str(idx)))
+    output=[]
+    for sentence in sentences:
+      output.append(' '.join([lemmatizer.lemmatize(word) for word in sentence.split() if word not in stop_words])+'@'+padding+str(idx))
+    return output
+
+def get_pre_processed_data(size):
+  sentences=[]
+  for idx,x in enumerate(df['Plot'].head(size).tolist()):
+     sentences.extend(preprocess(idx,x,df.shape[0]))
+  return sentences
+
+def get_top_k_matches(np_array,k,sentences):
+   indices=[]
+   for idx in heapsort(np_array,k):
+     i=len(sentences[idx])-1                  #based on the index find the sentence- reason for storing idx but not sentence
+     count=1
+     number=0
+     while sentences[idx][i]!='@':                     #o(8-10 digits)- o(1)
+       number=number+count*int(sentences[idx][i])
+       count*=10
+       i-=1
+     indices.append(number)
+   #print(indices)
+   return indices
+    
+
+import pandas as pd
+df=pd.DataFrame({"Title":dataset['train']['Title'],"Plot":dataset['train']['Overview']})
+
+def getOutput(text, size):
+    sentences=get_pre_processed_data(int(size))
+    np_array=getSimilarity(sentences,[text])
+    s=df.iloc[get_top_k_matches(np_array,2,sentences)]['Title']
+    return f"{s}."
+
+iface = gr.Interface(fn=getOutput, inputs=[gr.inputs.Textbox(label="Text"), gr.inputs.Number(label="size")], outputs="text")
+iface.launch(debug=True)

requirements.txt

@@ -0,0 +1,4 @@
+datasets
+sentence_transformers
+InstructorEmbedding
+