twitter iframe

app.py

@@ -6,6 +6,7 @@ from PIL import Image
 import requests
 import transformers
 import text2image
+import zeroshot
 import tokenizers
 from io import BytesIO
 import streamlit as st
@@ -24,6 +25,7 @@ st.sidebar.title("Explore our PLIP Demo")
 PAGES = {
     "Introduction": home,
     "Text to Image": text2image,
+    "Image Prediction": zeroshot,
 }
 
 page = st.sidebar.radio("", list(PAGES.keys()))

home.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 import streamlit as st
+import streamlit.components.v1 as components
 
 
 def read_markdown_file(markdown_file):
@@ -8,4 +9,14 @@ def read_markdown_file(markdown_file):
 
 def app():
     intro_markdown = read_markdown_file("introduction.md")
-    st.markdown(intro_markdown, unsafe_allow_html=True)
+    st.markdown(intro_markdown, unsafe_allow_html=True)
+
+    st.text('An example of twitter:')
+    components.html('''
+        <blockquote class="twitter-tweet">
+            <a href="https://twitter.com/xxx/status/1580753362059788288"></a>
+        </blockquote>
+        <script async src="https://platform.twitter.com/widgets.js" charset="utf-8">
+        </script>
+        ''',
+    height=600)

text2image.py

@@ -4,19 +4,17 @@ from plip_support import embed_text
 import numpy as np
 from PIL import Image
 import requests
-import transformers
 import tokenizers
 from io import BytesIO
-import streamlit as st
-from transformers import CLIPModel
-import clip
 import torch
 from transformers import (
     VisionTextDualEncoderModel,
     AutoFeatureExtractor,
-    AutoTokenizer
+    AutoTokenizer,
+    CLIPModel,
+    AutoProcessor
 )
-from transformers import AutoProcessor
+import streamlit.components.v1 as components
 
 
 def embed_texts(model, texts, processor):
@@ -51,7 +49,8 @@ def load_path_clip():
 def app():
     st.title('PLIP Image Search')
 
-    plip_dataset = pd.read_csv("tweet_eval_retrieval.tsv", sep="\t")
+    plip_imgURL = pd.read_csv("tweet_eval_retrieval.tsv", sep="\t")
+    plip_weblink = pd.read_csv("tweet_eval_retrieval_twlnk.tsv", sep="\t")
 
     model, processor = load_path_clip()
 
@@ -59,16 +58,58 @@ def app():
 
     query = st.text_input('Search Query', '')
 
-
     if query:
 
         text_embedding = embed_texts(model, [query], processor)[0].detach().cpu().numpy()
 
         text_embedding = text_embedding/np.linalg.norm(text_embedding)
+        
+        # Sort IDs by cosine-similarity from high to low
+        similarity_scores = text_embedding.dot(image_embedding.T)
+        id_sorted = np.argsort(similarity_scores)[::-1]
+
+
+        best_id = id_sorted[0]
+        score = similarity_scores[best_id]
+        target_url = plip_imgURL.iloc[best_id]["imageURL"]
+        target_weblink = plip_weblink.iloc[best_id]["weblink"]
+
+        st.caption('Most relevant image (similarity = %.4f)' % score)
+        #response = requests.get(target_url)
+        #img = Image.open(BytesIO(response.content))
+        #st.image(img)
+
+
+        components.html('''
+            <blockquote class="twitter-tweet">
+                <a href="%s"></a>
+            </blockquote>
+            <script async src="https://platform.twitter.com/widgets.js" charset="utf-8">
+            </script>
+            ''' % target_weblink,
+        height=600)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
-        best_id = np.argmax(text_embedding.dot(image_embedding.T))
-        url = (plip_dataset.iloc[best_id]["imageURL"])
 
-        response = requests.get(url)
-        img = Image.open(BytesIO(response.content))
-        st.image(img)