added duration controls

README.md

@@ -4,7 +4,7 @@ emoji: 🌅🎶
 colorFrom: green
 colorTo: purple
 sdk: gradio
-sdk_version: 3.9.1
+sdk_version: 3.15.0
 app_file: app.py
 pinned: false
 ---

app.py

@@ -1,4 +1,11 @@
+import time
+import base64
 import gradio as gr
+from sentence_transformers import SentenceTransformer
+
+import httpx
+import json
+
 import os
 import requests
 import urllib
@@ -7,7 +14,6 @@ from os import path
 from pydub import AudioSegment
 
 img_to_text = gr.Blocks.load(name="spaces/pharma/CLIP-Interrogator")
-text_to_music = gr.Interface.load("spaces/fffiloni/text-2-music")
 
 from share_btn import community_icon_html, loading_icon_html, share_js
 
@@ -15,22 +21,59 @@ def get_prompts(uploaded_image):
   
   prompt = img_to_text(uploaded_image, "ViT-L (best for Stable Diffusion 1.*)", "fast", fn_index=1)[0]
   
-  music_result = get_music(prompt)
+  music_result = generate_track_by_prompt(prompt, duration, gen_intensity, audio_format)
   
-  return music_result
+  return music_result[0], gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)
 
-def get_music(prompt):
-  
-  result = text_to_music(prompt, fn_index=0)
-  
-  print(f"""—————
-  NEW RESULTS
-  prompt : {prompt}
-  music : {result}
-  ———————
-  """)
-  
-  url = result
+from utils import get_tags_for_prompts, get_mubert_tags_embeddings, get_pat
+
+minilm = SentenceTransformer('all-MiniLM-L6-v2')
+mubert_tags_embeddings = get_mubert_tags_embeddings(minilm)
+
+
+def get_track_by_tags(tags, pat, duration, gen_intensity, maxit=20, loop=False):
+    if loop:
+        mode = "loop"
+    else:
+        mode = "track"
+    r = httpx.post('https://api-b2b.mubert.com/v2/RecordTrackTTM',
+                   json={
+                       "method": "RecordTrackTTM",
+                       "params": {
+                           "pat": pat,
+                           "duration": duration,
+                           "format": "wav",
+                           "intensity":gen_intensity,
+                           "tags": tags,
+                           "mode": mode
+                       }
+                   })
+
+    rdata = json.loads(r.text)
+    assert rdata['status'] == 1, rdata['error']['text']
+    trackurl = rdata['data']['tasks'][0]['download_link']
+
+    print('Generating track ', end='')
+    for i in range(maxit):
+        r = httpx.get(trackurl)
+        if r.status_code == 200:
+            return trackurl
+        time.sleep(1)
+
+
+def generate_track_by_prompt(prompt, duration, gen_intensity):
+    try:
+        pat = get_pat("[email protected]")
+        _, tags = get_tags_for_prompts(minilm, mubert_tags_embeddings, [prompt, ])[0]
+        result = get_track_by_tags(tags, pat, int(duration), gen_intensity, loop=False)
+        print(result)
+        return result, ",".join(tags), "Success"
+    except Exception as e:
+        return None, "", str(e)
+
+def convert_mp3_to_wav(mp3_filepath):
+ 
+  url = mp3_filepath
   save_as = "file.mp3"
   
   data = urllib.request.urlopen(url)
@@ -44,7 +87,7 @@ def get_music(prompt):
   sound = AudioSegment.from_mp3(save_as)
   sound.export(wave_file, format="wav")
   
-  return wave_file, gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)
+  return wave_file
 
 css = """
 #col-container {max-width: 700px; margin-left: auto; margin-right: auto;}
@@ -79,39 +122,42 @@ a {text-decoration-line: underline; font-weight: 600;}
 """
 
 with gr.Blocks(css=css) as demo:
-  with gr.Column(elem_id="col-container"):
-    gr.HTML("""<div style="text-align: center; max-width: 700px; margin: 0 auto;">
-              <div
-              style="
-                  display: inline-flex;
-                  align-items: center;
-                  gap: 0.8rem;
-                  font-size: 1.75rem;
-              "
-              >
-              <h1 style="font-weight: 900; margin-bottom: 7px; margin-top: 5px;">
-                  Image to Music
-              </h1>
-              </div>
-              <p style="margin-bottom: 10px; font-size: 94%">
-              Sends an image in to <a href="https://huggingface.co/spaces/pharma/CLIP-Interrogator" target="_blank">CLIP Interrogator</a>
-              to generate a text prompt which is then run through 
-              <a href="https://huggingface.co/Mubert" target="_blank">Mubert</a> text-to-music to generate music from the input image!
-              </p>
-          </div>""")
+    with gr.Column(elem_id="col-container"):
+        gr.HTML("""<div style="text-align: center; max-width: 700px; margin: 0 auto;">
+                <div
+                style="
+                    display: inline-flex;
+                    align-items: center;
+                    gap: 0.8rem;
+                    font-size: 1.75rem;
+                "
+                >
+                <h1 style="font-weight: 900; margin-bottom: 7px; margin-top: 5px;">
+                    Image to Music
+                </h1>
+                </div>
+                <p style="margin-bottom: 10px; font-size: 94%">
+                Sends an image in to <a href="https://huggingface.co/spaces/pharma/CLIP-Interrogator" target="_blank">CLIP Interrogator</a>
+                to generate a text prompt which is then run through 
+                <a href="https://huggingface.co/Mubert" target="_blank">Mubert</a> text-to-music to generate music from the input image!
+                </p>
+            </div>""")
     
     
     input_img = gr.Image(type="filepath", elem_id="input-img")
+    with gr.Row():
+        track_duration = gr.Slider(minimum=20, maximum=120, value=30, step=5, label="Track duration", elem_id="duration-inp")
+        gen_intensity = gr.Dropdown(choices=["low", "medium", "high"], value="high", label="Complexity")
     generate = gr.Button("Generate Music from Image")
   
     music_output = gr.Audio(label="Result", type="filepath", elem_id="music-output")
     
     with gr.Group(elem_id="share-btn-container"):
-      community_icon = gr.HTML(community_icon_html, visible=False)
-      loading_icon = gr.HTML(loading_icon_html, visible=False)
-      share_button = gr.Button("Share to community", elem_id="share-btn", visible=False)
+        community_icon = gr.HTML(community_icon_html, visible=False)
+        loading_icon = gr.HTML(loading_icon_html, visible=False)
+        share_button = gr.Button("Share to community", elem_id="share-btn", visible=False)
       
-  generate.click(get_prompts, inputs=[input_img], outputs=[music_output, share_button, community_icon, loading_icon], api_name="i2m")
-  share_button.click(None, [], [], _js=share_js)
+    generate.click(get_prompts, inputs=[input_img,track_duration,gen_intensity], outputs=[music_output, share_button, community_icon, loading_icon], api_name="i2m")
+    share_button.click(None, [], [], _js=share_js)
 
 demo.queue(max_size=32, concurrency_count=20).launch()

constants.py

@@ -0,0 +1,9 @@
+import numpy as np
+import os
+
+MUBERT_LICENCE = os.environ.get('MUBERT_LICENCE')
+MUBERT_TOKEN = os.environ.get('MUBERT_TOKEN')
+
+MUBERT_MODE = "loop"
+MUBERT_TAGS_STRING = 'tribal,action,kids,neo-classic,run 130,pumped,jazz / funk,ethnic,dubtechno,reggae,acid jazz,liquidfunk,funk,witch house,tech house,underground,artists,mystical,disco,sensorium,r&b,agender,psychedelic trance / psytrance,peaceful,run 140,piano,run 160,setting,meditation,christmas,ambient,horror,cinematic,electro house,idm,bass,minimal,underscore,drums,glitchy,beautiful,technology,tribal house,country pop,jazz & funk,documentary,space,classical,valentines,chillstep,experimental,trap,new jack swing,drama,post-rock,tense,corporate,neutral,happy,analog,funky,spiritual,sberzvuk special,chill hop,dramatic,catchy,holidays,fitness 90,optimistic,orchestra,acid techno,energizing,romantic,minimal house,breaks,hyper pop,warm up,dreamy,dark,urban,microfunk,dub,nu disco,vogue,keys,hardcore,aggressive,indie,electro funk,beauty,relaxing,trance,pop,hiphop,soft,acoustic,chillrave / ethno-house,deep techno,angry,dance,fun,dubstep,tropical,latin pop,heroic,world music,inspirational,uplifting,atmosphere,art,epic,advertising,chillout,scary,spooky,slow ballad,saxophone,summer,erotic,jazzy,energy 100,kara mar,xmas,atmospheric,indie pop,hip-hop,yoga,reggaeton,lounge,travel,running,folk,chillrave & ethno-house,detective,darkambient,chill,fantasy,minimal techno,special,night,tropical house,downtempo,lullaby,meditative,upbeat,glitch hop,fitness,neurofunk,sexual,indie rock,future pop,jazz,cyberpunk,melancholic,happy hardcore,family / kids,synths,electric guitar,comedy,psychedelic trance & psytrance,edm,psychedelic rock,calm,zen,bells,podcast,melodic house,ethnic percussion,nature,heavy,bassline,indie dance,techno,drumnbass,synth pop,vaporwave,sad,8-bit,chillgressive,deep,orchestral,futuristic,hardtechno,nostalgic,big room,sci-fi,tutorial,joyful,pads,minimal 170,drill,ethnic 108,amusing,sleepy ambient,psychill,italo disco,lofi,house,acoustic guitar,bassline house,rock,k-pop,synthwave,deep house,electronica,gabber,nightlife,sport & fitness,road trip,celebration,electro,disco house,electronic'
+MUBERT_TAGS = np.array(MUBERT_TAGS_STRING.split(','))

requirements.txt

@@ -1,3 +1,8 @@
+httpx
+sentence-transformers
+ffmpeg
+audio2numpy
+
 pydub
 ffmpeg
 requests

utils.py

@@ -0,0 +1,50 @@
+import json
+import numpy as np
+import httpx
+import os
+
+from constants import MUBERT_TAGS, MUBERT_MODE, MUBERT_LICENCE, MUBERT_TOKEN
+
+def get_mubert_tags_embeddings(w2v_model):
+    return w2v_model.encode(MUBERT_TAGS)
+
+
+def get_pat(email: str):
+    r = httpx.post('https://api-b2b.mubert.com/v2/GetServiceAccess',
+                   json={
+                       "method": "GetServiceAccess",
+                       "params": {
+                           "email": email,
+                           "license": MUBERT_LICENCE,
+                           "token": MUBERT_TOKEN,
+                           "mode": MUBERT_MODE,
+                       }
+                   })
+
+    rdata = json.loads(r.text)
+    assert rdata['status'] == 1, "probably incorrect e-mail"
+    pat = rdata['data']['pat']
+    return pat
+
+
+def find_similar(em, embeddings, method='cosine'):
+    scores = []
+    for ref in embeddings:
+        if method == 'cosine':
+            scores.append(1 - np.dot(ref, em) / (np.linalg.norm(ref) * np.linalg.norm(em)))
+        if method == 'norm':
+            scores.append(np.linalg.norm(ref - em))
+    return np.array(scores), np.argsort(scores)
+
+
+def get_tags_for_prompts(w2v_model, mubert_tags_embeddings, prompts, top_n=3, debug=False):
+    prompts_embeddings = w2v_model.encode(prompts)
+    ret = []
+    for i, pe in enumerate(prompts_embeddings):
+        scores, idxs = find_similar(pe, mubert_tags_embeddings)
+        top_tags = MUBERT_TAGS[idxs[:top_n]]
+        top_prob = 1 - scores[idxs[:top_n]]
+        if debug:
+            print(f"Prompt: {prompts[i]}\nTags: {', '.join(top_tags)}\nScores: {top_prob}\n\n\n")
+        ret.append((prompts[i], list(top_tags)))
+    return ret