add story extension

package-lock.json

@@ -8,8 +8,8 @@
       "name": "@jbilcke-hf/ai-stories-factory",
       "version": "0.0.0",
       "dependencies": {
-        "@aitube/clap": "0.0.19",
-        "@aitube/client": "0.0.28",
+        "@aitube/clap": "0.0.22",
+        "@aitube/client": "0.0.31",
         "@huggingface/hub": "^0.15.0",
         "@radix-ui/react-accordion": "^1.1.2",
         "@radix-ui/react-avatar": "^1.0.4",
@@ -54,10 +54,10 @@
         "react-device-frameset": "^1.3.4",
         "react-dom": "18.3.1",
         "react-hook-consent": "^3.5.3",
-        "react-icons": "^5.2.0",
-        "sharp": "^0.33.2",
-        "sonner": "^1.4.0",
-        "tailwind-merge": "^2.2.1",
+        "react-icons": "^5.2.1",
+        "sharp": "^0.33.4",
+        "sonner": "^1.4.41",
+        "tailwind-merge": "^2.3.0",
         "tailwindcss": "^3.4.3",
         "tailwindcss-animate": "^1.0.7",
         "ts-node": "^10.9.2",
@@ -65,14 +65,14 @@
         "use-file-picker": "^2.1.2",
         "usehooks-ts": "^2.14.0",
         "uuid": "^9.0.1",
-        "yaml": "^2.4.1",
+        "yaml": "^2.4.2",
         "zustand": "^4.5.2"
       }
     },
     "node_modules/@aitube/clap": {
-      "version": "0.0.19",
-      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.19.tgz",
-      "integrity": "sha512-2Xe9EDY06a09Pp5/pjbm8IQnFCBD/y48WzQqKo6b/1i7lBxzxJ9g5J8b0TM3Jd3JdEApvJ+62tnT7rFrGGLMRQ==",
+      "version": "0.0.22",
+      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.22.tgz",
+      "integrity": "sha512-oN+tfoy0fp95AiwNMVy6pbhkK9k/h/gZ3FTZPpF06bVLig4rZ/Bthyot+wy4E/hr9ERLeyGpfZVW8blRqHCAaw==",
       "dependencies": {
         "pure-uuid": "^1.8.1",
         "yaml": "^2.4.2"
@@ -82,14 +82,14 @@
       }
     },
     "node_modules/@aitube/client": {
-      "version": "0.0.28",
-      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.28.tgz",
-      "integrity": "sha512-yFejJ9j4VcYIhOO/wUhbzJ8qU66KuSOXnPLWio4w4E7c0VwtkD46dae4i6Ic7l4nyVe0avjRIKZlzAlTphFSvA==",
+      "version": "0.0.31",
+      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.31.tgz",
+      "integrity": "sha512-xjdPYRruuEWe8KdmWpMYB1ELylBRF2M6+BYr0K3VAf/fv4qcRqda+F9amitJ27h0t5rvagA7+MLLKFqlF7BT/A==",
       "dependencies": {
         "query-string": "^9.0.0"
       },
       "peerDependencies": {
-        "@aitube/clap": "0.0.19"
+        "@aitube/clap": "0.0.22"
       }
     },
     "node_modules/@alloc/quick-lru": {
@@ -3059,11 +3059,11 @@
       }
     },
     "node_modules/braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
       "dependencies": {
-        "fill-range": "^7.0.1"
+        "fill-range": "^7.1.1"
       },
       "engines": {
         "node": ">=8"
@@ -3177,9 +3177,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001620",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001620.tgz",
-      "integrity": "sha512-WJvYsOjd1/BYUY6SNGUosK9DUidBPDTnOARHp3fSmFO1ekdxaY6nKRttEVrfMmYi80ctS0kz1wiWmm14fVc3ew==",
+      "version": "1.0.30001621",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001621.tgz",
+      "integrity": "sha512-+NLXZiviFFKX0fk8Piwv3PfLPGtRqJeq2TiNoUff/qB5KJgwecJTvCXDpmlyP/eCI/GUEmp/h/y5j0yckiiZrA==",
       "funding": [
         {
           "type": "opencollective",
@@ -3801,9 +3801,9 @@
       "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
     },
     "node_modules/electron-to-chromium": {
-      "version": "1.4.774",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.774.tgz",
-      "integrity": "sha512-132O1XCd7zcTkzS3FgkAzKmnBuNJjK8WjcTtNuoylj7MYbqw5eXehjQ5OK91g0zm7OTKIPeaAG4CPoRfD9M1Mg=="
+      "version": "1.4.777",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.777.tgz",
+      "integrity": "sha512-n02NCwLJ3wexLfK/yQeqfywCblZqLcXphzmid5e8yVPdtEcida7li0A5WQKghHNG0FeOMCzeFOzEbtAh5riXFw=="
     },
     "node_modules/emoji-regex": {
       "version": "9.2.2",
@@ -4472,9 +4472,9 @@
       }
     },
     "node_modules/fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
       "dependencies": {
         "to-regex-range": "^5.0.1"
       },
@@ -5487,9 +5487,9 @@
       }
     },
     "node_modules/language-subtag-registry": {
-      "version": "0.3.22",
-      "resolved": "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.22.tgz",
-      "integrity": "sha512-tN0MCzyWnoz/4nHS6uxdlFWoUZT7ABptwKPQ52Ea7URk6vll88bWBVhodtnlfEuCcKWNGoc+uGbw1cwa9IKh/w=="
+      "version": "0.3.23",
+      "resolved": "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.23.tgz",
+      "integrity": "sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ=="
     },
     "node_modules/language-tags": {
       "version": "1.0.9",
@@ -5607,11 +5607,11 @@
       }
     },
     "node_modules/micromatch": {
-      "version": "4.0.5",
-      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
-      "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==",
+      "version": "4.0.7",
+      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz",
+      "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==",
       "dependencies": {
-        "braces": "^3.0.2",
+        "braces": "^3.0.3",
         "picomatch": "^2.3.1"
       },
       "engines": {
@@ -6310,9 +6310,9 @@
       }
     },
     "node_modules/postcss-selector-parser": {
-      "version": "6.0.16",
-      "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.16.tgz",
-      "integrity": "sha512-A0RVJrX+IUkVZbW3ClroRWurercFhieevHB38sr2+l9eUClMqome3LmEmnhlNy+5Mr2EYN6B2Kaw9wYdd+VHiw==",
+      "version": "6.1.0",
+      "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.0.tgz",
+      "integrity": "sha512-UMz42UD0UY0EApS0ZL9o1XnLhSTtvvvLe5Dc2H2O56fvRZi+KulDyf5ctDhhtYJBGKStV2FL1fy6253cmLgqVQ==",
       "dependencies": {
         "cssesc": "^3.0.0",
         "util-deprecate": "^1.0.2"

package.json

@@ -9,8 +9,8 @@
     "lint": "next lint"
   },
   "dependencies": {
-    "@aitube/clap": "0.0.19",
-    "@aitube/client": "0.0.28",
+    "@aitube/clap": "0.0.22",
+    "@aitube/client": "0.0.31",
     "@huggingface/hub": "^0.15.0",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-avatar": "^1.0.4",
@@ -55,10 +55,10 @@
     "react-device-frameset": "^1.3.4",
     "react-dom": "18.3.1",
     "react-hook-consent": "^3.5.3",
-    "react-icons": "^5.2.0",
-    "sharp": "^0.33.2",
-    "sonner": "^1.4.0",
-    "tailwind-merge": "^2.2.1",
+    "react-icons": "^5.2.1",
+    "sharp": "^0.33.4",
+    "sonner": "^1.4.41",
+    "tailwind-merge": "^2.3.0",
     "tailwindcss": "^3.4.3",
     "tailwindcss-animate": "^1.0.7",
     "ts-node": "^10.9.2",
@@ -66,7 +66,7 @@
     "use-file-picker": "^2.1.2",
     "usehooks-ts": "^2.14.0",
     "uuid": "^9.0.1",
-    "yaml": "^2.4.1",
+    "yaml": "^2.4.2",
     "zustand": "^4.5.2"
   }
 }

src/app/main.tsx

@@ -29,9 +29,11 @@ export function Main() {
   const { storyPromptDraft, setStoryPromptDraft, promptDraftRef } = useStoryPromptDraft()
   const  { isBusy } = useIsBusy()
   const { orientation, toggleOrientation } = useOrientation()
-  const { handleSubmit } = useProcessors()
+  const { handleCreateStory, handleExtendStory } = useProcessors()
   useQueryStringParams()
-    
+
+  const currentVideo = useStore(s => s.currentVideo)
+
   const showAuthWall = useStore(s => s.showAuthWall)
   const { isLoggedIn, enableOAuthWall } = useOAuth()
 
@@ -282,7 +284,7 @@ export function Main() {
                         </div>
                         {/* END OF ORIENTATION SWITCH */}
                         <Button
-                          onClick={handleSubmit}
+                          onClick={currentVideo ? handleExtendStory : handleCreateStory}
                           disabled={!storyPromptDraft || isBusy || !isLoggedIn}
                           // variant="ghost"
                           className={cn(
@@ -293,7 +295,9 @@ export function Main() {
                             storyPromptDraft ? "opacity-100" : "opacity-80"
                           )}
                         >
-                        <span className="mr-1.5">Create</span><span className="hidden md:inline">👉</span><span className="inline md:hidden">👇</span>
+                        <span className="mr-1.5">{
+                          currentVideo ? 'Extend' : 'Create'
+                        }</span><span className="hidden md:inline">👉</span><span className="inline md:hidden">👇</span>
                         </Button>
                       </div>
 

src/app/server/aitube/createClap.ts

@@ -7,7 +7,7 @@ import { ClapProject, ClapMediaOrientation } from "@aitube/clap"
 import { createClap as apiCreateClap } from "@aitube/client"
 
 import { getToken } from "./getToken"
-import { RESOLUTION_LONG, RESOLUTION_SHORT } from "../config"
+import { RESOLUTION_LONG, RESOLUTION_SHORT, MAX_PROMPT_LENGTH_IN_CHARS } from "../config"
 import { getRateLimit } from "../redis/getRateLimit"
 
 const rateLimit = getRateLimit()
@@ -40,7 +40,7 @@ export async function createClap({
   */
 
   const clap: ClapProject = await apiCreateClap({
-    prompt: prompt.slice(0, 512),
+    prompt: prompt.slice(0, MAX_PROMPT_LENGTH_IN_CHARS),
 
     height: orientation === ClapMediaOrientation.PORTRAIT ? RESOLUTION_LONG : RESOLUTION_SHORT,
     width: orientation === ClapMediaOrientation.PORTRAIT ? RESOLUTION_SHORT : RESOLUTION_LONG,

src/app/server/aitube/editClapStory.ts

@@ -5,6 +5,7 @@ import { editClapStory as apiEditClapStory, ClapCompletionMode } from "@aitube/c
 
 import { getToken } from "./getToken"
 import { Workaround } from "./types"
+import { MAX_PROMPT_LENGTH_IN_CHARS } from "../config"
 
 export async function editClapStory({
   clap,
@@ -22,7 +23,7 @@ export async function editClapStory({
   async function promise() {
     return await apiEditClapStory({
       clap,
-      prompt,
+      prompt: `${prompt || ""}`.slice(0, MAX_PROMPT_LENGTH_IN_CHARS),
       startTimeInMs,
       endTimeInMs,
       completionMode: ClapCompletionMode.MERGE,

src/app/server/aitube/exportClapToVideo.ts

@@ -4,6 +4,7 @@ import { ClapProject } from "@aitube/clap"
 import { exportClapToVideo as apiExportClapToVideo } from "@aitube/client"
 
 import { getToken } from "./getToken"
+import { removeFinalVideos } from "@/lib/utils/removeFinalVideos"
 
 export async function exportClapToVideo({
   clap,
@@ -12,6 +13,15 @@ export async function exportClapToVideo({
   clap: ClapProject
   turbo?: boolean
 }): Promise<string> {
+
+  // TODO move this safety into apiExportClapToVideo
+
+  // one last precaustion, we make sure to we remove any existing render beforehand, as this eats up space
+  clap.segments = removeFinalVideos(clap)
+
+  // we have to leave the rest it, however..
+  // ours final render needs all the video and audio clips!
+
   // TODO: maybe we should return a blob instead,
   // as this could be big eg. a few megabytes
   // or maybe we should convert it to an object id

src/app/server/config.ts

@@ -9,3 +9,5 @@ export const RESOLUTION_LONG = 896 // 832 // 768
 export const RESOLUTION_SHORT = 512 // 448 // 384
 
 // ValueError: `height` and `width` have to be divisible by 8 but are 512 and 1.
+
+export const MAX_PROMPT_LENGTH_IN_CHARS = 1024

src/app/store.ts

@@ -33,7 +33,14 @@ export const useStore = create<{
   finalGenerationStatus: TaskStatus
   isBusy: boolean
 
-  currentClap?: ClapProject
+  // a clap file stripped of all its image, video and audio assets
+  // this lightweight clap (which can still grow large)
+  // is best suited for doing API calls
+  skeletonClap?: ClapProject
+
+  // the full clap file, with all the binary assets
+  fullClap?: ClapProject
+
   currentVideo: string
 
   // orientation of the currently loaded video (which can be different from `orientation`)
@@ -60,7 +67,8 @@ export const useStore = create<{
   setVideoGenerationStatus: (videoGenerationStatus: TaskStatus) => void
   setFinalGenerationStatus: (finalGenerationStatus: TaskStatus) => void
   syncStatusAndStageState: () => void
-  setCurrentClap: (currentClap?: ClapProject) => void
+  setSkeletonClap: (fullClap?: ClapProject) => void
+  setFullClap: (fullClap?: ClapProject) => void
 
   // note: this will preload the video, and compute the orientation too
   setCurrentVideo: (currentVideo: string) => Promise<void>
@@ -92,7 +100,8 @@ export const useStore = create<{
   videoGenerationStatus: "idle",
   finalGenerationStatus: "idle",
   isBusy: false,
-  currentClap: undefined,
+  skeletonClap: undefined,
+  fullClap: undefined,
   currentVideo: "",
   currentVideoOrientation: ClapMediaOrientation.PORTRAIT,
   progress: 0,
@@ -216,7 +225,8 @@ export const useStore = create<{
 
     set({ isBusy, stage, statusMessage })
   },
-  setCurrentClap: (currentClap?: ClapProject) => { set({ currentClap }) },
+  setSkeletonClap: (skeletonClap?: ClapProject) => { set({ skeletonClap }) },
+  setFullClap: (fullClap?: ClapProject) => { set({ fullClap }) },
   setCurrentVideo: async (currentVideo: string): Promise<void> => {
     set({
       currentVideo,
@@ -244,10 +254,10 @@ export const useStore = create<{
 
     if (!currentVideo) { throw new Error(`cannot save a video.. if there is no video`) }
 
-    const currentClapBlob: Blob = await fetch(currentVideo).then(r => r.blob())
+    const fullClapBlob: Blob = await fetch(currentVideo).then(r => r.blob())
    
     // Create an object URL for the compressed clap blob
-    const objectUrl = URL.createObjectURL(currentClapBlob)
+    const objectUrl = URL.createObjectURL(fullClapBlob)
   
     // Create an anchor element and force browser download
     const anchor = document.createElement("a")
@@ -269,24 +279,24 @@ export const useStore = create<{
     document.body.removeChild(anchor)
   },
   saveClap: async (): Promise<void> => {
-    const { currentClap , storyPrompt, currentVideo } = get()
+    const { fullClap , storyPrompt, currentVideo } = get()
 
-    if (!currentClap) { throw new Error(`cannot save a clap.. if there is no clap`) }
+    if (!fullClap) { throw new Error(`cannot save a clap.. if there is no clap`) }
 
-    currentClap.meta.description = storyPrompt
+    fullClap.meta.description = storyPrompt
 
     // make sure we update the total duration
-    for (const s of currentClap.segments) {
-      if (s.endTimeInMs > currentClap.meta.durationInMs) {
-        currentClap.meta.durationInMs = s.endTimeInMs
+    for (const s of fullClap.segments) {
+      if (s.endTimeInMs > fullClap.meta.durationInMs) {
+        fullClap.meta.durationInMs = s.endTimeInMs
       }
     }
 
-    const alreadyAnEmbeddedFinalVideo = currentClap.segments.filter(s =>
+    const alreadyAnEmbeddedFinalVideo = fullClap.segments.filter(s =>
       s.category === ClapSegmentCategory.VIDEO &&
       s.status === "completed" &&
       s.startTimeInMs === 0 &&
-      s.endTimeInMs === currentClap.meta.durationInMs &&
+      s.endTimeInMs === fullClap.meta.durationInMs &&
       s.assetUrl).at(0)
 
     // inject the final mp4 video file into the .clap
@@ -295,21 +305,21 @@ export const useStore = create<{
       alreadyAnEmbeddedFinalVideo.assetUrl = currentVideo
     } else {
       console.log(`editing the clap to add a new final video`)
-      currentClap.segments.push(newSegment({
+      fullClap.segments.push(newSegment({
         category: ClapSegmentCategory.VIDEO,
         status: "completed",
         startTimeInMs: 0,
-        endTimeInMs: currentClap.meta.durationInMs,
+        endTimeInMs: fullClap.meta.durationInMs,
         assetUrl: currentVideo,
-        assetDurationInMs: currentClap.meta.durationInMs,
+        assetDurationInMs: fullClap.meta.durationInMs,
         assetSourceType: getClapAssetSourceType(currentVideo),
         outputGain: 1.0,
       }))
     }
-    const currentClapBlob: Blob = await serializeClap(currentClap)
+    const fullClapBlob: Blob = await serializeClap(fullClap)
 
     // Create an object URL for the compressed clap blob
-    const objectUrl = URL.createObjectURL(currentClapBlob)
+    const objectUrl = URL.createObjectURL(fullClapBlob)
   
     // Create an anchor element and force browser download
     const anchor = document.createElement("a")
@@ -338,31 +348,31 @@ export const useStore = create<{
       throw new Error(`missing blob`)
     }
 
-    const currentClap: ClapProject | undefined = await parseClap(blob)
+    const fullClap: ClapProject | undefined = await parseClap(blob)
 
-    if (!currentClap) { throw new Error(`failed to import the clap`) }
+    if (!fullClap) { throw new Error(`failed to import the clap`) }
 
-    const storyPrompt = currentClap.meta.description.split("||").pop() || ""
+    const storyPrompt = fullClap.meta.description.split("||").pop() || ""
 
     putTextInTextAreaElement(
       document.getElementById("story-prompt-draft") as HTMLTextAreaElement,
       storyPrompt
     )
 
-    const orientation = parseMediaOrientation(currentClap.meta.orientation)
+    const orientation = parseMediaOrientation(fullClap.meta.orientation)
 
-    currentClap.meta.height = orientation === ClapMediaOrientation.LANDSCAPE ? RESOLUTION_SHORT : RESOLUTION_LONG
-    currentClap.meta.width = orientation === ClapMediaOrientation.PORTRAIT ? RESOLUTION_SHORT : RESOLUTION_LONG
+    fullClap.meta.height = orientation === ClapMediaOrientation.LANDSCAPE ? RESOLUTION_SHORT : RESOLUTION_LONG
+    fullClap.meta.width = orientation === ClapMediaOrientation.PORTRAIT ? RESOLUTION_SHORT : RESOLUTION_LONG
 
-    const embeddedFinalVideoAssetUrl = currentClap.segments.filter(s =>
+    const embeddedFinalVideoAssetUrl = fullClap.segments.filter(s =>
       s.category === ClapSegmentCategory.VIDEO &&
       s.status === "completed" &&
       s.startTimeInMs === 0 &&
-      s.endTimeInMs === currentClap.meta.durationInMs &&
+      s.endTimeInMs === fullClap.meta.durationInMs &&
       s.assetUrl).map(s => s.assetUrl).at(0)
 
     set({
-      currentClap,
+      fullClap,
       storyPrompt,
       orientation,
       currentVideo: embeddedFinalVideoAssetUrl || get().currentVideo,
@@ -370,7 +380,7 @@ export const useStore = create<{
     })
 
     return {
-      clap: currentClap,
+      clap: fullClap,
       regenerateVideo: !embeddedFinalVideoAssetUrl,
     }
   },

src/components/interface/characters.tsx

@@ -2,14 +2,14 @@ import { useStore } from "@/app/store"
 import { CharacterButton } from "./character-button"
 
 export function Characters() {
-  const currentClap = useStore(s => s.currentClap)
+  const fullClap = useStore(s => s.fullClap)
   
 
   return (
     <div className="flex flex-row space-x-0">
-      {currentClap && currentClap.entities?.length > 0
+      {fullClap && fullClap.entities?.length > 0
         // now: we only support displaying ONE entity for now
-        ? currentClap.entities.slice(0, 1).map(entity =>
+        ? fullClap.entities.slice(0, 1).map(entity =>
           <CharacterButton key={entity.id} entity={entity} />
         )
         : <CharacterButton />

src/components/interface/load-clap-button.tsx

@@ -29,8 +29,8 @@ export function LoadClapButton() {
           storyPromptDraft ? "opacity-100" : "opacity-80"
         )}
       >
-        <span className="hidden xl:inline mr-1">Load</span>
-        <span className="inline xl:hidden mr-1">Load</span>
+        <span className="hidden xl:inline">Load</span>
+        <span className="inline xl:hidden">Load</span>
       </Button></TooltipTrigger>
     <TooltipContent side="top">
       <p className="text-xs font-normal text-stone-100/90 text-center">

src/components/interface/save-clap-button.tsx

@@ -12,7 +12,7 @@ import { useIsBusy, useStoryPromptDraft } from "@/lib/hooks"
 
 export function SaveClapButton() {
   const { isBusy } = useIsBusy()
-  const currentClap = useStore(s => s.currentClap)
+  const fullClap = useStore(s => s.fullClap)
   const saveClap = useStore(s => s.saveClap)
   const { storyPromptDraft } = useStoryPromptDraft()
 
@@ -20,7 +20,7 @@ export function SaveClapButton() {
       <Tooltip>
         <TooltipTrigger asChild><Button
       onClick={() => saveClap()}
-      disabled={!currentClap || isBusy}
+      disabled={!fullClap || isBusy}
       // variant="ghost"
       className={cn(
         `text-xs md:text-sm lg:text-base`,
@@ -30,8 +30,8 @@ export function SaveClapButton() {
         storyPromptDraft ? "opacity-100" : "opacity-80"
       )}
     >
-      <span className="hidden xl:inline mr-1">Save</span>
-      <span className="inline xl:hidden mr-1">Save</span>
+      <span className="hidden xl:inline">Save</span>
+      <span className="inline xl:hidden">Save</span>
     </Button></TooltipTrigger>
       <TooltipContent side="top">
         <p className="text-xs font-normal text-stone-100/90 text-center">

src/lib/hooks/useImportClap.ts

@@ -9,7 +9,7 @@ import { useProcessors } from "./useProcessors"
 export function useImportClap() {
 
   const setError = useStore(s => s.setError)
-  const setCurrentClap = useStore(s => s.setCurrentClap)
+  const setFullClap = useStore(s => s.setFullClap)
   const loadClap = useStore(s => s.loadClap)
 
   const {
@@ -65,13 +65,13 @@ export function useImportClap() {
         
         // clap = await generateSounds(clap)
 
-        // setCurrentClap(clap)
+        // setFullClap(clap)
 
-        console.log("loadClap(): clap = ", clap)
+        console.log("importClap: clap = ", clap)
 
         // it is important to skip regeneration if we already have a video
         if (regenerateVideo) {
-          console.log(`regenerating music and videos..`)
+          console.log(`importClap: regenerating music and videos..`)
           const claps = await Promise.all([
             generateMusic(clap),
             generateVideos(clap)
@@ -87,12 +87,12 @@ export function useImportClap() {
           }
 
 
-          setCurrentClap(clap)
+          setFullClap(clap)
 
           await generateFinalVideo(clap)
 
         } else {
-          console.log(`skipping music and video regeneration`)
+          console.log(`importClap: skipping music and video regeneration`)
         }
 
         setStatus("finished")

src/lib/hooks/useProcessors.ts

@@ -1,7 +1,7 @@
 "use client"
 
 import React, { useState, useTransition } from "react"
-import { ClapProject, ClapSegmentCategory, getClapAssetSourceType, newEntity, updateClap } from "@aitube/clap"
+import { ClapProject, ClapSegmentCategory, filterAssets, getClapAssetSourceType, newEntity, parseClap, serializeClap, updateClap } from "@aitube/clap"
 
 import { logImage } from "@/lib/utils"
 import { useIsBusy, useStoryPromptDraft } from "@/lib/hooks"
@@ -11,6 +11,7 @@ import { useToast } from "@/components/ui/use-toast"
 import { createClap } from "@/app/server/aitube/createClap"
 import { editClapEntities } from "@/app/server/aitube/editClapEntities"
 import { editClapDialogues } from "@/app/server/aitube/editClapDialogues"
+import { editClapStory } from "@/app/server/aitube/editClapStory"
 import { editClapStoryboards } from "@/app/server/aitube/editClapStoryboards"
 import { editClapSounds } from "@/app/server/aitube/editClapSounds"
 import { editClapMusic } from "@/app/server/aitube/editClapMusic"
@@ -19,6 +20,7 @@ import { exportClapToVideo } from "@/app/server/aitube/exportClapToVideo"
 
 import { useStore } from "../../app/store"
 import { useOAuth } from "../oauth/useOAuth"
+import { removeFinalVideos } from "../utils/removeFinalVideos"
 
 export function useProcessors() {
   const [isLocked, setLocked] = useState(false)
@@ -30,7 +32,9 @@ export function useProcessors() {
   const mainCharacterImage = useStore(s => s.mainCharacterImage)
   const mainCharacterVoice = useStore(s => s.mainCharacterVoice)
 
-  const currentClap = useStore(s => s.currentClap)
+  const skeletonClap = useStore(s => s.skeletonClap)
+  const fullClap = useStore(s => s.fullClap)
+  const setSkeletonClap = useStore(s => s.setSkeletonClap)
   const setStoryPrompt = useStore(s => s.setStoryPrompt)
   const setMainCharacterImage = useStore(s => s.setMainCharacterImage)
   const setMainCharacterVoice = useStore(s => s.setMainCharacterVoice)
@@ -48,7 +52,7 @@ export function useProcessors() {
   const setImageGenerationStatus = useStore(s => s.setImageGenerationStatus)
   const setVideoGenerationStatus = useStore(s => s.setVideoGenerationStatus)
   const setFinalGenerationStatus = useStore(s => s.setFinalGenerationStatus)
-  const setCurrentClap = useStore(s => s.setCurrentClap)
+  const setFullClap = useStore(s => s.setFullClap)
   const setCurrentVideo = useStore(s => s.setCurrentVideo)
   const setProgress = useStore(s => s.setProgress)
 
@@ -84,11 +88,11 @@ export function useProcessors() {
       console.log(`generateStory():  copying over entities from the previous clap`)
 
       console.log(`generateStory(): later we can add button(s) to clear the project and/or the character(s)`)
-      const { currentClap } = useStore.getState()
+      const { fullClap } = useStore.getState()
 
-      clap.entities = Array.isArray(currentClap?.entities) ? currentClap.entities : []
+      clap.entities = Array.isArray(fullClap?.entities) ? fullClap.entities : []
 
-      setCurrentClap(clap)
+      setFullClap(clap)
       setStoryGenerationStatus("finished")
 
       console.log("---------------- GENERATED STORY ----------------")
@@ -106,6 +110,46 @@ export function useProcessors() {
     }
   }
 
+  const extendStory = async (clap: ClapProject): Promise<ClapProject> => {
+    try {
+      setStoryGenerationStatus("generating")
+
+      const prompt = promptDraftRef.current.slice(0, 1024)
+
+      clap = await editClapStory({
+        clap,
+        prompt,
+        // startTimeInMs: 0,
+        // endTimeInMs: 0,
+        // generating entities requires a "smart" LLM
+        turbo: false,
+        // turbo: true,
+      }).then(r => r.promise)
+
+      if (!clap) { throw new Error(`failed to create the clap`) }
+
+      if (clap.segments.length <= 1) { throw new Error(`failed to generate more than one segments`) }
+
+      console.log(`generateStory(): received a clap with more shots = `, clap)
+
+      setFullClap(clap)
+      setStoryGenerationStatus("finished")
+
+      console.log("---------------- EXTENDED STORY ----------------")
+      console.table(clap.segments, [
+        // 'startTimeInMs',
+        'endTimeInMs',
+        // 'track',
+        'category',
+        'prompt'
+      ])
+      return clap
+    } catch (err) {
+      setStoryGenerationStatus("error")
+      throw err
+    }
+  }
+
   const generateEntities = async (clap: ClapProject): Promise<ClapProject> => {
     try {
       // setProgress(20)
@@ -163,7 +207,7 @@ export function useProcessors() {
 
       if (!clap) { throw new Error(`failed to edit the sound`) }
 
-      console.log(`generateSounds(): received a clap with sound = `, clap)
+      // console.log(`generateSounds(): received a clap with sound = `, clap)
       setSoundGenerationStatus("finished")
       console.log("---------------- GENERATED SOUND ----------------")
       console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.SOUND), [
@@ -190,7 +234,7 @@ export function useProcessors() {
 
       if (!clap) { throw new Error(`failed to edit the music`) }
 
-      console.log(`generateMusic(): received a clap with music = `, clap)
+      // console.log(`generateMusic(): received a clap with music = `, clap)
       setMusicGenerationStatus("finished")
       console.log("---------------- GENERATED MUSIC ----------------")
       console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.MUSIC), [
@@ -259,7 +303,7 @@ export function useProcessors() {
 
       if (!clap) { throw new Error(`failed to edit the videos`) }
 
-      console.log(`handleSubmit(): received individual video clips = `, clap)
+      console.log(`handleCreateStory(): received individual video clips = `, clap)
       setVideoGenerationStatus("finished")
       console.log("---------------- GENERATED VIDEOS ----------------")
       console.table(clap.segments.filter(s => s.category === ClapSegmentCategory.VIDEO), [
@@ -331,7 +375,44 @@ export function useProcessors() {
     }
   }
   
-  const handleSubmit = async () => {
+  const injectCharacters = async (clap: ClapProject): Promise<void> => {
+    const storyboards = clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD)
+
+    let mainCharacter = clap.entities.at(0)
+
+    // let's do something basic for now: we only support 1 entity (character)
+    // and we apply it to *all* the storyboards (we can always improve this later)
+    if (mainCharacter) {
+      console.log(`injectCharacters(): we use the clap's main character's face on all storyboards`)
+      storyboards.forEach(storyboard => { storyboard.entityId = mainCharacter!.id })
+      logImage(mainCharacter.imageId, 0.35)
+    } else if (mainCharacterImage) {
+      console.log(`injectCharacters(): declaring a new entity for our main character`)
+      const entityName = "person"
+      mainCharacter = newEntity({
+        category: ClapSegmentCategory.CHARACTER,
+        triggerName: entityName,
+        label: entityName,
+        description: entityName,
+        author: "auto",
+        thumbnailUrl: mainCharacterImage,
+  
+        imagePrompt: "",
+        imageSourceType: getClapAssetSourceType(mainCharacterImage),
+        imageEngine: "", 
+        imageId: mainCharacterImage,
+        audioPrompt: "",
+      })
+
+      clap.entities.push(mainCharacter!)
+      console.log(`injectCharacters(): we use the main character's face on all storyboards`)
+      
+      storyboards.forEach(storyboard => { storyboard.entityId = mainCharacter!.id })
+      logImage(mainCharacterImage, 0.35)
+    }
+  }
+
+  const handleCreateStory = async () => {
 
     if (busyRef.current) { return }
 
@@ -347,46 +428,13 @@ export function useProcessors() {
       setStatus("generating")
       busyRef.current = true
   
-      console.log(`handleSubmit(): generating a clap using prompt = "${promptDraftRef.current}" `)
+      console.log(`handleCreateStory(): generating a clap using prompt = "${promptDraftRef.current}" `)
 
       try {
         let clap = await generateStory()
-        setCurrentClap(clap)
-
-        const storyboards = clap.segments.filter(s => s.category === ClapSegmentCategory.STORYBOARD)
-
-        let mainCharacter = clap.entities.at(0)
-
-        // let's do something basic for now: we only support 1 entity (character)
-        // and we apply it to *all* the storyboards (we can always improve this later)
-        if (mainCharacter) {
-          console.log(`handleSubmit(): we use the clap's main character's face on all storyboards`)
-          storyboards.forEach(storyboard => { storyboard.entityId = mainCharacter!.id })
-          logImage(mainCharacter.imageId, 0.35)
-        } else if (mainCharacterImage) {
-          console.log(`handleSubmit(): declaring a new entity for our main character`)
-          const entityName = "person"
-          mainCharacter = newEntity({
-            category: ClapSegmentCategory.CHARACTER,
-            triggerName: entityName,
-            label: entityName,
-            description: entityName,
-            author: "auto",
-            thumbnailUrl: mainCharacterImage,
-      
-            imagePrompt: "",
-            imageSourceType: getClapAssetSourceType(mainCharacterImage),
-            imageEngine: "", 
-            imageId: mainCharacterImage,
-            audioPrompt: "",
-          })
+        setFullClap(clap)
 
-          clap.entities.push(mainCharacter!)
-          console.log(`handleSubmit(): we use the main character's face on all storyboards`)
-          
-          storyboards.forEach(storyboard => { storyboard.entityId = mainCharacter!.id })
-          logImage(mainCharacterImage, 0.35)
-        }
+        await injectCharacters(clap)
 
         const tasks = [
           generateMusic(clap),
@@ -402,7 +450,7 @@ export function useProcessors() {
             overwriteMeta: false,
             inlineReplace: true,
           })
-          setCurrentClap(clap)
+          setFullClap(clap)
         }
 
         /*
@@ -420,7 +468,7 @@ export function useProcessors() {
 
         /*
         if (mainCharacterImage) {
-          console.log("handleSubmit(): User specified a main character image")
+          console.log("handleCreateStory(): User specified a main character image")
           // various strategies here, for instance we can assume that the first character is the main character,
           // or maybe a more reliable way is to count the number of occurrences.
           // there is a risk of misgendering, so ideally we should add some kind of UI to do this,
@@ -436,8 +484,8 @@ export function useProcessors() {
      
         
         
-        console.log("final clap: ", clap)
-        setCurrentClap(clap)
+        console.log("handleCreateStory(): final clap: ", clap)
+        setFullClap(clap)
         await generateFinalVideo(clap)
 
         setStatus("finished")
@@ -462,6 +510,83 @@ export function useProcessors() {
       }
     })
   }
+
+  const handleExtendStory = async () => {
+
+    if (busyRef.current) { return }
+
+    if (enableOAuthWall && !isLoggedIn) {
+      setShowAuthWall(true)
+      return
+    }
+
+    setStatus("generating")
+    busyRef.current = true
+
+    startTransition(async () => {
+      setStatus("generating")
+      setProgress(0)
+      busyRef.current = true
+  
+      let { fullClap } = useStore.getState()
+
+      if (!fullClap) {
+        setStatus("error")
+        setError(`cannot extend the story if there is no current clap file`)
+        return
+      }
+
+      try {
+        console.log(`handleExtendStory(): we strip the clap from its final video (don't worry, it will be re-generated)`)
+    
+        fullClap.segments = removeFinalVideos(fullClap)
+
+        let clap = await extendStory(fullClap)
+
+        if (!clap) {
+          setStatus("error")
+          setError(`failed to extend the story (received an empty clap)`)
+          return
+        }
+
+        await injectCharacters(clap)
+
+        console.log(`handleExtendStory(): new clap with extended story = `, clap)
+
+        const tasks = [
+          generateMusic(clap),
+          generateStoryboardsThenVideos(clap)
+        ]
+
+        const claps = await Promise.all(tasks)
+
+        console.log(`finished processing ${tasks.length} tasks in parallel`)
+
+        for (const newerClap of claps) {
+          clap = await updateClap(clap, newerClap, {
+            overwriteMeta: false,
+            inlineReplace: true,
+          })
+          setFullClap(clap)
+        }
+
+        console.log("handleExtendStory(): calling generateFinalVideo(clap)")
+            
+        await generateFinalVideo(clap)
+
+        setFullClap(clap)
+
+        setStatus("finished")
+        setError("")
+      } catch (err) {
+        console.error(`handleExtendStory(): error: ${err}`)
+        setStoryGenerationStatus("error")
+        setStatus("error")
+      } finally {
+        busyRef.current = false
+      }
+    })
+  }
   
   return {
     generateDialogues,
@@ -473,6 +598,7 @@ export function useProcessors() {
     generateStoryboards,
     generateStoryboardsThenVideos,
     generateVideos,
-    handleSubmit,
+    handleCreateStory,
+    handleExtendStory,
   }
 }

src/lib/hooks/useQueryStringParams.ts

@@ -11,7 +11,7 @@ import { useProcessors } from "./useProcessors"
 export function useQueryStringParams() {
   const { storyPromptDraft, setStoryPromptDraft, promptDraftRef } = useStoryPromptDraft()
   const { busyRef } = useIsBusy()
-  const { handleSubmit } = useProcessors()
+  const { handleCreateStory } = useProcessors()
   
   const setOrientation = useStore(s => s.setOrientation)
   // this is how we support query string parameters
@@ -43,7 +43,7 @@ export function useQueryStringParams() {
       // note: during development we will be called twice,
       // which is why we have a guard on  busyRef.current
       if (maybeAutorun === "true" || maybeAutorun === "1" && !busyRef.current) {
-        handleSubmit()
+        handleCreateStory()
       }
     }
   }, [queryStringPrompt, queryStringAutorun, queryStringOrientation])

src/lib/utils/removeFinalVideos.ts

@@ -0,0 +1,36 @@
+import { ClapProject, ClapSegment, ClapSegmentCategory } from "@aitube/clap"
+
+export function removeFinalVideos(clap: ClapProject): ClapSegment[] {
+  const alreadyAnEmbeddedFinalVideo = clap.segments.filter(s =>
+    s.category === ClapSegmentCategory.VIDEO &&
+    s.status === "completed" &&
+    s.startTimeInMs === 0 &&
+    s.endTimeInMs === clap.meta.durationInMs &&
+    s.assetUrl).at(0)
+
+  let ignoreThisVideoSegmentId = ""
+
+  if (alreadyAnEmbeddedFinalVideo) {
+    ignoreThisVideoSegmentId = alreadyAnEmbeddedFinalVideo?.id || ""
+
+    /*
+    you know what.. let's just ignore it, and re-generate fresh content
+    because most probably the user made an honest mistake
+
+    const outputFilePath = await writeBase64ToFile(
+      alreadyAnEmbeddedFinalVideo.assetUrl,
+      join(outputDir, `existing_final_video`)
+    )
+
+    return {
+      tmpWorkDir: outputDir,
+      outputFilePath
+    }
+    */
+  }
+
+  // we remove the final video from the full list of segments
+  const allCategoriesOfSegments = clap.segments.filter(s => s.id !== ignoreThisVideoSegmentId)
+
+  return allCategoriesOfSegments
+}