Create user query to script generation pipeline

allynee · allynee · commit ab209f07b2fc · 2023-10-14T23:04:33.000+08:00
diff --git a/agent/coder.py b/agent/coder.py
@@ -118,7 +118,7 @@ def _generate_function(self, task, examples):
 
         The task you have to create a function for is: {{task}}. 
                          
-        Here is an example of similar functions that worked:\n {{examples}}
+        Here are examples of similar functions which may or may not be applicable to your task:\n {{examples}}
         
         Your output should strictly follow the format below. Do NOT include any other information in your output.
         public void Method1()
@@ -132,7 +132,7 @@ def _generate_function(self, task, examples):
         ''')
         resp = coder(task=task, examples=examples)
         return resp["function"]
-    
+         
     def _generate_script(self, task, plan, functions):
         guidance.llm = guidance.llms.OpenAI("gpt-3.5-turbo-16k")
         coder = guidance('''
@@ -182,7 +182,7 @@ def _generate_script(self, task, plan, functions):
         ```
         {{~/user}}
         {{#assistant~}}
-        {{gen "script" temperature=0 max_tokens=3200}}
+        {{gen "script" temperature=0 max_tokens=5000}}
         {{~/assistant}}
         ''')  
         resp = coder(task=task, plan=plan, functions=functions)
diff --git a/agent/planner.py b/agent/planner.py
@@ -10,9 +10,63 @@ def __init__(self, model_name="gpt-3.5-turbo", temperature=0, resume=False, ckpt
         guidance.llm = guidance.llms.OpenAI(model_name, temperature=temperature)
         self.llm=guidance.llm
 
-    #TODO: Create plan function
-    def _generate_plan(self, user_query):
-        return ""
+    #TODO: Create plan function. Separate each point with new line character \n
+    def _generate_plan(self, task, examples):
+        planner = guidance('''
+        {{#system~}}
+        You are an efficient, direct and helpful Assistant tasked with helping shape a ubicomp space. 
+        {{~/system}}
+        {{#user~}}
+        Based on the user's prompt, create a set of instruction in which each task is actionable and will result in a visible change noticeable by the user in the 3D ubicomp space/scene. 
+        For tasks that will create an interaction, the change would only be noticed by the user, when they trigger the interaction.
+        There are three Task Types and each step of the instruction must only use one of the following types.
+        1. Create: A new object is created/added to the scene. During this task, you may also indicate the starting position and rotation of the object.
+            1.1. Find the object type from the list, closest to the one requested by the user. If there are no objects remotely close to what the user asked, instructions should be set to null.
+            1.2. When writing the task, use the exact case-sensitive name.
+        2. Edit: An existing object's properties are changed. These properties can be one of the following: Position, Rotation, Size, Color, Illumination (Whether the object eminates light), Luminous Intensity (The brightness of the light between 1 and 10), Levitation (When an object is not levitated, it follows the rules of gravity, and when levitated, it floats).
+            2.1 Do not come up with your own numbers when editing the position, rotation, or size. You must always use relative numbers corresponding to the properties of the user, object(s), or scene. For example, if the user asks for an object to be placed close to the wall, instead of saying:
+            "Place the Lamp 10 cm away from the position of the left wall" 
+            say: 
+            "Place the Lamp <X% * room's width> away from the wall the user can see in their point of view" (where you would replace X by an appropriate number.)  
+            2.2 For colors, always use rgba amounts. 
+            2.3 When editing, always look at the value before change, and based on that value, make the edit. 
+        3. Interact: When a trigger event happens, object(s) are Edited. 
+            3.1 For touch and point triggers, there are already made functions that you may use.
+            3.2 For all other triggers, you must create a "void Update()" method and check for the trigger there. Make sure you always get values in the update, before triggering the event.
+
+
+        When creating the instruction, break the user prompt into actionable tasks that will be done by the order you put them in. Each task will:
+        - result in a visible change in the 3D scene.
+        - be one of three task types (Create, Edit, or Interact)
+        - give direct, actionable instruction without any explanations.
+        
+        The user instruction is {{task}}
+                           
+        Here are examples of similar functions which may or may not be applicable to your task:\n {{examples}}
+
+        Your output should strictly follow the format below. Do NOT include any other information in your output.
+        1. Instruction 1\n
+        2. Instruction 2\n
+        ...
+        
+        Final Notes:
+        a. Stick to coding conventions, avoiding GUI terms like 'drag' or 'click'.
+        b. Be precise in your instruction. Derive numbers from the room and objects unless specified the user.
+        c. Translate vague user terms (e.g., 'small') into value-based calculations based on the properties of the scene and objects.
+        d. Use specific math expressions for vague terms, e.g., instead of "close to the desk", use "smaller than <math expression based on room and object size>".
+        e. Adjust the orientation of objects placed on non-horizontal surfaces such as walls to fit that surface.
+        f. Every object can be illuminated, so you can use any of them as lights.
+        g. Your instruction should not be in a code format. These instruction should be easy to understand.
+        h. The instructions must be numbered and in each number only one action. 
+        h. You must not respond anything other than the JSON. Do not add any text to before or after the JSON.
+        i. If at any point the user mentions an object, and there are no objects remotely close to what they said in the list of current objects in the room given to you, you should make the instruction null and explain in the message.
+        {{~/user}}
+        {{#assistant~}}
+        {{gen "plan" max_tokens=1000 temperature=0}}
+        {{~/assistant}}
+        ''')
+        resp = planner(task=task, examples=examples)
+        return resp["plan"]
     
     def _orchestrate(self, user_query):
         task_type = self._identify_task_type(user_query)
diff --git a/gui/downloaded_code.cs b/gui/downloaded_code.cs
@@ -0,0 +1,36 @@
+using UnityEngine;
+using UnityEngine.Events;
+using UnityEngine.XR.Interaction.Toolkit;
+using System;
+using System.Collections.Generic;
+
+public class TurnOffCeilingLights : SolutionClass
+{
+    private SceneAPI sceneAPI;
+    private Object3D ceilingLight;
+
+    public void TurnOffAllCeilingLights()
+    {
+        sceneAPI = GetSceneAPI();
+        ceilingLight = sceneAPI.FindObject3DByName("Ceiling LED Light");
+
+        if (ceilingLight != null)
+        {
+            EditCeilingLEDLightIllumination();
+        }
+        else
+        {
+            HandleCeilingLEDLightNotFound();
+        }
+    }
+
+    private void EditCeilingLEDLightIllumination()
+    {
+        ceilingLight.Illuminate(0);
+    }
+
+    private void HandleCeilingLEDLightNotFound()
+    {
+        Debug.Log("Ceiling LED Light object not found in the scene.");
+    }
+}
diff --git a/gui/pages/0👤Unity agent.py b/gui/pages/0👤Unity agent.py
@@ -1,22 +1,66 @@
+import base64
 import streamlit as st
+import os
 import sys
 sys.path.append("/Users/allyne/Documents/GitHub/Unity-Agent/")
 import agent as A
 
+def edit_code_string(code_string):
+    try:
+        first_index = code_string.index("using")
+        last_index = code_string.rindex("}")  
+        return code_string[first_index:last_index+1]  
+    except ValueError:
+        st.write("Invalid code: 'using' or '}' not found.")
+
+def create_and_download_cs_file(code_string):
+    code_string = edit_code_string(code_string)
+    file_name = "generated_script.cs"
+    with open(file_name, "w", encoding="utf-8") as file:
+        file.write(code_string)
+    
+    with open(file_name, "rb") as file:
+        btn = st.download_button(
+            label="Download .cs file",
+            data=file,
+            file_name=file_name,
+            mime='text/plain',
+        )
+
+    if btn:
+        os.remove(file_name)
+
 def initialize_components():
     planner = A.Planner()
     coder = A.Coder()
     memorymanager = A.MemoryManager()
     # TODO: Critic
     return planner, coder, memorymanager
 
-def generate_initial_script(instruction):
+def generate_initial_script(task):
+    st.write(f"- Received your task to generate a script for: {task}")
+    st.write("- Intializing components...")
     planner, coder, memorymanager = initialize_components()
-    plan = planner._generate_plan(instruction)
+    st.write("- Retrieving similar plans...")
+    plan_examples = memorymanager._get_plan(task)
+    st.write("- Generating plan...")
+    plan = planner._generate_plan(task, plan_examples)
+    st.write("Here is the generated plan!")
     plans = plan.split("\n")
+    st.write(plans)
     functions = []
+    st.write("Generating functions...")
     for plan in plans:
-        functions.append(coder._generate_function(plan))
+        function_examples = memorymanager._get_code(plan)
+        functions.append(coder._generate_function(plan, function_examples))
+    st.write("Here are the generated functions!")
+    st.write(functions)
+    st.write("Generating script...")
+    script = coder._generate_script(task, plan, functions)
+    st.write("Here is the generated script!")
+    st.write(script)
+    st.write("\n\nDownload the script here:")
+    create_and_download_cs_file(script)
     # TODO: Finish
 
 def simulate_self_repair():
@@ -32,4 +76,13 @@ def refine_strategy_pipeline():
 def add_new_experience():
     pass
 
-# TODO: May need to store a class of generated outputs to ensure learning can take place
+# TODO: May need to store a class of generated outputs to ensure learning can take place
+
+st.title("Testing entire pipeline")
+
+st.write("1. Generate initial script")
+task = st.text_area(f"Enter task here", key="task")
+if st.button("Run", key="generate_script"):
+    with st.spinner("Processing"):
+        generate_initial_script(task)
+        st.success("Process done!")
