fix: refine the prompt to force complete code & refine the logic of running (#1069)

* change refine prompt for full code

* fix: fix the logic of running

* refine prompt

* fix some bugs

* fix

* add two guidelines

* refactor the code

* make costeer evaluator more logical

* refine eval prompt

* make costeer eval prompt markdown

* update code diff prompt

* correct pipeline

* feat: add apply_patch utility and update ret.py with patch functionality (#1071)

* restore to the right version

* fix the docstring

* fix extract_output fcn

* add inplace parameter to apply patch

* remove enable_runner_iteration and make the eval prompt same as main

* refine runner eval prompt based on main

* Update rdagent/scenarios/data_science/dev/runner/prompts.yaml

* add wait_retry

* refactor: move enable_runner_code_diff to DSRunnerCoSTEERSettings as diff_mode

* reformat and remove enable_runner_code_diff

---------

Co-authored-by: yuanteli <[email protected]>
Co-authored-by: Xu <[email protected]>
Co-authored-by: Jensen Lee <[email protected]>
Co-authored-by: you-n-g <[email protected]>
Co-authored-by: Qizheng Li <[email protected]>

Author: 5 people

rdagent/core/experiment.py

@@ -184,7 +184,7 @@ def link_all_files_in_folder_to_workspace(data_path: Path, workspace_path: Path)
             workspace_data_file_path = workspace_path / data_file_path.name
             if workspace_data_file_path.exists():
                 workspace_data_file_path.unlink()
-            if platform.system() == "Linux":
+            if platform.system() in ("Linux", "Darwin"):
                 os.symlink(data_file_path, workspace_data_file_path)
             if platform.system() == "Windows":
                 os.link(data_file_path, workspace_data_file_path)

rdagent/scenarios/data_science/dev/runner/__init__.py

@@ -21,8 +21,9 @@
 from rdagent.log import rdagent_logger as logger
 from rdagent.oai.llm_utils import APIBackend, md5_hash
 from rdagent.scenarios.data_science.dev.runner.eval import DSCoSTEERCoSTEEREvaluator
-from rdagent.utils.agent.ret import PythonBatchEditOut
+from rdagent.utils.agent.ret import PythonBatchEditOut, PythonBatchPatchOut
 from rdagent.utils.agent.tpl import T
+from rdagent.utils.workflow import wait_retry
 
 
 class DSRunnerCoSTEERSettings(DSCoderCoSTEERSettings):
@@ -33,40 +34,57 @@ class Config:
 
     max_seconds: int = DS_RD_SETTING.full_timeout
     env_type: str = "docker"
+    diff_mode: bool = False
     # TODO: extract a function for env and conf.
 
 
 class DSRunnerMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
+    @wait_retry(retry_n=5)
     def implement_one_task(
         self,
         target_task: CoSTEERTask,
         queried_knowledge: CoSTEERQueriedKnowledge | None = None,
         workspace: FBWorkspace | None = None,
         prev_task_feedback: CoSTEERSingleFeedback | None = None,
     ) -> dict[str, str]:
+
         if prev_task_feedback is None:
-            # if no prev_tak_feedback, it is the first loop; we do not make any changes and goto evaluators directly.
+            # if no prev_task_feedback, it is the first loop; we do not make any changes and goto evaluators directly.
             return {}
+
+        # Output Agent Map
+        output_map = {
+            True: (PythonBatchPatchOut.get_spec(), PythonBatchPatchOut.extract_output),
+            False: (
+                PythonBatchEditOut.get_spec(with_del=False),
+                PythonBatchEditOut.extract_output,
+            ),
+        }
+        output_spec, extract_output_fn = output_map[self.settings.diff_mode]
+
         if prev_task_feedback.hyperparameter_tuning_decision:
-            task_information_str = target_task.get_task_information()
-            # 1. code
+            # Use system_refine for hyperparameter tuning
             system_prompt = T(".prompts:DSCoSTEER.system_refine").r(
-                out_spec=PythonBatchEditOut.get_spec(with_del=False),
+                out_spec=output_spec,
+                diff_mode=self.settings.diff_mode,
             )
         else:
             task_information_str = target_task.get_task_information()
-            # 1. code
-            system_prompt = T(".prompts:DSCoSTEER.system_debugger").r(
+            # Use system_debugger for error fixing and debugging
+            system_prompt = T(".prompts:DSCoSTEER.system_refine").r(
                 task_desc=task_information_str,
-                out_spec=PythonBatchEditOut.get_spec(with_del=False),
+                out_spec=output_spec,
+                diff_mode=self.settings.diff_mode,
             )
+
+        # Generate user prompt for both cases
         user_prompt = T(".prompts:DSCoSTEER.user").r(
             code=workspace.all_codes,
             feedback=prev_task_feedback,
             hyperparameter_tuning_suggestion=prev_task_feedback.hyperparameter_tuning_suggestion,
         )
 
-        batch_edit = PythonBatchEditOut.extract_output(
+        batch_edit = extract_output_fn(
             APIBackend().build_messages_and_create_chat_completion(
                 user_prompt=user_prompt,
                 system_prompt=system_prompt,
@@ -112,7 +130,7 @@ def __init__(
         settings = DSRunnerCoSTEERSettings()
         es = DSRunnerMultiProcessEvolvingStrategy(scen=scen, settings=settings)
 
-        # In runner, we don't need very big loops, so we set max_loop to 3
+        # In runner, we don't need very big loops, so we set max_loop to runner_max_loop
         super().__init__(
             *args,
             settings=settings,

rdagent/scenarios/data_science/dev/runner/prompts.yaml

@@ -1,57 +1,60 @@
 DSCoSTEER_eval:
   system: |-
-    You are a data scientist responsible for evaluating all the code.
-
-    ## Target Task Description
-    The user is trying to build a data science solution in the following scenario:
+    {% include "scenarios.data_science.share:scen.role" %}
+    {% if is_sub_enabled %}
+    You will be provided with:
+    1. `Code base`: The code base of the solution
+    2. `The stdout of code execution and testing`: The generated stdout when executing the code base and corresponding testing
+    3, `The time spent on code execution`: The time spent on the code execution
+    4. `The timeout of code execution`: the time limitation of the code execution
+    5. `The percent of timeout used`: the percentage of the time limitation used
+    Your task is to perform the following evaluation(s):
+
+    # Evalution 1: Code Correctness
+    ## Scenario
+    The code is focusing on the following scenario:
     {{ scenario }}
 
-    The task is as follows:
+    ## Target Task Description
+    The code is focusing on the following task
     {{ task_desc }}
 
+    ## Runtime Environment
     You have following environment to run the code:
     {{ runtime_environment }}
 
-    The whole workflow includes multiple stages, such as:
-    - Data loading
-    - Feature engineering
-    - Model training
-    - Ensembling
-
-    ## You'll be provided with the following information about a solution to the Target Task
-    `code base`: The code base of the solution
-    `the stdout of code execution and testing`: The generated stdout when executing the code base and corresponding testing
-    `the time spent on code execution`: The time spent on the code execution
-    `the timeout of code execution`: the time limitation of the code execution
-    `the percent of timeout used`: the percentage of the time limitation used
-   
-    ## Your task is to provide feedback on the solution to the Target Task
-    In the feedback response, 
-    Evaluate the code base based on several aspects, including execution, return checking, and code quality. After your evaluation, make a clear decision to either accept or reject the solution in the `final_decision` section.
-
+    ## Evaluation Guidelines
+    1. Evaluate the code base based on several aspects, including execution correctness, return checking, and code quality.
+    2. Ensure the code does not contain any incorrect, fabricated, or deceptive operations, such as mocking data, scores, or results.
+    3. Confirm that the prediction file (`submission.csv`) is generated using only the test dataset, and its format matches the sample submission.
+    If the code does not satisfy the requirements:
+    - Set "final_decision" to false.
+    - set "hyperparameter_tuning_decision" to false.
+    - Set "hyperparameter_tuning_suggestion" to an empty string.
+    If the code satisfy the requirements:
+    - Proceed to the next evaluation.
+
+    # Evaluation 2: Hyperparameter
+    ## Evaluation Description
     The user will provide you the time spent on the whole code execution and the timeout of the code execution. You should decide whether the hyperparameter is reasonable based on the time.
     For example, if the code uses only a very small portion of the allowed time, and hyperparameters like `n_estimators` or `epochs` have low values, with early stopping not being triggered and possible signs of underfitting, you should suggest increasing these hyperparameters.
-
     You should also notice other resources utilization hyper-parameters,
     For example, if you are using a GPU with large memory, and the batch size is set very low, you should suggest increasing the batch size if it is not reasonable.
 
-    Please provide your feedback in two key-value pairs:
-    "hyperparameter_tuning_decision": <true/false>
-    "hyperparameter_tuning_suggestion": <suggestion in plain text for hyperparameter tuning, e.g., increase n_estimators to 1000, increase epochs to 100, increase batch size to 64, give an empty string if decide not to tune the hyperparameter>
-    [Notice]
-    - You should only suggest the hyperparameter tuning if the code applies early stopping strategy because increasing the training time blindly may lead to overfitting. Once you found the code didn't apply early stopping strategy, you should not suggest to tune the hyperparameter.
-    - Your suggestion should be reasonable and include not only the target hyperparameter but also the hyperparameter sets.
-    - Your suggestion should have a strong chance of improving the model's performance. Focus on the most obvious and impactful opportunities for quick improvement by leveraging more training time. Don't explore hyperparameters with low confidence.  If there are no obvious and impactful opportunities and the code runs well, please accept it.
-    - Once you decide to tune the hyperparameter you should set "final_decision" to false.
-    [Format]
-    - "hyperparameter_tuning_suggestion" should begin with a clear observation, followed by your suggestion. For example: "[Observation] The maximum number of epochs was reached, but the validation loss is still going down and early stopping was not activated. Only 15% of the allowed time was used. [Suggestion] We recommend increasing epochs to 100 to avoid underfitting and further improve model performance."
-
-    {% if is_sub_enabled %}
-    The user will provide you the whole code base, some logs generated during the execution of the whole workflow. Your evaluation scope includes whether the workflow code:
-    1. Executes successfully, correctly organizing components and generating a final submission.
-    2. Generates predictions in the correct format, ensuring they align with the **sample submission** structure!
-    
-    Please respond with your feedback in the following JSON format and order
+    ## Evaluation Guidelines
+    1. The code execution time or resource utilization suggest that there is room for improvement in the hyperparameters.
+    2. The code must apply early stopping strategy already (in order to prevent overfitting).
+    3. Your suggestion should have a strong chance of improving the model's performance. Focus on the most obvious and impactful opportunities for quick improvement by leveraging more training time. Don't explore hyperparameters with low confidence.  If there are no obvious and impactful opportunities and the code runs well, please accept it.
+    If the code satisfy the requirements:
+    - Set "hyperparameter_tuning_decision" to true.
+    - Set "final_decision" to false.
+    - Provide a reasonable suggestion in "hyperparameter_tuning_suggestion". The "hyperparameter_tuning_suggestion" should begin with a clear observation, followed by your suggestion. For example: "[Observation] The maximum number of epochs was reached, but the validation loss is still going down and early stopping was not activated. Only 15% of the allowed time was used. [Suggestion] We recommend increasing epochs to 100 to avoid underfitting and further improve model performance."
+    If the code does not satisfy the requirements:
+    - Set "hyperparameter_tuning_decision" to false.
+    - Set "hyperparameter_tuning_suggestion" to an empty string.
+
+    ## Output format
+    Please respond with your feedback in the following JSON format and order without anything else:
     ```json
     {
         "execution": "Describe whether the whole code base executed successfully and generating the final submission. Include any errors or issues encountered, and retain all error messages and traceback details.",
@@ -81,62 +84,90 @@ DSCoSTEER_eval:
 # NOTE: when is_sub_enabled == False, we don't have any checking about the return. So it is just placeholder currently
 
   user: |-
-    --------- code base ---------
+    # Code base
     {{ code }}
-    --------- the stdout of code execution and testing ---------
+
+    ## Stdout of code execution and testing
     {{ stdout }}
-    --------- the time spent on code execution ---------
+
+    # The time spend on code execution and timeout
     {{ time_spent }}
-    --------- the timeout of code execution ---------
+
+    ## The timeout of code execution
     {{ timeout }}
-    --------- the percent of timeout used ---------
+
+    ## The percent of timeout used
     {{ percent_of_timeout_used }}
 
 DSCoSTEER:
   system_debugger: |-
-    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
-    You have finished the implementation of the whole workflow which has executed well on a sampled dataset. However, the user has reported that the workflow failed to execute on the full dataset.
-
-    Your current job is to debug the whole code base, try to correct the errors, and ensure that the workflow can execute successfully on the full dataset.
-    The user will provide your the whole code base and some feedback generated during the execution of the whole workflow. Please identify the issues and provide the corrected code.
-
-    Task description:
+    {% include "scenarios.data_science.share:scen.role" %}
+    You have finished the implementation of the whole workflow which has executed well on a sampled dataset. Now we are working on the full dataset.
+    The user has reported that the workflow failed to execute on the full dataset.
+    Your will be provided with:
+    1. Code base.
+    2. Task description, which is the task the code is trying to solve.
+    3. Feedback generated during the execution of the whole workflow.
+    4. Suggestions for hyperparameter tuning.
+    Your job is to debug the whole code base, try to correct the errors, and ensure that the workflow can execute successfully on the full dataset.
+
+    ## Task description
     {{ task_desc }}
 
-    Your modified code should follow the minimal changes principle. You should only modify the code that is necessary to fix the issues but not affect any other parts of the code. Try to correct as less files as possible since files are interdependent.
+    ## Instructions
+    1. Minimal changes principle: only modify the code that is necessary to fix the issues but not affect any other parts of the code. Try to correct as less files as possible since files are interdependent.
+    {% if diff_mode %}
+    2. You must output in Code Diff format. The detailed format specification is as follows.
+    {% else %}
+    2. You must output the COMPLETE and FULL code. Do not truncate, summarize, or omit any parts of the code. Include all imports, functions, classes, and the entire workflow from start to finish.
+    {% endif %}
 
     ## Output Format
     {% if out_spec %}
     {{ out_spec }}
     {% else %}
-    Please response the code in the following json format. Here is an example structure for the JSON output:
+    Please response the code in the following JSON format without anything else.
     {
         "code": "The Python code as a string."
     }
     {% endif %}
-  system_refine: |-
-    You are a world-class data scientist and machine learning engineer with deep expertise in statistics, mathematics, and computer science.
-    You have finished the implementation of the whole workflow which has executed well on a sampled dataset. However, the user has reported that the hyperparameters are not reasonable and the code didn't make the best use of the time limit.
-    Your current job is to refine the whole code base, try to refine the hyperparameters.
 
-    The user will provide your the whole code base, some feedback generated during the execution of the whole workflow and some suggestions for hyperparameter tuning.
-    Your modified code should follow the minimal changes principle. Only modify the hyperparameters that is necessary.
+  system_refine: |-
+    {% include "scenarios.data_science.share:scen.role" %}
+    You have finished the implementation of the whole workflow which has executed well on a sampled dataset. Now we are working on the full dataset.
+    The user has reported that the hyperparameters are not reasonable and the code didn't make the best use of the time limit.
+    Your will be provided with:
+    1. Code base.
+    2. Feedback generated during the execution of the whole workflow.
+    3. Suggestions for hyperparameter tuning.
+    Your task is to refine the code base and modify the hyperparameters based on the feedback and suggestions.
+
+    ## Instructions
+    1. Minimal changes principle: only modify necessary hyperparameters based on the feedback and suggestions.
+    {% if diff_mode %}
+    2. You must output in Code Diff format. The detailed format specification is as follows.
+    {% else %}
+    2. You must output the COMPLETE and FULL code. Do not truncate, summarize, or omit any parts of the code. Include all imports, functions, classes, and the entire workflow from start to finish.
+    {% endif %}
 
     ## Output Format
     {% if out_spec %}
     {{ out_spec }}
     {% else %}
-    Please response the code in the following json format. Here is an example structure for the JSON output:
+    Please response the code in the following JSON format without anything else.
     {
         "code": "The Python code as a string."
     }
     {% endif %}
+
   user: |-
-    --------- code base ---------
+    # Code Base
     {{ code }}
-    --------- feedback ---------
+
+    ## Feedback
     {{ feedback }}
+
     {% if hyperparameter_tuning_suggestion is not none %}
-    --------- hyperparameter tuning suggestions ---------
+    ## Hyperparameter Tuning Suggestion
     {{ hyperparameter_tuning_suggestion }}
     {% endif %}