diff --git a/bases/rsptx/book_server_api/routers/coach.py b/bases/rsptx/book_server_api/routers/coach.py index 0848a2f0e..1f518db94 100644 --- a/bases/rsptx/book_server_api/routers/coach.py +++ b/bases/rsptx/book_server_api/routers/coach.py @@ -48,6 +48,7 @@ from rsptx.db.crud.crud import fetch_api_token from rsptx.db.crud.course import fetch_course from rsptx.db.crud.question import fetch_question +from rsptx.db.crud import fetch_source_code # .. _APIRouter config: # @@ -241,6 +242,79 @@ def _extract_suffix_code_from_htmlsrc(htmlsrc): return "" +# A quoted name with a file extension, e.g. open("scores.csv") or +# new File("words.txt") -- a possible data file to look up in source_code. +# Single spaces inside the name are allowed, since uploaded datafiles keep +# theirs (open("my data.csv")). +_FILENAME_LITERAL_RE = re.compile( + r"""["']([\w\-./]+(?: [\w\-./]+)*\.[A-Za-z0-9]{1,8})["']""" +) +# Bounds the source_code lookups that guessed filename literals can trigger. +# Names the question declares in data-datafile are always kept. +_MAX_DATA_FILE_CANDIDATES = 20 + + +def _data_file_candidates(htmlsrc, *code_sources): + """ + Filenames the question's program may open. Livecode (JOBE) questions list + them in data-datafile, but Python questions don't declare them anywhere -- + Skulpt resolves open() at runtime -- so also take every quoted + filename-like literal from the question's code, its tests and the + student's code. Names that turn out not to be data files simply miss in + the source_code lookup. + """ + declared = [] + htmlsrc = htmlsrc or "" + match = re.search(r'data-datafile\s*=\s*"([^"]*)"', htmlsrc) + if match: + declared.extend(name.strip() for name in match.group(1).split(",")) + declared = list(dict.fromkeys(name for name in declared if name)) + textarea = re.search(r"]*>(.*?)", htmlsrc, flags=re.DOTALL) + question_code = html.unescape(textarea.group(1)) if textarea else "" + inferred = [] + for text in (question_code, *code_sources): + inferred.extend(_FILENAME_LITERAL_RE.findall(text or "")) + inferred = [name for name in dict.fromkeys(inferred) if name not in declared] + return declared + inferred[:_MAX_DATA_FILE_CANDIDATES] + + +async def _fetch_data_files(candidates, course_scopes): + """ + Resolve candidate filenames against source_code -- the same store + Skulpt's file reader and livecode.js fall back to -- so the files can be + shipped to JOBE with every CodeTailor test run. + + course_scopes is a list of (base_course, course_name) pairs tried in + order: the requesting course first, then the book the question came from + (which differs for cloned courses and cross-book selectquestions). + Returns dicts in the shape evaluate_fixed_code._push_data_files expects. + """ + data_files = [] + for name in candidates: + for base_course, course_name in course_scopes: + try: + row = await fetch_source_code( + base_course=base_course, course_name=course_name, filename=name + ) + except Exception: + # fetch_source_code raises when nothing matches + row = None + if row and row.main_code is not None: + filename = row.filename or name + # data-datafile names the acid while the code names the file, + # so the same file can be resolved twice -- ship it once + if filename not in {data_file["filename"] for data_file in data_files}: + data_files.append( + { + "filename": filename, + "content": row.main_code, + "is_binary": bool(row.is_binary), + } + ) + break + return data_files + + def _build_static_parsons_response( parsonsexample_code, parsons_attrs, personalization_level ): @@ -640,6 +714,25 @@ async def parsons_scaffolding( if language and language.lower() == "python": internal_test_case = clean_python_testcase(internal_test_case) + # Files the program opens (datafiles) must travel with each JOBE test run, + # or every generated solution fails with a missing-file error. + data_files = await _fetch_data_files( + _data_file_candidates(question.htmlsrc, internal_test_case, student_code), + list( + dict.fromkeys( + [ + (basecourse, course_name), + (question.base_course, question.base_course), + ] + ) + ), + ) + if data_files: + rslogger.info( + f"CodeTailor: '{problem_id}' uses data files " + f"{[data_file['filename'] for data_file in data_files]}" + ) + # Comments that ship with the starter code ("# write code here") are not # the student's subgoal labels, so CodeTailor should not keep them. student_code = _drop_starter_comments( @@ -693,6 +786,7 @@ def parsons_help( # this can be returned verbatim as the fixed/example code when # LLM personalization falls back (see end_to_end.generate_example_solution) "CF (Code)": student_code, + "Data_Files": data_files, } return get_parsons_help(api_token, language, input_dict, personalization_level) @@ -717,6 +811,7 @@ def parsons_help( language, problem_description, internal_test_case, + data_files, ) if not example_code: return ( diff --git a/bases/rsptx/book_server_api/routers/personalized_parsons/end_to_end.py b/bases/rsptx/book_server_api/routers/personalized_parsons/end_to_end.py index d0aac287f..9b327bb0a 100644 --- a/bases/rsptx/book_server_api/routers/personalized_parsons/end_to_end.py +++ b/bases/rsptx/book_server_api/routers/personalized_parsons/end_to_end.py @@ -26,7 +26,7 @@ def get_parsons_help(api_token, language, dict_buggy_code, personalize_level): Args: api_token: The API token for GenerativeAI -- now only support OpenAI language: The programming language of the code - dict_buggy_code: A dictionary containing the current buggy code, problem description, unittest code, and a predefined example + dict_buggy_code: A dictionary containing the current buggy code, problem description, unittest code, a predefined example, and (optionally) the question's data files personalize_level: The level of personalization to apply (e.g., "Solution", "Multiple") Returns: A tuple containing the final fixed code, the final Parsons block, the solution generation type, and the generation result type. @@ -39,6 +39,8 @@ def get_parsons_help(api_token, language, dict_buggy_code, personalize_level): unittest_code = dict_buggy_code["Unittest_Code"].replace("\\n", "\n") # This can be "LLM-example" or a predefined example code predefined_example = dict_buggy_code["Example"].replace("\\n", "\n") + # files the program opens (e.g. datafiles) -- shipped with every test run + data_files = dict_buggy_code.get("Data_Files") # check if language is java, keep it as it is, otherwise, assign it to "python" as default if language.lower() == "java" or language.lower() == "python": language = language.lower() @@ -55,6 +57,7 @@ def get_parsons_help(api_token, language, dict_buggy_code, personalize_level): default_test_code, predefined_example, unittest_code, + data_files=data_files, ) final_fixed_code = cleaned_fixed_code.lstrip() else: @@ -77,6 +80,7 @@ def get_parsons_help(api_token, language, dict_buggy_code, personalize_level): default_start_code, default_test_code, unittest_code, + data_files=data_files, ) elif personalize_level == "Multiple": # generate Parsons puzzles with personalization at both solution and multiple levels with paired distractors and potentially settled block lines @@ -89,6 +93,7 @@ def get_parsons_help(api_token, language, dict_buggy_code, personalize_level): default_start_code, default_test_code, unittest_code, + data_files=data_files, ) else: return "Error: invalid personalize_level" @@ -121,6 +126,7 @@ def request_fixed_code_from_openai( situation, failure_reason, unittest_result, + data_files=None, ): """ Request a fixed code from GenerativeAI based on the buggy code and other parameters. @@ -139,6 +145,7 @@ def request_fixed_code_from_openai( situation (str): The situation for the request, e.g., "a correct answer" failure_reason (str): The reason for the failure, e.g., "not correct" unittest_result (bool): The result of the unittest evaluation for the buggy code + data_files (list): The question's data files the code may open Outputs: cleaned_fixed_code (str): The cleaned fixed code generated by the LLM generation_result_type (str): The type of generation result, e.g., "AI_personalized", "example_more_personalized", "example_solution" @@ -165,6 +172,7 @@ def request_fixed_code_from_openai( default_start_code, default_test_code, unittest_code, + data_files=data_files, ) print("this-round-result:", unittest_result, cleaned_fixed_code) @@ -217,11 +225,17 @@ def request_fixed_code_from_openai( situation="a correct answer", failure_reason="not correct", unittest_result=False, + data_files=data_files, ) def generate_example_solution( - api_token, language, problem_description, unittest_code, predefined_example + api_token, + language, + problem_description, + unittest_code, + predefined_example, + data_files=None, ): """ Generate or retrieve an example solution based on the provided parameters. @@ -231,12 +245,13 @@ def generate_example_solution( problem_description (str): textual question description unittest_code (str): The unittest code for the question predefined_example (str): A predefined example solution or the keyword "LLM-example" to generate one using LLM + data_files (list): The question's data files the code may open Outputs: example_solution (str): The example solution, either generated by LLM or the predefined one """ if predefined_example == "LLM-example": example_solution = get_example_solution( - api_token, language, problem_description, unittest_code + api_token, language, problem_description, unittest_code, data_files ) else: example_solution = predefined_example @@ -253,6 +268,7 @@ def generate_personalized_fixed_code( predefined_example, unittest_code, API_attempt=0, + data_files=None, ): """ Generate a personalized fixed code based on the buggy code and other parameters. @@ -266,6 +282,7 @@ def generate_personalized_fixed_code( predefined_example (str): A predefined example solution or the keyword "LLM-example" to generate one using LLM unittest_code (str): The unittest code for the question API_attempt (int): The current API attempt number -- starts from 0 (reserved in case we want to use different API keys in the future) + data_files (list): The question's data files the code may open Outputs: cleaned_fixed_code (str): The cleaned fixed code generated by the LLM generation_result_type (str): The type of generation result, e.g., "AI_personalized", "example_more_personalized", "example_solution", "written_code_correct", "empty_beginning" @@ -279,6 +296,7 @@ def generate_personalized_fixed_code( default_start_code, default_test_code, unittest_case=unittest_code, + data_files=data_files, ) if unittest_result is not True: # If the code is not correct, we will TRY to get an example solution first @@ -288,6 +306,7 @@ def generate_personalized_fixed_code( problem_description, unittest_code, predefined_example, + data_files=data_files, ) try: @@ -307,6 +326,7 @@ def generate_personalized_fixed_code( situation="", failure_reason="", unittest_result="", + data_files=data_files, ) return result_personalized @@ -320,7 +340,12 @@ def generate_personalized_fixed_code( # If the code is empty, directly return the example solution else: example_solution = generate_example_solution( - api_token, language, problem_description, unittest_code, predefined_example + api_token, + language, + problem_description, + unittest_code, + predefined_example, + data_files=data_files, ) return example_solution.lstrip(), "example_solution" @@ -335,6 +360,7 @@ def generate_multi_personalized_Parsons_blocks( default_start_code, default_test_code, unittest_code, + data_files=None, ): """ Generate personalized Parsons blocks based on the student buggy code and fixed code. @@ -347,6 +373,7 @@ def generate_multi_personalized_Parsons_blocks( default_start_code (str): The default starting code for the question, current "" default_test_code (str): The default test code for the question, current "" unittest_code (str): The unittest code for the question + data_files (list): The question's data files the code may open Outputs: personalized_Parsons_block (str): The generated personalized Parsons block code """ @@ -388,6 +415,7 @@ def generate_multi_personalized_Parsons_blocks( default_start_code, default_test_code, unittest_code, + data_files=data_files, ) # If the code with distractors passes the unittest, we will remove the distractor from the distractors list if unittest_flag: diff --git a/bases/rsptx/book_server_api/routers/personalized_parsons/evaluate_fixed_code.py b/bases/rsptx/book_server_api/routers/personalized_parsons/evaluate_fixed_code.py index f81052e39..5f745d0d3 100644 --- a/bases/rsptx/book_server_api/routers/personalized_parsons/evaluate_fixed_code.py +++ b/bases/rsptx/book_server_api/routers/personalized_parsons/evaluate_fixed_code.py @@ -67,8 +67,38 @@ def _ensure_file_on_jobe( CHECK_PROXY = "/ns/rsproxy/jobeCheckFile/" +def _push_data_files(sess, data_files): + """ + Upload the question's data files to JOBE so the program under test can + open() them, mirroring how livecode.js ships datafiles with a run. + Inputs: + sess: requests session with the JOBE headers set + data_files (list): dicts with "filename", "content" and "is_binary"; + binary contents are already base64 (as stored in source_code) + Output: list: [file_id, filename] pairs for the run_spec file_list + """ + file_list = [] + for data_file in data_files or []: + filename = data_file["filename"] + content = data_file["content"] + if data_file.get("is_binary"): + b64 = content + else: + b64 = _b64_text_utf8(content) + file_id = _runestone_file_id(filename, content) + file_url = settings.jobe_server + "/jobe/index.php/restapi/files/" + file_id + if sess.head(file_url, timeout=10).status_code != 204: + put = sess.put(file_url, json={"file_contents": b64}, timeout=10) + if put.status_code != 204: + raise RuntimeError( + f"Failed to push data file {filename} to JOBE: {put.status_code}" + ) + file_list.append([file_id, filename]) + return file_list + + # modified from rsproxy.py and livecode.js logic -def load_and_run_java_tests(java_code, test_code): +def load_and_run_java_tests(java_code, test_code, data_files=None): """ Compile and run Java code with test cases. Inputs: @@ -76,6 +106,7 @@ def load_and_run_java_tests(java_code, test_code): test_code (str): The Java test cases -- a JUnit test class (extends CodeTestHelper, methods annotated with @Test, no main()), matching the standard Runestone suffix_code convention used for regular activecode runs. + data_files (list): the question's data files to place beside the program (see _push_data_files) Output: bool: True if all tests pass, False otherwise. """ @@ -142,6 +173,8 @@ def extract_class_name(code): "body": put.text[:500], } + data_file_list = _push_data_files(sess, data_files) + # Mirrors livecode.js: run the JUnit test class via JUnitCore rather than # calling a main() the test class doesn't have. runner_code = f"""import org.junit.runner.JUnitCore; @@ -167,7 +200,8 @@ def extract_class_name(code): "file_list": [ [student_id, student_filename], [test_id, test_filename], - ], + ] + + data_file_list, } resp = sess.post(runs_url, json={"run_spec": runspec}, timeout=10) @@ -199,7 +233,7 @@ def wasSuccessful(self) -> bool: return self._passed -def load_and_run_tests(unittest_case, code_to_test, time_limit=6): +def load_and_run_tests(unittest_case, code_to_test, time_limit=6, data_files=None): """ Run Python test cases against the provided code via JOBE. @@ -207,6 +241,7 @@ def load_and_run_tests(unittest_case, code_to_test, time_limit=6): unittest_case (str): unittest source (class myTests(unittest.TestCase): ...) code_to_test (str): the Python solution code to validate time_limit (int): JOBE wall-clock time limit in seconds + data_files (list): the question's data files to place beside the program (see _push_data_files) Output: _JobeTestResult with wasSuccessful() method """ # Suppress __main__ guards in student code — JOBE runs as top-level script @@ -230,6 +265,8 @@ def load_and_run_tests(unittest_case, code_to_test, time_limit=6): "sourcefilename": "solution.py", "parameters": {"timelimitsecs": time_limit}, } + if data_files: + runspec["file_list"] = _push_data_files(sess, data_files) resp = sess.post(runs_url, json={"run_spec": runspec}, timeout=time_limit + 10) try: result = resp.json() @@ -372,20 +409,25 @@ def remove_java_comments(code): def unittest_evaluation( - language, fixed_code, starting_code, default_test_code, unittest_case + language, + fixed_code, + starting_code, + default_test_code, + unittest_case, + data_files=None, ): if language == "java": return java_unittest_evaluation( - fixed_code, starting_code, default_test_code, unittest_case + fixed_code, starting_code, default_test_code, unittest_case, data_files ) else: return python_unittest_evaluation( - fixed_code, starting_code, default_test_code, unittest_case + fixed_code, starting_code, default_test_code, unittest_case, data_files ) def java_unittest_evaluation( - fixed_code, starting_code, default_test_code, unittest_case + fixed_code, starting_code, default_test_code, unittest_case, data_files=None ): try: fixed_code.split("\n") @@ -397,7 +439,9 @@ def java_unittest_evaluation( try: print("fixed_code_test", fixed_code) - java_test_result = load_and_run_java_tests(fixed_code, unittest_case) + java_test_result = load_and_run_java_tests( + fixed_code, unittest_case, data_files + ) print("java_results\n", java_test_result) return java_test_result, fixed_code except Exception: @@ -405,7 +449,7 @@ def java_unittest_evaluation( def python_unittest_evaluation( - fixed_code, starting_code, default_test_code, unittest_case + fixed_code, starting_code, default_test_code, unittest_case, data_files=None ): """ Load and run Python test cases against the provided code. @@ -414,6 +458,7 @@ def python_unittest_evaluation( starting_code (str): The default starting code provided to the student. Now it's "" default_test_code (str): The default test code provided to the student. Now it's "". unittest_case (str): The Python test cases. The test code is automatically reformatted based on the unittest_code provided by instructors in the RST file. + data_files (list): The question's data files the code may open (see _push_data_files). Output: bool: True if all tests pass, False otherwise. str: The cleaned fixed code after removing comments and empty lines. @@ -429,13 +474,15 @@ def python_unittest_evaluation( return False, fixed_code try: ##print("fixed_code_first attempt", fixed_code) - results = load_and_run_tests(unittest_case, fixed_code) + results = load_and_run_tests(unittest_case, fixed_code, data_files=data_files) print("results.wasSuccessful()\n", results.wasSuccessful()) return results.wasSuccessful(), fixed_code except Exception: try: fixed_code = fix_indentation(fixed_code) - results = load_and_run_tests(unittest_case, fixed_code) + results = load_and_run_tests( + unittest_case, fixed_code, data_files=data_files + ) # print("fix_indentation", fixed_code) if contain_default_starting_code(starting_code, fixed_code): # print("results.wasSuccessful()\n", results.wasSuccessful()) @@ -447,7 +494,12 @@ def python_unittest_evaluation( def code_distractor_unittest_evaluation( - language, code_with_distrator, starting_code, default_test_code, unittest_case + language, + code_with_distrator, + starting_code, + default_test_code, + unittest_case, + data_files=None, ): """ Evaluate the code with distractors using unit tests. @@ -457,6 +509,7 @@ def code_distractor_unittest_evaluation( starting_code (str): The default starting code provided to the student. Now it's "" default_test_code (str): The default test code provided to the student. Now it's "". unittest_case (str): The Python/Java test cases. The test code is automatically reformatted based on the unittest_code provided by instructors in the RST file. + data_files (list): The question's data files the code may open (see _push_data_files). Output: bool: True if all tests pass, False otherwise. str: The cleaned code with distractors after removing comments and empty lines. @@ -464,14 +517,16 @@ def code_distractor_unittest_evaluation( if language == "java": try: java_test_result = load_and_run_java_tests( - code_with_distrator, unittest_case + code_with_distrator, unittest_case, data_files ) return java_test_result, code_with_distrator except Exception: return False, code_with_distrator else: try: - results = load_and_run_tests(unittest_case, code_with_distrator) + results = load_and_run_tests( + unittest_case, code_with_distrator, data_files=data_files + ) if contain_default_starting_code(starting_code, code_with_distrator): return results.wasSuccessful(), code_with_distrator else: @@ -479,7 +534,9 @@ def code_distractor_unittest_evaluation( except Exception: try: code_with_distrator = fix_indentation(code_with_distrator) - results = load_and_run_tests(unittest_case, code_with_distrator) + results = load_and_run_tests( + unittest_case, code_with_distrator, data_files=data_files + ) if contain_default_starting_code(starting_code, code_with_distrator): return results.wasSuccessful(), code_with_distrator else: diff --git a/bases/rsptx/book_server_api/routers/personalized_parsons/get_personalized_solution.py b/bases/rsptx/book_server_api/routers/personalized_parsons/get_personalized_solution.py index f5468bde8..614e90842 100644 --- a/bases/rsptx/book_server_api/routers/personalized_parsons/get_personalized_solution.py +++ b/bases/rsptx/book_server_api/routers/personalized_parsons/get_personalized_solution.py @@ -234,7 +234,9 @@ def get_fixed_code( return fixed_code_response -def get_example_solution(api_token, language, problem_description, unittest_code): +def get_example_solution( + api_token, language, problem_description, unittest_code, data_files=None +): """ Get an example solution for the coding problem using the LLM, called when we do not have an instructor-provided example solution. Inputs: @@ -242,6 +244,7 @@ def get_example_solution(api_token, language, problem_description, unittest_code language (str): The programming language of the code ("python" or "java"). problem_description (str): The description of the coding problem. unittest_code (str): The unittest code to validate the fixed code. + data_files (list): The question's data files the code may open. Output: str: The generated example solution code. Or an empty string if the LLM-generated code does not pass the unittest. """ @@ -288,7 +291,12 @@ def get_example_solution(api_token, language, problem_description, unittest_code # test if the LLM_example_code is correct remove all potential # LLM_example_code = LLM_example_code.lstrip("#").rstrip("#").strip() unittest_result, cleaned_LLM_example_code = unittest_evaluation( - language, LLM_example_code, "", "", unittest_case=unittest_code + language, + LLM_example_code, + "", + "", + unittest_case=unittest_code, + data_files=data_files, ) if unittest_result: # LLM_example_code is correct diff --git a/components/rsptx/db/crud/rsfiles.py b/components/rsptx/db/crud/rsfiles.py index f22a8db74..06fd3d1cb 100644 --- a/components/rsptx/db/crud/rsfiles.py +++ b/components/rsptx/db/crud/rsfiles.py @@ -172,6 +172,8 @@ async def fetch_source_code( ), ) ) + # A cloned course's own copy of a file overrides the base book's + query = query.order_by((SourceCode.course_id == course_name).desc()) async with async_session() as session: res = await session.execute(query) return SourceCodeValidator.from_orm(res.scalars().first()) diff --git a/test/components/rsptx/db/test_rsfiles.py b/test/components/rsptx/db/test_rsfiles.py index 5497b4843..6710333bc 100644 --- a/test/components/rsptx/db/test_rsfiles.py +++ b/test/components/rsptx/db/test_rsfiles.py @@ -70,3 +70,51 @@ async def test_update_source_code_changes_is_binary_on_existing_row(init_test_db ) assert row.is_binary is True assert row.main_code == "second" + + +async def test_fetch_source_code_prefers_cloned_course_copy(init_test_db): + """A cloned course's own copy of a file wins over the base book's. + + The base row is inserted first, so an unordered query would most likely + return it. + """ + await update_source_code( + acid="clone_override.csv", + filename="clone_override.csv", + course_id="test_base_book", + main_code="base", + ) + await update_source_code( + acid="clone_override.csv", + filename="clone_override.csv", + course_id="test_clone_course", + main_code="clone", + ) + by_filename = await fetch_source_code( + base_course="test_base_book", + course_name="test_clone_course", + filename="clone_override.csv", + ) + assert by_filename.main_code == "clone" + by_acid = await fetch_source_code( + base_course="test_base_book", + course_name="test_clone_course", + acid="clone_override.csv", + ) + assert by_acid.main_code == "clone" + + +async def test_fetch_source_code_falls_back_to_base_book(init_test_db): + """A clone without its own copy still gets the base book's file.""" + await update_source_code( + acid="base_only.csv", + filename="base_only.csv", + course_id="test_base_book", + main_code="base", + ) + row = await fetch_source_code( + base_course="test_base_book", + course_name="test_clone_course", + filename="base_only.csv", + ) + assert row.main_code == "base"