Skip to content

Commit

Permalink
fix testing
Browse files Browse the repository at this point in the history
  • Loading branch information
@yrobla
yrobla committed Nov 8, 2024
1 parent 87d300c commit 6033650
Show file tree
Hide file tree
Showing 3 changed files with 112 additions and 111 deletions.
188 changes: 92 additions & 96 deletions coverage.xml
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
<?xml version="1.0" ?>
<coverage version="7.6.4" timestamp="1731067565017" lines-valid="554" lines-covered="260" line-rate="0.4693" branches-covered="0" branches-valid="0" branch-rate="0" complexity="0">
<coverage version="7.6.4" timestamp="1731078959805" lines-valid="550" lines-covered="258" line-rate="0.4691" branches-covered="0" branches-valid="0" branch-rate="0" complexity="0">
<!-- Generated by coverage.py: https://coverage.readthedocs.io/en/7.6.4 -->
<!-- Based on https://raw.githubusercontent.com/cobertura/web/master/htdocs/xml/coverage-04.dtd -->
<sources>
<source>/Users/yolanda/development/promptwright/promptwright</source>
</sources>
<packages>
<package name="." line-rate="0.4693" branch-rate="0" complexity="0">
<package name="." line-rate="0.4691" branch-rate="0" complexity="0">
<classes>
<class name="__init__.py" filename="__init__.py" complexity="0" line-rate="1" branch-rate="0">
<methods/>
Expand Down Expand Up @@ -107,7 +107,7 @@
<line number="216" hits="0"/>
</lines>
</class>
<class name="engine.py" filename="engine.py" complexity="0" line-rate="0.6833" branch-rate="0">
<class name="engine.py" filename="engine.py" complexity="0" line-rate="0.6864" branch-rate="0">
<methods/>
<lines>
<line number="1" hits="1"/>
Expand All @@ -125,16 +125,16 @@
<line number="19" hits="1"/>
<line number="20" hits="0"/>
<line number="23" hits="1"/>
<line number="25" hits="0"/>
<line number="26" hits="0"/>
<line number="27" hits="0"/>
<line number="25" hits="1"/>
<line number="26" hits="1"/>
<line number="27" hits="1"/>
<line number="28" hits="0"/>
<line number="30" hits="0"/>
<line number="31" hits="0"/>
<line number="33" hits="0"/>
<line number="35" hits="0"/>
<line number="30" hits="1"/>
<line number="31" hits="1"/>
<line number="33" hits="1"/>
<line number="35" hits="1"/>
<line number="37" hits="0"/>
<line number="39" hits="0"/>
<line number="39" hits="1"/>
<line number="40" hits="0"/>
<line number="41" hits="0"/>
<line number="44" hits="1"/>
Expand All @@ -160,13 +160,13 @@
<line number="71" hits="1"/>
<line number="79" hits="1"/>
<line number="81" hits="1"/>
<line number="83" hits="1"/>
<line number="84" hits="1"/>
<line number="85" hits="1"/>
<line number="83" hits="0"/>
<line number="84" hits="0"/>
<line number="85" hits="0"/>
<line number="86" hits="0"/>
<line number="87" hits="1"/>
<line number="87" hits="0"/>
<line number="90" hits="0"/>
<line number="91" hits="1"/>
<line number="91" hits="0"/>
<line number="93" hits="0"/>
<line number="94" hits="0"/>
<line number="97" hits="0"/>
Expand All @@ -176,8 +176,8 @@
<line number="103" hits="1"/>
<line number="110" hits="1"/>
<line number="111" hits="1"/>
<line number="113" hits="1"/>
<line number="114" hits="1"/>
<line number="113" hits="0"/>
<line number="114" hits="0"/>
<line number="118" hits="1"/>
<line number="120" hits="1"/>
<line number="128" hits="1"/>
Expand Down Expand Up @@ -219,137 +219,133 @@
<line number="190" hits="1"/>
<line number="191" hits="1"/>
<line number="192" hits="1"/>
<line number="193" hits="0"/>
<line number="195" hits="0"/>
<line number="196" hits="0"/>
<line number="198" hits="0"/>
<line number="193" hits="1"/>
<line number="194" hits="1"/>
<line number="196" hits="1"/>
<line number="197" hits="1"/>
<line number="199" hits="0"/>
<line number="200" hits="0"/>
<line number="202" hits="0"/>
<line number="203" hits="0"/>
<line number="206" hits="0"/>
<line number="207" hits="0"/>
<line number="210" hits="0"/>
<line number="201" hits="0"/>
<line number="203" hits="1"/>
<line number="204" hits="1"/>
<line number="207" hits="1"/>
<line number="208" hits="0"/>
<line number="211" hits="0"/>
<line number="212" hits="0"/>
<line number="213" hits="0"/>
<line number="215" hits="1"/>
<line number="216" hits="1"/>
<line number="217" hits="1"/>
<line number="218" hits="1"/>
<line number="219" hits="1"/>
<line number="220" hits="1"/>
<line number="222" hits="1"/>
<line number="224" hits="0"/>
<line number="213" hits="1"/>
<line number="214" hits="1"/>
<line number="216" hits="0"/>
<line number="217" hits="0"/>
<line number="218" hits="0"/>
<line number="219" hits="0"/>
<line number="220" hits="0"/>
<line number="221" hits="0"/>
<line number="223" hits="0"/>
<line number="225" hits="0"/>
<line number="226" hits="0"/>
<line number="227" hits="0"/>
<line number="228" hits="0"/>
<line number="230" hits="0"/>
<line number="229" hits="0"/>
<line number="231" hits="0"/>
<line number="232" hits="0"/>
<line number="233" hits="0"/>
<line number="234" hits="0"/>
<line number="236" hits="1"/>
<line number="235" hits="0"/>
<line number="237" hits="1"/>
<line number="238" hits="1"/>
<line number="240" hits="1"/>
<line number="242" hits="1"/>
<line number="244" hits="1"/>
<line number="239" hits="1"/>
<line number="241" hits="1"/>
<line number="243" hits="1"/>
<line number="245" hits="1"/>
<line number="246" hits="1"/>
<line number="247" hits="1"/>
<line number="248" hits="1"/>
<line number="249" hits="1"/>
<line number="250" hits="1"/>
<line number="251" hits="1"/>
<line number="252" hits="1"/>
<line number="253" hits="1"/>
<line number="254" hits="1"/>
<line number="256" hits="1"/>
<line number="262" hits="1"/>
<line number="250" hits="0"/>
<line number="251" hits="0"/>
<line number="252" hits="0"/>
<line number="253" hits="0"/>
<line number="254" hits="0"/>
<line number="255" hits="1"/>
<line number="257" hits="1"/>
<line number="263" hits="1"/>
<line number="264" hits="1"/>
<line number="267" hits="1"/>
<line number="269" hits="1"/>
<line number="265" hits="1"/>
<line number="268" hits="1"/>
<line number="270" hits="1"/>
<line number="272" hits="1"/>
<line number="271" hits="1"/>
<line number="273" hits="1"/>
<line number="274" hits="0"/>
<line number="275" hits="1"/>
<line number="277" hits="1"/>
<line number="274" hits="1"/>
<line number="275" hits="0"/>
<line number="276" hits="1"/>
<line number="278" hits="1"/>
<line number="279" hits="1"/>
<line number="281" hits="0"/>
<line number="280" hits="1"/>
<line number="282" hits="0"/>
<line number="283" hits="0"/>
<line number="284" hits="0"/>
<line number="286" hits="1"/>
<line number="285" hits="0"/>
<line number="287" hits="1"/>
<line number="288" hits="0"/>
<line number="289" hits="1"/>
<line number="291" hits="1"/>
<line number="293" hits="1"/>
<line number="295" hits="1"/>
<line number="299" hits="1"/>
<line number="288" hits="1"/>
<line number="289" hits="0"/>
<line number="290" hits="1"/>
<line number="292" hits="1"/>
<line number="294" hits="1"/>
<line number="296" hits="1"/>
<line number="300" hits="1"/>
<line number="301" hits="1"/>
<line number="303" hits="1"/>
<line number="302" hits="1"/>
<line number="304" hits="1"/>
<line number="305" hits="1"/>
<line number="306" hits="1"/>
<line number="308" hits="1"/>
<line number="307" hits="1"/>
<line number="309" hits="1"/>
<line number="310" hits="1"/>
<line number="312" hits="1"/>
<line number="311" hits="1"/>
<line number="313" hits="1"/>
<line number="315" hits="1"/>
<line number="322" hits="1"/>
<line number="329" hits="1"/>
<line number="314" hits="1"/>
<line number="316" hits="1"/>
<line number="323" hits="1"/>
<line number="330" hits="1"/>
<line number="331" hits="1"/>
<line number="332" hits="0"/>
<line number="334" hits="0"/>
<line number="335" hits="0"/>
<line number="337" hits="0"/>
<line number="332" hits="1"/>
<line number="334" hits="1"/>
<line number="335" hits="1"/>
<line number="336" hits="1"/>
<line number="338" hits="0"/>
<line number="339" hits="0"/>
<line number="341" hits="1"/>
<line number="342" hits="1"/>
<line number="344" hits="1"/>
<line number="345" hits="1"/>
<line number="348" hits="1"/>
<line number="349" hits="1"/>
<line number="350" hits="1"/>
<line number="351" hits="1"/>
<line number="352" hits="1"/>
<line number="355" hits="1"/>
<line number="340" hits="0"/>
<line number="342" hits="0"/>
<line number="343" hits="0"/>
<line number="345" hits="0"/>
<line number="346" hits="0"/>
<line number="349" hits="0"/>
<line number="350" hits="0"/>
<line number="351" hits="0"/>
<line number="352" hits="0"/>
<line number="353" hits="0"/>
<line number="356" hits="1"/>
<line number="357" hits="1"/>
<line number="360" hits="1"/>
<line number="358" hits="1"/>
<line number="361" hits="1"/>
<line number="362" hits="1"/>
<line number="364" hits="1"/>
<line number="363" hits="1"/>
<line number="365" hits="1"/>
<line number="366" hits="1"/>
<line number="367" hits="1"/>
<line number="369" hits="0"/>
<line number="368" hits="1"/>
<line number="370" hits="0"/>
<line number="371" hits="0"/>
<line number="372" hits="0"/>
<line number="373" hits="0"/>
<line number="374" hits="0"/>
<line number="375" hits="0"/>
<line number="376" hits="0"/>
<line number="376" hits="1"/>
<line number="377" hits="1"/>
<line number="378" hits="1"/>
<line number="379" hits="1"/>
<line number="380" hits="1"/>
<line number="381" hits="0"/>
<line number="382" hits="0"/>
<line number="384" hits="0"/>
<line number="385" hits="0"/>
<line number="386" hits="0"/>
<line number="387" hits="0"/>
<line number="388" hits="0"/>
<line number="389" hits="0"/>
<line number="391" hits="0"/>
<line number="381" hits="1"/>
<line number="383" hits="1"/>
</lines>
</class>
<class name="hf_hub.py" filename="hf_hub.py" complexity="0" line-rate="0.3" branch-rate="0">
Expand Down
16 changes: 4 additions & 12 deletions promptwright/engine.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ def create_data( # noqa: PLR0912
data_creation_prompt=data_creation_prompt,
num_example_demonstrations=num_example_demonstrations,
subtopics_list=path,
)
)
prompts.append(sample_prompt)

for attempt in range(self.args.max_retries):
Expand All @@ -188,6 +188,7 @@ def create_data( # noqa: PLR0912
)

samples = []
print(responses)
for r in responses:
response_content = r.choices[0].message.content
parsed_json = validate_json_response(response_content)
Expand Down Expand Up @@ -326,13 +327,13 @@ def create_questions( # noqa: PLR0912
)

# Parse and validate
samples = []
for r in responses:
response_content = r.choices[0].message.content
parsed_json = validate_json_response(response_content)

if parsed_json:
samples.append(parsed_json)
questions_list.append(parsed_json)
success_count += 1
else:
self.failed_samples.append(response_content)
failure_type = self.analyze_failure(response_content)
Expand Down Expand Up @@ -368,19 +369,10 @@ def create_questions( # noqa: PLR0912

except KeyboardInterrupt:
print("\nGeneration interrupted by user.")
self.save_dataset("interrupted_dataset.jsonl")

except Exception as e:
print(f"\nUnexpected error: {str(e)}")
self.save_dataset("error_dataset.jsonl")
raise

finally:
# Save failure log if there were any failures
if self.failed_samples:
with open("generation_failures.json", "w") as f:
json.dump(self.failed_samples, f, indent=2)

total_duration = time.time() - start_time
print("\nGeneration complete:")
print(f"Total samples: {success_count}/{num_questions}")
Expand Down
19 changes: 16 additions & 3 deletions tests/test_engine.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def test_create_data_no_steps(data_engine):


@patch("promptwright.engine.litellm.batch_completion")
def test_create_data_success(mock_batch_completion, data_engine):
def test_create_data_success(mock_batch_completion):
# Mock valid JSON responses to match the expected structure for 10 samples
mock_batch_completion.return_value = [
MagicMock(
Expand Down Expand Up @@ -80,7 +80,20 @@ def test_engine_create_questions(mock_batch_completion):
assert len(questions) == 10


def test_engine_validation(data_engine):
@patch("promptwright.engine.litellm.batch_completion")
def test_engine_validation(mock_batch_completion, data_engine):
mock_batch_completion.return_value = [
MagicMock(
choices=[
MagicMock(
message=MagicMock(
content='{"messages": [{"role": "user", "content": "example"}, {"role": "assistant", "content": "response"}]}'
)
)
]
)
] * 10

topic_tree = MagicMock()
topic_tree.tree_paths = [
"path1",
Expand All @@ -99,7 +112,7 @@ def test_engine_validation(data_engine):
expected_num_samples = 10

# Generate the data
dataset = data_engine.create_data(num_steps=1, batch_size=10, topic_tree=topic_tree)
dataset = data_engine.create_data(num_steps=1, batch_size=10, topic_tree=topic_tree)

# Assert that the dataset contains exactly the expected number of samples
assert len(dataset.samples) == expected_num_samples
Expand Down

0 comments on commit 6033650

Please sign in to comment.