Skip to content

Commit aa40db7

Browse files
committed
[red-knot] Add fuzzer to catch panics for invalid syntax
1 parent f1b2e85 commit aa40db7

File tree

4 files changed

+166
-9
lines changed

4 files changed

+166
-9
lines changed

fuzz/Cargo.toml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ libfuzzer = ["libfuzzer-sys/link_libfuzzer"]
1717
cargo-fuzz = true
1818

1919
[dependencies]
20+
red_knot_python_semantic = { path = "../crates/red_knot_python_semantic" }
21+
red_knot_vendored = { path = "../crates/red_knot_vendored" }
22+
ruff_db = { path = "../crates/ruff_db" }
2023
ruff_linter = { path = "../crates/ruff_linter" }
2124
ruff_python_ast = { path = "../crates/ruff_python_ast" }
2225
ruff_python_codegen = { path = "../crates/ruff_python_codegen" }
@@ -26,12 +29,18 @@ ruff_python_formatter = { path = "../crates/ruff_python_formatter"}
2629
ruff_text_size = { path = "../crates/ruff_text_size" }
2730

2831
libfuzzer-sys = { git = "https://github.com/rust-fuzz/libfuzzer", default-features = false }
32+
salsa = { git = "https://github.com/salsa-rs/salsa.git", rev = "254c749b02cde2fd29852a7463a33e800b771758" }
2933
similar = { version = "2.5.0" }
34+
tracing = { version = "0.1.40" }
3035

3136
# Prevent this from interfering with workspaces
3237
[workspace]
3338
members = ["."]
3439

40+
[[bin]]
41+
name = "red_knot_check_invalid_syntax"
42+
path = "fuzz_targets/red_knot_check_invalid_syntax.rs"
43+
3544
[[bin]]
3645
name = "ruff_parse_simple"
3746
path = "fuzz_targets/ruff_parse_simple.rs"

fuzz/corpus/red_knot_simple

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ruff_fix_validity
Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
//! Fuzzer harness that runs the type checker to catch for panics for source code containing
2+
//! syntax errors.
3+
4+
#![no_main]
5+
6+
use libfuzzer_sys::{fuzz_target, Corpus};
7+
8+
use red_knot_python_semantic::types::check_types;
9+
use red_knot_python_semantic::{
10+
Db as SemanticDb, Program, ProgramSettings, PythonVersion, SearchPathSettings,
11+
};
12+
use ruff_db::files::{system_path_to_file, File, Files};
13+
use ruff_db::system::{DbWithTestSystem, System, SystemPathBuf, TestSystem};
14+
use ruff_db::vendored::VendoredFileSystem;
15+
use ruff_db::{Db as SourceDb, Upcast};
16+
use ruff_python_parser::{parse_unchecked, Mode};
17+
18+
/// Database that can be used for testing.
19+
///
20+
/// Uses an in memory filesystem and it stubs out the vendored files by default.
21+
#[salsa::db]
22+
struct TestDb {
23+
storage: salsa::Storage<Self>,
24+
files: Files,
25+
system: TestSystem,
26+
vendored: VendoredFileSystem,
27+
events: std::sync::Arc<std::sync::Mutex<Vec<salsa::Event>>>,
28+
}
29+
30+
impl TestDb {
31+
fn new() -> Self {
32+
Self {
33+
storage: salsa::Storage::default(),
34+
system: TestSystem::default(),
35+
vendored: red_knot_vendored::file_system().clone(),
36+
events: std::sync::Arc::default(),
37+
files: Files::default(),
38+
}
39+
}
40+
}
41+
42+
#[salsa::db]
43+
impl SourceDb for TestDb {
44+
fn vendored(&self) -> &VendoredFileSystem {
45+
&self.vendored
46+
}
47+
48+
fn system(&self) -> &dyn System {
49+
&self.system
50+
}
51+
52+
fn files(&self) -> &Files {
53+
&self.files
54+
}
55+
}
56+
57+
impl DbWithTestSystem for TestDb {
58+
fn test_system(&self) -> &TestSystem {
59+
&self.system
60+
}
61+
62+
fn test_system_mut(&mut self) -> &mut TestSystem {
63+
&mut self.system
64+
}
65+
}
66+
67+
impl Upcast<dyn SourceDb> for TestDb {
68+
fn upcast(&self) -> &(dyn SourceDb + 'static) {
69+
self
70+
}
71+
fn upcast_mut(&mut self) -> &mut (dyn SourceDb + 'static) {
72+
self
73+
}
74+
}
75+
76+
#[salsa::db]
77+
impl SemanticDb for TestDb {
78+
fn is_file_open(&self, file: File) -> bool {
79+
!file.path(self).is_vendored_path()
80+
}
81+
}
82+
83+
#[salsa::db]
84+
impl salsa::Database for TestDb {
85+
fn salsa_event(&self, event: &dyn Fn() -> salsa::Event) {
86+
let event = event();
87+
tracing::trace!("event: {:?}", event);
88+
let mut events = self.events.lock().unwrap();
89+
events.push(event);
90+
}
91+
}
92+
93+
fn setup_db() -> TestDb {
94+
let db = TestDb::new();
95+
96+
let src_root = SystemPathBuf::from("/src");
97+
db.memory_file_system()
98+
.create_directory_all(&src_root)
99+
.unwrap();
100+
101+
Program::from_settings(
102+
&db,
103+
&ProgramSettings {
104+
target_version: PythonVersion::default(),
105+
search_paths: SearchPathSettings::new(src_root),
106+
},
107+
)
108+
.expect("Valid search path settings");
109+
110+
db
111+
}
112+
113+
fn do_fuzz(case: &[u8]) -> Corpus {
114+
let Ok(code) = std::str::from_utf8(case) else {
115+
return Corpus::Reject;
116+
};
117+
118+
let parsed = parse_unchecked(code, Mode::Module);
119+
if parsed.is_valid() {
120+
return Corpus::Reject;
121+
}
122+
123+
let mut db = setup_db();
124+
db.write_file("/src/a.py", code).unwrap();
125+
let file = system_path_to_file(&db, "/src/a.py").unwrap();
126+
check_types(&db, file);
127+
128+
Corpus::Keep
129+
}
130+
131+
fuzz_target!(|case: &[u8]| -> Corpus { do_fuzz(case) });

fuzz/init-fuzzer.sh

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,32 @@ fi
1111

1212
if [ ! -d corpus/ruff_fix_validity ]; then
1313
mkdir -p corpus/ruff_fix_validity
14-
read -p "Would you like to build a corpus from a python source code dataset? (this will take a long time!) [Y/n] " -n 1 -r
15-
echo
16-
cd corpus/ruff_fix_validity
17-
if [[ $REPLY =~ ^[Yy]$ ]]; then
18-
curl -L 'https://zenodo.org/record/3628784/files/python-corpus.tar.gz?download=1' | tar xz
14+
15+
(
16+
cd corpus/ruff_fix_validity
17+
18+
read -p "Would you like to build a corpus from a python source code dataset? (this will take a long time!) [Y/n] " -n 1 -r
19+
echo
20+
if [[ $REPLY =~ ^[Yy]$ ]]; then
21+
curl -L 'https://zenodo.org/record/3628784/files/python-corpus.tar.gz?download=1' | tar xz
22+
fi
23+
24+
# Build a smaller corpus in addition to the (optional) larger corpus
25+
curl -L 'https://github.com/python/cpython/archive/refs/tags/v3.12.0b2.tar.gz' | tar xz
26+
cp -r "../../../crates/red_knot_workspace/resources/test/corpus" "red_knot_workspace"
27+
cp -r "../../../crates/ruff_linter/resources/test/fixtures" "ruff_linter"
28+
cp -r "../../../crates/ruff_python_formatter/resources/test/fixtures" "ruff_python_formatter"
29+
cp -r "../../../crates/ruff_python_parser/resources" "ruff_python_parser"
30+
31+
# Delete all non-Python files
32+
find . -type f -not -name "*.py" -delete
33+
)
34+
35+
if [[ "$OSTYPE" == "darwin"* ]]; then
36+
cargo +nightly fuzz cmin ruff_fix_validity -- -timeout=5
37+
else
38+
cargo fuzz cmin -s none ruff_fix_validity -- -timeout=5
1939
fi
20-
curl -L 'https://github.com/python/cpython/archive/refs/tags/v3.12.0b2.tar.gz' | tar xz
21-
cp -r "../../../crates/ruff_linter/resources/test" .
22-
cd -
23-
cargo fuzz cmin -s none ruff_fix_validity -- -timeout=5
2440
fi
2541

2642
echo "Done! You are ready to fuzz."

0 commit comments

Comments
 (0)