Skip to content

Commit 57096fe

Browse files
henrymercerhenrymercer
committed
Add a PR check to validate that ML-powered queries are run correctly
1 parent b0ddf36 commit 57096fe

File tree

11 files changed

+407
-0
lines changed

11 files changed

+407
-0
lines changed

‎.github/workflows/__ml-powered-queries.yml

+119
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pr-checks/checks/ml-powered-queries.yml

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: "ML-powered queries"
2+
description: "Tests that ML-powered queries are run with the security-extended suite and that they produce alerts on a test DB"
3+
versions: [
4+
# Latest release in 2.7.x series
5+
"stable-20220120",
6+
"cached",
7+
"latest",
8+
"nightly-latest",
9+
]
10+
# Test on all three platforms since ML-powered queries use native code
11+
os: ["ubuntu-latest", "macos-latest", "windows-latest"]
12+
steps:
13+
- uses: ./../action/init
14+
with:
15+
languages: javascript
16+
queries: security-extended
17+
source-root: ./../action/tests/ml-powered-queries-repo
18+
tools: ${{ steps.prepare-test.outputs.tools-url }}
19+
20+
- uses: ./../action/analyze
21+
with:
22+
output: "${{ runner.temp }}/results"
23+
upload-database: false
24+
env:
25+
TEST_MODE: true
26+
27+
- name: Upload SARIF
28+
uses: actions/upload-artifact@v2
29+
with:
30+
name: ml-powered-queries-${{ matrix.os }}-${{ matrix.version }}.sarif.json
31+
path: "${{ runner.temp }}/results/javascript.sarif"
32+
retention-days: 7
33+
34+
- name: Check results
35+
env:
36+
IS_WINDOWS: ${{ matrix.os == 'windows-latest' }}
37+
shell: bash
38+
run: |
39+
cd "$RUNNER_TEMP/results"
40+
# We should run at least the ML-powered queries in `expected_rules`.
41+
expected_rules="js/ml-powered/nosql-injection js/ml-powered/path-injection js/ml-powered/sql-injection js/ml-powered/xss"
42+
43+
for rule in ${expected_rules}; do
44+
found_rule=$(jq --arg rule "${rule}" '[.runs[0].tool.extensions[].rules | select(. != null) |
45+
flatten | .[].id] | any(. == $rule)' javascript.sarif)
46+
echo "Did find rule '${rule}': ${found_rule}"
47+
if [[ "${found_rule}" != "true" && "${IS_WINDOWS}" != "true" ]]; then
48+
echo "Expected SARIF output to contain rule '${rule}', but found no such rule."
49+
exit 1
50+
elif [[ "${found_rule}" == "true" && "${IS_WINDOWS}" == "true" ]]; then
51+
echo "Found rule '${rule}' in the SARIF output which shouldn't have been part of the analysis."
52+
exit 1
53+
fi
54+
done
55+
56+
# We should have at least one alert from an ML-powered query.
57+
num_alerts=$(jq '[.runs[0].results[] |
58+
select(.properties.score != null and (.rule.id | startswith("js/ml-powered/")))] | length' \
59+
javascript.sarif)
60+
echo "Found ${num_alerts} alerts from ML-powered queries.";
61+
if [[ "${num_alerts}" -eq 0 && "${IS_WINDOWS}" != "true" ]]; then
62+
echo "Expected to find at least one alert from an ML-powered query but found ${num_alerts}."
63+
exit 1
64+
elif [[ "${num_alerts}" -ne 0 && "${IS_WINDOWS}" == "true" ]]; then
65+
echo "Expected not to find any alerts from an ML-powered query but found ${num_alerts}."
66+
exit 1
67+
fi

‎tests/ml-powered-queries-repo/add-note.js

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
const mongoose = require('mongoose');
2+
3+
Logger = require('./logger').Logger;
4+
Note = require('./models/note').Note;
5+
6+
(async () => {
7+
if (process.argv.length != 5) {
8+
Logger.log("Creates a private note. Usage: node add-note.js <token> <title> <body>")
9+
return;
10+
}
11+
12+
// Open the default mongoose connection
13+
await mongoose.connect('mongodb://localhost:27017/notes', { useFindAndModify: false });
14+
15+
const [userToken, title, body] = process.argv.slice(2);
16+
await Note.create({ title, body, userToken });
17+
18+
Logger.log(`Created private note with title ${title} and body ${body} belonging to user with token ${userToken}.`);
19+
20+
await mongoose.connection.close();
21+
})();

‎tests/ml-powered-queries-repo/app.js

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
const bodyParser = require('body-parser');
2+
const express = require('express');
3+
const mongoose = require('mongoose');
4+
5+
const notesApi = require('./notes-api');
6+
const usersApi = require('./users-api');
7+
8+
const addSampleData = module.exports.addSampleData = async () => {
9+
const [userA, userB] = await User.create([
10+
{
11+
name: "A",
12+
token: "tokenA"
13+
},
14+
{
15+
name: "B",
16+
token: "tokenB"
17+
}
18+
]);
19+
20+
await Note.create([
21+
{
22+
title: "Public note belonging to A",
23+
body: "This is a public note belonging to A",
24+
isPublic: true,
25+
ownerToken: userA.token
26+
},
27+
{
28+
title: "Public note belonging to B",
29+
body: "This is a public note belonging to B",
30+
isPublic: true,
31+
ownerToken: userB.token
32+
},
33+
{
34+
title: "Private note belonging to A",
35+
body: "This is a private note belonging to A",
36+
ownerToken: userA.token
37+
},
38+
{
39+
title: "Private note belonging to B",
40+
body: "This is a private note belonging to B",
41+
ownerToken: userB.token
42+
}
43+
]);
44+
}
45+
46+
module.exports.startApp = async () => {
47+
// Open the default mongoose connection
48+
await mongoose.connect('mongodb://mongo:27017/notes', { useFindAndModify: false });
49+
// Drop contents of DB
50+
mongoose.connection.dropDatabase();
51+
// Add some sample data
52+
await addSampleData();
53+
54+
const app = express();
55+
56+
app.use(bodyParser.json());
57+
app.use(bodyParser.urlencoded());
58+
59+
app.get('/', async (_req, res) => {
60+
res.send('Hello World');
61+
});
62+
63+
app.use('/api/notes', notesApi.router);
64+
app.use('/api/users', usersApi.router);
65+
66+
app.listen(3000);
67+
Logger.log('Express started on port 3000');
68+
};

‎tests/ml-powered-queries-repo/index.js

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
const startApp = require('./app').startApp;
2+
3+
Logger = require('./logger').Logger;
4+
Note = require('./models/note').Note;
5+
User = require('./models/user').User;
6+
7+
startApp();

‎tests/ml-powered-queries-repo/logger.js

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module.exports.Logger = class {
2+
log(message, ...objs) {
3+
console.log(message, objs);
4+
}
5+
};

‎tests/ml-powered-queries-repo/models/note.js

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
const mongoose = require('mongoose');
2+
3+
module.exports.Note = mongoose.model('Note', new mongoose.Schema({
4+
title: String,
5+
body: String,
6+
ownerToken: String,
7+
isPublic: Boolean
8+
}));

‎tests/ml-powered-queries-repo/models/user.js

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
const mongoose = require('mongoose');
2+
3+
module.exports.User = mongoose.model('User', new mongoose.Schema({
4+
name: String,
5+
token: String
6+
}));

‎tests/ml-powered-queries-repo/notes-api.js

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
const express = require('express')
2+
3+
const router = module.exports.router = express.Router();
4+
5+
function serializeNote(note) {
6+
return {
7+
title: note.title,
8+
body: note.body
9+
};
10+
}
11+
12+
router.post('/find', async (req, res) => {
13+
const notes = await Note.find({
14+
ownerToken: req.body.token
15+
}).exec();
16+
res.json({
17+
notes: notes.map(serializeNote)
18+
});
19+
});
20+
21+
router.get('/findPublic', async (_req, res) => {
22+
const notes = await Note.find({
23+
isPublic: true
24+
}).exec();
25+
res.json({
26+
notes: notes.map(serializeNote)
27+
});
28+
});
29+
30+
router.post('/findVisible', async (req, res) => {
31+
const notes = await Note.find({
32+
$or: [
33+
{
34+
isPublic: true
35+
},
36+
{
37+
ownerToken: req.body.token
38+
}
39+
]
40+
}).exec();
41+
res.json({
42+
notes: notes.map(serializeNote)
43+
});
44+
});

‎tests/ml-powered-queries-repo/read-notes.js

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
const mongoose = require('mongoose');
2+
3+
Logger = require('./logger').Logger;
4+
Note = require('./models/note').Note;
5+
User = require('./models/user').User;
6+
7+
(async () => {
8+
if (process.argv.length != 3) {
9+
Logger.log("Outputs all notes visible to a user. Usage: node read-notes.js <token>")
10+
return;
11+
}
12+
13+
// Open the default mongoose connection
14+
await mongoose.connect('mongodb://localhost:27017/notes', { useFindAndModify: false });
15+
16+
const ownerToken = process.argv[2];
17+
18+
const user = await User.findOne({
19+
token: ownerToken
20+
}).exec();
21+
22+
const notes = await Note.find({
23+
$or: [
24+
{ isPublic: true },
25+
{ ownerToken }
26+
]
27+
}).exec();
28+
29+
notes.map(note => {
30+
Logger.log("Title:" + note.title);
31+
Logger.log("By:" + user.name);
32+
Logger.log("Body:" + note.body);
33+
Logger.log();
34+
});
35+
36+
await mongoose.connection.close();
37+
})();

0 commit comments

Comments
 (0)