Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updated for datafusion 44.0.0 #301

Merged
merged 4 commits into from
Feb 7, 2025
Merged

Updated for datafusion 44.0.0 #301

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4d15624
updated for datafusion release 44.0.0
pmcgleenon Feb 4, 2025
7c0a8ed
fixed regex sytax by adding additional escapes to: www\. and \1
pmcgleenon Feb 5, 2025
b31ef39
reinstate 'read -r' for regex query
pmcgleenon Feb 5, 2025
8455754
reran with Q28 update for datafusion-44.0.0
pmcgleenon Feb 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion datafusion/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ The benchmark should be completed in under an hour. On-demand pricing is $0.6 pe
1. `cd ClickBench/datafusion`
1. `vi benchmark.sh` and modify following line to target Datafusion version
```
git checkout 43.0.0
git checkout 44.0.0
```
1. `bash benchmark.sh`

Expand Down
2 changes: 1 addition & 1 deletion datafusion/benchmark.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ sudo yum install gcc -y
# Install DataFusion main branch
git clone https://github.com/apache/arrow-datafusion.git
cd arrow-datafusion/datafusion-cli
git checkout 43.0.0
git checkout 44.0.0
CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" cargo build --release
export PATH="`pwd`/target/release:$PATH"
cd ../..
Expand Down
2 changes: 1 addition & 1 deletion datafusion/queries.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY to_timestamp
SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "SearchPhrase" LIMIT 10;
SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY to_timestamp_seconds("EventTime"), "SearchPhrase" LIMIT 10;
SELECT "CounterID", AVG(length("URL")) AS l, COUNT(*) AS c FROM hits WHERE "URL" <> '' GROUP BY "CounterID" HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;
SELECT REGEXP_REPLACE("Referer", '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length("Referer")) AS l, COUNT(*) AS c, MIN("Referer") FROM hits WHERE "Referer" <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;
SELECT REGEXP_REPLACE("Referer", '^https?://(?:www\\.)?([^/]+)/.*$', '\\1') AS k, AVG(length("Referer")) AS l, COUNT(*) AS c, MIN("Referer") FROM hits WHERE "Referer" <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;
SELECT SUM("ResolutionWidth"), SUM("ResolutionWidth" + 1), SUM("ResolutionWidth" + 2), SUM("ResolutionWidth" + 3), SUM("ResolutionWidth" + 4), SUM("ResolutionWidth" + 5), SUM("ResolutionWidth" + 6), SUM("ResolutionWidth" + 7), SUM("ResolutionWidth" + 8), SUM("ResolutionWidth" + 9), SUM("ResolutionWidth" + 10), SUM("ResolutionWidth" + 11), SUM("ResolutionWidth" + 12), SUM("ResolutionWidth" + 13), SUM("ResolutionWidth" + 14), SUM("ResolutionWidth" + 15), SUM("ResolutionWidth" + 16), SUM("ResolutionWidth" + 17), SUM("ResolutionWidth" + 18), SUM("ResolutionWidth" + 19), SUM("ResolutionWidth" + 20), SUM("ResolutionWidth" + 21), SUM("ResolutionWidth" + 22), SUM("ResolutionWidth" + 23), SUM("ResolutionWidth" + 24), SUM("ResolutionWidth" + 25), SUM("ResolutionWidth" + 26), SUM("ResolutionWidth" + 27), SUM("ResolutionWidth" + 28), SUM("ResolutionWidth" + 29), SUM("ResolutionWidth" + 30), SUM("ResolutionWidth" + 31), SUM("ResolutionWidth" + 32), SUM("ResolutionWidth" + 33), SUM("ResolutionWidth" + 34), SUM("ResolutionWidth" + 35), SUM("ResolutionWidth" + 36), SUM("ResolutionWidth" + 37), SUM("ResolutionWidth" + 38), SUM("ResolutionWidth" + 39), SUM("ResolutionWidth" + 40), SUM("ResolutionWidth" + 41), SUM("ResolutionWidth" + 42), SUM("ResolutionWidth" + 43), SUM("ResolutionWidth" + 44), SUM("ResolutionWidth" + 45), SUM("ResolutionWidth" + 46), SUM("ResolutionWidth" + 47), SUM("ResolutionWidth" + 48), SUM("ResolutionWidth" + 49), SUM("ResolutionWidth" + 50), SUM("ResolutionWidth" + 51), SUM("ResolutionWidth" + 52), SUM("ResolutionWidth" + 53), SUM("ResolutionWidth" + 54), SUM("ResolutionWidth" + 55), SUM("ResolutionWidth" + 56), SUM("ResolutionWidth" + 57), SUM("ResolutionWidth" + 58), SUM("ResolutionWidth" + 59), SUM("ResolutionWidth" + 60), SUM("ResolutionWidth" + 61), SUM("ResolutionWidth" + 62), SUM("ResolutionWidth" + 63), SUM("ResolutionWidth" + 64), SUM("ResolutionWidth" + 65), SUM("ResolutionWidth" + 66), SUM("ResolutionWidth" + 67), SUM("ResolutionWidth" + 68), SUM("ResolutionWidth" + 69), SUM("ResolutionWidth" + 70), SUM("ResolutionWidth" + 71), SUM("ResolutionWidth" + 72), SUM("ResolutionWidth" + 73), SUM("ResolutionWidth" + 74), SUM("ResolutionWidth" + 75), SUM("ResolutionWidth" + 76), SUM("ResolutionWidth" + 77), SUM("ResolutionWidth" + 78), SUM("ResolutionWidth" + 79), SUM("ResolutionWidth" + 80), SUM("ResolutionWidth" + 81), SUM("ResolutionWidth" + 82), SUM("ResolutionWidth" + 83), SUM("ResolutionWidth" + 84), SUM("ResolutionWidth" + 85), SUM("ResolutionWidth" + 86), SUM("ResolutionWidth" + 87), SUM("ResolutionWidth" + 88), SUM("ResolutionWidth" + 89) FROM hits;
SELECT "SearchEngineID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "ClientIP" ORDER BY c DESC LIMIT 10;
SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10;
Expand Down
88 changes: 44 additions & 44 deletions datafusion/results/partitioned.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,56 +3,56 @@
"date": "2024-11-15",
"machine": "c6a.4xlarge, 500gb gp2",
"cluster_size": 1,
"comment": "v43.0.0 (88f58bf)",
"comment": "v44.0.0 (3cc3fca)",

"tags": ["Rust", "column-oriented", "embedded", "stateless"],

"load_time": 0,
"data_size": 14779976446,

"result": [
[0.051, 0.019, 0.019],
[0.091, 0.035, 0.035],
[0.189, 0.085, 0.088],
[0.383, 0.081, 0.077],
[1.071, 0.811, 0.803],
[0.944, 0.801, 0.805],
[0.078, 0.030, 0.030],
[0.103, 0.037, 0.037],
[1.313, 1.205, 1.201],
[1.357, 1.034, 1.025],
[0.511, 0.255, 0.253],
[0.634, 0.295, 0.301],
[1.016, 0.856, 0.879],
[2.615, 1.421, 1.374],
[1.131, 0.931, 0.918],
[1.051, 0.952, 0.958],
[2.672, 2.031, 2.066],
[2.592, 1.879, 1.887],
[5.549, 4.226, 4.335],
[0.254, 0.078, 0.075],
[9.967, 1.098, 1.092],
[11.248, 1.329, 1.327],
[21.868, 2.820, 2.818],
[55.458, 10.286, 10.609],
[2.678, 0.488, 0.486],
[0.802, 0.352, 0.354],
[2.672, 0.507, 0.498],
[9.614, 1.513, 1.507],
[8.368, 3.394, 3.521],
[0.535, 0.418, 0.439],
[2.362, 0.854, 0.861],
[5.957, 0.910, 0.914],
[4.780, 3.806, 3.871],
[10.168, 3.654, 3.586],
[10.090, 3.645, 3.546],
[1.775, 1.644, 1.660],
[0.364, 0.199, 0.183],
[0.183, 0.078, 0.075],
[0.290, 0.128, 0.123],
[0.619, 0.376, 0.376],
[0.148, 0.053, 0.044],
[0.142, 0.042, 0.042],
[0.155, 0.065, 0.053]
[0.061, 0.018, 0.018],
[0.108, 0.036, 0.033],
[0.188, 0.084, 0.085],
[2.097, 0.078, 0.079],
[2.648, 0.830, 0.817],
[2.707, 0.802, 0.803],
[0.081, 0.030, 0.029],
[0.102, 0.037, 0.036],
[2.493, 0.899, 0.910],
[3.445, 1.039, 1.043],
[2.078, 0.259, 0.244],
[2.470, 0.295, 0.270],
[2.916, 0.850, 0.858],
[5.908, 1.347, 1.340],
[2.918, 0.802, 0.810],
[2.081, 0.961, 0.973],
[5.740, 1.737, 1.739],
[5.654, 1.632, 1.643],
[10.074, 3.593, 3.575],
[1.433, 0.072, 0.073],
[20.760, 1.050, 1.078],
[23.608, 1.293, 1.335],
[45.123, 2.632, 2.540],
[111.820, 9.779, 10.107],
[6.605, 0.441, 0.437],
[2.607, 0.355, 0.351],
[6.611, 0.506, 0.516],
[20.174, 1.486, 1.484],
[17.954, 9.277, 9.326],
[0.499, 0.416, 0.417],
[5.743, 0.767, 0.780],
[13.208, 0.924, 0.906],
[10.116, 3.444, 3.523],
[20.379, 3.615, 3.595],
[20.457, 3.616, 3.621],
[1.437, 1.192, 1.204],
[0.327, 0.183, 0.182],
[0.172, 0.076, 0.074],
[0.250, 0.114, 0.111],
[0.569, 0.338, 0.337],
[0.154, 0.043, 0.045],
[0.130, 0.040, 0.040],
[0.137, 0.052, 0.053]
]
}
88 changes: 44 additions & 44 deletions datafusion/results/single.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,56 +3,56 @@
"date": "2024-11-15",
"machine": "c6a.4xlarge, 500gb gp2",
"cluster_size": 1,
"comment": "v43.0.0 (88f58bf)",
"comment": "v44.0.0 (3cc3fca)",

"tags": ["Rust", "column-oriented", "embedded", "stateless"],

"load_time": 0,
"data_size": 14779976446,

"result": [
[0.093, 0.055, 0.056],
[0.138, 0.070, 0.070],
[0.206, 0.120, 0.117],
[0.346, 0.118, 0.114],
[0.979, 0.867, 0.871],
[1.030, 0.902, 0.904],
[0.125, 0.064, 0.077],
[0.143, 0.083, 0.078],
[1.304, 1.169, 1.240],
[1.533, 1.104, 1.100],
[0.475, 0.272, 0.278],
[0.562, 0.309, 0.315],
[1.165, 0.931, 0.965],
[2.643, 1.402, 1.490],
[1.143, 0.997, 0.983],
[1.106, 0.991, 0.993],
[2.727, 2.161, 2.098],
[2.578, 1.954, 1.947],
[5.530, 4.311, 4.253],
[0.319, 0.105, 0.107],
[9.732, 1.155, 1.149],
[11.337, 1.468, 1.407],
[22.055, 3.678, 3.663],
[55.942, 10.017, 10.014],
[2.561, 0.557, 0.577],
[0.809, 0.510, 0.519],
[2.579, 0.634, 0.620],
[9.630, 1.618, 1.655],
[8.645, 3.565, 3.699],
[0.584, 0.493, 0.485],
[2.285, 0.978, 0.991],
[5.690, 1.046, 1.006],
[4.468, 3.833, 3.885],
[10.123, 3.663, 3.654],
[10.114, 3.672, 3.685],
[1.743, 1.597, 1.659],
[0.389, 0.242, 0.230],
[0.266, 0.155, 0.170],
[0.369, 0.161, 0.180],
[0.659, 0.446, 0.416],
[0.190, 0.084, 0.085],
[0.177, 0.078, 0.079],
[0.164, 0.103, 0.088]
[0.094, 0.065, 0.057],
[0.143, 0.070, 0.071],
[0.220, 0.120, 0.114],
[1.794, 0.118, 0.117],
[2.429, 0.883, 0.860],
[2.616, 0.907, 0.906],
[0.119, 0.077, 0.065],
[0.136, 0.082, 0.083],
[2.404, 0.953, 0.932],
[3.222, 1.093, 1.099],
[1.920, 0.278, 0.268],
[2.293, 0.312, 0.310],
[2.759, 0.932, 0.939],
[5.665, 1.363, 1.361],
[2.813, 0.895, 0.886],
[1.992, 0.982, 1.004],
[5.626, 1.856, 1.870],
[5.548, 1.707, 1.715],
[9.787, 3.610, 3.623],
[1.394, 0.109, 0.107],
[20.622, 1.145, 1.168],
[23.507, 1.449, 1.522],
[45.338, 3.505, 3.571],
[112.371, 10.242, 10.180],
[6.294, 0.574, 0.555],
[2.560, 0.509, 0.522],
[6.269, 0.628, 0.646],
[20.105, 1.590, 1.599],
[19.270, 10.050, 9.949],
[0.530, 0.458, 0.445],
[5.463, 0.914, 0.895],
[12.601, 1.037, 0.996],
[9.671, 3.516, 3.500],
[20.402, 3.748, 3.752],
[20.388, 3.791, 3.799],
[1.392, 1.247, 1.227],
[0.389, 0.231, 0.229],
[0.237, 0.154, 0.160],
[0.312, 0.164, 0.160],
[0.600, 0.418, 0.394],
[0.181, 0.083, 0.081],
[0.163, 0.077, 0.080],
[0.182, 0.087, 0.121]
]
}