Skip to content

Commit

Permalink
Add hardware-accelerated encryption to rosbags (#1)
Browse files Browse the repository at this point in the history
* Implement Bag encryption/decryption. (ros#1206)

Ported to 1.12.12

* Python Bag class supports encryption; all the rosbag tools work for encrypted bags. (ros#1206)

* Improve exception messages raised when a public key is missing.

* Randomize initialization vectors for encrypt/decrypt.

* Fix bag encryption routine (ros#1310)

Bag encryption routine was truncating the recorded block by the size of the IV.

* Drop const qualifier from *::decryptChunk methods

Since decryption can change EVP's context, these methods
can't be const anymore.

* Move encryption to from openssl software aes to EVP API

* Check EVP API results

* Add EncryptionOptions to Recorder

* Parse encryption and encryption-param options in the record executable

* Fix gtests

* Fix rostests

- With ninja, when `_rostest_ARGS` is empty, the space right before it
gets escaped, and the command that ultimately gets executed has a
trailing slash.
- rospy.log testing fails because our ROSCONSOLE_FORMAT does not print
severity
- bag.py had a bug in get_info_str() that has been fixed upstream
- bz2 performs a few bytes better than expected, failing the rosbag
compression test
- roswtf tests had an outdated dependency list (TBH I don't understand
what this list is)
  • Loading branch information
madsciencetist authored Feb 22, 2018
1 parent ab76f34 commit 902ea3a
Show file tree
Hide file tree
Showing 25 changed files with 1,757 additions and 102 deletions.
2 changes: 1 addition & 1 deletion test/test_rosbag/test/test_bag.py
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ def test_get_compression_info(self):
# the value varies each run, I suspect based on rand, but seems
# to generally be around 960 to 980 on my comp
self.assertLess(info.compressed, 1000)
self.assertGreater(info.compressed, 900)
self.assertGreater(info.compressed, 850)

def test_get_time(self):
fn = '/tmp/test_get_time.bag'
Expand Down
1 change: 1 addition & 0 deletions test/test_roscpp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ catkin_package(
)

if(CATKIN_ENABLE_TESTING)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
add_subdirectory(test)
add_subdirectory(perf)
add_subdirectory(test_serialization)
Expand Down
6 changes: 3 additions & 3 deletions test/test_rospy/test/rostest/test_rospy_client_online.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,17 +88,17 @@ def test_log(self):
self.assert_("test 1" in sys.stdout.getvalue())

rospy.logwarn("test 2")
self.assert_("[WARN]" in sys.stderr.getvalue())
#self.assert_("[WARN]" in sys.stderr.getvalue())
self.assert_("test 2" in sys.stderr.getvalue())

sys.stderr = StringIO()
rospy.logerr("test 3")
self.assert_("[ERROR]" in sys.stderr.getvalue())
#self.assert_("[ERROR]" in sys.stderr.getvalue())
self.assert_("test 3" in sys.stderr.getvalue())

sys.stderr = StringIO()
rospy.logfatal("test 4")
self.assert_("[FATAL]" in sys.stderr.getvalue())
#self.assert_("[FATAL]" in sys.stderr.getvalue())
self.assert_("test 4" in sys.stderr.getvalue())

# logXXX_throttle
Expand Down
11 changes: 11 additions & 0 deletions tools/rosbag/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ target_link_libraries(record rosbag)
add_executable(play src/play.cpp)
target_link_libraries(play rosbag)

if(NOT WIN32)
add_executable(encrypt src/encrypt.cpp)
target_link_libraries(encrypt ${catkin_LIBRARIES})
endif()

install(DIRECTORY include/${PROJECT_NAME}/
DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION}
FILES_MATCHING PATTERN "*.h")
Expand All @@ -49,6 +54,12 @@ install(TARGETS record play
ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION})
if(NOT WIN32)
install(TARGETS encrypt
ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION})
endif()
catkin_install_python(PROGRAMS
scripts/bag2png.py
scripts/bagsort.py
Expand Down
2 changes: 2 additions & 0 deletions tools/rosbag/include/rosbag/recorder.h
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ struct ROSBAG_DECL RecorderOptions
bool snapshot;
bool verbose;
CompressionType compression;
std::string encryption;
std::string encryption_param;
std::string prefix;
std::string name;
boost::regex exclude_regex;
Expand Down
2 changes: 2 additions & 0 deletions tools/rosbag/package.xml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
<run_depend>boost</run_depend>
<run_depend>genmsg</run_depend>
<run_depend>genpy</run_depend>
<run_depend>python-crypto</run_depend>
<run_depend>python-gnupg</run_depend>
<run_depend>python-rospkg</run_depend>
<run_depend>rosbag_storage</run_depend>
<run_depend>rosconsole</run_depend>
Expand Down
198 changes: 198 additions & 0 deletions tools/rosbag/src/encrypt.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
/*********************************************************************
* Software License Agreement (BSD License)
*
* Copyright (c) 2017, Open Source Robotics Foundation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials provided
* with the distribution.
* * Neither the name of Willow Garage, Inc. nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*********************************************************************/

#include <iostream>

#include <boost/scoped_ptr.hpp>
#include <boost/program_options.hpp>
#include <boost/progress.hpp>
#include <boost/regex.hpp>

#include <ros/ros.h>

#include "rosbag/bag.h"
#include "rosbag/view.h"

namespace po = boost::program_options;

struct EncryptorOptions
{
EncryptorOptions() : quiet(false), compression(rosbag::compression::Uncompressed) { }

void buildOutbagName();

bool quiet;
std::string plugin;
std::string param;
rosbag::CompressionType compression;
std::string inbag;
std::string outbag;
};

void EncryptorOptions::buildOutbagName()
{
if (!outbag.empty())
return;
if (inbag.empty())
throw ros::Exception("Input bag is not specified.");
std::string::size_type pos = inbag.find_last_of('.');
if (pos == std::string::npos)
throw ros::Exception("Input bag name has no extension.");
outbag = inbag.substr(0, pos) + std::string(".out") + inbag.substr(pos);
}

//! Parse the command-line arguments for encrypt options
EncryptorOptions parseOptions(int argc, char** argv)
{
EncryptorOptions opts;

po::options_description desc("Allowed options");

desc.add_options()
("help,h", "produce help message")
("quiet,q", "suppress console output")
("plugin,p", po::value<std::string>()->default_value("rosbag/AesCbcEncryptor"), "encryptor name")
("param,r", po::value<std::string>()->default_value("*"), "encryptor parameter")
("bz2,j", "use BZ2 compression")
("lz4", "use lz4 compression")
("inbag", po::value<std::string>(), "bag file to encrypt")
("outbag,o", po::value<std::string>(), "bag file encrypted")
;

po::positional_options_description p;
p.add("inbag", -1);

po::variables_map vm;

try
{
po::store(po::command_line_parser(argc, argv).options(desc).positional(p).run(), vm);
}
catch (boost::program_options::invalid_command_line_syntax& e)
{
throw ros::Exception(e.what());
}
catch (boost::program_options::unknown_option& e)
{
throw ros::Exception(e.what());
}

if (vm.count("help"))
{
std::cout << desc << std::endl;
exit(0);
}

if (vm.count("quiet"))
opts.quiet = true;
if (vm.count("plugin"))
opts.plugin = vm["plugin"].as<std::string>();
if (vm.count("param"))
opts.param = vm["param"].as<std::string>();
if (vm.count("bz2"))
opts.compression = rosbag::compression::BZ2;
if (vm.count("lz4"))
opts.compression = rosbag::compression::LZ4;
if (vm.count("inbag"))
opts.inbag = vm["inbag"].as<std::string>();
else
throw ros::Exception("You must specify bag to encrypt.");
if (vm.count("outbag"))
opts.outbag = vm["outbag"].as<std::string>();
opts.buildOutbagName();

return opts;
}

std::string getStringCompressionType(rosbag::CompressionType compression)
{
switch(compression)
{
case rosbag::compression::Uncompressed: return "none";
case rosbag::compression::BZ2: return "bz2";
case rosbag::compression::LZ4: return "lz4";
default: return "Unknown";
}
}

int encrypt(EncryptorOptions const& options)
{
if (!options.quiet)
{
std::cout << "Output bag: " << options.outbag << "\n";
std::cout << "Encryption: " << options.plugin << ":" << options.param << "\n";
std::cout << "Compression: " << getStringCompressionType(options.compression) << "\n";
}
rosbag::Bag inbag(options.inbag, rosbag::bagmode::Read);
rosbag::Bag outbag(options.outbag, rosbag::bagmode::Write);
// Compression type is per chunk, and cannot be retained.
// If chunk-by-chunk encryption is implemented, compression type could be honored.
outbag.setEncryptorPlugin(options.plugin, options.param);
outbag.setCompression(options.compression);
rosbag::View view(inbag);
boost::scoped_ptr<boost::progress_display> progress;
if (!options.quiet)
progress.reset(new boost::progress_display(view.size(), std::cout, "Progress:\n ", " ", " "));
for (rosbag::View::const_iterator it = view.begin(); it != view.end(); ++it)
{
outbag.write(it->getTopic(), it->getTime(), *it, it->getConnectionHeader());
if (progress)
++(*progress);
}
outbag.close();
inbag.close();
return 0;
}

int main(int argc, char** argv)
{
// Parse the command-line options
EncryptorOptions opts;
try
{
opts = parseOptions(argc, argv);
}
catch (ros::Exception const& ex)
{
ROS_ERROR("Error reading options: %s", ex.what());
return 1;
}
catch(boost::regex_error const& ex)
{
ROS_ERROR("Error reading options: %s\n", ex.what());
return 1;
}

return encrypt(opts);
}
24 changes: 22 additions & 2 deletions tools/rosbag/src/record.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,9 @@ rosbag::RecorderOptions parseOptions(int argc, char** argv) {
("topic", po::value< std::vector<std::string> >(), "topic to record")
("size", po::value<uint64_t>(), "The maximum size of the bag to record in MB.")
("duration", po::value<std::string>(), "Record a bag of maximum duration in seconds, unless 'm', or 'h' is appended.")
("node", po::value<std::string>(), "Record all topics subscribed to by a specific node.");

("node", po::value<std::string>(), "Record all topics subscribed to by a specific node.")
("encryption", po::value<std::string>(), "Encryption plugin to use.")
("encryption-param", po::value<std::string>(), "Parameter to the encryption plugin");

po::positional_options_description p;
p.add("topic", -1);
Expand Down Expand Up @@ -260,6 +261,25 @@ rosbag::RecorderOptions parseOptions(int argc, char** argv) {
opts.record_all = true;
}

bool encryption_set = false;
if (vm.count("encryption"))
{
opts.encryption = vm["encryption"].as<std::string>();
std::cout << "Using " << opts.encryption << " encryption plugin." << std::endl;
encryption_set = true;
}

if (vm.count("encryption-param"))
{
if (!encryption_set)
{
throw ros::Exception("Can only set encryption params if encryption is set.");
}

opts.encryption_param = vm["encryption-param"].as<std::string>();
std::cout << "Using " << opts.encryption_param << " as encryption plugin param." << std::endl;
}

return opts;
}

Expand Down
3 changes: 3 additions & 0 deletions tools/rosbag/src/recorder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ RecorderOptions::RecorderOptions() :
snapshot(false),
verbose(false),
compression(compression::Uncompressed),
encryption("rosbag/NoEncryptor"),
encryption_param("*"),
prefix(""),
name(""),
exclude_regex(),
Expand Down Expand Up @@ -380,6 +382,7 @@ void Recorder::snapshotTrigger(std_msgs::Empty::ConstPtr trigger) {

void Recorder::startWriting() {
bag_.setCompression(options_.compression);
bag_.setEncryptorPlugin(options_.encryption, options_.encryption_param);
bag_.setChunkThreshold(options_.chunk_size);

updateFilenames();
Expand Down
Loading

0 comments on commit 902ea3a

Please sign in to comment.