From ed773ce60da782a55e9b03e8e023b14cc0d6ba23 Mon Sep 17 00:00:00 2001 From: Benedikt Christoph Wolters Date: Sun, 27 Nov 2016 17:58:47 +0100 Subject: [PATCH 1/4] Introduced named network namespace for replayshell this allows the creation of a a custom resolv.conf which is independent of the mahimahi host resolv.conf --- src/frontend/replayshell.cc | 24 +++-- src/util/Makefile.am | 3 +- src/util/network_namespace.cc | 168 ++++++++++++++++++++++++++++++++++ src/util/network_namespace.hh | 27 ++++++ 4 files changed, 213 insertions(+), 9 deletions(-) create mode 100644 src/util/network_namespace.cc create mode 100644 src/util/network_namespace.hh diff --git a/src/frontend/replayshell.cc b/src/frontend/replayshell.cc index 33cb49bc..37683ef7 100644 --- a/src/frontend/replayshell.cc +++ b/src/frontend/replayshell.cc @@ -1,10 +1,9 @@ /* -*-mode:c++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -#include -#include - #include #include +#include +#include +#include #include "util.hh" #include "netdevice.hh" @@ -16,6 +15,7 @@ #include "http_response.hh" #include "dns_server.hh" #include "exception.hh" +#include "network_namespace.hh" #include "http_record.pb.h" @@ -61,22 +61,31 @@ int main( int argc, char *argv[] ) /* get working directory */ const string working_directory { get_working_directory() }; + /* chdir to result of getcwd just in case */ SystemCall( "chdir", chdir( working_directory.c_str() ) ); + const string netns_name = string("mahimahi.") + to_string( getpid() ); + + NetworkNamespace network_namespace( netns_name ); + + /* Setup our own resolvconf with nameserver 8.8.8.8 */ + network_namespace.create_resolvconf( "8.8.8.8" ); + + /* Switch to the newly created network namespace */ + network_namespace.enter(); + /* what command will we run inside the container? */ vector< string > command; if ( argc == 2 ) { command.push_back( shell_path() ); } else { + for ( int i = 2; i < argc; i++ ) { command.push_back( argv[ i ] ); } } - /* create a new network namespace */ - SystemCall( "unshare", unshare( CLONE_NEWNET ) ); - /* bring up localhost */ interface_ioctl( SIOCSIFFLAGS, "lo", [] ( ifreq &ifr ) { ifr.ifr_flags = IFF_UP; } ); @@ -143,7 +152,6 @@ int main( int argc, char *argv[] ) const string interface_name = "nameserver" + to_string( server_num ); add_dummy_interface( interface_name, nameservers.at( server_num ) ); } - /* start dnsmasq */ event_loop.add_child_process( start_dnsmasq( dnsmasq_args ) ); diff --git a/src/util/Makefile.am b/src/util/Makefile.am index ab48fde7..4e8a05e1 100644 --- a/src/util/Makefile.am +++ b/src/util/Makefile.am @@ -5,7 +5,8 @@ noinst_LIBRARIES = libutil.a libutil_a_SOURCES = exception.hh ezio.cc ezio.hh \ file_descriptor.hh file_descriptor.cc netdevice.cc netdevice.hh \ - timestamp.cc timestamp.hh \ + timestamp.cc timestamp.hh \ + network_namespace.cc network_namespace.hh \ child_process.hh child_process.cc signalfd.hh signalfd.cc \ socket.cc socket.hh address.cc address.hh \ system_runner.hh system_runner.cc nat.hh nat.cc \ diff --git a/src/util/network_namespace.cc b/src/util/network_namespace.cc new file mode 100644 index 00000000..f699c93f --- /dev/null +++ b/src/util/network_namespace.cc @@ -0,0 +1,168 @@ +/* -*-mode:c++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +#include "network_namespace.hh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "system_runner.hh" +#include "exception.hh" +#include "file_descriptor.hh" + +using namespace std; + +NetworkNamespace::NetworkNamespace( const std::string & name ) + : name_(name), has_own_resolvconf_(false), has_entered_(false) +{ + /* create a new network namespace */ + run( { IP, "netns", "add", name_ } ); +} + + +NetworkNamespace::~NetworkNamespace() +{ + if ( has_entered_ ) { + + char etc_netns_path[PATH_MAX]; + char netns_name[PATH_MAX]; + char etc_name[PATH_MAX]; + struct dirent *entry; + DIR *dir; + + snprintf( etc_netns_path, sizeof( etc_netns_path ), "%s/%s", NETNS_ETC_DIR.c_str(), name_.c_str() ); + dir = opendir( etc_netns_path ); + if (!dir ) + return; + + while ( ( entry = readdir(dir) ) != NULL ) { + + if ( strcmp(entry->d_name, ".") == 0 ) + continue; + if ( strcmp(entry->d_name, "..") == 0 ) + continue; + + snprintf(netns_name, sizeof(netns_name), "%s/%s", etc_netns_path, entry->d_name); + snprintf(etc_name, sizeof(etc_name), "/etc/%s", entry->d_name); + + if ( umount( etc_name ) < 0 ) { + throw runtime_error( string("Unmounting... ") + netns_name + " -> " + etc_name + " failed: " + strerror(errno) + "\n" ); + } + } + + } + + if ( has_own_resolvconf_ ) { + const string netns_dir = NETNS_ETC_DIR + "/" + name_; + const string resolvconf_path = netns_dir + "/resolv.conf"; + + SystemCall("unlink", unlink ( resolvconf_path.c_str() ) ); + SystemCall("unlink", rmdir ( netns_dir.c_str() ) ); + } + + /*char netns_path[PATH_MAX]; + snprintf(netns_path, sizeof(netns_path), "%s/%s", NETNS_DIR.c_str(), name_.c_str()); + umount2(netns_path, MNT_FORCE); + if (unlink(netns_path) < 0) { + throw runtime_error(string("Cannot remove namespace file manually \"")+netns_path+"\":" + strerror(errno) + "\n" ); + }*/ + + + run( { IP, "netns", "delete", name_ } ); +} + + +void NetworkNamespace::create_resolvconf( const std::string & nameserver ) +{ + has_own_resolvconf_ = true; + /* create an individual resolv.conf for this network namespace */ + const string netns_dir = NETNS_ETC_DIR + "/" + name_; + const string resolvconf_path = netns_dir + "/resolv.conf"; + + SystemCall("mkdir", mkdir( netns_dir.c_str() , S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) ); + + FileDescriptor fd( SystemCall( "open", open( resolvconf_path.c_str(), + O_WRONLY | O_CREAT | O_TRUNC, + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) ) ) ; + + fd.write( "nameserver " + nameserver + "\n" ); +} + + +void NetworkNamespace::enter() +{ + char net_path[PATH_MAX]; + int netns; + + snprintf( net_path, sizeof(net_path), "%s/%s", NETNS_DIR.c_str(), name_.c_str() ); + netns = open( net_path, O_RDONLY | O_CLOEXEC ); + + if ( netns < 0 ) { + throw runtime_error( "Cannot open network namespace \"" + name_ + "\": " + strerror(errno) + "\n " ); + } + + if ( setns(netns, CLONE_NEWNET) < 0 ) { + throw runtime_error( "setting the network namespace \"" + name_ + "\" failed: " + strerror(errno) + "\n " ); + } + close( netns ); + + has_entered_ = true; + + + if ( unshare(CLONE_NEWNS) < 0 ) { + throw runtime_error( string("unshare failed: ") + strerror(errno) + "\n " ); + } + + /* Don't let any mounts propagate back to the parent */ + /* + if ( mount( "", "/", "none", MS_SLAVE | MS_REC, NULL ) ) { + throw runtime_error( string("\"mount --make-rslave /\" failed: ") + strerror(errno) + "\n " ); + }*/ + + /* Mount a version of /sys that describes the network namespace */ + if ( umount2( "/sys", MNT_DETACH) < 0 ) { + throw runtime_error( string("umount of /sys failed: ") + strerror(errno) + "\n" ); + } + + if ( mount( name_.c_str(), "/sys", "sysfs", 0, NULL ) < 0 ) { + throw runtime_error( string("mount of /sys failed: ") + strerror(errno) + "\n" ); + } + + + /* Setup bind mounts for config files in /etc */ + char etc_netns_path[PATH_MAX]; + char netns_name[PATH_MAX]; + char etc_name[PATH_MAX]; + struct dirent *entry; + DIR *dir; + + snprintf( etc_netns_path, sizeof( etc_netns_path ), "%s/%s", NETNS_ETC_DIR.c_str(), name_.c_str() ); + dir = opendir( etc_netns_path ); + if (!dir ) + return; + + while ( ( entry = readdir(dir) ) != NULL ) { + + if ( strcmp(entry->d_name, ".") == 0 ) + continue; + if ( strcmp(entry->d_name, "..") == 0 ) + continue; + + snprintf(netns_name, sizeof(netns_name), "%s/%s", etc_netns_path, entry->d_name); + snprintf(etc_name, sizeof(etc_name), "/etc/%s", entry->d_name); + + if ( mount( netns_name, etc_name, "none", MS_BIND, NULL ) < 0 ) { + throw runtime_error( string("Bind ") + netns_name + " -> " + etc_name + " failed: " + strerror(errno) + "\n" ); + } + } + + closedir( dir ); + + +} diff --git a/src/util/network_namespace.hh b/src/util/network_namespace.hh new file mode 100644 index 00000000..27b1de64 --- /dev/null +++ b/src/util/network_namespace.hh @@ -0,0 +1,27 @@ +/* -*-mode:c++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#ifndef NETNAMESPACE_HH +#define NETNAMESPACE_HH + +#include + +class NetworkNamespace +{ +private: + std::string name_; + bool has_own_resolvconf_; + bool has_entered_; + +public: + + const std::string NETNS_DIR = "/var/run/netns"; + const std::string NETNS_ETC_DIR = "/etc/netns"; + + NetworkNamespace( const std::string & name ); + ~NetworkNamespace(); + + void create_resolvconf( const std::string & nameserver ); + void enter( void ); +}; + +#endif From 4771f36c3009e33dceddeb1cf0f63bf1e6f3bc42 Mon Sep 17 00:00:00 2001 From: Benedikt Christoph Wolters Date: Mon, 28 Nov 2016 15:17:20 +0100 Subject: [PATCH 2/4] got rid of the named network ns, resolv.conf as tmpfile still lacks proper raii cleanup for namespace --- src/frontend/replayshell.cc | 2 +- src/util/network_namespace.cc | 135 +++++++--------------------------- src/util/network_namespace.hh | 9 ++- 3 files changed, 35 insertions(+), 111 deletions(-) diff --git a/src/frontend/replayshell.cc b/src/frontend/replayshell.cc index 37683ef7..754ea74f 100644 --- a/src/frontend/replayshell.cc +++ b/src/frontend/replayshell.cc @@ -67,7 +67,7 @@ int main( int argc, char *argv[] ) const string netns_name = string("mahimahi.") + to_string( getpid() ); - NetworkNamespace network_namespace( netns_name ); + NetworkNamespace network_namespace; /* Setup our own resolvconf with nameserver 8.8.8.8 */ network_namespace.create_resolvconf( "8.8.8.8" ); diff --git a/src/util/network_namespace.cc b/src/util/network_namespace.cc index f699c93f..8c4688d4 100644 --- a/src/util/network_namespace.cc +++ b/src/util/network_namespace.cc @@ -11,6 +11,8 @@ #include #include +#include + #include "config.h" #include "system_runner.hh" #include "exception.hh" @@ -18,151 +20,68 @@ using namespace std; -NetworkNamespace::NetworkNamespace( const std::string & name ) - : name_(name), has_own_resolvconf_(false), has_entered_(false) +NetworkNamespace::NetworkNamespace( ) + : has_own_resolvconf_(false), resolvconf_file_(nullptr), has_entered_(false) { - /* create a new network namespace */ - run( { IP, "netns", "add", name_ } ); } NetworkNamespace::~NetworkNamespace() { - if ( has_entered_ ) { - - char etc_netns_path[PATH_MAX]; - char netns_name[PATH_MAX]; - char etc_name[PATH_MAX]; - struct dirent *entry; - DIR *dir; - - snprintf( etc_netns_path, sizeof( etc_netns_path ), "%s/%s", NETNS_ETC_DIR.c_str(), name_.c_str() ); - dir = opendir( etc_netns_path ); - if (!dir ) - return; - - while ( ( entry = readdir(dir) ) != NULL ) { - - if ( strcmp(entry->d_name, ".") == 0 ) - continue; - if ( strcmp(entry->d_name, "..") == 0 ) - continue; + /* If we want to cleanup resolv.conf tempfile we need to make sure it is unmounted first */ + if ( has_entered_ && has_own_resolvconf_ ) { - snprintf(netns_name, sizeof(netns_name), "%s/%s", etc_netns_path, entry->d_name); - snprintf(etc_name, sizeof(etc_name), "/etc/%s", entry->d_name); - - if ( umount( etc_name ) < 0 ) { - throw runtime_error( string("Unmounting... ") + netns_name + " -> " + etc_name + " failed: " + strerror(errno) + "\n" ); - } + if ( umount( "/etc/resolv.conf" ) < 0 ) { + //TODO (worenga): proper RAII for mounts + std::cerr << string("Unmounting... ") << resolvconf_file_->name() << " -> " << "/etc/resolv.conf" << " failed: " << strerror(errno) << "\n" << std::endl; } - - } - - if ( has_own_resolvconf_ ) { - const string netns_dir = NETNS_ETC_DIR + "/" + name_; - const string resolvconf_path = netns_dir + "/resolv.conf"; - - SystemCall("unlink", unlink ( resolvconf_path.c_str() ) ); - SystemCall("unlink", rmdir ( netns_dir.c_str() ) ); } - /*char netns_path[PATH_MAX]; - snprintf(netns_path, sizeof(netns_path), "%s/%s", NETNS_DIR.c_str(), name_.c_str()); - umount2(netns_path, MNT_FORCE); - if (unlink(netns_path) < 0) { - throw runtime_error(string("Cannot remove namespace file manually \"")+netns_path+"\":" + strerror(errno) + "\n" ); - }*/ - + //TODO (worenga): How can we redo the (mount/bind) namespace in a RAII manner? - run( { IP, "netns", "delete", name_ } ); } void NetworkNamespace::create_resolvconf( const std::string & nameserver ) { - has_own_resolvconf_ = true; - /* create an individual resolv.conf for this network namespace */ - const string netns_dir = NETNS_ETC_DIR + "/" + name_; - const string resolvconf_path = netns_dir + "/resolv.conf"; - - SystemCall("mkdir", mkdir( netns_dir.c_str() , S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) ); - - FileDescriptor fd( SystemCall( "open", open( resolvconf_path.c_str(), - O_WRONLY | O_CREAT | O_TRUNC, - S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) ) ) ; + if ( has_entered_ ) + { + throw runtime_error( string("Cannot create resolvconf after namespace has been entered.\n") ); + } - fd.write( "nameserver " + nameserver + "\n" ); + resolvconf_file_.reset(std::move( new TempFile( "resolvconf" ) )); + has_own_resolvconf_ = true; + resolvconf_file_->write( "# Ephemeral resolv.conf(5) file for glibc resolver(3) generated by mahimahi\n" ); + resolvconf_file_->write( "# DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE LOST\n" ); + resolvconf_file_->write( "nameserver " + nameserver + "\n" ); } void NetworkNamespace::enter() { - char net_path[PATH_MAX]; - int netns; - - snprintf( net_path, sizeof(net_path), "%s/%s", NETNS_DIR.c_str(), name_.c_str() ); - netns = open( net_path, O_RDONLY | O_CLOEXEC ); - - if ( netns < 0 ) { - throw runtime_error( "Cannot open network namespace \"" + name_ + "\": " + strerror(errno) + "\n " ); - } - - if ( setns(netns, CLONE_NEWNET) < 0 ) { - throw runtime_error( "setting the network namespace \"" + name_ + "\" failed: " + strerror(errno) + "\n " ); - } - close( netns ); - - has_entered_ = true; - if ( unshare(CLONE_NEWNS) < 0 ) { throw runtime_error( string("unshare failed: ") + strerror(errno) + "\n " ); } + /* Don't let any mounts propagate back to the parent */ - /* if ( mount( "", "/", "none", MS_SLAVE | MS_REC, NULL ) ) { throw runtime_error( string("\"mount --make-rslave /\" failed: ") + strerror(errno) + "\n " ); - }*/ - - /* Mount a version of /sys that describes the network namespace */ - if ( umount2( "/sys", MNT_DETACH) < 0 ) { - throw runtime_error( string("umount of /sys failed: ") + strerror(errno) + "\n" ); } - if ( mount( name_.c_str(), "/sys", "sysfs", 0, NULL ) < 0 ) { - throw runtime_error( string("mount of /sys failed: ") + strerror(errno) + "\n" ); + if ( unshare(CLONE_NEWNET) < 0 ) { + throw runtime_error( string("unshare failed: ") + strerror(errno) + "\n " ); } + has_entered_ = true; - /* Setup bind mounts for config files in /etc */ - char etc_netns_path[PATH_MAX]; - char netns_name[PATH_MAX]; - char etc_name[PATH_MAX]; - struct dirent *entry; - DIR *dir; - - snprintf( etc_netns_path, sizeof( etc_netns_path ), "%s/%s", NETNS_ETC_DIR.c_str(), name_.c_str() ); - dir = opendir( etc_netns_path ); - if (!dir ) - return; - - while ( ( entry = readdir(dir) ) != NULL ) { - - if ( strcmp(entry->d_name, ".") == 0 ) - continue; - if ( strcmp(entry->d_name, "..") == 0 ) - continue; - - snprintf(netns_name, sizeof(netns_name), "%s/%s", etc_netns_path, entry->d_name); - snprintf(etc_name, sizeof(etc_name), "/etc/%s", entry->d_name); + if ( has_own_resolvconf_ ) { - if ( mount( netns_name, etc_name, "none", MS_BIND, NULL ) < 0 ) { - throw runtime_error( string("Bind ") + netns_name + " -> " + etc_name + " failed: " + strerror(errno) + "\n" ); + if ( mount( resolvconf_file_->name().c_str() , "/etc/resolv.conf" , "none", MS_BIND, NULL ) < 0 ) { + throw runtime_error( string("Bind ") + resolvconf_file_->name() + " -> " + "/etc/resolv.conf" + " failed: " + strerror(errno) + "\n" ); } - } - - closedir( dir ); + } } diff --git a/src/util/network_namespace.hh b/src/util/network_namespace.hh index 27b1de64..dedb652d 100644 --- a/src/util/network_namespace.hh +++ b/src/util/network_namespace.hh @@ -4,12 +4,17 @@ #define NETNAMESPACE_HH #include +#include + +#include "temp_file.hh" class NetworkNamespace { + private: - std::string name_; bool has_own_resolvconf_; + std::unique_ptr resolvconf_file_; + bool has_entered_; public: @@ -17,7 +22,7 @@ public: const std::string NETNS_DIR = "/var/run/netns"; const std::string NETNS_ETC_DIR = "/etc/netns"; - NetworkNamespace( const std::string & name ); + NetworkNamespace(); ~NetworkNamespace(); void create_resolvconf( const std::string & nameserver ); From 416297293d3688d094fc6779b1fff77dcd2c4417 Mon Sep 17 00:00:00 2001 From: Benedikt Christoph Wolters Date: Wed, 30 Nov 2016 11:02:38 +0100 Subject: [PATCH 3/4] cs --- src/util/network_namespace.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/util/network_namespace.cc b/src/util/network_namespace.cc index 8c4688d4..d05eb86e 100644 --- a/src/util/network_namespace.cc +++ b/src/util/network_namespace.cc @@ -44,12 +44,11 @@ NetworkNamespace::~NetworkNamespace() void NetworkNamespace::create_resolvconf( const std::string & nameserver ) { - if ( has_entered_ ) - { + if ( has_entered_ ) { throw runtime_error( string("Cannot create resolvconf after namespace has been entered.\n") ); } - resolvconf_file_.reset(std::move( new TempFile( "resolvconf" ) )); + resolvconf_file_.reset( std::move( new TempFile( "resolvconf" ) ) ); has_own_resolvconf_ = true; resolvconf_file_->write( "# Ephemeral resolv.conf(5) file for glibc resolver(3) generated by mahimahi\n" ); resolvconf_file_->write( "# DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE LOST\n" ); From 9901f0ddbb4848307ca633c24f1c0af879b08d90 Mon Sep 17 00:00:00 2001 From: Benedikt Christoph Wolters Date: Wed, 30 Nov 2016 11:02:52 +0100 Subject: [PATCH 4/4] fixes permission for temp resolv.conf file --- src/util/network_namespace.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/util/network_namespace.cc b/src/util/network_namespace.cc index d05eb86e..e12d62a3 100644 --- a/src/util/network_namespace.cc +++ b/src/util/network_namespace.cc @@ -50,6 +50,12 @@ void NetworkNamespace::create_resolvconf( const std::string & nameserver ) resolvconf_file_.reset( std::move( new TempFile( "resolvconf" ) ) ); has_own_resolvconf_ = true; + + char mode[] = "0644"; + if ( chmod (resolvconf_file_->name().c_str(), strtol(mode, 0, 8) ) < 0 ) { + throw runtime_error( string("chmod ") + mode + " for " + resolvconf_file_->name() +" failed: " + strerror(errno) + "\n " ); + } + resolvconf_file_->write( "# Ephemeral resolv.conf(5) file for glibc resolver(3) generated by mahimahi\n" ); resolvconf_file_->write( "# DO NOT EDIT THIS FILE BY HAND -- CHANGES WILL BE LOST\n" ); resolvconf_file_->write( "nameserver " + nameserver + "\n" );