diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfdb8b7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/71-nvidia.rules b/71-nvidia.rules new file mode 100644 index 0000000..e17dad1 --- /dev/null +++ b/71-nvidia.rules @@ -0,0 +1,31 @@ +# Tag the device as master-of-seat so that logind is happy +# (see LP: #1365336) +SUBSYSTEM=="pci", ATTRS{vendor}=="0x10de", DRIVERS=="nvidia", TAG+="seat", TAG+="master-of-seat" + +# Start and stop nvidia-persistenced on power on and power off +# respectively +ACTION=="add" DEVPATH=="/bus/acpi/drivers/NVIDIA ACPI Video Driver" SUBSYSTEM=="drivers" RUN+="/bin/systemctl start --no-block nvidia-persistenced.service" +ACTION=="remove" DEVPATH=="/bus/acpi/drivers/NVIDIA ACPI Video Driver" SUBSYSTEM=="drivers" RUN+="/bin/systemctl stop --no-block nvidia-persistenced" + +# Start and stop nvidia-persistenced when loading and unloading +# the driver +ACTION=="add" DEVPATH=="/module/nvidia" SUBSYSTEM=="module" RUN+="/bin/systemctl start --no-block nvidia-persistenced.service" +ACTION=="remove" DEVPATH=="/module/nvidia" SUBSYSTEM=="module" RUN+="/bin/systemctl stop --no-block nvidia-persistenced" + +# Load and unload nvidia-modeset module +ACTION=="add" DEVPATH=="/module/nvidia" SUBSYSTEM=="module" RUN+="/opt/bin/nvidia-insmod.sh nvidia-modeset.ko" +ACTION=="remove" DEVPATH=="/module/nvidia" SUBSYSTEM=="module" RUN+="/usr/sbin/rmmod -r nvidia-modeset" + +# Load and unload nvidia-drm module +ACTION=="add" DEVPATH=="/module/nvidia" SUBSYSTEM=="module" RUN+="/opt/bin/nvidia-insmod.sh nvidia-drm.ko" +ACTION=="remove" DEVPATH=="/module/nvidia" SUBSYSTEM=="module" RUN+="/usr/sbin/rmmod nvidia-drm" + +# Load and unload nvidia-uvm module +ACTION=="add" DEVPATH=="/module/nvidia" SUBSYSTEM=="module" RUN+="/opt/bin/nvidia-insmod.sh nvidia-uvm.ko" +ACTION=="remove" DEVPATH=="/module/nvidia" SUBSYSTEM=="module" RUN+="/usr/sbin/rmmod -r nvidia-uvm" + +# This will create the device nvidia device nodes +ACTION=="add" DEVPATH=="/module/nvidia" SUBSYSTEM=="module" RUN+="/opt/bin/nvidia-smi" + +# Create the device node for the nvidia-uvm module +ACTION=="add" DEVPATH=="/module/nvidia_uvm" SUBSYSTEM=="module" RUN+="/opt/bin/create-uvm-dev-node.sh" diff --git a/README.md b/README.md index 2c37485..ce7737c 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ started from a machine running _any_ kind of Linux distribution: it ## Usage: -build.sh [--keep] DRIVER_VERSION CHANNEL COREOS_VERSION +build.sh [--keep] DRIVER_VERSION [CHANNEL] [COREOS_VERSION] e.g. @@ -33,6 +33,7 @@ helpful for debugging purposes. The scripts will then create three archives: ``` libraries-[DRIVER_VERSION].tar.bz2 +libraries-tls-[DRIVER_VERSION].tar.bz2 tools-[DRIVER_VERSION].tar.bz2 modules-[COREOS_VERSION]-[DRIVER_VERSION].tar.bz2 ``` diff --git a/_export.sh b/_export.sh index 98dadb7..6b6aa4d 100755 --- a/_export.sh +++ b/_export.sh @@ -11,5 +11,6 @@ nvidia-persistenced nvidia-settings" # Create archives with no paths tar -C ${ARTIFACT_DIR} -cvj $(basename -a ${ARTIFACT_DIR}/*.so.*) > libraries-${VERSION}.tar.bz2 +tar -C ${ARTIFACT_DIR}/tls -cvj $(basename -a ${ARTIFACT_DIR}/tls/*.so.*) > libraries-tls-${VERSION}.tar.bz2 tar -C ${ARTIFACT_DIR} -cvj ${TOOLS} > tools-${VERSION}.tar.bz2 tar -C ${ARTIFACT_DIR}/kernel -cvj $(basename -a ${ARTIFACT_DIR}/kernel/*.ko) > modules-${COMBINED_VERSION}.tar.bz2 diff --git a/build.sh b/build.sh index f3032d2..bea49ca 100755 --- a/build.sh +++ b/build.sh @@ -26,9 +26,20 @@ done echo "Keeping container around after build: ${KEEP_CONTAINER}" echo "Additional flags: ${EMERGE_SOURCES}" -DRIVER_VERSION=${1:-367.57} -COREOS_TRACK=${2:-beta} -COREOS_VERSION=${3:-1185.5.0} +COREOS_TRACK_DEFAULT=beta +COREOS_VERSION_DEFAULT=1185.5.0 +# If we are on CoreOS by default build for the current CoreOS version +if [[ -f /etc/lsb-release && -f /etc/coreos/update.conf ]]; then + source /etc/lsb-release + source /etc/coreos/update.conf + + COREOS_TRACK_DEFAULT=$GROUP + COREOS_VERSION_DEFAULT=$DISTRIB_RELEASE +fi + +DRIVER_VERSION=${1:-375.20} +COREOS_TRACK=${2:-$COREOS_TRACK_DEFAULT} +COREOS_VERSION=${3:-$COREOS_VERSION_DEFAULT} DRIVER_ARCHIVE=NVIDIA-Linux-x86_64-${DRIVER_VERSION} DRIVER_ARCHIVE_PATH=${PWD}/nvidia_installers/${DRIVER_ARCHIVE}.run diff --git a/create-uvm-dev-node.sh b/create-uvm-dev-node.sh new file mode 100755 index 0000000..58a3d4d --- /dev/null +++ b/create-uvm-dev-node.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# Get the major device number for nvidia-uvm and create the node +echo "Set up NVIDIA UVM" +major=`grep nvidia-uvm /proc/devices | awk '{print $1}'` +if [ -n "$major" ]; then + mknod -m 666 /dev/nvidia-uvm c $major 0 +fi diff --git a/nvidia-docker.service b/nvidia-docker.service new file mode 100644 index 0000000..ed16e4a --- /dev/null +++ b/nvidia-docker.service @@ -0,0 +1,26 @@ +[Unit] +Description=NVIDIA Docker plugin +Documentation=https://github.com/NVIDIA/nvidia-docker/wiki +After=local-fs.target network.target nvidia-start.service +Wants=docker.service + +[Service] +Environment="SOCK_DIR=/var/lib/nvidia-docker" +Environment="SPEC_FILE=/etc/docker/plugins/nvidia-docker.spec" +Environment="PATH=/opt/bin/:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin" + +#User=nvidia-docker +User=root +#PermissionsStartOnly=true +Restart=on-failure +RestartSec=1 +TimeoutStartSec=0 +TimeoutStopSec=20 + +ExecStart=/opt/bin/nvidia-docker-plugin -s $SOCK_DIR +ExecStartPost=/bin/sh -c '/bin/mkdir -p $( dirname $SPEC_FILE )' +ExecStartPost=/bin/sh -c '/bin/echo unix://$SOCK_DIR/nvidia-docker.sock > $SPEC_FILE' +ExecStopPost=/bin/rm -f $SPEC_FILE + +[Install] +WantedBy=multi-user.target diff --git a/nvidia-insmod.sh b/nvidia-insmod.sh new file mode 100755 index 0000000..8992401 --- /dev/null +++ b/nvidia-insmod.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +/usr/sbin/insmod /opt/lib/modules/$(uname -r)/video/$1 diff --git a/nvidia-persistenced.service b/nvidia-persistenced.service new file mode 100644 index 0000000..a11b975 --- /dev/null +++ b/nvidia-persistenced.service @@ -0,0 +1,8 @@ +[Unit] +Description=NVIDIA Persistence Daemon +Wants=local-fs.target + +[Service] +Type=forking +ExecStart=/opt/bin/nvidia-persistenced --user nvidia-persistenced --no-persistence-mode --verbose +ExecStopPost=/bin/rm -rf /var/run/nvidia-persistenced diff --git a/nvidia-start.service b/nvidia-start.service new file mode 100644 index 0000000..4bae9bc --- /dev/null +++ b/nvidia-start.service @@ -0,0 +1,9 @@ +[Unit] +Description=Load NVIDIA module +After=local-fs.target + +[Service] +ExecStart=/opt/bin/nvidia-start.sh + +[Install] +WantedBy=multi-user.target diff --git a/nvidia-start.sh b/nvidia-start.sh new file mode 100755 index 0000000..3365e2c --- /dev/null +++ b/nvidia-start.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +/opt/bin/nvidia-insmod.sh nvidia.ko + +# Start the first devices +/usr/bin/mknod -m 666 /dev/nvidiactl c 195 255 2>/dev/null +/usr/bin/mknod -m 666 /dev/nvidia0 c 195 0 2>/dev/null diff --git a/nvidia_docker_install.sh b/nvidia_docker_install.sh new file mode 100755 index 0000000..7c789a1 --- /dev/null +++ b/nvidia_docker_install.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +NVIDIA_DOCKER_VERSION=${1:-1.0.1} + +if [[ $(uname -r) != *"-coreos-"* ]]; then + echo "OS is not CoreOS" + exit 1 +fi + +wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v${NVIDIA_DOCKER_VERSION}/nvidia-docker_${NVIDIA_DOCKER_VERSION}_amd64.tar.xz +tar --strip-components=1 -C /opt/bin -xvf /tmp/nvidia-docker*.tar.xz && rm /tmp/nvidia-docker*.tar.xz +echo "Setting up permissions" +chown root:root /opt/bin/nvidia-docker* +setcap cap_fowner+pe /opt/bin/nvidia-docker-plugin + +echo "Configuring nvidia user" +id -u nvidia-docker >/dev/null 2>&1 || \ +useradd -r -M -d /var/lib/nvidia-docker -s /usr/sbin/nologin -c "NVIDIA Docker plugin" nvidia-docker +mkdir -p /var/lib/nvidia-docker 2>/dev/null +chown nvidia-docker: /var/lib/nvidia-docker + +systemctl enable nvidia-docker.service +systemctl start nvidia-docker.service diff --git a/nvidia_install.sh b/nvidia_install.sh new file mode 100755 index 0000000..9c436d3 --- /dev/null +++ b/nvidia_install.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +if [[ $(uname -r) != *"-coreos-"* ]]; then + echo "OS is not CoreOS" + exit 1 +fi + +COREOS_TRACK_DEFAULT=beta +COREOS_VERSION_DEFAULT=1185.5.0 +# If we are on CoreOS by default use the current CoreOS version +if [[ -f /etc/lsb-release && -f /etc/coreos/update.conf ]]; then + source /etc/lsb-release + source /etc/coreos/update.conf + + COREOS_TRACK_DEFAULT=$GROUP + COREOS_VERSION_DEFAULT=$DISTRIB_RELEASE + if [[ $DISTRIB_ID != *"CoreOS"* ]]; then + echo "Distribution is not CoreOS" + exit 1 + fi +fi + +DRIVER_VERSION=${1:-375.20} +COREOS_TRACK=${2:-$COREOS_TRACK_DEFAULT} +COREOS_VERSION=${3:-$COREOS_VERSION_DEFAULT} + +# this is where the modules go +release=$(uname -r) + +mkdir -p /opt/lib64/tls 2>/dev/null +mkdir -p /opt/bin 2>/dev/null +ln -sfT lib64 /opt/lib 2>/dev/null +mkdir -p /opt/lib64/modules/$release/video/ + +tar xvf libraries-$DRIVER_VERSION.tar.bz2 -C /opt/lib64/ +tar xvf libraries-tls-$DRIVER_VERSION.tar.bz2 -C /opt/lib64/tls/ +tar xvf modules-$COREOS_VERSION-$DRIVER_VERSION.tar.bz2 -C /opt/lib64/modules/$release/video/ +tar xvf tools-$DRIVER_VERSION.tar.bz2 -C /opt/bin/ + +install -m 755 create-uvm-dev-node.sh /opt/bin/ +install -m 755 nvidia-start.sh /opt/bin/ +install -m 755 nvidia-insmod.sh /opt/bin/ +cp -f 71-nvidia.rules /etc/udev/rules.d/ +udevadm control --reload-rules + +mkdir -p /etc/ld.so.conf.d/ 2>/dev/null +echo "/opt/lib64" > /etc/ld.so.conf.d/nvidia.conf +ldconfig + +echo "Configuring nvidia persistence user" +id -u nvidia-persistenced >/dev/null 2>&1 || \ +useradd --system --home '/' --shell '/sbin/nologin' -c 'NVIDIA Persistence Daemon' nvidia-persistenced + +cp *.service /etc/systemd/system/ +systemctl daemon-reload +systemctl enable nvidia-start.service +systemctl start nvidia-start.service