Upgrade nvidia-docker to nvidia-docker2

Author: alfpark

contrib/packer/ubuntu-16.04-GPU+IB/bootstrap.sh

@@ -62,39 +62,13 @@ EOF
     chmod 755 $nvdriver
     $nvdriver -s
     # install nvidia-docker
-    curl -OfSsL https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/nvidia-docker_1.0.1-1_amd64.deb
-    dpkg -i nvidia-docker_1.0.1-1_amd64.deb
-    rm nvidia-docker_1.0.1-1_amd64.deb
-    # do not auto-enable nvidia docker service
-    systemctl disable nvidia-docker.service
-    systemctl start nvidia-docker.service
-    systemctl status nvidia-docker.service
-    # get driver version
-    nvdriverver=`cat /proc/driver/nvidia/version | grep "Kernel Module" | cut -d ' ' -f 9`
-    echo nvidia driver version $nvdriverver detected
-    # create the docker volume now to avoid volume driver conflicts for
-    # tasks. run this in a loop as it can fail if triggered too quickly
-    # after start
-    NV_START=$(date -u +"%s")
-    set +e
-    while :
-    do
-        echo "Attempting to create nvidia-docker volume with version $nvdriverver"
-        docker volume create -d nvidia-docker --name nvidia_driver_$nvdriverver
-        if [ $? -eq 0 ]; then
-            break
-        else
-            NV_NOW=$(date -u +"%s")
-            NV_DIFF=$((($NV_NOW-$NV_START)/60))
-            # fail after 5 minutes of attempts
-            if [ $NV_DIFF -ge 5 ]; then
-                echo "could not create nvidia-docker volume"
-                exit 1
-            fi
-            sleep 1
-        fi
-    done
-    set -e
+    curl -fSsL https://nvidia.github.io/nvidia-docker/gpgkey | apt-key add -
+    curl -fSsL https://nvidia.github.io/nvidia-docker/ubuntu16.04/amd64/nvidia-docker.list | \
+        tee /etc/apt/sources.list.d/nvidia-docker.list
+    apt-get update
+    apt-get install nvidia-docker2
+    pkill -SIGHUP dockerd
+    nvidia-docker version
 fi
 set -e
 

contrib/packer/ubuntu-16.04-GPU/bootstrap.sh

@@ -60,38 +60,12 @@ EOF
     chmod 755 $nvdriver
     $nvdriver -s
     # install nvidia-docker
-    curl -OfSsL https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/nvidia-docker_1.0.1-1_amd64.deb
-    dpkg -i nvidia-docker_1.0.1-1_amd64.deb
-    rm nvidia-docker_1.0.1-1_amd64.deb
-    # do not auto-enable nvidia docker service
-    systemctl disable nvidia-docker.service
-    systemctl start nvidia-docker.service
-    systemctl status nvidia-docker.service
-    # get driver version
-    nvdriverver=`cat /proc/driver/nvidia/version | grep "Kernel Module" | cut -d ' ' -f 9`
-    echo nvidia driver version $nvdriverver detected
-    # create the docker volume now to avoid volume driver conflicts for
-    # tasks. run this in a loop as it can fail if triggered too quickly
-    # after start
-    NV_START=$(date -u +"%s")
-    set +e
-    while :
-    do
-        echo "Attempting to create nvidia-docker volume with version $nvdriverver"
-        docker volume create -d nvidia-docker --name nvidia_driver_$nvdriverver
-        if [ $? -eq 0 ]; then
-            break
-        else
-            NV_NOW=$(date -u +"%s")
-            NV_DIFF=$((($NV_NOW-$NV_START)/60))
-            # fail after 5 minutes of attempts
-            if [ $NV_DIFF -ge 5 ]; then
-                echo "could not create nvidia-docker volume"
-                exit 1
-            fi
-            sleep 1
-        fi
-    done
-    set -e
+    curl -fSsL https://nvidia.github.io/nvidia-docker/gpgkey | apt-key add -
+    curl -fSsL https://nvidia.github.io/nvidia-docker/ubuntu16.04/amd64/nvidia-docker.list | \
+        tee /etc/apt/sources.list.d/nvidia-docker.list
+    apt-get update
+    apt-get install nvidia-docker2
+    pkill -SIGHUP dockerd
+    nvidia-docker version
 fi
 set -e

convoy/fleet.py

@@ -63,30 +63,6 @@
 _REQUEST_CHUNK_SIZE = 4194304
 _ROOT_PATH = pathlib.Path(__file__).resolve().parent.parent
 _RESOURCES_PATH = None
-__NVIDIA_DOCKER_RPM = {
-    'url': (
-        'https://github.com/NVIDIA/nvidia-docker/releases/download/'
-        'v1.0.1/nvidia-docker-1.0.1-1.x86_64.rpm'
-    ),
-    'sha256': (
-        'f05dfe7fe655ed39c399db0d6362e351b059f2708c3e6da17f590a000237ec3a'
-    ),
-    'target': 'nvidia-docker.rpm'
-}
-_NVIDIA_DOCKER = {
-    'ubuntuserver': {
-        'url': (
-            'https://github.com/NVIDIA/nvidia-docker/releases/download/'
-            'v1.0.1/nvidia-docker_1.0.1-1_amd64.deb'
-        ),
-        'sha256': (
-            '9fbfd98f87ef2fd2e2137e3ba59431890dde6caf96f113ea0a1bd15bb3e51afa'
-        ),
-        'target': 'nvidia-docker.deb'
-    },
-    'centos': __NVIDIA_DOCKER_RPM,
-    'centos-hpc': __NVIDIA_DOCKER_RPM,
-}
 _NVIDIA_DRIVER = {
     'compute': {
         'url': (
@@ -304,36 +280,6 @@ def _setup_nvidia_driver_package(blob_client, config, vm_size):
     return pkg
 
 
-def _setup_nvidia_docker_package(blob_client, config):
-    # type: (azure.storage.blob.BlockBlobService, dict) -> pathlib.Path
-    """Set up the nvidia docker package
-    :param azure.storage.blob.BlockBlobService blob_client: blob client
-    :param dict config: configuration dict
-    :rtype: pathlib.Path
-    :return: package path
-    """
-    offer = settings.pool_offer(config, lower=True)
-    pkg = _RESOURCES_PATH / _NVIDIA_DOCKER[offer]['target']
-    # check to see if package is downloaded
-    if (not pkg.exists() or
-            util.compute_sha256_for_file(pkg, False) !=
-            _NVIDIA_DOCKER[offer]['sha256']):
-        # download package
-        logger.debug('downloading NVIDIA docker to {}'.format(
-            _NVIDIA_DOCKER[offer]['target']))
-        response = requests.get(_NVIDIA_DOCKER[offer]['url'], stream=True)
-        with pkg.open('wb') as f:
-            for chunk in response.iter_content(chunk_size=_REQUEST_CHUNK_SIZE):
-                if chunk:
-                    f.write(chunk)
-        logger.debug('wrote {} bytes to {}'.format(pkg.stat().st_size, pkg))
-        # check sha256
-        if (util.compute_sha256_for_file(pkg, False) !=
-                _NVIDIA_DOCKER[offer]['sha256']):
-            raise RuntimeError('sha256 mismatch for {}'.format(pkg))
-    return pkg
-
-
 def _generate_azure_mount_script_name(
         batch_account_name, pool_id, is_file_share, is_windows):
     # type: (str, str, bool, bool) -> pathlib.Path
@@ -1022,12 +968,9 @@ def _construct_pool_object(
             gpu_type = settings.get_gpu_type_from_vm_size(
                 pool_settings.vm_size)
             gpu_driver = pathlib.Path(_NVIDIA_DRIVER[gpu_type]['target'])
-        gpupkg = _setup_nvidia_docker_package(blob_client, config)
-        _rflist.append((gpupkg.name, gpupkg))
-        gpu_env = '{}:{}:{}'.format(
+        gpu_env = '{}:{}'.format(
             settings.is_gpu_visualization_pool(pool_settings.vm_size),
-            gpu_driver.name,
-            gpupkg.name)
+            gpu_driver.name)
     else:
         gpu_env = None
     # get container registries
@@ -2006,8 +1949,9 @@ def _adjust_settings_for_pool_creation(config):
             config, vm_size=pool.vm_size)
     if not allowed and util.is_none_or_empty(node_agent):
         raise ValueError(
-            ('unsupported Docker Host VM Config, publisher={} offer={} '
-             'sku={} vm_size={}').format(publisher, offer, sku, pool.vm_size))
+            ('unsupported Docker (and/or GPU) Host VM Config, publisher={} '
+             'offer={} sku={} vm_size={}').format(
+                 publisher, offer, sku, pool.vm_size))
     # ensure HPC offers are matched with RDMA sizes
     if (not is_windows and (
             (offer == 'centos-hpc' or offer == 'sles-hpc') and

docs/63-batch-shipyard-custom-images.md

@@ -167,9 +167,10 @@ be invocable as root with default path and permissions. The Docker socket
 (`/var/run/docker.sock`) must be available (it is available by default).
 
 **Important Note:** If you have modified the Docker Root directory to
-mount on the node local temporary disk, then you must not enable the
+mount on the node local temporary disk, then you must disable the
 service to run on boot due to potential races with the disk not being
-set up properly before the service starts.
+set up before the service starts. Batch Shipyard will take care of properly
+starting the service on boot.
 
 #### SSH Server
 An SSH server should be installed and operational on port 22. You can
@@ -182,14 +183,9 @@ In order to utilize the GPUs available on compute nodes that have them
 (e.g., N-series VMs), the NVIDIA driver must be installed and loaded upon
 boot.
 
-Additionally, [nvidia-docker](https://github.com/NVIDIA/nvidia-docker)
+Additionally, [nvidia-docker2](https://github.com/NVIDIA/nvidia-docker)
 must be installed.
 
-**Important Note:** If you have modified the Docker Root directory to
-mount on the node local temporary disk, then you must not enable the
-nvidia-docker service to run on boot due to potential races with the disk
-not being set up properly before the service starts.
-
 #### Infiniband/RDMA-enabled Compute Nodes
 The host VM Infiniband/RDMA stack must be enabled with the proper drivers
 and the required user-land software for Infiniband installed. It is best to