El registro de la memoria del dispositivo de caracteres de Linux mapeado con cudaHostRegister da como resultado un argumento no válido
Estoy tratando de aumentar la transferencia de datos DMA <-> CPU <-> GPU: 1. Mapeando mi dispositivo (propietario) Linux Kernel asignó memoria al espacio de usuario 2. Registrando el último (memoria mapeada) a Cuda con la función API cudaHostRegister.
Mientras se asigna la memoria asignada del espacio de usuario asignada a mi dispositivo DMA y luego registrada en Cuda con cudaHostRegister funciona bien, al intentar registrar la memoria "kmalloced" se produce un error de "argumento no válido" devuelto por cudaHostRegister.
Primero, pensé que el problema era con la alineación o que el controlador de mi dispositivo complicaba la gestión de la agrupación de memoria, así que escribí un dispositivo de caracteres más simple que implementa .mmap () donde el búfer kzalloced de 10Kb se reasigna con remap_pfn_range y el problema sigue en pie.
Desafortunadamente, no encontré preguntas similares en la red, así que sinceramente espero encontrar una respuesta aquí.
Parte de la información del sistema y el controlador del kernel <-> código de aplicación de espacio de usuario + información de registro de tiempo de ejecución:
CUDA : 8.0
OS Dist : Ubuntu 14.04
Kernel : 3.16.0-31-generic
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 375.26 Driver Version: 375.26 |
|-------------------------------+----------------------+----------------------+
| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |
|===============================+======================+======================|
| 0 GeForce GTX 770 Off | 0000:83:00.0 N/A | N/A |
| 26% 32C P8 N/A / N/A | 79MiB / 1997MiB | N/A Default |
+-------------------------------+----------------------+----------------------+
Código de mmap () del dispositivo de caracteres:
#define MEM_CHUNK_SIZE 4 * _K
#define MEM_POOL_SIZE 10 * _K
/**/
static int chdv_mmap(struct file *filp, struct vm_area_struct *vma)
{
unsigned int pages_per_buf = ( MEM_CHUNK_SIZE >> PAGE_SHIFT ) ;
unsigned long pfn, vsize;
/*make sure the buffer is allocated*/
if((NULL == g_membuff) &&
(NULL == (g_membuff = kzalloc(MEM_POOL_SIZE , GFP_KERNEL))))
{
kdbgprintln("Error: Not enough memory");
return -ENOMEM;
}
vsize = vma->vm_end - vma->vm_start ;
kdbgprintln("MEM_CHUNK_SIZE %u, pages_per_buf %u, vsize %lu vma->vm_pgoff %lu",
MEM_CHUNK_SIZE,
pages_per_buf,
vsize,
vma->vm_pgoff);
if(vsize > MEM_POOL_SIZE)
{
kdbgprintln("Error: vsize %lu > MEM_POOL_SIZE %u", vsize, MEM_POOL_SIZE);
return -EINVAL;
}
/* We allow only mapping of one whole buffer so offset must be multiple
* of pages_per_buf and size must be equal to dma_buf_size.
*/
if( vma->vm_pgoff % pages_per_buf )
{
kdbgprintln("Error:Mapping DMA buffers is allowed only from beginning");
return -EINVAL ;
}
vma->vm_flags = vma->vm_flags | (VM_DONTEXPAND | VM_LOCKED | VM_IO);
/*Get the PFN for remap*/
pfn = page_to_pfn(virt_to_page((unsignedcudaHostRegister char *)g_membuff));
kdbgprintln("PFN : %lu", pfn);
if(remap_pfn_range(vma, vma->vm_start, pfn, vsize, vma->vm_page_prot))
{
kdbgprintln("Error:Failed to remap memory");
return -EINVAL;
}
/*Sealing data header & footer*/
*((unsigned long *)g_membuff) = 0xCDFFFFFFFFFFFFAB;
*((unsigned long *)g_membuff + 1) = 0xAB000000000000EF;
*(unsigned long *)((unsigned char *)g_membuff + vsize - sizeof(unsigned long)) = 0xEF0000000C0000AA;
kdbgprintln("Mapped 'kalloc' buffer" \
"\n\t\tFirst 8 bytes: %lX" \
"\n\t\tSecond 8 bytes: %lX" \
"\n\t\tLast 8 bytes: %lX",
*((unsigned long *)g_membuff),
*((unsigned long *)g_membuff + 1),
*(unsigned long *)((unsigned char *)g_membuff + vsize - sizeof(unsigned long)));
return 0;
}
Código de aplicación de prueba:
static unsigned long map_mem_size;
int main(int argc, char** argv)
{
int fd;
const char dev_name[] = "/dev/chardev";
void * address = NULL;
long page_off = 0;
cudaError_t cudarc;
switch(argc)
{
case 2:
page_off = atoi(argv[1]) * getpagesize();
break;
default:
page_off = 0;
break;
}
map_mem_size = 2 * getpagesize();
printf("Opening %s file\n", dev_name);
errno = 0;
if(0 > (fd = open(dev_name, O_RDWR) ))
{
printf("Error %d - %s\n", errno, strerror(errno));
}
else
{
printf("About to map %lu bytes of %s device memory\n", map_mem_size, dev_name);
errno = 0;
if(MAP_FAILED == (address = mmap(NULL, map_mem_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, page_off)))
{
printf("Error %d - %s\n", errno, strerror(errno));
}
else
{
printf("mapped %s driver 'kmalloc' memory" \
"\n\t\tFirst 8 bytes : %lX" \
"\n\t\tSecond 8 bytes: %lX" \
"\n\t\tLast 8 bytes: %lX\n",
dev_name,
*((unsigned long *)address),
*((unsigned long *)address + 1),
*(unsigned long *)((unsigned char *)address + map_mem_size - sizeof(unsigned long)));
if (cudaSuccess != (cudarc = cudaHostRegister(address, map_mem_size, cudaHostRegisterDefault)))
{
printf("Error: Failed cudaHostRegister: %s, address %p\n", cudaGetErrorString(cudarc), address);
}
}
}
/*Release resources block*/
return EXIT_SUCCESS;
}
Información de depuración en tiempo de ejecución:
Espacio de usuario:
./chrdev_test
Opening /dev/chardev file
About to map 8192 bytes of /dev/chardev device memory
mapped /dev/chardev driver 'kmalloc' memory
First 8 bytes : CDFFFFFFFFFFFFAB
Second 8 bytes: AB000000000000EF
Last 8 bytes: EF0000000C0000AA
Error: Failed cudaHostRegister: invalid argument
Unmapping /dev/chardev file
Closing /dev/chardev file
Espacio del kernel (tail -f / var / log / syslog):
[ 4814.119537] [chardev] chardev.c, chdv_mmap, line 292:MEM_CHUNK_SIZE 4096, pages_per_buf 1, vsize 8192 vma->vm_pgoff 0
[ 4814.119538] [chardev] chardev.c, chdv_mmap, line 311:PFN : 16306184
[ 4814.119543] [chardev] chardev.c, chdv_mmap, line 330:Mapped 'kzalloced' buffer
[ 4814.119543] First 8 bytes: CDFFFFFFFFFFFFAB
[ 4814.119543] Second 8 bytes: AB000000000000EF
[ 4814.119543] Last 8 bytes: EF0000000C0000AA
Gracias por delante