Problem using aio functions and D

[Stef Bon <stefbon at gmail dot com>]

Hello,

I'm writing a fuse fs, and I want it to make use of aio. Now it's
using normal blocking reads and writes,
and this leads to blocking behaviour.

Since I want my fs to play an important role for a "GoboLinux" like
setup, see for screenshots:

http://linux.bononline.nl/wiki/index.php/Changes_and_issues


Now using various implementations of aio (first libaio) and now posix
aio, I never have been able
to make it run.

To try it I now have a simple program using signalfd, epoll and aio.
It's attached.
This program works very good, at least when it's compiled like:


gcc -Wall -D_GNU_SOURCE -lrt testsignal.c -o testsignal


It's a advanced version of the test program found in the manpage of signalfd.

now test it:

./testsignal
mainloop: adding signalfd 4 to epoll
^CGot SIGINT: start the test aio read.


the progress in logfile:

Jan 28 11:58:08 clfs20091030 testsignal: aio_read_file
Jan 28 11:58:08 clfs20091030 testsignal: aio_read return: 0
Jan 28 11:58:08 clfs20091030 testsignal: mainloop aio read, no error,
nbytes: 14196767
Jan 28 11:58:08 clfs20091030 testsignal: aio_write_file
Jan 28 11:58:08 clfs20091030 testsignal: aio_write return: 0
Jan 28 11:58:08 clfs20091030 testsignal: mainloop aio write, no error,
nbytes: 14196767


This is as expected.

Now when I compile it using the -D_FILE_OFFSET_BITS=64


gcc -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -lrt testsignal.c -o testsignal


the log when Ctrl-C is pressed (and the aio read is strarted:)

Jan 28 11:50:27 clfs20091030 testsignal: aio_read_file
Jan 28 11:50:27 clfs20091030 testsignal: aio_read return: 0
Jan 28 11:50:27 clfs20091030 testsignal: unknown aio opcode 128


Someone an idea??

Thanks in advance. This is maybe the problem with making aio work with fuse.
Fuse must be compiled using the -D_FILE_... flag.

Stef

#include <sys/signalfd.h>
#include <signal.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <syslog.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <aio.h>
#include <string.h>

#include <sys/epoll.h>


#define handle_error(msg) \
           do { perror(msg); exit(EXIT_FAILURE); } while (0)

#define MAXPOLLSIZE 1024
#define MAX_NUM_EVENTS 128

static unsigned pagesize;


// #define INOTIFY_EVENT_SIZE (sizeof (struct inotify_event))

// #define INOTIFY_BUF_LEN (1024 * (INOTIFY_EVENT_SIZE + 16))

static int aio_read_file(const char *path)
{
	int nreturn=0, res;
	struct aiocb *raiocb;
	struct stat st;
	unsigned fd;
	size_t size;
	size_t aiosize;
	char *buff;

	fd=open(path, O_RDONLY | O_LARGEFILE | O_NONBLOCK );

	if ( fd==-1 ) {

	  nreturn=-errno;
	  goto out;

	}

	res=fstat(fd, &st);

	if ( res==-1 ) {

	  nreturn=-errno;
	  goto out;

	}

	size=st.st_size;

	// to define a memalignied temporary buffer
	// size hase to be multiple of pagesize

	aiosize=((size-1)/pagesize + 1)*pagesize;

	syslog(LOG_DEBUG, "aio_read_file, aiosize: %lu", aiosize);

	raiocb=malloc(sizeof(struct aiocb));

	if ( raiocb ) {

	    memset(raiocb, 0, sizeof(struct aiocb));

	} else {

	    nreturn=-ENOMEM;
	    goto out;

	}

	// setup file options

	raiocb->aio_fildes=fd;

	nreturn=posix_memalign(&buff, pagesize, aiosize);

	if ( nreturn!=0 ) {

	    nreturn=-abs(nreturn);
	    if ( raiocb ) free (raiocb);
	    goto out;

	}

	memset(buff, 0, aiosize);

	raiocb->aio_buf=buff;

	if ( ! raiocb->aio_buf ) {

	    nreturn=-ENOMEM;
	    if ( raiocb ) free(raiocb);
	    goto out;

	}

	raiocb->aio_nbytes=size;
	raiocb->aio_offset=0;

	// setup signal options

	raiocb->aio_sigevent.sigev_notify = SIGEV_SIGNAL;
	raiocb->aio_sigevent.sigev_signo = SIGIO;
	raiocb->aio_sigevent.sigev_value.sival_ptr = raiocb;

	// launch aio read

	nreturn=aio_read( raiocb );

	if ( nreturn==-1 ) nreturn=-errno;

	out:

	syslog(LOG_DEBUG, "aio_read return: %i", nreturn);

	return nreturn;

}

static int aio_write_file(char *buff, size_t size)
{
	int nreturn=0;
	struct aiocb *waiocb;
	unsigned fd;
	char *path="/tmp/test2";

	fd=open(path, O_WRONLY | O_CREAT , 0666);

	if ( fd==-1 ) {

	  nreturn=-errno;
	  goto out;

	}

	syslog(LOG_DEBUG, "aio_write_file");

	waiocb=malloc(sizeof(struct aiocb));


	if ( waiocb ) {

	    memset(waiocb, 0, sizeof(struct aiocb));

	} else {

	    nreturn=-ENOMEM;
	    goto out;

	}


	// setup file options

	waiocb->aio_fildes=fd;
	waiocb->aio_buf=buff;

	if ( ! waiocb->aio_buf ) {

	    nreturn=-ENOMEM;
	    if ( waiocb ) free(waiocb);
	    goto out;

	}

	waiocb->aio_nbytes=size;
	waiocb->aio_offset=0;

	// setup signal options

	waiocb->aio_sigevent.sigev_notify = SIGEV_SIGNAL;
	waiocb->aio_sigevent.sigev_signo = SIGIO;
	waiocb->aio_sigevent.sigev_value.sival_ptr = waiocb;

	// launch aio read

	nreturn=aio_write( waiocb );

	if ( nreturn==-1 ) nreturn=-errno;

	out:

	syslog(LOG_DEBUG, "aio_write return: %i", nreturn);

	return nreturn;

}



int main(int argc, char *argv[])
{
  sigset_t mask;
  int signal_fd, epoll_fd;
  struct signalfd_siginfo fdsi;
  ssize_t s;
  struct epoll_event epoll_events[MAXPOLLSIZE];
  int i, res, nerror;
  struct aiocb *testaiocb;
  char testfile[]="/tmp/test1";


  epoll_fd=epoll_create(MAXPOLLSIZE);


  sigemptyset(&mask);
  
  sigaddset(&mask, SIGINT);
  sigaddset(&mask, SIGQUIT);
  sigaddset(&mask, SIGIO);
  sigaddset(&mask, SIGHUP);
  sigaddset(&mask, SIGTERM);
  sigaddset(&mask, SIGPIPE);
  
  
  /* Block signals so that they aren't handled
    according to their default dispositions */

  if (sigprocmask(SIG_BLOCK, &mask, NULL) == -1) handle_error("sigprocmask");
  
  
  signal_fd = signalfd(-1, &mask, 0);
  if (signal_fd == -1) handle_error("signalfd");

  
  if ( signal_fd>0 ) {

    // there is an signal fd
    // add to the list of fd's for epoll to listen to

    fprintf(stdout, "mainloop: adding signalfd %i to epoll\n", signal_fd);

    static struct epoll_event ee_signalfd;
    ee_signalfd.events=EPOLLIN;
    ee_signalfd.data.fd=signal_fd;

    res=epoll_ctl(epoll_fd, EPOLL_CTL_ADD, signal_fd, &ee_signalfd);

    if ( res==-1 ) {

      res=-errno;
      exit(1);

    }

  } 
  
  pagesize=getpagesize();
  
  for (;;) {
  
    int number_of_fds=epoll_wait(epoll_fd, epoll_events, MAXPOLLSIZE, -1);

    
    for (i=0; i<number_of_fds; i++) {

      int fd=epoll_events[i].data.fd;


      if ( signal_fd>0 && fd == signal_fd ) {

	// read the signal

	s = read(signal_fd, &fdsi, sizeof(struct signalfd_siginfo));
    
	if (s != sizeof(struct signalfd_siginfo)) handle_error("read");


	if (fdsi.ssi_signo == SIGINT) {

	  fprintf(stdout, "Got SIGINT: start the test aio read.\n");

	  // start the aio read of a test file

	  aio_read_file(testfile);
     
	} else if (fdsi.ssi_signo == SIGQUIT) {
	  

	  fprintf(stdout, "Got SIGQUIT\n");
	  exit(EXIT_SUCCESS);
      
	} else if ( fdsi.ssi_signo == SIGIO ) {



	  testaiocb=(struct aiocb *) (uintptr_t) fdsi.ssi_ptr;


	  if ( ! testaiocb ) {

	    syslog(LOG_DEBUG, "error, aiocb not set....");
	    continue;

	  }


	  //
	  // process the request here
	  //


	  nerror=aio_error(testaiocb);


	  if ( testaiocb->aio_lio_opcode==LIO_READ ) {

	    // aio read 

	    if ( nerror==0 ) {

	      // no error

	      syslog(LOG_DEBUG, "mainloop aio read, no error, nbytes: %zi", aio_return(testaiocb));

	      // turn the aio read in a write

	      aio_write_file(testaiocb->aio_buf, testaiocb->aio_nbytes);


	    } else {

	      // error

	      syslog(LOG_DEBUG, "aio read, error %i", nerror);


	    }

	    // if ( testaiocb->aio_buf ) free(testaiocb->aio_buf);
	    // if ( testaiocb->aio_fildes>0 ) close(testaiocb->aio_fildes);



	  } else if ( testaiocb->aio_lio_opcode==LIO_WRITE ) {


	    // aio write 

	    if ( nerror==0 ) {

	      // no error

	      syslog(LOG_DEBUG, "mainloop aio write, no error, nbytes: %zu", aio_return(testaiocb));


	    } else {

	      // error

	      syslog(LOG_DEBUG, "aio write, error %i", nerror);


	    }

	    if ( testaiocb->aio_fildes>0 ) close(testaiocb->aio_fildes);
	    if ( testaiocb->aio_buf ) free(testaiocb->aio_buf);

	  } else {

	    syslog(LOG_DEBUG, "unknown aio opcode %i", testaiocb->aio_lio_opcode);

	  }

	  // free the aiocb

	  free(testaiocb);

	}

      }

    }

  }

  exit(1);

}