aoe-sancheck.c

/* Copyright Coraid, Inc. 2010.  All Rights Reserved. */

#include <stdio.h>
#include <sched.h>
#include <stdlib.h>
#include <unistd.h>
#include <signal.h>
#include <string.h>
#include <time.h>
#include <stdlib.h>
#include <sys/ioctl.h>
#include <pthread.h>
#include <poll.h>
#include <errno.h>

#include <netpacket/packet.h>
#include <net/ethernet.h>
#include <sys/types.h>
#include <net/if.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/socket.h>

#define	nelem(x)	(sizeof(x)/sizeof((x)[0]))
#define nil NULL
#define vprintf(...) if (qflag) ; else fprintf(stderr, __VA_ARGS__)
#define mintu(x) ((x) > 60 ? x : 60)

typedef unsigned char uchar;
typedef unsigned long ulong;

typedef struct Aoe Aoe;
typedef struct Qc Qc;
typedef struct Ata Ata;
typedef struct Lun Lun;
typedef struct Eth Eth;
typedef struct Targ Targ;
typedef struct Mac Mac;

struct Aoe {
	uchar dst[6];
	uchar src[6];
	uchar type[2];
	uchar flags;
	uchar error;
	uchar major[2];
	uchar minor;
	uchar cmd;
	uchar tag[4];
};

struct Qc {
	Aoe h;
	uchar bufcnt[2];
	uchar firmware[2];
	uchar scnt;
	uchar vercmd;
	uchar len[2];
//	uchar data[1024];
};


struct Ata {
	Aoe h;
	uchar aflag;
	uchar err;
	uchar scnt;
	uchar cmd;
	uchar lba[6];
	uchar res[2];
};

struct Lun {
	Lun *next;
	int state;
	char ea[6];
	int major;
	int minor;
	int nsect;
	int maxsect;
	int id;
};

struct Eth {
	Lun *luns;
	int fd;
	char *name;
	char ea[6];
	int mtu;
	int up;
	uchar pkt[16*1024];
};

struct Targ {
	Targ *next;
	int major;
	int minor;
};

struct Mac {
	Mac *next;
	char ea[6];
};

enum {
	Neth= 16,
	Nstack= 16*1024,
	Nws= 5,

	Lnew= 0,
	Lprobe,

	Arsp= (1<<3),
	Aerr= (1<<2),

	Cata= 0,
	Cqc= 1,

	ETaoe= 0x88a2,
};

int ethlist(char **, int);
int ethopen(Eth *);
void *jcheck(void *);
int jinput(Eth *);
Lun *findlun(Eth *e, Aoe *a);
void jprobe(Eth *e, Lun *lun);
void printlist(void);
void printsancheck(void);
ulong nhgetl(uchar *);
ushort nhgets(uchar *);
void hnputs(uchar *, ushort);
void hnputl(uchar *, ulong);
void *mallocz(int);
void inserttarg(int, int);
void insertmac(Mac **, char *);
void sancheck(int, int);
void ifsummary(void);
int ifup(char *);
void inserteth(char **, int, char *);
char *getpciid(char *, int, char *);

Eth eth[Neth];
int waitsecs = Nws;
pthread_t threads[Neth];
Targ *targlist;

int
main(int argc, char **argv)
{
	int n, i;
	char *ethnames[Neth];
	Eth *e;

	memset(ethnames, 0, sizeof ethnames);
	printf("Probing...");
	fflush(0);
	n = ethlist(ethnames, nelem(ethnames));
	for (i=0; i<n; i++) {
		e = &eth[i];
		e->name = ethnames[i];
		e->up = ifup(ethnames[i]);
		if (pthread_create(&threads[i], 0, jcheck, e)) {
			fprintf(stderr, "pthread_create failed.\n");
			break;
		}
	}
	n = i;
	for (i=0; i<n; i++)
		pthread_join(threads[i], 0);
	printf("done.\n");
	printsancheck();
	if (argc == 2 && strcmp(argv[1], "-v") == 0)
		printlist();
	return 0;
}

char *
cea(char *op, char *ea)
{
	int i;
	char *hex = "0123456789abcdef", *p;

	p = op;
	for (i=0; i<6; i++) {
		*p++ = hex[((ea[i] >> 4) & 0xf)];
		*p++ = hex[ea[i] & 0xf];
	}
	*p = 0;
	return op;
}

void
printlist(void)
{
	Eth *e;
	Lun *lun;
	char mac[13];

	for (e=eth; e->name; e++) {
		printf("%s:\n", e->name);
		for (lun=e->luns; lun; lun=lun->next)
			printf("e%d.%d %s %d\n", lun->major, lun->minor, cea(mac, lun->ea), lun->maxsect);
		printf("\n");
	}
}

void
timewait(int secs)  /* arrange for a sig_alarm signal after `secs' seconds */
{
    struct sigaction sa;
	void catch(int);

    memset(&sa, 0, sizeof sa);
    sa.sa_handler = catch;
    sa.sa_flags = SA_RESETHAND;
    sigaction(SIGALRM, &sa, NULL);
    alarm(secs);
}

int
discover(Eth *e)
{
	Aoe *a;
	Qc *q;

	memset(e->pkt, 0, sizeof e->pkt);
	a = (Aoe *) e->pkt;
	memset(a->dst, 0xff, 6);
	memmove(a->src, e->ea, 6);
	hnputs(a->type, ETaoe);
	hnputl(a->tag, 1<<31);
	hnputs(a->major, 0xffff);
	a->minor = 0xff;
	a->cmd = Cqc;
	a->flags = 0x10;
	if (write(e->fd, a, mintu(sizeof *q)) <= 0)
		return -1;
	return 0;
}

void *
jcheck(void *v)
{
	Eth *e = v;
	int n;
	time_t t, nt;
	struct pollfd pd;

	if (ethopen(e) < 0)
		return 0;
	if (discover(e) < 0) {
		fprintf(stderr, "skipping %s, discover failure: %s\n", e->name, strerror(errno));
		return 0;
	}
		
	pd.fd = e->fd;
	pd.events = POLLIN;
	t = time(0);
	for (;;) {
		nt = time(0);
		if (nt-t >= waitsecs)
			return 0;
		if (poll(&pd, 1, waitsecs*1000) > 0)
		if ((n = read(e->fd, e->pkt, sizeof e->pkt)) > 0)
		if (jinput(e))
			t = time(0);
	}
}

/* return 1 == useful, 0 == not useful */
int
jinput(Eth *e)
{
	Aoe *a;
	Qc *q;
	Lun *lun;
	ulong tag;
	int n;

	a = (Aoe *) e->pkt;
	if ((a->flags & (Arsp|Aerr)) != Arsp)
		return 0;
	if ((a->tag[0] & 0x80) == 0)
		return 0;
	tag = nhgetl(a->tag);
	switch (a->cmd) {
	case Cqc:
		q = (Qc *) a;
		lun = findlun(e, a);
		if (lun->state == Lnew) {
			lun->nsect = q->scnt;
			jprobe(e, lun);
			lun->state = Lprobe;
			break;
		}
		return 0;
	case Cata:
		lun = findlun(e, a);
		if (lun == nil)
			return 0;
		if (lun->id != tag>>16) {
			printf("lun->id %d != tag %ld for %d.%d\n", lun->id, tag>>16, lun->major, lun->minor);
			return 0;
		}
		n = tag & 0xff;
		if (n > lun->maxsect)
			lun->maxsect = n;
		break;
	default:
		return 0;
	}
	return 1;
}

void
hnputl(uchar *p, ulong n)
{
	*p++ = n >> 24;
	*p++ = n >> 16;
	*p++ = n >> 8;
	*p = n;
}

void
hnputs(uchar *p, ushort s)
{
	*p++ = s >> 8;
	*p = s;
}

ushort
nhgets(uchar *p)
{
	ushort s;

	s = *p++;
	s <<= 8;
	s += *p++ & 0xff;
	return s;
}

ulong
nhgetl(uchar *p)
{
	ulong n;

	n = *p++;
	n <<= 8;
	n += *p++ & 0xff;
	n <<= 8;
	n += *p++ & 0xff;
	n <<= 8;
	n += *p++ & 0xff;
	return n;
}

void
jprobe(Eth *e, Lun *lun)
{
	Aoe *a;
	Ata *aa;
	int n;

	memset(e->pkt, 0, sizeof e->pkt);
	a = (Aoe *) e->pkt;
	aa = (Ata *) a;
	memcpy(a->dst, lun->ea, 6);
	memcpy(a->src, e->ea, 6);
	hnputs(a->type, ETaoe);
	hnputs(a->major, lun->major);
	a->minor = lun->minor;
	a->flags = 0x10;
	hnputl(a->tag, lun->id<<16);
	aa->cmd = 0xec;
	aa->scnt = 1;
	n = e->mtu - sizeof *aa;
	for (n &= ~511; n > 0; n -= 512) {
		a->tag[3] = n/512;
		if (write(e->fd, a, sizeof *aa + n) <= 0) {
			printf("write failed\n");
		}
		usleep(100);
	}
}

Lun *
findlun(Eth *e, Aoe *a)
{
	Lun *p, **pp;
	int maj, n;
	static int id;

	maj = nhgets(a->major);
	pp = &e->luns;
	for (; (p=*pp); pp=&p->next) {
		if (maj < p->major)
			continue;
		if (maj > p->major)
			break;
		if (a->minor < p->minor)
			continue;
		if (a->minor > p->minor)
			break;
		n = memcmp(p->ea, a->src, 6);
		if (n < 0)
			continue;
		if (n > 0)
			break;
		return p;
	}
	if (a->cmd == Cata)
		return nil;
	p = mallocz(sizeof *p);
	p->major = maj;
	p->minor = a->minor;
	memmove(p->ea, a->src, 6);
	p->next = *pp;
	p->id = 0x8000 | id++;
	inserttarg(p->major, p->minor);
	return *pp = p;
}	
	

void
catch(int sig)
{
}

int
getindx(int sfd, char *name)  // return the index of device 'name'
{
	struct ifreq xx;
	int n;

    strcpy(xx.ifr_name, name);
    n = ioctl(sfd, SIOCGIFINDEX, &xx);
    if (n == -1)
        return -1;
    return xx.ifr_ifindex;
}

int
getmtu(Eth *e)
{
    struct ifreq xx;
    int n;

    strcpy(xx.ifr_name, e->name);
    n = ioctl(e->fd, SIOCGIFMTU, &xx);
    if (n == -1) {
        perror("Can't get mtu");
        return 1500;
    }
    return xx.ifr_mtu;
}

int
ethopen(Eth *e)	  // get us a raw connection to an interface
{   
	int n, sfd, rbsz, sbsz;
	struct sockaddr_ll sa;
	struct ifreq xx;
	
	rbsz = 64*1024*1024;
	sbsz = 64*1024*1024; 
	memset(&sa, 0, sizeof sa);
	memset(&xx, 0, sizeof xx);
	sfd = socket(PF_PACKET, SOCK_RAW, htons(ETaoe));
	if (sfd == -1) {
		perror("got bad socket");
		return -1;
	}
	if (setsockopt(sfd, SOL_SOCKET, SO_RCVBUFFORCE, &rbsz, sizeof rbsz) < 0)
		fprintf(stderr, "Failed to set socket rcvbuf size\n");
	if (setsockopt(sfd, SOL_SOCKET, SO_SNDBUFFORCE, &sbsz, sizeof sbsz) < 0)
		fprintf(stderr, "Failed to set socket sndbuf size\n");
	n = getindx(sfd, e->name);
	sa.sll_family = AF_PACKET;
	sa.sll_protocol = htons(ETaoe);
	sa.sll_ifindex = n;
	n = bind(sfd, (struct sockaddr *)&sa, sizeof sa);
	if (n == -1) {
		perror("bind funky");
		return -1;
	}
	strcpy(xx.ifr_name, e->name);
	n = ioctl(sfd, SIOCGIFHWADDR, &xx);
	if (n == -1) { 
		perror("Can't get hw addr");
		return -1;
	}
	memmove(e->ea, xx.ifr_hwaddr.sa_data, 6);
	e->fd = sfd;
	e->mtu = getmtu(e);
	return 0;
}

int
ethlist(char **ifs, int nifs)
{
	int i, s, n;
	struct ifreq ifr;

	s = socket(AF_INET, SOCK_STREAM, 0);
	if (s < 0)
		return 0;
	n = 0;
	for (i=0; i<nifs; i++) {
		memset(&ifr, 0, sizeof ifr);
		ifr.ifr_ifindex = i;
		if (ioctl(s, SIOCGIFNAME, &ifr) < 0)
			continue;
		if (strncmp(ifr.ifr_name, "eth", 3))
			continue;
		inserteth(ifs, nifs, ifr.ifr_name);
		n++;
	}
	close(s);
	return n;
}

void *
mallocz(int sz)
{
	void *p;

	p = malloc(sz);
	memset(p, 0, sz);
	return p;
}

void
inserttarg(int maj, int min)
{
	Targ *nt, *t, *p;

	nt = mallocz(sizeof *nt);
	nt->major = maj;
	nt->minor = min;
	if (targlist == NULL) {
		targlist = nt;
		return;
	}
	if (nt->major < targlist->major) { 
		nt->next = targlist;
		targlist = nt;
		return;
	} else if (nt->major == targlist->major && nt->minor < targlist->minor) {
		nt->next = targlist;
		targlist = nt;
		return;
	} else if (nt->major == targlist->major && nt->minor == targlist->minor) 
		return;
	for (p = targlist,t = targlist->next; t; p = t,t = t->next) {
		if (nt->major == t->major && nt->minor == t->minor)
			return;
		if (nt->major < t->major) { 
			p->next = nt;
			nt->next = t;
			return;
		} else if (nt->major == t->major && nt->minor < t->minor) {
				p->next = nt;
				nt->next = t;
				return;
		}
	}
	p->next = nt;
}

void
printsancheck()
{
	Targ *t;

	printf("==========================================\n");
	printf("INTERFACE SUMMARY\n");
	printf("==========================================\n");
	printf("Name\tStatus\tMTU\tPCI ID\n");
	ifsummary();
	printf("==========================================\n");
	printf("DEVICE SUMMARY\n");
	printf("==========================================\n");
	printf("Device\tMacs\tPayload\tLocal Interfaces\n");
	for (t = targlist; t; t = t->next) {
		sancheck(t->major, t->minor);
	}
}

void
ifsummary()
{
	Eth *e;
	char buf[32];
	char *p;

	for (e=eth; e->name; e++) { 
		p = getpciid(buf, sizeof buf, e->name);
		printf("%s\t%s\t%d\t%s\n", e->name, (e->up ? "UP" : "DN"), e->mtu, (p ? p : ""));
	}

}

void
sancheck(int maj, int min)
{
	Eth *e;
	Lun *l;
	Mac *ml, *m;
	int a, found;
	int ps, nsect, nea, nloc;
	int mtu;
	char buf[128];
	char mac[13];

	nsect = mtu = 0;
	nloc = nea = 0;
	a = 0;
	ps = 0;
	memset(buf, 0, sizeof buf);
	ml = NULL;

	for (e=eth; e->name; e++) {
		found = 0;
		for (l = e->luns; l; l = l->next) {
			if (!(l->major == maj && l->minor == min))
				continue;
			found = 1;
			if (mtu == 0)
				mtu = e->mtu;
			else if (mtu != e->mtu)
				mtu = (e->mtu > mtu ? e->mtu : mtu);

			insertmac(&ml, l->ea);
			if (ps == 0)
				ps = l->maxsect;
			else
				ps = (l->maxsect < ps ? l->maxsect : ps);
			nsect = l->nsect;
		}
		if (found) {
			snprintf(buf + a,(sizeof buf) - a, "%s%s", (a ? ",": ""), e->name);
			a += strlen(e->name) + (a ? 1 : 0);
			nloc++;
		}
	}
	for (m = ml; m; ml = m, nea++) {
		m = m->next;
		free(ml);
	}
	printf("e%d.%d\t%4d\t%d\t%s\n", maj, min, nea, nsect*512, buf); 
	if (nea != nloc)
		printf("   Mismatched number of local vs remote interfaces\n");
	for (e=eth; e->name; e++) {
		found = 0;
		for (l = e->luns; l; l = l->next)
			if (l->major == maj && l->minor == min) {
				found = 1;
				if (l->maxsect < (e->mtu-32)/512)
					printf("   The path %s->%s is only capable of %d byte payloads\n", e->name, cea(mac, l->ea), l->maxsect*512);
			}
		if (found) {
			if (e->mtu < nsect*512 + 36)
				printf("   %s: MTU (%d) not set optimally for device's capable payload\n",e->name, e->mtu);
			if (e->mtu < mtu)
				printf("   %s: MTU different from other interfaces (%d vs. %d)\n", e->name, e->mtu, mtu);
		}
	}
}

void
insertmac(Mac **ml, char *m)
{
	Mac *nm,*p, *pp;

	for (p = *ml, pp=NULL; p; pp = p, p = p->next)
		if (memcmp(p->ea, m, 6) == 0)
			return;
	nm = mallocz(sizeof *nm);
	memcpy(nm->ea, m, 6);
	if (pp)
		pp->next = nm;
	else
		*ml = nm;	
}

int
ifup(char *ethname)
{
	struct ifreq ifr;
	int r, s;

	memset(&ifr, 0, sizeof ifr);
	s = socket(AF_INET, SOCK_STREAM, 0);
	if (s < 0)
		return 0;
	strcpy(ifr.ifr_name, ethname);
	if (ioctl(s, SIOCGIFFLAGS, &ifr) < 0) {
		close(s);
		return 0;
	}
	r = ifr.ifr_flags & IFF_UP;
	close(s);
	return r;
}

void
inserteth(char **ifs, int nifs, char *n)
{
	int i, j;
	char a[64], b[64];
	
	memset(a, 0, sizeof a);
	memset(b, 0, sizeof b);
	for (i=0; i < nifs; i++) {
		if (ifs[i] == 0) { 
			ifs[i] = strdup(n);
			break;
		}
		j = strcmp(n, ifs[i]);
		if (j < 0) {
			strcpy(a, n);
			for (; ifs[i]; i++) {
				strcpy(b, ifs[i]);
				free(ifs[i]);
				ifs[i] = strdup(a); 
				strcpy(a, b);
			}
			ifs[i] = strdup(a);
			break;
		} else if (j == 0)
			break;
		else if (j > 0) 
			continue;
	}
}

char *
getpciid(char *b, int blen, char *n)
{
	FILE *fd;
	char dev[8];
	char ven[8];
	char path[128];

	memset(dev, 0, sizeof dev);
	memset(ven, 0, sizeof ven);
	memset(path, 0, sizeof path);
	sprintf(path, "/sys/class/net/%s/device/vendor", n);
	if ((fd = fopen(path, "r")) == NULL)
		return NULL;
	fseek(fd, 2, SEEK_SET);
	if (fread(ven, 1, 4, fd) <= 0) {
		fclose(fd);
		return NULL;
	}
	fclose(fd);
	        sprintf(path, "/sys/class/net/%s/device/device", n);
        if ((fd = fopen(path, "r")) == NULL)
                return NULL;
        fseek(fd, 2, SEEK_SET);
        if (fread(dev, 1, 4, fd) <= 0) {
                fclose(fd);
                return NULL;
        }
        fclose(fd);
	snprintf(b, blen, "%s:%s", ven, dev);	
	return b;
}