From 592224193b16925c056439e8f3c05b190dc50c99 Mon Sep 17 00:00:00 2001 From: JoAnne Levatin Date: Thu, 25 Apr 2019 14:56:09 -0700 Subject: [PATCH 1/4] Modified code to take both short and long options. Did not implement -w and -r yet, but added a -n for number of files. --- src/dfilemaker/dfilemaker.c | 232 +++++++++++++++++++++++++++++------- src/dfilemaker/makefile | 19 ++- 2 files changed, 207 insertions(+), 44 deletions(-) diff --git a/src/dfilemaker/dfilemaker.c b/src/dfilemaker/dfilemaker.c index 2cdde73b0..e627f8283 100644 --- a/src/dfilemaker/dfilemaker.c +++ b/src/dfilemaker/dfilemaker.c @@ -2,6 +2,7 @@ #include #include #include +#include #include //#include "handle_args.h" #include "mfu.h" @@ -244,17 +245,37 @@ int getnum(const char* fname) } //----------------------------------- -// put nwds random ints into buffer +// put nwds ints into buffer //------------------------------------ -void fillbuff(int* ibuff, int nwds) +void fillbuff(int* ibuff, size_t nwds, int kft) { int i; - for (i = 0; i < nwds; i++) { - ibuff[i] = rand(); + switch (kft) + { + case 0: + for (i = 0; i < nwds; i++) { + ibuff[i] = rand(); + } + break; + case 1: + for (i = 0; i < nwds; i++) { + ibuff[i] = 0xFFFFFFFF; + } + break; + case 2: + for (i = 0; i < nwds; i++) { + ibuff[i] = 0x00000000; + } + break; + case 3: + for (i = 0; i < nwds; i++) { + ibuff[i] = 0xAAAAAAAA; + } + break; } } -size_t bufsize = 1024 * 1024; +size_t bufsize = 1024 * 1024 * 4; char* buf; size_t size, isize; int nnum; @@ -262,19 +283,20 @@ int nnum; /*----------------------------------------------*/ /* add content to a node created by create_file */ /*----------------------------------------------*/ -static int write_file(mfu_flist list, uint64_t idx) +static int write_file(mfu_flist list, uint64_t idx, int kft) { int rc = 0; /* get destination name */ const char* dest_path = mfu_flist_file_get_name(list, idx); + + /* get destination size */ uint64_t fsize = mfu_flist_file_get_size(list, idx); size = fsize; - isize = (size + 1) / 2; - //printf("writing file %s, fsize = %li, size = %li\n",dest_path,fsize,size); nnum = getnum(dest_path); srand(nnum); - fillbuff((int*)buf, isize); + isize = bufsize/4; + fillbuff((int*)buf, isize, kft); /* open file */ int fd = mfu_open(dest_path, O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR); @@ -288,9 +310,8 @@ static int write_file(mfu_flist list, uint64_t idx) /* determine amount to write */ size_t left = fsize; size_t remaining = size - written; - if (remaining < fsize) { - left = remaining; - } + if (remaining < fsize) left = remaining; + if (left > isize) left = isize; /* write data to file */ ssize_t n = mfu_write(dest_path, fd, ptr, left); @@ -320,7 +341,7 @@ static int write_file(mfu_flist list, uint64_t idx) /*----------------------------------------------*/ /* add content to nodes created by create_files */ /*----------------------------------------------*/ -static int write_files(mfu_flist list) +static int write_files(mfu_flist list, int kft) { int rc = 0; @@ -343,7 +364,7 @@ static int write_files(mfu_flist list) /* process files and links */ if (type == MFU_TYPE_FILE) { /* TODO: skip file if it's not readable */ - write_file(list, idx); + write_file(list, idx, kft); } } @@ -506,15 +527,44 @@ void lnamunsort(char** buff, char** tarray, int* lind, int nitems) } } +/*-----------------------*/ +/* Print help message */ +/*-----------------------*/ +static void print_usage(void) +{ + printf("\n"); + printf("Usage: dfilemaker [options] \n"); + printf("\n"); + printf("Options:\n"); + printf(" -i, --seed=*integer* - seed to use for random number generation\n"); + printf(" -f, --fill=*filltype* - filltype = random, true, false, or alternate\n"); + printf(" -d, --depth=*min*-*max* - directory depth, integers min and max\n"); + printf(" -n, --nitems=*min*-*max* - number of items, integers min and max (subs for -r and -w)\n"); + printf(" -r, --ratio=*min*-*max* - ratio of files to directories as a percent (not implemented)\n"); + printf(" -s, --size=*min*-*max* - file size, integers min and max followed by MB or GB\n"); + printf(" -w, --width=*min*-*max* - directory width, integers min and max (not implemented)\n"); + printf(" -v, --verbose - print version number\n"); + printf(" -h, --help - print usage\n"); + printf("\n"); + printf("For more information see https://mpifileutils.readthedocs.io.\n"); + printf("\n"); + fflush(stdout); +} +/*-----------------------------------------------------------------*/ +/* Extract min and max strings from arguments to certain options */ +/*-----------------------------------------------------------------*/ +void mmparse(char* optarg,char** minterm,char** maxterm) +{ + char delim[5]="-\n"; + //printf("optarg = %s\n",optarg); + *minterm=strtok(optarg,delim); + *maxterm=strtok(0,delim); +} /***************************************************************** * * Create trees of directories, files, links - * Usage: dfilemaker - * where - * numitems = total number of dirs, files, links - * relmaxdepth = maximum directory depth rel to start - * maxflength = maximum length of any regular file + * Usage: dfilemaker [options] * ****************************************************************/ int main(int narg, char** arg) @@ -523,11 +573,11 @@ int main(int narg, char** arg) uint64_t i, j, ifst, ilst; int namlen; long int* ftypes, *flens; - long int maxflen = 1024L; + long int maxflen = 2000L; int ifrac, *nfiles; // nfiles is number of files at levels from 0 top - int ntotal; + int ntotal=3000; int nfsum = 0; - int nlevels = 2, nsum, ilev; // number of levels with top (./) + int nlevels = 5, nsum, ilev; // number of levels with top (./) int outlevels, outmin; mfu_flist* outlists; char* cp; @@ -556,7 +606,7 @@ int main(int narg, char** arg) char** larray; char** tarray; char* lnames; // global lists of link names - char** tnames; // global lists of items as targets over all levels + //char** tnames; // global lists of items as targets over all levels int nlink, *linksg; int nitem, *itemsg; uint64_t* targIDs; // global indices of things that links point to for a dir level @@ -566,6 +616,28 @@ int main(int narg, char** arg) int* lind; // list of ints in order to resort things int initsum, noff; + int kft=0,ifac=0; // kft is index of filltype + double ratio=0.; + int longind=0; + char *minterm,*maxterm; + int depmin=0,depmax=0; + int nmin=0,nmax=0; + int sizemin=0,sizemax=0; + int widmin=0,widmax=0; + static char *filltype[]={"random","true","false","alternate"}; + static struct option long_options[] = { + {"seed", 1, 0, 'i'}, + {"fill", 1, 0, 'f'}, + {"depth", 1, 0, 'd'}, + {"nitems", 1, 0, 'n'}, + {"ratio", 1, 0, 'r'}, + {"size", 1, 0, 's'}, + {"width", 1, 0, 'w'}, + {"version", 0, 0, 'v'}, + {"help", 0, 0, 'h'}, + {0, 0, 0, 0} + }; + /*-------------------------- * initialize mfu and MPI *--------------------------*/ @@ -580,25 +652,100 @@ int main(int narg, char** arg) MPI_Datatype dirname_type; MPI_Datatype tname_type; - /*---------------------------------------------- - * get nfiles = number of files to create basic - *----------------------------------------------*/ - if (narg < 4) { - if (rank == 0) { - printf("Usage: dfilemaker [-i seed]\n"); - } - MPI_Finalize(); - exit(0); - } - ntotal = atoi(arg[1]); - nlevels = atoi(arg[2]); - maxflen = atoi(arg[3]); - c=getopt(narg,arg,"i:"); - if (narg > 1) + /*---------------------------------------------- + * get nfiles = number of files to create basic + *----------------------------------------------*/ + + /*--------------------- + * loop over options + *---------------------*/ + while (1) { - if (c=='i') jseed = atoi(optarg); - if (jseed) iseed=jseed; - } + /*--------------------------------------------------------------------- + * shortopts below are followed by a colon if they take an argument + *---------------------------------------------------------------------*/ + c=getopt_long(narg,arg,"i:f:d:n:r:s:w:vh",long_options,&longind); + if (c <= 0) break; + minterm=(char*)MFU_MALLOC(10*sizeof(char)); + maxterm=(char*)MFU_MALLOC(10*sizeof(char)); + switch (c) { + case 'i': + jseed = atoi(optarg); + if (jseed) iseed=jseed; + break; + case 'f': + for (kft=0;kft<4;kft++) if (strcmp(optarg,filltype[kft])==0) break; + if (kft==4 && rank==0) + { + printf("%s not a fill option\n",optarg); + MPI_Finalize(); + } + break; + case 'd': + // range for nlevels + mmparse(optarg,&minterm,&maxterm); + depmin=atoi(minterm); + depmax=atoi(maxterm); + break; + case 'n': + // range for ntotal + mmparse(optarg,&minterm,&maxterm); + nmin=atoi(minterm); + nmax=atoi(maxterm); + break; + case 'r': + ratio = atof(optarg); + printf("ratio = %f\n",ratio); + break; + case 's': + // range for maxflen + mmparse(optarg,&minterm,&maxterm); + int lterm=strlen(minterm); + if (isdigit(minterm[lterm-1])) sizemin=atoi(minterm); + else if (strchr(minterm,'B')||strchr(minterm,'b')) + { + minterm[lterm-1]='\0'; + if (strchr(minterm,'K')||strchr(minterm,'k')) ifac=1000; + else if (strchr(minterm,'M')||strchr(minterm,'m')) ifac=1000000; + else if (strchr(minterm,'G')||strchr(minterm,'g')) ifac=1000000000; + minterm[lterm-2]='\0'; + sizemin=ifac*atoi(minterm); + } + if (isdigit(maxterm[lterm-1])) sizemax=atoi(maxterm); + else if (strchr(maxterm,'B')||strchr(maxterm,'b')) + { + maxterm[lterm-1]='\0'; + if (strchr(maxterm,'K')||strchr(maxterm,'k')) ifac=1000; + else if (strchr(maxterm,'M')||strchr(maxterm,'m')) ifac=1000000; + else if (strchr(maxterm,'G')||strchr(maxterm,'g')) ifac=1000000000; + maxterm[lterm-2]='\0'; + sizemax=ifac*atoi(maxterm); + } + break; + case 'w': + mmparse(optarg,&minterm,&maxterm); + widmin=atoi(minterm); + widmax=atoi(maxterm); + break; + case 'v': + if (rank == 0) printf("verbose is on\n"); + break; + case 'h': + if (rank == 0) print_usage(); + break; + default: + break; + } + }; + if (nmax > 0) ntotal=nmin+rand()%(1+nmax-nmin); + if (depmax > 0) nlevels=depmin+rand()%(1+depmax-depmin); + if (sizemax > 0) maxflen=sizemin+rand()%(1+sizemax-sizemin); + if (rank == 0 ) + { + printf("ntotal = %d\n",ntotal); + printf("nlevels = %d\n",nlevels); + printf("maxflen = %d\n",maxflen); + } /*------------------------------------------------------- * each level has nfiles[0] more than the one above @@ -881,7 +1028,7 @@ int main(int narg, char** arg) //--------------------------------- mfu_flist_mkdir(mybflist); mfu_flist_mknod(mybflist); - write_files(mybflist); + write_files(mybflist, kft); //------------------------------------ // reset statistics at this point @@ -902,6 +1049,7 @@ int main(int narg, char** arg) // //***************************************************************************** char* itemnames; // local to proc + char** tnames; // global lists of items as targets over all levels tnames = (char**) MFU_MALLOC(nlevels * sizeof(char*)); itemsg = (int*) MFU_MALLOC(nrank * sizeof(int)); const char* item_name = (char*) MFU_MALLOC(PATH_MAX + 1); diff --git a/src/dfilemaker/makefile b/src/dfilemaker/makefile index d629ea759..4681fdbcb 100644 --- a/src/dfilemaker/makefile +++ b/src/dfilemaker/makefile @@ -5,14 +5,29 @@ # #*************************************************************** -BASEDIR = /usr/tce/packages/mpifileutils/mpifileutils-0.8 +BASEDIR = /usr/tce/packages/mpifileutils/mpifileutils-0.9 INCLOC = ${BASEDIR}/install/include LIBLOC = ${BASEDIR}/install/lib MPCC = mpicc +CC = gcc dfilemaker: dfilemaker.o - ${MPCC} -o dfilemaker dfilemaker.o -L ${LIBLOC} -ldtcmp -lcircle -larchive -L${BASEDIR}/lib -lmfu -Wl,-rpath=${BASEDIR}/install/lib -Wl,-rpath=${BASEDIR}/lib + ${MPCC} -o dfilemaker dfilemaker.o -L ${LIBLOC} -ldtcmp -lcircle -larchive -L${BASEDIR}/lib64 -lmfu -Wl,-rpath=${BASEDIR}/install/lib -Wl,-rpath=${BASEDIR}/lib dfilemaker.o: dfilemaker.c ${MPCC} -c -DHAVE_BYTESWAP_H dfilemaker.c -I../common -I ${INCLOC} -I/usr/include clean: dfilemaker rm dfilemaker + +goexam: goexam.c + $(CC) -g -o goexam goexam.c +golly: golly.c + $(CC) -g -o golly golly.c +atest: atest.c + $(CC) -g -o atest atest.c +btest: btest.c + $(CC) -g -o btest btest.c +wf: wf.o + ${MPCC} -o wf wf.o -L ${LIBLOC} -L${BASEDIR}/lib64 -lmfu -Wl,-rpath=${BASEDIR}/install/lib -Wl,-rpath=${BASEDIR}/lib +wf.o: wf.c + $(MPCC) -DHAVE_BYTESWAP_H -I../common -I ${INCLOC} -I/usr/include -g -c wf.c + From 0988c192746fdbf1feebeb20d25a5329ab6e90c4 Mon Sep 17 00:00:00 2001 From: JoAnne Levatin Date: Thu, 25 Apr 2019 15:03:26 -0700 Subject: [PATCH 2/4] Test production deck for dfilemaker in dfilemaker3. --- src/dfilemaker/rundf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/dfilemaker/rundf diff --git a/src/dfilemaker/rundf b/src/dfilemaker/rundf new file mode 100644 index 000000000..fe49bcefc --- /dev/null +++ b/src/dfilemaker/rundf @@ -0,0 +1,18 @@ +#!/bin/csh +#SBATCH -A cbronze ## bank +#SBATCH -N 1 ## number of nodes +#SBATCH -t 10:00 ## length of run in hr:min +#SBATCH -p pdebug ## queue (pdebug or pbatch) +#SBATCH -o outdf ## name of output file +#SBATCH -e errdf ## name of error file +#SBATCH --export=ALL ## verbose to + +# set directory to run in +cd /p/lustre2/levatin/dfile3 +# next line may not be needed. it sets the path to shared libraries needed by your code, if any +#setenv LD_LIBRARY_PATH /usr/tce/packages/mpifileutils/mpifileutils-0.8/install/lib:/usr/tce/packages/mpifileutils/mpifileutils-0.8/lib +setenv LD_LIBRARY_PATH /usr/tce/packages/mpifileutils/mpifileutils-0.9/install/lib:/usr/tce/packages/mpifileutils/mpifileutils-0.9/lib64 +# set full path to code to run +set codepath = /usr/workspace/levatin/nfs/tmp2/moody2/mpifileutils/src/dfilemaker/dfilemaker +# parallel run with 5 cpus, do not need srun if serial code +time srun -n5 $codepath -i 12 -f true -s 4mb-10mb -n 500-1000 From 903bcf03fa0bf980326a6c543ccea26b008c2e14 Mon Sep 17 00:00:00 2001 From: JoAnne Levatin Date: Fri, 26 Apr 2019 15:40:55 -0700 Subject: [PATCH 3/4] removing makefile and rundf from dfilemaker distribution --- src/dfilemaker/makefile | 33 --------------------------------- src/dfilemaker/rundf | 18 ------------------ 2 files changed, 51 deletions(-) delete mode 100644 src/dfilemaker/makefile delete mode 100644 src/dfilemaker/rundf diff --git a/src/dfilemaker/makefile b/src/dfilemaker/makefile deleted file mode 100644 index 4681fdbcb..000000000 --- a/src/dfilemaker/makefile +++ /dev/null @@ -1,33 +0,0 @@ -#*************************************************************** -# -# This makefile is just for dfilemaker. -# Please do not remove. For testing only. -# -#*************************************************************** - -BASEDIR = /usr/tce/packages/mpifileutils/mpifileutils-0.9 -INCLOC = ${BASEDIR}/install/include -LIBLOC = ${BASEDIR}/install/lib -MPCC = mpicc -CC = gcc - -dfilemaker: dfilemaker.o - ${MPCC} -o dfilemaker dfilemaker.o -L ${LIBLOC} -ldtcmp -lcircle -larchive -L${BASEDIR}/lib64 -lmfu -Wl,-rpath=${BASEDIR}/install/lib -Wl,-rpath=${BASEDIR}/lib -dfilemaker.o: dfilemaker.c - ${MPCC} -c -DHAVE_BYTESWAP_H dfilemaker.c -I../common -I ${INCLOC} -I/usr/include -clean: dfilemaker - rm dfilemaker - -goexam: goexam.c - $(CC) -g -o goexam goexam.c -golly: golly.c - $(CC) -g -o golly golly.c -atest: atest.c - $(CC) -g -o atest atest.c -btest: btest.c - $(CC) -g -o btest btest.c -wf: wf.o - ${MPCC} -o wf wf.o -L ${LIBLOC} -L${BASEDIR}/lib64 -lmfu -Wl,-rpath=${BASEDIR}/install/lib -Wl,-rpath=${BASEDIR}/lib -wf.o: wf.c - $(MPCC) -DHAVE_BYTESWAP_H -I../common -I ${INCLOC} -I/usr/include -g -c wf.c - diff --git a/src/dfilemaker/rundf b/src/dfilemaker/rundf deleted file mode 100644 index fe49bcefc..000000000 --- a/src/dfilemaker/rundf +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/csh -#SBATCH -A cbronze ## bank -#SBATCH -N 1 ## number of nodes -#SBATCH -t 10:00 ## length of run in hr:min -#SBATCH -p pdebug ## queue (pdebug or pbatch) -#SBATCH -o outdf ## name of output file -#SBATCH -e errdf ## name of error file -#SBATCH --export=ALL ## verbose to - -# set directory to run in -cd /p/lustre2/levatin/dfile3 -# next line may not be needed. it sets the path to shared libraries needed by your code, if any -#setenv LD_LIBRARY_PATH /usr/tce/packages/mpifileutils/mpifileutils-0.8/install/lib:/usr/tce/packages/mpifileutils/mpifileutils-0.8/lib -setenv LD_LIBRARY_PATH /usr/tce/packages/mpifileutils/mpifileutils-0.9/install/lib:/usr/tce/packages/mpifileutils/mpifileutils-0.9/lib64 -# set full path to code to run -set codepath = /usr/workspace/levatin/nfs/tmp2/moody2/mpifileutils/src/dfilemaker/dfilemaker -# parallel run with 5 cpus, do not need srun if serial code -time srun -n5 $codepath -i 12 -f true -s 4mb-10mb -n 500-1000 From 72b3a6d51e1e590d6fc2122f495a746c28b4be68 Mon Sep 17 00:00:00 2001 From: JoAnne Levatin Date: Mon, 29 Apr 2019 11:11:30 -0700 Subject: [PATCH 4/4] Modified -s option to use mfu abtoull to parse terms with KB, MB, GB. Adam please check. This version gives repeatable results for different numbers of processors. With GB size files may have problem setting size term in elements of linked list to the value as it takes long long. --- src/dfilemaker/dfilemaker.c | 44 +++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/dfilemaker/dfilemaker.c b/src/dfilemaker/dfilemaker.c index e627f8283..dbf5deb2f 100644 --- a/src/dfilemaker/dfilemaker.c +++ b/src/dfilemaker/dfilemaker.c @@ -617,12 +617,13 @@ int main(int narg, char** arg) int initsum, noff; int kft=0,ifac=0; // kft is index of filltype + unsigned long long sizeminl,sizemaxl; // for mfu_abtoul + long int sizemin=0,sizemax=0; double ratio=0.; int longind=0; char *minterm,*maxterm; int depmin=0,depmax=0; int nmin=0,nmax=0; - int sizemin=0,sizemax=0; int widmin=0,widmax=0; static char *filltype[]={"random","true","false","alternate"}; static struct option long_options[] = { @@ -700,27 +701,32 @@ int main(int narg, char** arg) case 's': // range for maxflen mmparse(optarg,&minterm,&maxterm); - int lterm=strlen(minterm); - if (isdigit(minterm[lterm-1])) sizemin=atoi(minterm); - else if (strchr(minterm,'B')||strchr(minterm,'b')) + + /* read file size */ + if (mfu_abtoull(minterm, &sizeminl) != MFU_SUCCESS) { - minterm[lterm-1]='\0'; - if (strchr(minterm,'K')||strchr(minterm,'k')) ifac=1000; - else if (strchr(minterm,'M')||strchr(minterm,'m')) ifac=1000000; - else if (strchr(minterm,'G')||strchr(minterm,'g')) ifac=1000000000; - minterm[lterm-2]='\0'; - sizemin=ifac*atoi(minterm); + if (rank == 0) + { + printf("Could not interpret %s as file size\n", minterm); + fflush(stdout); + } + mfu_finalize(); + MPI_Finalize(); + return 1; } - if (isdigit(maxterm[lterm-1])) sizemax=atoi(maxterm); - else if (strchr(maxterm,'B')||strchr(maxterm,'b')) + sizemin=(int)sizeminl; + if (mfu_abtoull(maxterm, &sizemaxl) != MFU_SUCCESS) { - maxterm[lterm-1]='\0'; - if (strchr(maxterm,'K')||strchr(maxterm,'k')) ifac=1000; - else if (strchr(maxterm,'M')||strchr(maxterm,'m')) ifac=1000000; - else if (strchr(maxterm,'G')||strchr(maxterm,'g')) ifac=1000000000; - maxterm[lterm-2]='\0'; - sizemax=ifac*atoi(maxterm); + if (rank == 0) + { + printf("Could not interpret %s as file size\n", maxterm); + fflush(stdout); + } + mfu_finalize(); + MPI_Finalize(); + return 1; } + sizemax=(int)sizemaxl; break; case 'w': mmparse(optarg,&minterm,&maxterm); @@ -737,6 +743,7 @@ int main(int narg, char** arg) break; } }; + srand(iseed); if (nmax > 0) ntotal=nmin+rand()%(1+nmax-nmin); if (depmax > 0) nlevels=depmin+rand()%(1+depmax-depmin); if (sizemax > 0) maxflen=sizemin+rand()%(1+sizemax-sizemin); @@ -816,7 +823,6 @@ int main(int narg, char** arg) //------------------------------------------------------ ftypes = (long int*) MFU_MALLOC(nfiles[0] * sizeof(long int)); flens = (long int*) MFU_MALLOC(nfiles[0] * sizeof(long int)); - srand(iseed); for (i = 0; i < ifst; i++) { rand(); }