Pwalk.c
From OSNEXUS Online Documentation Site
/*
* pwalk.c Parrallel Walk a file system and report file meta data
Copyright (C) 2013 John F Dey Copyright (C) 2013 Steve Umbehocker
This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* pwalk is inspired by du but designed to be used with large * file systems ( > 10 million files ) * * History: dir.c, walk.c, walkv2,3,4, pwalkfs.c * * Example of using the directory call, opendir and readdir to * simulate ls. * * 1997.03.20 John Dey Although this is the first documented date for * this file I have versions that date from 1988. * 2002.09.04 John Dey walk the directory and gather stats * 2002.09.06 John Dey make to look like du -a * 2004.07.06 John Dey add -a and -k * 2008.04.01 John Dey CSV output for database use * 2009.04.12 John Dey v3.1 * replaced constants with "FILENAME_MAX", * Size of directory is size of all files in directory plus itself * Added printStat function * print file count on line with direcories * 2009.05.18 check for control charaters and double qutoes in file names; * escape the double quotes and print bad file names to stderr * 2009.12.30 size for dir should just be dir size; Fix; count returns 0 * for normal files and count of just the local directory; Previously count * return the recursive file count for the whole tree. * 2010.01.08 john dey; New field to output: file name extension. Extension is defined as the last part of the name after a Dot "." if no dot is found extension is empty "" new feature: accepts multible dirctory names as cmd line argument
This line of code has been replaced
if ( f.st_mode & S_IFDIR && (f.st_mode & S_IFMT != S_IFLNK) ) {
With this new line of code:
if ( S_ISDIR(f.st_mode) ) { Or I could have done: if ( (f.st_mode & S_IFDIR) == S_IFDIR )
2010.01.11 John Dey
Complete re-write of walkv4 transforming it into pwalk.
pwalk is a threaded version of walkv4.
pwalk will call fileDir as a new thread until MAXTHRDS is reached.
2010.02.01 pwalk v1 did not detach nor did it join the theads; v2
fixes this short comming;
2010.03.24 john dey; New physical hardware is available to run pwalk.
16 threads are only using about 20% CPU with 10% IO wait. Based on this
the thread count will be doubled to 32.
2010.11.29 Add mutex for printStat
2012.10.09 --NoSnap flag added. ignore directories that have the name
.snapshot
2013.08.02 john f dey; Add GNU license, --version flag added
2013.10.15 steve umbehocker; added verbose --debug spam mode
added compare and copy capabilities
added retain= option to backup only recently modified files
added cusomizable maxthreads
added summary total display at end (disable with --nototals)
updated and expanded usage section
fixed segfault in walker
2013.10.16 steve umbehocker; added --log mode logging
added purge mode hooks
added dryrun/force flags
2013.10.21 steve umbehocker; added --status logging
added purge w/ retain
2013.10.22 steve umbehocker; added check so purge cannot compare against emty dir
added execute operation logic to do cp/unlink operations outside
of the mutex so that they're parallelized
fixed sync/copy to create empty leaf directories
2014.11.21 steve umbehocker; added fix for files with spaces and special chars
2015.06.12 steve umbehocker; added fix for --help and --version
- /
- include <stdio.h>
- include <stdlib.h>
- include <string.h>
- include <dirent.h>
- include <sys/types.h>
- include <sys/stat.h>
- include <utime.h>
- include <time.h>
- include <errno.h>
- include <pthread.h>
- undef THRD_DEBUG
static char *Version = "v3.16 - 06/12/2015 - John F Dey john@fuzzdog.com, Steve U. eng@osnexus.com"; static char *whoami = "pwalk";
int SKIP_SNAPSHOTS =0; /* if set ignore directories called .snapshot */ int TARGETDIR =0; /* if set copy the files to the target path */ int COMPAREDIR =0; /* if set diff the files to the compare dir to determine where the holee are then purge */ int DEBUG_MODE =0; /* verbose output */ int CMP_ATIME =0; /* indiates that compare operations should look at atime */ int CMP_MTIME =1; int CMP_CTIME =0; int SHOWTOTALS =1; /* show totals at the end via atexit() */ int DOTS =0; /* print little dots to show progress */ int QUIET =0; /* quiet gives the best backup performance */ int PURGE =0; int LOG =0; int NODIRPERMS =0; unsigned int DU_MODE =0; int STATUS =0; int RETENTION_DAYS =60; size_t MAXTHRDS =32; size_t RETAIN =0; size_t DRYRUN =0; size_t FORCE =0; int MOVE_MODE =0; unsigned long long DU_TOTAL =4096L;
typedef enum operationType { OP_NONE, OP_COPY, OP_DELETE } operationType_t; operationType_t opType; char opCommand[FILENAME_MAX+FILENAME_MAX+1];
struct fileData {
char dname[FILENAME_MAX+1]; /* full path and basename */ int THRDslot; /* slot ID 0 - MAXTHRDS */ int THRDid; /* unique ID increaments with each new THRD */ int flag; /* 0 if thread; recursion > 0 */ pthread_t thread_id; /* system assigned */ pthread_attr_t tattr; };
int ThreadCNT =0; /* ThreadCNT < MAXTHRDS */ int totalTHRDS =0; struct fileData *fdslot; pthread_mutex_t mutexFD; pthread_mutex_t mutexPrintStat;
char targetdir[FILENAME_MAX+1]; char comparedir[FILENAME_MAX+1]; char logfile[FILENAME_MAX+1]; char statusfile[FILENAME_MAX+1]; int rootdirlen =0;
/* counters for summing up and printing totals at the end */ size_t scancount =0; size_t copycount =0; size_t modifiedcount =0; size_t createdcount =0; size_t errorcount =0; size_t skippedcount =0; size_t deletedcount =0; time_t currenttime =0; time_t laststatustime =0;
void printVersion( ) {
fprintf( stderr, "%s version %s\n", whoami, Version ); fprintf( stderr, "%s Copyright (C) 2013 John F Dey & Steve Umbehocker\n", whoami ); fprintf( stderr, " * pwalk comes with ABSOLUTELY NO WARRANTY;\n" ); fprintf( stderr, " * This is free software, you can redistribute it and/or modify it under the terms of the GNU General Public License\n" ); fprintf( stderr, " * as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.\n" );
}
void printHelp( ) {
fprintf( stderr, "%s version %s\n\n", whoami, Version ); fprintf( stderr, "Usage : \n"); fprintf( stderr, "%s --help --version\n", whoami ); fprintf( stderr, " Common Args :\n"); fprintf( stderr, " --dryrun : use this to test commands\n" ); fprintf( stderr, " without making any changes to the system\n" ); fprintf( stderr, " --maxthreads=N : indicates the number of threads (default=32)\n" ); fprintf( stderr, " --nototals : disables printing of totals after the scan\n" ); fprintf( stderr, " --dots : prints a dot and total every 1000 files scanned.\n" ); fprintf( stderr, " --quiet : no chatter, speeds up the scan.\n" ); fprintf( stderr, " --nosnap : Ignore directories with name .snapshot\n"); fprintf( stderr, " --debug : Verbose debug spam\n"); fprintf( stderr, " Output Format : CSV\n" ); fprintf( stderr, " Fields : DateStamp,\"inode\",\"filename\",\"fileExtension\",\"UID\",\n"); fprintf( stderr, " \"GID\",\"st_size\",\"st_blocks\",\"st_mode\",\"atime\",\n" ); fprintf( stderr, " \"mtime\",\"ctime\",\"File Count\",\"Directory Size\"\n"); fprintf( stderr, "\n" ); fprintf( stderr, "Walk Usage : \n"); fprintf( stderr, "%s SOURCEDIR\n", whoami); fprintf( stderr, " Command Args :\n"); fprintf( stderr, " SOURCEDIR : Fully qualified path to the directory to walk\n"); fprintf( stderr, "\n" ); fprintf( stderr, "Copy/Backup Usage : \n"); fprintf( stderr, "%s --targetdir=TARGETDIR SOURCEDIR\n", whoami); fprintf( stderr, "%s --retain=30 --targetdir=TARGETDIR SOURCEDIR\n", whoami); fprintf( stderr, " Command Args :\n" ); fprintf( stderr, " --move : move files rather than copy\n" ); fprintf( stderr, " --nodirperms : don't copy directory permissions\n" ); fprintf( stderr, " --targetdir : copy files to specified TARGETDIR\n" ); fprintf( stderr, " --atime : copy if access time change (default=no atime)\n" ); fprintf( stderr, " --backuplog=LOGFILE : log all files that were copied.\n" ); fprintf( stderr, " --status=STATUSFILE : write periodic status updates to specified file\n" ); fprintf( stderr, " --retain : copy if file ctime or mtime within retention period\n" ); fprintf( stderr, " specified in days. eg: --retain=60\n" ); //fprintf( stderr, " --noctime : copy if create time change (default=no ctime)\n" ); fprintf( stderr, " --nomtime : ignore mtime (default=use mtime)\n" ); fprintf( stderr, " SOURCEDIR : Fully qualified path to the directory to walk\n"); fprintf( stderr, "\n" ); fprintf( stderr, "Delete/Purge Usage : \n"); fprintf( stderr, "%s --purge [--force] --comparedir=COMPAREDIR PURGEDIR\n", whoami); fprintf( stderr, "%s --purge [--force] --retain=N PURGEDIR\n", whoami); fprintf( stderr, " Command Args :\n" ); fprintf( stderr, " --comparedir : compare against this dir but dont touch any files\n" ); fprintf( stderr, " in it. comparedir is usually the SOURCEDIR from\n" ); fprintf( stderr, " a prior copy/sync stage.\n" ); fprintf( stderr, " --purge : !!WARNING!! this deletes files older than the\n" ); fprintf( stderr, " retain period -OR- if retain is not specified\n" ); fprintf( stderr, " --comparedir is required. The comparedir is\n" ); fprintf( stderr, " compared against the specified dir and any files\n" ); fprintf( stderr, " not found in the comparedir are purged.\n" ); fprintf( stderr, " --force : !NOTE! default is a *dry-run* for purge, you must\n"); fprintf( stderr, " specify --force option to actually purge files\n" ); fprintf( stderr, " --atime : keep if access time within retain period\n" ); fprintf( stderr, " --retain : keep if file ctime or mtime within retention period\n" ); fprintf( stderr, " specified in days. eg: --retain=60\n" ); fprintf( stderr, "\n");
}
void updateStatus(char *message) {
time_t now;
time(&now);
size_t elapsed = (size_t)difftime(now, currenttime);
size_t hours = elapsed/3600;
size_t minutes = (elapsed%3600)/60;
size_t seconds = elapsed%60;
FILE *statusFp = fopen(statusfile, "w");
if(statusFp != NULL)
{
fprintf(statusFp, " status: %s\n", message);
fprintf(statusFp, " scanned: %lu\n", scancount);
fprintf(statusFp, " copied: %lu\n", copycount);
fprintf(statusFp, " modified: %lu\n", modifiedcount);
fprintf(statusFp, " created: %lu\n", createdcount);
fprintf(statusFp, " skipped: %lu\n", skippedcount);
fprintf(statusFp, " deleted: %lu\n", deletedcount);
fprintf(statusFp, " errors: %lu\n", errorcount);
fprintf(statusFp, " elapsed: %02lu:%02lu:%02lu\n", hours, minutes, seconds);
fclose(statusFp);
}
}
void printTotals( ) {
if(STATUS)
{
updateStatus("completed");
}
if(DU_MODE)
{
fprintf(stderr, "TOTAL: %llu\n", DU_TOTAL);
}
if(!SHOWTOTALS)
return;
time_t now; time(&now); size_t elapsed = (size_t)difftime(now, currenttime); size_t hours = elapsed/3600; size_t minutes = (elapsed%3600)/60; size_t seconds = elapsed%60;
fprintf( stderr, "\n\n%12s %12s %12s %12s %12s %12s %12s %12s\n", "Scanned", "Copied", "Modified", "Created", "Deleted", "Skipped", "Errors", "Elapsed (HH:MM:SS)"); fprintf( stderr, "%12lu %12lu %12lu %12lu %12lu %12lu %12lu %02lu:%02lu:%02lu\n", scancount, copycount, modifiedcount, createdcount, deletedcount, skippedcount, errorcount, hours, minutes, seconds);
}
void getDirPath(char *dirPath, char *fullPath, size_t bufferLen) {
strncpy(dirPath, fullPath, FILENAME_MAX+FILENAME_MAX);
size_t pos = strlen(dirPath)-1;
while(pos > 0 && dirPath[pos] != '/')
{
dirPath[pos]=0;
pos--;
}
}
void logPath(char *targetPath) {
if(!LOG || (logfile[0] == 0))
return;
FILE *fp = fopen(logfile, "a");
if(fp != NULL)
{
fprintf(fp, "%s\n", targetPath);
fclose(fp);
}
}
int dirContainsFiles(char *targetPath) {
struct dirent *ep = NULL;
DIR *dp = opendir(targetPath);
if (dp != NULL)
{
while(ep = readdir(dp))
{
if(ep->d_name[0] == '.')
continue;
//fprintf(stderr, "INFO: directory entry '%s'", ep->d_name);
closedir(dp);
return 1;
}
closedir(dp);
}
return 0;
}
/*
* this needs to be in a crital secion (and it is!) */
void printStat( char *fname, char *exten, struct stat *f, long fileCnt, long dirSz ) {
char sourcePath[FILENAME_MAX+FILENAME_MAX]; char sourcePathDir[FILENAME_MAX+FILENAME_MAX]; char targetPath[FILENAME_MAX+FILENAME_MAX]; char targetPathDir[FILENAME_MAX+FILENAME_MAX]; char outFile[FILENAME_MAX+FILENAME_MAX];
char *s, *t = sourcePath; int cnt = 0; char Sep=','; /* this was added to help with debugging */
opType = OP_NONE;
cnt =0;
/* fix bad file name is moved inside printStat to make it thread safe */
s = fname;
while ( *s ) {
if ( *s == '"' )
*t++ = '\\';
//if ( *s < 32 ) {
// s++;
// cnt++;
//} else
*t++ = *s++;
}
*t++ = *s++;
char badfile=0;
if ( cnt )
{
fprintf( stderr, "Bad File: %s\n", fname );
badfile=1;
}
if(!DOTS && !QUIET)
{
sprintf ( outFile, "\"%ld\",\"%s\",\"%s\",\"%ld\",\"%ld\",\"%ld\",\"%ld\",\"%07o\",\"%ld\",\"%ld\",\"%ld\",\"%ld\",\"%ld\"\n",
(long)f->st_ino, sourcePath, (exten)? exten:"", (long)f->st_uid,
(long)f->st_gid, (long)f->st_size, (long)f->st_blocks, (int)f->st_mode,
(long)f->st_atime, (long)f->st_mtime, (long)f->st_ctime, fileCnt, dirSz );
fputs( outFile, stdout );
}
if(STATUS && (scancount%256==0))
{
time_t now;
time(&now);
size_t elapsed = (size_t)difftime(now, laststatustime);
if(elapsed >= 10)
{
laststatustime = now;
updateStatus("running");
}
}
if(DOTS)
{
if(scancount%1000==0)
printf("%ld", scancount);
else if(scancount%100==0)
printf(".");
}
scancount++;
if(PURGE)
{
if(S_ISDIR(f->st_mode))
{
if(DEBUG_MODE) printf("INFO: Purge skipping DIR: '%s'\n", sourcePath);
skippedcount++;
return;
}
if(!RETAIN && COMPAREDIR)
{
//Add a trailing / to the base path if it's not there.
size_t comparedirLen = strlen(comparedir);
if((comparedirLen+1 < FILENAME_MAX+FILENAME_MAX) && comparedir[comparedirLen-1] != '/')
{
comparedir[comparedirLen] = '/';
comparedir[comparedirLen+1] = 0;
}
//Build the full path to the target file
size_t offset = rootdirlen+1;
strncpy(targetPath, comparedir, FILENAME_MAX+FILENAME_MAX);
strncat(targetPath, &sourcePath[offset], FILENAME_MAX+FILENAME_MAX);
//Build the full path to the target directory by trimming from the end of the full file path
getDirPath(targetPathDir, targetPath, FILENAME_MAX+FILENAME_MAX);
if(DEBUG_MODE)
printf("comparing: %s : %s\n", targetPath, sourcePath);
struct stat statTarget = {0};
if(stat(targetPath, &statTarget) == -1 && S_ISREG(f->st_mode))
{
deletedcount++;
if(DEBUG_MODE)
printf("INFO: PURGE [%s] : %s\n", (DRYRUN || !FORCE) ? "dryrun" : "UNLINK", sourcePath);
if(!DRYRUN && FORCE)
{
opType = OP_DELETE;
strncpy(opCommand, sourcePath, FILENAME_MAX+FILENAME_MAX);
if(DOTS)
printf("!");
}
}
else
skippedcount++;
return;
}
if(RETAIN && (currenttime > f->st_ctime) && (
(!CMP_CTIME || (difftime(currenttime, f->st_ctime) > RETAIN)) &&
(!CMP_MTIME || (difftime(currenttime, f->st_mtime) > RETAIN)) &&
(!CMP_ATIME || (difftime(currenttime, f->st_atime) > RETAIN))))
{
deletedcount++;
if(DEBUG_MODE)
{
printf("INFO: PURGE [%s] (%lu days old): %s\n", (DRYRUN || !FORCE) ? "dryrun" : "UNLINK", (size_t)difftime(currenttime, f->st_mtime)/(3600*24), sourcePath);
}
if(!DRYRUN && FORCE)
{
//we want to just save the path so we can do the purge outside of this criticial section
//this way our delete is parallelized
strncpy(opCommand, sourcePath, FILENAME_MAX+FILENAME_MAX);
opType = OP_DELETE;
if(DOTS)
printf("!");
}
}
else
skippedcount++;
return;
}
//If a target dir was specified then we need to do file copy operations
if(TARGETDIR && !badfile)
{
//Add a trailing / to the base path if it's not there.
size_t targetdirLen = strlen(targetdir);
if((targetdirLen+1 < FILENAME_MAX+FILENAME_MAX) && targetdir[targetdirLen-1] != '/')
{
targetdir[targetdirLen] = '/';
targetdir[targetdirLen+1] = 0;
}
//Build the full path to the target file
size_t offset = rootdirlen+1;
strncpy(targetPath, targetdir, FILENAME_MAX+FILENAME_MAX);
strncat(targetPath, &sourcePath[offset], FILENAME_MAX+FILENAME_MAX);
//Build the full path to the target directory by trimming from the end of the full file path
getDirPath(targetPathDir, targetPath, FILENAME_MAX+FILENAME_MAX);
struct stat statTargetDir = {0};
if(stat(targetPathDir, &statTargetDir) == -1 || !S_ISDIR(statTargetDir.st_mode))
{
getDirPath(sourcePathDir, sourcePath, FILENAME_MAX+FILENAME_MAX);
struct stat statSourceDir = {0};
if(!stat(sourcePathDir, &statSourceDir) && S_ISDIR(statSourceDir.st_mode))
{
char command[FILENAME_MAX+FILENAME_MAX];
command[0]='\0';
snprintf(command, FILENAME_MAX+FILENAME_MAX, "mkdir -p \"%s\"", targetPathDir);
if(DEBUG_MODE)
printf("INFO: making DIR: %s\n", command);
if(!DRYRUN)
{
system(command);
struct utimbuf times;
times.actime = statSourceDir.st_atime;
times.modtime = statSourceDir.st_mtime;
utime(targetPathDir, ×);
}
//preserve the permissions for the directories
if(!NODIRPERMS)
snprintf(command, FILENAME_MAX+FILENAME_MAX, "rsync -dAXogtpx --no-recursive \"%s/\" \"%s\"", sourcePathDir, targetPathDir);
if(DEBUG_MODE)
printf("INFO: updating DIR perms: %s\n", command);
if(!DRYRUN)
{
system(command);
}
}
else
{
fprintf(stderr, "ERR: Failed to create target directory '%s'\n", targetPathDir);
errorcount++;
return;
}
}
if(S_ISDIR(f->st_mode))
{
struct stat statTargetDir = {0};
if(stat(targetPath, &statTargetDir) == -1)
{
char command[FILENAME_MAX+FILENAME_MAX];
command[0]='\0';
snprintf(command, FILENAME_MAX+FILENAME_MAX, "mkdir -p \"%s\"", targetPath);
if(DEBUG_MODE)
printf("INFO: making DIR: %s\n", command);
if(!DRYRUN)
{
system(command);
struct utimbuf times;
times.actime = f->st_atime;
times.modtime = f->st_mtime;
utime(targetPath, ×);
}
//preserve the permissions for the directories
if(!NODIRPERMS)
snprintf(command, FILENAME_MAX+FILENAME_MAX, "rsync -dAXogtpx --no-recursive \"%s/\" \"%s\"", sourcePath, targetPath);
if(DEBUG_MODE)
printf("INFO: updating DIR perms: %s\n", command);
if(!DRYRUN)
{
system(command);
}
createdcount++;
if(DEBUG_MODE)
printf("INFO: Created DIR: '%s'\n", sourcePath);
}
return;
}
struct stat statTarget = {0};
char doCopy = 0;
char doCompare = 0;
if(stat(targetPath, &statTarget) == -1 || MOVE_MODE)
{
doCopy = 1;
}
else if(S_ISDIR(statTarget.st_mode))
{
fprintf(stderr, "ERR: Target is a directory '%s'\n", targetPath);
errorcount++;
}
else if(!S_ISREG(statTarget.st_mode))
{
fprintf(stderr, "ERR: Target not regular file '%s'\n", targetPath);
errorcount++;
}
else
{
doCompare = 1;
}
if(doCompare)
{
if(DEBUG_MODE)
{
sprintf ( outFile, "INFO: comparing with [\"%ld\",\"%s\",\"%s\",\"%ld\",\"%ld\",\"%ld\",\"%ld\",\"%07o\",\"%ld\",\"%ld\",\"%ld\",\"%ld\",\"%ld\"]\n",
(long)statTarget.st_ino, targetPath, (exten)? exten:"", (long)statTarget.st_uid,
(long)statTarget.st_gid, (long)statTarget.st_size, (long)statTarget.st_blocks, (int)statTarget.st_mode,
(long)statTarget.st_atime, (long)statTarget.st_mtime, (long)statTarget.st_ctime, fileCnt, dirSz );
fputs( outFile, stdout );
}
if(MOVE_MODE){ doCopy=1; }
else if(statTarget.st_mode != f->st_mode)
{
if(DEBUG_MODE) printf("INFO: File has MODE CHANGE, copying\n");
doCopy = 1;
}
else if(statTarget.st_size != f->st_size)
{
if(DEBUG_MODE) printf("INFO: File has SIZE CHANGE, copying\n");
doCopy = 1;
}
// the target file will always have a different create time so we don't
// want to use == for ctime compare, rather we just want to see if the source
// was created after the target, if so, need to recopy
else if(CMP_CTIME && (statTarget.st_ctime < f->st_ctime))
{
if(DEBUG_MODE) printf("INFO: CTIME on target is older than source, copying\n");
doCopy = 1;
}
else if(CMP_MTIME && (statTarget.st_mtime != f->st_mtime))
{
if(DEBUG_MODE) printf("INFO: File has MTIME CHANGE, copying\n");
doCopy = 1;
}
if(!doCopy && CMP_ATIME && (statTarget.st_atime != f->st_atime))
{
if(DEBUG_MODE) printf("INFO: File has only ATIME CHANGE, updating timestamps\n");
struct utimbuf times;
times.actime = f->st_atime;
times.modtime = f->st_mtime;
utime(targetPath, ×);
}
}
if(doCopy)
{
if(RETAIN)
{
if((currenttime < f->st_ctime) ||
(CMP_CTIME && (difftime(currenttime, f->st_ctime) < RETAIN)) ||
(CMP_MTIME && (difftime(currenttime, f->st_mtime) < RETAIN)) ||
(CMP_ATIME && (difftime(currenttime, f->st_atime) < RETAIN)))
{
if(DEBUG_MODE)
printf("INFO: INSIDE RETAIN (%lu days old): %s\n", (size_t)difftime(currenttime, f->st_mtime)/(3600*24), sourcePath);
}
else
{
if(DEBUG_MODE)
printf("INFO: OUTSIDE RETAIN (%lu days old), skipping: %s\n", (size_t)difftime(currenttime, f->st_mtime)/(3600*24), sourcePath);
skippedcount++;
return;
}
}
copycount++;
if(doCompare)
modifiedcount++;
else
createdcount++;
char command[FILENAME_MAX+FILENAME_MAX];
command[0]='\0';
if(MOVE_MODE)
snprintf(command, FILENAME_MAX+FILENAME_MAX, "mv \"%s\" \"%s\"", sourcePath, targetPath);
else
snprintf(command, FILENAME_MAX+FILENAME_MAX, "cp --no-dereference --preserve=all \"%s\" \"%s\"", sourcePath, targetPath);
if(DEBUG_MODE)
printf("INFO: %s\n", command);
if(!DRYRUN)
{
//we want to just save the command so we can do the copy outside of this criticial section
//this way our copy op is parallelized
opType = OP_COPY;
strncpy(opCommand, command, FILENAME_MAX+FILENAME_MAX);
}
if(LOG)
logPath(targetPath);
}
}
}
void execOperation(operationType_t doOperation, char *doCommand) {
if(doOperation == OP_COPY)
system(doCommand);
if(doOperation == OP_DELETE)
unlink(doCommand);
}
/*
* Open a directory and read the conents. Call stat with each * file name. * * Recursively call self for each sub dir. * * print inode meta data for each file, one line per file in CSV format */
void
- fileDir( void *arg )
{
char *s, *t, *u, *dot, *end_dname; char fname[FILENAME_MAX+1]; int slot, id, found; DIR *dirp; long localCnt =0; /* number of files in a specific directory */ unsigned long long localSz =0; /* byte cnt of files in the local directory 2010.07 */ struct dirent *d; struct stat f; struct fileData *fd, local;
fd = (struct fileData *) arg;
- ifdef THRD_DEBUG
printf( "Start %2d%5d %2d %s\n", fd->THRDslot, fd->THRDid, fd->flag, fd->dname );
- endif /* THRD_DEBUG */
if ( (dirp = opendir( fd->dname )) == NULL ) {
exit ( 1 );
}
/* find the end of fs->name and put '/' at the end <end_dname>
points to char after '/' */
s = fd->dname + strlen(fd->dname);
*s++ = '/';
end_dname = s;
while ( (d = readdir( dirp )) != NULL ) {
if ( strcmp(".",d->d_name) == 0 ) continue;
if ( strcmp("..",d->d_name) == 0 ) continue;
localCnt++;
s = d->d_name;
t = end_dname;
while ( *s )
*t++ = *s++;
*t = '\0';
if ( lstat ( fd->dname, &f ) == -1 ) {
fprintf( stderr, "error %2d%5d %2d %s\n", fd->THRDslot, fd->THRDid, fd->flag, fd->dname );
continue;
}
/* Follow Sub dirs recursivly but don't follow links */
localSz += f.st_size;
DU_TOTAL += f.st_size;
if ( S_ISDIR(f.st_mode) ) {
if ( SKIP_SNAPSHOTS && !strcmp( ".snapshot", d->d_name ) ) {
fprintf( stderr, "SnapShot: %s\n", d->d_name );
continue;
}
pthread_mutex_lock (&mutexFD);
if ( ThreadCNT < MAXTHRDS ) {
ThreadCNT++;
id = totalTHRDS++;
slot =0; found = -1;
while ( slot < MAXTHRDS ) {
if ( fdslot[slot].THRDslot == -1 ) {
found = slot;
break;
}
slot++;
}
if ( found == -1 )
fprintf( stderr, "SlotE %2d%5d %2d %s\n", fd->THRDslot, fd->THRDid, fd->flag, "no available threads" );
else
fdslot[slot].THRDslot = slot;
} else
slot = -1;
pthread_mutex_unlock (&mutexFD);
if ( slot != -1 ) {
strcpy( fdslot[slot].dname, (const char*)fd->dname );
fdslot[slot].THRDid = id;
fdslot[slot].flag = 0;
pthread_create( &fdslot[slot].thread_id, &fdslot[0].tattr,
fileDir, (void*)&fdslot[slot] );
} else {
strcpy( local.dname, (const char*)fd->dname );
local.THRDslot = fd->THRDslot;
local.THRDid = fd->THRDid;
local.flag = fd->flag + 1;
fileDir( (void*) &local );
}
} else {
s = end_dname + 1; dot = '\0';
while ( *s )
{
if (*s == '.')
dot = s+1;
s++;
}
pthread_mutex_lock (&mutexPrintStat);
printStat( fd->dname, dot, &f, (long)0, (long)0 );
operationType_t doOperation = opType;
opType = OP_NONE;
char doCommand[FILENAME_MAX+FILENAME_MAX+1];
strncpy(doCommand, opCommand, FILENAME_MAX+FILENAME_MAX);
pthread_mutex_unlock (&mutexPrintStat);
execOperation(doOperation, doCommand);
}
}
closedir( dirp );
*--end_dname = '\0';
- ifdef THRD_DEBUG
printf( "Ending %2d%5d %2d<%s>\n", fd->THRDslot, fd->THRDid, fd->flag, fd->dname );
- endif /* THRD_DEBUG */
if ( lstat ( fd->dname, &f ) == -1 ) {
fprintf( stderr, "ERROR %2d%5d %2d %s\n", fd->THRDslot, fd->THRDid, fd->flag, fd->dname );
}
else {
s = end_dname - 1; dot = '\0';
while ( *s != '/' && (s != fd->dname))
{
if (*s == '.')
dot = s+1;
s--;
}
if ( s+2 == dot ) /* this dot is next to the slash like /.R */
dot = '\0';
pthread_mutex_lock (&mutexPrintStat);
printStat( fd->dname, dot, &f, localCnt, localSz );
operationType_t doOperation = opType;
opType = OP_NONE;
char doCommand[FILENAME_MAX+FILENAME_MAX+1];
strncpy(doCommand, opCommand, FILENAME_MAX+FILENAME_MAX);
pthread_mutex_unlock (&mutexPrintStat);
execOperation(doOperation, doCommand);
}
if ( fd->flag == 0 ) { /* this instance of fileDir is a thread */
pthread_mutex_lock ( &mutexFD );
--ThreadCNT;
fd->THRDslot = -1;
pthread_mutex_unlock ( &mutexFD );
pthread_exit( EXIT_SUCCESS );
}
/* return ; */
}
int
main( int argc, char* argv[] )
{
int error, i;
char *s, *c;
void *status;
if ( argc < 2 || argv == NULL)
{
printHelp( );
exit( EXIT_FAILURE );
}
argc--; argv++;
while(argc > 1)
{
if (!strncmp( *argv, "--maxthreads=", 13 ))
{
MAXTHRDS = (size_t)atoi(&((*argv)[strlen("--maxthreads=")]));
if(MAXTHRDS > 64) MAXTHRDS = 64;
if(MAXTHRDS <= 2) MAXTHRDS = 2;
if(DEBUG_MODE) printf("INFO: MAXTHRDS='%lu'\n", MAXTHRDS);
}
else if (!strncmp( *argv, "--retain=", 9 ))
{
RETAIN = (size_t)atoi(&((*argv)[strlen("--retain=")]))*24*3600;
if(DEBUG_MODE) printf("INFO: RETAIN='%lu'\n", RETAIN);
}
else if ( !strcmp( *argv, "--move" ) )
MOVE_MODE = 1;
else if ( !strcmp( *argv, "--debug" ) )
DEBUG_MODE = 1;
else if ( !strcmp( *argv, "--du" ) )
{
DU_MODE = 1;
SHOWTOTALS = 0;
QUIET = 1;
}
else if ( !strcmp( *argv, "--nototals" ) )
SHOWTOTALS = 0;
else if ( !strcmp( *argv, "--noatime" ) )
CMP_ATIME = 0;
else if ( !strcmp( *argv, "--atime" ) )
CMP_ATIME = 1;
else if ( !strcmp( *argv, "--nomtime" ) )
CMP_MTIME = 0;
else if ( !strcmp( *argv, "--mtime" ) )
CMP_MTIME = 1;
else if ( !strcmp( *argv, "--noctime" ) )
CMP_CTIME = 0;
else if ( !strcmp( *argv, "--ctime" ) )
CMP_CTIME = 1;
else if ( !strcmp( *argv, "--NoSnap" ) || !strcmp( *argv, "--nosnap" ) )
SKIP_SNAPSHOTS = 1;
else if ( !strcmp( *argv, "--dryrun" ) )
DRYRUN = 1;
else if ( !strcmp( *argv, "--force" ) )
FORCE = 1;
else if ( !strcmp( *argv, "--nodirperms" ) )
NODIRPERMS = 1;
else if ( !strcmp( *argv, "--purge" ) )
{
PURGE = 1;
}
else if ( !strncmp( *argv, "--backuplog=", 12 ) )
{
LOG = 1;
memset(logfile, 0, sizeof(logfile));
strncpy(logfile, &((*argv)[strlen("--backuplog=")]), sizeof(logfile));
if(strlen(logfile) == 0)
{
fprintf( stderr, "ERR: Invalid log file name.");
LOG=0;
}
if(DEBUG_MODE) printf("INFO: LOGFILE='%s'\n", logfile);
}
else if ( !strncmp( *argv, "--status=", 9 ) )
{
STATUS = 1;
memset(statusfile, 0, sizeof(statusfile));
strncpy(statusfile, &((*argv)[strlen("--status=")]), sizeof(statusfile));
if(strlen(statusfile) == 0)
{
fprintf( stderr, "ERR: Invalid status log file name.");
STATUS=0;
}
if(DEBUG_MODE) printf("INFO: STATUS='%s'\n", statusfile);
}
else if ( !strncmp( *argv, "--targetdir=", 12 ) )
{
TARGETDIR = 1;
memset(targetdir, 0, sizeof(targetdir));
strncpy(targetdir, &((*argv)[strlen("--targetdir=")]), sizeof(targetdir));
if(DEBUG_MODE)
printf("INFO: TARGETDIR='%s'\n", targetdir);
}
else if ( !strncmp( *argv, "--comparedir=", 13 ) )
{
COMPAREDIR = 1;
memset(comparedir, 0, sizeof(comparedir));
strncpy(comparedir, &((*argv)[strlen("--comparedir=")]), sizeof(comparedir));
if(DEBUG_MODE) printf("INFO: COMPAREDIR='%s'\n", comparedir);
if(!dirContainsFiles(comparedir))
{
printf("ERR: Cannot run purge operation against a comparison directory with no files '%s'\n", comparedir);
exit( EXIT_FAILURE );
}
}
else if ( !strcmp( *argv, "--dots" ) )
DOTS = 1;
else if ( !strcmp( *argv, "--quiet" ) )
QUIET = 1;
argc--; argv++;
}
if (!strcmp( *argv, "--help" ))
{
printHelp( );
exit( 0 );
}
if (!strcmp( *argv, "--version" ) || !strcmp( *argv, "-v" ))
{
printVersion( );
exit( 0 );
}
time(¤ttime);
fdslot = malloc(sizeof(struct fileData) * MAXTHRDS);
for ( i=0; i<MAXTHRDS; i++ )
{
fdslot[i].THRDslot = -1;
if ( (error = pthread_attr_init( &fdslot[i].tattr )) )
fprintf( stderr, "Failed to create pthread attr: %s\n",
strerror(error));
else if ( (error = pthread_attr_setdetachstate( &fdslot[i].tattr,
PTHREAD_CREATE_DETACHED)
) )
fprintf( stderr, "failed to set attribute detached: %s\n",
strerror(error));
}
pthread_mutex_init(&mutexFD, NULL);
strcpy( fdslot[0].dname, (const char*) *argv );
rootdirlen = strlen(fdslot[0].dname);
//Trim the trailing slash so we don't get // in the path
if(fdslot[0].dname[strlen(fdslot[0].dname)-1] == '/')
{
fdslot[0].dname[strlen(fdslot[0].dname)-1] = '\0';
rootdirlen--;
}
fdslot[0].THRDslot = ThreadCNT++;
fdslot[0].THRDid = totalTHRDS++;
fdslot[0].flag = 0;
pthread_create( &(fdslot[0].thread_id), &fdslot[0].tattr, fileDir,
(void*)&fdslot[0] );
if(SHOWTOTALS || STATUS || DU_MODE)
atexit(printTotals);
pthread_exit( NULL );
}
