-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathparfu_main.hh
149 lines (127 loc) · 5.4 KB
/
parfu_main.hh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
////////////////////////////////////////////////////////////////////////////////
//
// University of Illinois/NCSA Open Source License
// http://otm.illinois.edu/disclose-protect/illinois-open-source-license
//
// Parfu is copyright © 2017-2020,
// by The Trustees of the University of Illinois.
// All rights reserved.
//
// Parfu was developed by:
// The University of Illinois
// The National Center For Supercomputing Applications (NCSA)
// Blue Waters Science and Engineering Applications Support Team (SEAS)
// Craig P Steffen <csteffen@ncsa.illinois.edu>
// Roland Haas <rhaas@illinois.edu>
//
// https://github.com/ncsa/parfu_archive_tool
// http://www.ncsa.illinois.edu/People/csteffen/parfu/
//
// For full licnse text see the LICENSE file provided with the source
// distribution.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef PARFU_MAIN_HH_
#define PARFU_MAIN_HH_
#include <cstdio>
#include <sys/types.h>
#include <sys/stat.h>
#include <mpi.h>
#include <dirent.h>
#include <string.h>
#include <unistd.h>
#include <iostream>
#include <string>
#include <vector>
#include <list>
//#include <experimental/filesystem>
#include <algorithm>
#include <fstream>
///////////////////////
//
// User-adjustable build-time parameters
// These numerical constants can be adjusted (within reason) at build
// time to customize parfu's default behavior.
// WARNING: Setting this to a character that appears
// in filenames is utterly catastrophic for parfu. We
// used "tab" or '\t' because no reasonable person would
// put a tab in a filename. If for some reason tab doesn't
// work in your instance, substituting another non-printable
// ascii character might work, but test carefully and be cautious.
#define PARFU_ENTRY_SEPARATOR_CHARACTER '\t'
// WARNING: Setting this to anything other than \n is very
// dangerous. If you set it to a character that appears in
// filenames that would be catastrophic. It's not clear that
// any other character would actually ever work.
#define PARFU_LINE_SEPARATOR_CHARACTER '\n'
///////////////////////
//
// Users: Do not adjust values in the rest of the file
#define PARFU_OFFSET_INVALID (-1L)
#define PARFU_DEFAULT_LAST_TIME_SPIDERED (0)
#define PARFU_FILE_TYPE_INVALID (-1)
#define PARFU_FILE_TYPE_REGULAR (0)
#define PARFU_FILE_TYPE_DIRECTORY (1)
#define PARFU_FILE_TYPE_SYMLINK (2)
#define PARFU_FILE_TYPE_PAD (22)
#define PARFU_FILE_SIZE_EMPTY (0L)
// These legacy values should not be used in new .pfu files.
// These are to catch values from old-format parfu files.
#define PARFU_FILE_SIZE_LEGACY_INVALID (-1L)
#define PARFU_FILE_SIZE_LEGACY_SYMLINK (-2L)
#define PARFU_FILE_SIZE_LEGACY_DIR (-3L)
// These values to be used with new .pfu files
#define PARFU_FILE_SIZE_INVALID (-10L)
#define PARFU_FILE_SIZE_SYMLINK (-11L)
#define PARFU_FILE_SIZE_DIR (-12L)
#define PARFU_SPIDER_DIRECTORY_RETURN_ERROR (-1L)
#include "tarentry.hh"
#include "parfu_file_system_classes.hh"
#include "parfu_2021_legacy.hh"
#define PARFU_WHAT_IS_PATH_REGFILE 0x001
#define PARFU_WHAT_IS_PATH_SYMLINK 0x002
#define PARFU_WHAT_IS_PATH_DIR 0x004
#define PARFU_WHAT_IS_PATH_DOES_NOT_EXIST 0x100
#define PARFU_WHAT_IS_PATH_IGNORED_TYPE 0x200
#define PARFU_WHAT_IS_PATH_ERROR 0x400
#define PARFU_FILE_TYPE_REGULAR_CHAR 'F'
#define PARFU_FILE_TYPE_DIRECTORY_CHAR 'D'
#define PARFU_FILE_TYPE_SYMLINK_CHAR 'L'
#define PARFU_FILE_TYPE_INVALID_CHAR 'X'
using namespace std;
#include "parfu_2021_legacy.hh"
#include "parfu_data_transfer.hh"
#include "parfu_file_system_classes.hh"
#include "tarentry.hh"
#include "parfu_rank_move_data.hh"
#include "parfu_worker_node.hh"
#include "parfu_boss_functions.hh"
vector <string> *parfu_parse_args(unsigned nargs,
char *args[],
unsigned long *bucket_size,
unsigned *max_orders_per_bucket,
string *archive_file_name,
int *archive_file_multiplier);
void parfu_usage(void);
// classes to define for new structure of parfu
// Parfu_file: (contains information about target file file on disk)
// (also contains a list of one or more file slices)
// parfu files have a type, a "real" file or a "pad" file.
// Parfu_file_slice: specifies location of subfile within file, and also within container
// pad_file: subclass of subfile; specifies location of padding file within container
// dir: contains information about a directory
// whether the directory has been spidered
// if it has been spidered, a list of things within it
// contains a list of subdirs
// contains a list of subfiles
// may be serializable to be passed across MPI
// container_file: contains all the informationa about a container file
// file pointers, state, etc.
// contains list of objects who will reside in the container file
// when written to disk. So a list of subfiles and/or container slices
// actually probably contains a list of container slides, maybe instead of the above
// container_slice: nicely-sized chunk of container file. written to container file by
// one rank
// node_boss: contains a work order for the boss rank of a node to process
// node_worker: contains list that a thread on a node should process.
#endif // #ifndef PARFU_MAIN_HH_