Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #ifndef __ELEM_H
19 : #define __ELEM_H
20 :
21 : #include "util.h"
22 : #include "support.h"
23 : #include "tommyds/tommyhash.h"
24 : #include "tommyds/tommylist.h"
25 : #include "tommyds/tommytree.h"
26 : #include "tommyds/tommyhashdyn.h"
27 : #include "tommyds/tommyarray.h"
28 : #include "tommyds/tommyarrayblkof.h"
29 :
30 : /****************************************************************************/
31 : /* snapraid */
32 :
33 : /**
34 : * Number of measures of the operation progress.
35 : */
36 : #define PROGRESS_MAX 100
37 :
38 : /**
39 : * Max UUID length.
40 : */
41 : #define UUID_MAX 128
42 :
43 : /**
44 : * Invalid position.
45 : */
46 : #define POS_NULL ((block_off_t)-1)
47 :
48 : /**
49 : * Content file specification.
50 : */
51 : struct snapraid_content {
52 : char content[PATH_MAX]; /**< Path of the content file. */
53 : uint64_t device; /**< Device identifier. */
54 : void* context; /**< Context used for multithread operations. */
55 : tommy_node node; /**< Next node in the list. */
56 : };
57 :
58 : /**
59 : * Filter for paths.
60 : */
61 : struct snapraid_filter {
62 : char pattern[PATH_MAX]; /**< Filter pattern. */
63 : int is_disk; /**< If the pattern is a disk one. */
64 : int is_path; /**< If the pattern is only for the complete path. */
65 : int is_dir; /**< If the pattern is only for dir. */
66 : int direction; /**< If it's an inclusion (=1) or an exclusion (=-1). */
67 : tommy_node node; /**< Next node in the list. */
68 : };
69 :
70 : /**
71 : * Block pointer used to represent unused blocks.
72 : */
73 : #define BLOCK_NULL 0
74 :
75 : /**
76 : * This block is an empty one.
77 : * Note that an empty block is represent with ::BLOCK_NULL.
78 : */
79 : #define BLOCK_STATE_EMPTY 0
80 :
81 : /**
82 : * The block has both the hash and the parity computed.
83 : * This is the normal state of a saved block.
84 : *
85 : * The block hash field IS set.
86 : * The parity for this disk is updated.
87 : */
88 : #define BLOCK_STATE_BLK 1
89 :
90 : /**
91 : * The block is new and not yet hashed.
92 : * This happens when a new block overwrite a just removed block, or an empty space.
93 : *
94 : * The block hash field MAY be set and it represents the hash of the OLD data.
95 : * The hash may be also INVALID or ZERO.
96 : *
97 : * If the OLD block was empty, the hash is set to the special ZERO value.
98 : * If the OLD block was lost, the hash is set to the special INVALID value.
99 : *
100 : * The parity for this disk is not updated, but it contains the old data referenced by the hash.
101 : *
102 : * If the state is read from an incomplete sync, we don't really know if the hash is referring at the
103 : * data used to compute the parity, because the sync process was interrupted at an unknown point,
104 : * and the parity may or may not be updated.
105 : *
106 : * For this reason we clear all such hashes when reading the state from an incomplete sync before
107 : * starting a new sync, because sync is affected by such hashes, as sync updates the parity, only
108 : * if the new data read for CHG blocks has a mismatching hash.
109 : * Clearing is done setting the ::clear_past_hash flag before reading the state.
110 : * No clearing is done in other commands, as check and fix are instead able to work with unsynced
111 : * hashes, and scrub ignores CHG/DELETED blocks.
112 : */
113 : #define BLOCK_STATE_CHG 2
114 :
115 : /**
116 : * The block is new and hashed.
117 : * This happens when a new block overwrite a just removed block, or an empty space.
118 : *
119 : * Note that when the file copy heuristic is enabled, the REP blocks may be set
120 : * using this heuristic, meaning that the hash may be wrong.
121 : *
122 : * For this reason, when the ::force_nocopy flag is enabled in sync, we convert all the REP blocks
123 : * to CHG, invalidating the stored hash.
124 : * Clearing is done setting the ::clear_past_hash flag before reading the state.
125 : * No clearing is done in other commands, as they don't stop the process like in sync
126 : * when there is a false silent error.
127 : *
128 : * The block hash field IS set, and it represents the hash of the new data.
129 : * The parity for this disk is not updated.
130 : */
131 : #define BLOCK_STATE_REP 3
132 :
133 : /**
134 : * This block is a deleted one.
135 : * This happens when a file is deleted.
136 : *
137 : * The block hash field IS set, and it represents the hash of the previous data,
138 : * but only if it's different by all 0.
139 : * The parity for this disk is not updated, but it contains the old data referenced by the hash.
140 : *
141 : * If the state is read from an incomplete sync, we don't really know if the hash is referring at the
142 : * data used to compute the parity, because the sync process was interrupted at an unknown point,
143 : * and the parity may or may not be updated.
144 : *
145 : * A now the sync process is not affected by DELETED hash, so clearing won't be really needed,
146 : * but considering that we have to do it for CHG blocks, we do it also for DELETED ones,
147 : * clearing all the past hashes.
148 : * Clearing is done setting the ::clear_past_hash flag before reading the state.
149 : * No clearing is done in other commands, as check and fix are instead able to work with unsynced
150 : * hashes, and scrub ignores CHG/DELETED blocks.
151 : */
152 : #define BLOCK_STATE_DELETED 4
153 :
154 : /**
155 : * Block hash size.
156 : *
157 : * At max HASH_MAX.
158 : */
159 : extern int BLOCK_HASH_SIZE;
160 :
161 : /**
162 : * Block of a file.
163 : */
164 : struct snapraid_block {
165 : unsigned char state; /**< State of the block. */
166 :
167 : /**
168 : * Hash of the block.
169 : *
170 : * The effective stored size is BLOCK_HASH_SIZE.
171 : */
172 : unsigned char hash[HASH_MAX];
173 : };
174 :
175 : /**
176 : * If a file is present in the disk.
177 : * It's used only in scan to detect present and missing files.
178 : */
179 : #define FILE_IS_PRESENT 0x01
180 :
181 : /**
182 : * If it's an excluded file from the processing.
183 : * It's used in both check and fix to mark files to exclude from the processing.
184 : */
185 : #define FILE_IS_EXCLUDED 0x02
186 :
187 : /**
188 : * If a fix was attempted but it failed.
189 : * It's used only in fix to mark that some data is unrecoverable.
190 : */
191 : #define FILE_IS_DAMAGED 0x04
192 :
193 : /**
194 : * If a fix was done.
195 : * It's used only in fix to mark that some data was recovered.
196 : */
197 : #define FILE_IS_FIXED 0x08
198 :
199 : /**
200 : * If the file was originally missing, and it was created in the fix process.
201 : * It's used only in fix to mark files recovered from scratch,
202 : * meaning that they don't have any previous content.
203 : * This is important because it means that deleting them, you are not going
204 : * to lose something that cannot be recovered.
205 : * Note that excluded files won't ever get this flag.
206 : */
207 : #define FILE_IS_CREATED 0x10
208 :
209 : /**
210 : * If the file has completed its processing, meaning that it won't be opened anymore.
211 : * It's used only in fix to mark when we finish processing one file.
212 : * Note that excluded files won't ever get this flag.
213 : */
214 : #define FILE_IS_FINISHED 0x20
215 :
216 : /**
217 : * If the file hash was obtained from a file copy
218 : * identified by the same name, size and stamp.
219 : */
220 : #define FILE_IS_COPY 0x40
221 :
222 : /**
223 : * If the file was opened.
224 : * It's used in fix to detect if it's the first time a file is opened.
225 : */
226 : #define FILE_IS_OPENED 0x80
227 :
228 : /**
229 : * If the file is modified from the latest sync.
230 : * It's used in fix to store if the state of the file before being modified.
231 : */
232 : #define FILE_IS_UNSYNCED 0x100
233 :
234 : /**
235 : * If the file is without inode.
236 : * It could happen in file-system where inodes are not persistent,
237 : * or when restoring a full disk with "fix".
238 : * In such cases we have to clear any stored duplicate inode.
239 : * After the scan process completes, no file should have this flag set.
240 : */
241 : #define FILE_IS_WITHOUT_INODE 0x200
242 :
243 : /**
244 : * The file is deleted.
245 : * This happens when a file is deleted from the array,
246 : * but it's keep inside the parity until the next sync.
247 : *
248 : * During the file-system check we needs this information,
249 : * because deleted files may be present only partially.
250 : */
251 : #define FILE_IS_DELETED 0x400
252 :
253 : /**
254 : * The file is missing.
255 : * This happens in fix/check when a file is cannot be opened,
256 : * and marking it as such prevents to retry to open it again.
257 : */
258 : #define FILE_IS_MISSING 0x800
259 :
260 : #define FILE_IS_HARDLINK 0x1000 /**< If it's an hardlink. */
261 : #define FILE_IS_SYMLINK 0x2000 /**< If it's a file symlink. */
262 : #define FILE_IS_SYMDIR 0x4000 /**< If it's a dir symlink for Windows. Not yet supported. */
263 : #define FILE_IS_JUNCTION 0x8000 /**< If it's a junction for Windows. Not yet supported. */
264 : #define FILE_IS_LINK_MASK 0xF000 /**< Mask for link type. */
265 :
266 : /**
267 : * File.
268 : */
269 : struct snapraid_file {
270 : int64_t mtime_sec; /**< Modification time. */
271 : uint64_t inode; /**< Inode. */
272 : uint64_t physical; /**< Physical offset of the file. */
273 : data_off_t size; /**< Size of the file. */
274 : struct snapraid_block* blockvec; /**< All the blocks of the file. */
275 : int mtime_nsec; /**< Modification time nanoseconds. In the range 0 <= x < 1,000,000,000, or STAT_NSEC_INVALID if not present. */
276 : block_off_t blockmax; /**< Number of blocks. */
277 : unsigned flag; /**< FILE_IS_* flags. */
278 : char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
279 :
280 : /* nodes for data structures */
281 : tommy_node nodelist;
282 : tommy_hashdyn_node nodeset;
283 : tommy_hashdyn_node pathset;
284 : tommy_hashdyn_node stampset;
285 : };
286 :
287 : /**
288 : * Symbolic Link.
289 : */
290 : struct snapraid_link {
291 : unsigned flag; /**< FILE_IS_* flags. */
292 : char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
293 : char* linkto; /**< Link to. */
294 :
295 : /* nodes for data structures */
296 : tommy_node nodelist;
297 : tommy_hashdyn_node nodeset;
298 : };
299 :
300 : /**
301 : * Dir.
302 : */
303 : struct snapraid_dir {
304 : unsigned flag; /**< FILE_IS_* flags. */
305 : char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
306 :
307 : /* nodes for data structures */
308 : tommy_node nodelist;
309 : tommy_hashdyn_node nodeset;
310 : };
311 :
312 : /**
313 : * Chunk.
314 : *
315 : * A extent represents a fragment of a file mapped into the parity.
316 : */
317 : struct snapraid_extent {
318 : struct snapraid_file* file; /**< File containing this extent. */
319 : block_off_t parity_pos; /**< Parity position. */
320 : block_off_t file_pos; /**< Position in the file. */
321 : block_off_t count; /**< Number of sequential blocks in the file and parity. */
322 : tommy_tree_node parity_node; /**< Tree sorted by <parity_pos>. */
323 : tommy_tree_node file_node; /**< Tree sorter by <file,file_pos>. */
324 : };
325 :
326 : /**
327 : * Disk.
328 : */
329 : struct snapraid_disk {
330 : char name[PATH_MAX]; /**< Name of the disk. */
331 : char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */
332 : char smartctl[PATH_MAX]; /**< Custom command for smartctl. Empty means auto. */
333 : char uuid[UUID_MAX]; /**< UUID of the disk. */
334 :
335 : uint64_t device; /**< Device identifier. */
336 : block_off_t total_blocks; /**< Number of total blocks. */
337 : block_off_t free_blocks; /**< Number of free blocks at the last sync. */
338 :
339 : uint64_t tick; /**< Usage time. */
340 : uint64_t progress_tick[PROGRESS_MAX]; /**< Last ticks of progress. */
341 : unsigned cached; /**< Number of IO blocks cached. */
342 :
343 : /**
344 : * First free searching block.
345 : * Note that it doesn't necessarily point at the first free block,
346 : * but it just tell you that no free block is present before this position.
347 : */
348 : block_off_t first_free_block;
349 :
350 : int has_volatile_inodes; /**< If the underline file-system has not persistent inodes. */
351 : int has_volatile_hardlinks; /**< If the underline file-system has not syncronized metadata for hardlink (NTFS). */
352 : int has_unreliable_physical; /**< If the physical offset of files has duplicates. */
353 : int has_different_uuid; /**< If the disk has a different UUID, meaning that it is not the same file-system. */
354 : int has_unsupported_uuid; /**< If the disk doesn't report UUID, meaning it's not supported. */
355 : int had_empty_uuid; /**< If the disk had an empty UUID, meaning that it's a new disk. */
356 : int mapping_idx; /**< Index in the mapping vector. Used only as buffer when writing the content file. */
357 : int skip_access; /**< If the disk is inaccessible and it should be skipped. */
358 :
359 : #if HAVE_PTHREAD
360 : /**
361 : * Mutex for protecting the filesystem structure.
362 : *
363 : * Specifically, this protects ::fs_parity, ::fs_file, and ::fs_last,
364 : * meaning that it protects only extents.
365 : *
366 : * Files, links and dirs are not protected as they are not expected to
367 : * change during multithread processing.
368 : */
369 : pthread_mutex_t fs_mutex;
370 : #endif
371 :
372 : /**
373 : * Mapping of extents in the parity.
374 : * Sorted by <parity_pos> and by <file,file_pos>
375 : */
376 : tommy_tree fs_parity;
377 : tommy_tree fs_file;
378 :
379 : /**
380 : * Last extent we accessed.
381 : * It's used to optimize access of sequential blocks.
382 : */
383 : struct snapraid_extent* fs_last;
384 :
385 : /**
386 : * List of all the snapraid_file for the disk.
387 : */
388 : tommy_list filelist;
389 :
390 : /**
391 : * List of all the deleted file for the disk.
392 : *
393 : * These files are kept allocated, because the blocks are still referenced in
394 : * the ::blockarr.
395 : */
396 : tommy_list deletedlist;
397 :
398 : tommy_hashdyn inodeset; /**< Hashtable by inode of all the files. */
399 : tommy_hashdyn pathset; /**< Hashtable by path of all the files. */
400 : tommy_hashdyn stampset; /**< Hashtable by stamp (size and time) of all the files. */
401 : tommy_list linklist; /**< List of all the links. */
402 : tommy_hashdyn linkset; /**< Hashtable by name of all the links. */
403 : tommy_list dirlist; /**< List of all the empty dirs. */
404 : tommy_hashdyn dirset; /**< Hashtable by name of all the empty dirs. */
405 :
406 : /* nodes for data structures */
407 : tommy_node node;
408 : };
409 :
410 : /**
411 : * Disk mapping.
412 : */
413 : struct snapraid_map {
414 : char name[PATH_MAX]; /**< Name of the disk. */
415 : char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */
416 : block_off_t total_blocks; /**< Number of total blocks. */
417 : block_off_t free_blocks; /**< Number of free blocks at last 'sync'. */
418 : unsigned position; /**< Position of the disk in the parity. */
419 :
420 : /* nodes for data structures */
421 : tommy_node node;
422 : };
423 :
424 : /**
425 : * Max number of parity split.
426 : */
427 : #define SPLIT_MAX 8
428 :
429 : /**
430 : * Invalid parity size.
431 : *
432 : * This value is used to identify new parities,
433 : * like when you alter the configuration adding
434 : * a new parity level, creating it with 'fix'.
435 : * Given that 'fix' doesn't write the content file,
436 : * the new size will be written only at the next
437 : * 'sync'.
438 : */
439 : #define PARITY_SIZE_INVALID -1LL
440 :
441 : /**
442 : * Parity split.
443 : */
444 : struct snapraid_split {
445 : char path[PATH_MAX]; /**< Path of the parity file. */
446 : char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */
447 :
448 : /**
449 : * Size of the parity split.
450 : * Only the latest not zero size is allowed to grow.
451 : * If the value is unset, it's PARITY_SIZE_INVALID.
452 : */
453 : data_off_t size;
454 :
455 : uint64_t device; /**< Device identifier of the parity. */
456 : };
457 :
458 : /**
459 : * Parity.
460 : */
461 : struct snapraid_parity {
462 : struct snapraid_split split_map[SPLIT_MAX]; /**< Parity splits. */
463 : unsigned split_mac; /**< Number of parity splits. */
464 : char smartctl[PATH_MAX]; /**< Custom command for smartctl. Empty means auto. */
465 : block_off_t total_blocks; /**< Number of total blocks. */
466 : block_off_t free_blocks; /**< Number of free blocks at the last sync. */
467 : int is_excluded_by_filter; /**< If the parity is excluded by filters. */
468 : int skip_access; /**< If at least one of the parity disk is inaccessible and it should be skipped. */
469 : uint64_t tick; /**< Usage time. */
470 : uint64_t progress_tick[PROGRESS_MAX]; /**< Last cpu ticks of progress. */
471 : unsigned cached; /**< Number of IO blocks cached. */
472 : };
473 :
474 : /**
475 : * Info.
476 : */
477 : typedef uint32_t snapraid_info;
478 :
479 : /**
480 : * Allocate a content.
481 : */
482 : struct snapraid_content* content_alloc(const char* path, uint64_t dev);
483 :
484 : /**
485 : * Deallocate a content.
486 : */
487 : void content_free(struct snapraid_content* content);
488 :
489 : /**
490 : * Allocate a filter pattern for files and directories.
491 : */
492 : struct snapraid_filter* filter_alloc_file(int is_include, const char* pattern);
493 :
494 : /**
495 : * Allocate a filter pattern for disks.
496 : */
497 : struct snapraid_filter* filter_alloc_disk(int is_include, const char* pattern);
498 :
499 : /**
500 : * Deallocate an exclusion.
501 : */
502 : void filter_free(struct snapraid_filter* filter);
503 :
504 : /**
505 : * Filter type description.
506 : */
507 : const char* filter_type(struct snapraid_filter* filter, char* out, size_t out_size);
508 :
509 : /**
510 : * Filter hidden files.
511 : * Return !=0 if it matches and it should be excluded.
512 : */
513 3173014 : static inline int filter_hidden(int enable, struct dirent* dd)
514 : {
515 3173014 : if (enable && dirent_hidden(dd)) {
516 0 : return 1; /* filter out */
517 : }
518 :
519 3173014 : return 0;
520 : }
521 :
522 : /**
523 : * Filter a path using a list of filters.
524 : * For each element of the path all the filters are applied, until the first one that matches.
525 : * Return !=0 if it should be excluded.
526 : */
527 : int filter_path(tommy_list* filterlist, struct snapraid_filter** reason, const char* disk, const char* sub);
528 :
529 : /**
530 : * Filter a file/link/dir if missing.
531 : * This call imply a disk check for the file presence.
532 : * Return !=0 if the file is present and it should be excluded.
533 : */
534 : int filter_existence(int filter_missing, const char* dir, const char* sub);
535 :
536 : /**
537 : * Filter a file if bad.
538 : * Return !=0 if the file is correct and it should be excluded.
539 : */
540 : int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file);
541 :
542 : /**
543 : * Filter a dir using a list of filters.
544 : * For each element of the path all the filters are applied, until the first one that matches.
545 : * Thesesdir are always by included by default, to allow to apply rules at the contained files.
546 : * Return !=0 if should be excluded.
547 : */
548 : int filter_subdir(tommy_list* filterlist, struct snapraid_filter** reason, const char* disk, const char* sub);
549 :
550 : /**
551 : * Filter a dir using a list of filters.
552 : * For each element of the path all the filters are applied, until the first one that matches.
553 : * Return !=0 if should be excluded.
554 : */
555 : int filter_emptydir(tommy_list* filterlist, struct snapraid_filter** reason, const char* disk, const char* sub);
556 :
557 : /**
558 : * Filter a path if it's a content file.
559 : * Return !=0 if should be excluded.
560 : */
561 : int filter_content(tommy_list* contentlist, const char* path);
562 :
563 : /**
564 : * Check if the specified hash is invalid.
565 : *
566 : * An invalid hash is represented with all bytes at 0x00.
567 : *
568 : * If working with reduced hash lengths, this function always return 0.
569 : */
570 759521 : static inline int hash_is_invalid(const unsigned char* hash)
571 : {
572 : int i;
573 :
574 : /* if the hash is reduced, we cannot grant that it's a specific kind of hash */
575 759521 : if (BLOCK_HASH_SIZE != HASH_MAX)
576 0 : return 0;
577 :
578 1179390 : for (i = 0; i < BLOCK_HASH_SIZE; ++i)
579 1153326 : if (hash[i] != 0x00)
580 733457 : return 0;
581 :
582 26064 : return 1;
583 : }
584 :
585 : /**
586 : * Check if the specified hash represent the zero block.
587 : *
588 : * A zero hash is represented with all bytes at 0xFF.
589 : *
590 : * If working with reduced hash lengths, this function always return 0.
591 : */
592 760984 : static inline int hash_is_zero(const unsigned char* hash)
593 : {
594 : int i;
595 :
596 : /* if the hash is reduced, we cannot grant that it's a specific kind of hash */
597 760984 : if (BLOCK_HASH_SIZE != HASH_MAX)
598 0 : return 0;
599 :
600 995194 : for (i = 0; i < BLOCK_HASH_SIZE; ++i)
601 980725 : if (hash[i] != 0xFF)
602 746515 : return 0;
603 :
604 14469 : return 1;
605 : }
606 :
607 : /**
608 : * Check if the specified hash is unequivocally representing the data.
609 : *
610 : * If working with reduced hash lengths, this function always return 0.
611 : */
612 44272 : static inline int hash_is_unique(const unsigned char* hash)
613 : {
614 : /* if the hash is reduced, we cannot grant that it's a specific kind of hash */
615 44272 : if (BLOCK_HASH_SIZE != HASH_MAX)
616 0 : return 0;
617 :
618 44272 : return !hash_is_zero(hash) && !hash_is_invalid(hash);
619 : }
620 :
621 : /**
622 : * Set the hash to the special INVALID value.
623 : */
624 9234056 : static inline void hash_invalid_set(unsigned char* hash)
625 : {
626 9234056 : memset(hash, 0x00, BLOCK_HASH_SIZE);
627 9234056 : }
628 :
629 : /**
630 : * Set the hash to the special ZERO value.
631 : */
632 115543 : static inline void hash_zero_set(unsigned char* hash)
633 : {
634 115543 : memset(hash, 0xFF, BLOCK_HASH_SIZE);
635 115543 : }
636 :
637 : /**
638 : * Allocated space for block.
639 : */
640 58061578 : static inline size_t block_sizeof(void)
641 : {
642 58061578 : return 1 + BLOCK_HASH_SIZE;
643 : }
644 :
645 : /**
646 : * Get the state of the block.
647 : *
648 : * For this function, it's allowed to pass a NULL block
649 : * pointer than results in the BLOCK_STATE_EMPTY state.
650 : */
651 52302812 : static inline unsigned block_state_get(const struct snapraid_block* block)
652 : {
653 52302812 : if (block == BLOCK_NULL)
654 1721787 : return BLOCK_STATE_EMPTY;
655 :
656 50581025 : return block->state;
657 : }
658 :
659 : /**
660 : * Set the state of the block.
661 : */
662 19117243 : static inline void block_state_set(struct snapraid_block* block, unsigned state)
663 : {
664 19117243 : block->state = state;
665 19117243 : }
666 :
667 : /**
668 : * Check if the specified block has an updated hash.
669 : *
670 : * Note that EMPTY / CHG / DELETED return 0.
671 : */
672 2218840 : static inline int block_has_updated_hash(const struct snapraid_block* block)
673 : {
674 2218840 : unsigned state = block_state_get(block);
675 :
676 2218840 : return state == BLOCK_STATE_BLK || state == BLOCK_STATE_REP;
677 : }
678 :
679 : /**
680 : * Check if the specified block has a past hash,
681 : * i.e. the hash of the data that it's now overwritten or lost.
682 : *
683 : * Note that EMPTY / BLK / REP return 0.
684 : */
685 2542004 : static inline int block_has_past_hash(const struct snapraid_block* block)
686 : {
687 2542004 : unsigned state = block_state_get(block);
688 :
689 2542004 : return state == BLOCK_STATE_CHG || state == BLOCK_STATE_DELETED;
690 : }
691 :
692 : /**
693 : * Check if the specified block is part of a file.
694 : *
695 : * Note that EMPTY / DELETED return 0.
696 : */
697 14585890 : static inline int block_has_file(const struct snapraid_block* block)
698 : {
699 14585890 : unsigned state = block_state_get(block);
700 :
701 14587920 : return state == BLOCK_STATE_BLK
702 14587920 : || state == BLOCK_STATE_CHG || state == BLOCK_STATE_REP;
703 : }
704 :
705 : /**
706 : * Check if the block has an invalid parity than needs to be updated.
707 : *
708 : * Note that EMPTY / BLK return 0.
709 : */
710 12320776 : static inline int block_has_invalid_parity(const struct snapraid_block* block)
711 : {
712 12320776 : unsigned state = block_state_get(block);
713 :
714 12320776 : return state == BLOCK_STATE_DELETED
715 12320776 : || state == BLOCK_STATE_CHG || state == BLOCK_STATE_REP;
716 : }
717 :
718 : /**
719 : * Check if the block is part of a file with valid parity.
720 : *
721 : * Note that anything different than BLK return 0.
722 : */
723 220963 : static inline int block_has_file_and_valid_parity(const struct snapraid_block* block)
724 : {
725 220963 : unsigned state = block_state_get(block);
726 :
727 220963 : return state == BLOCK_STATE_BLK;
728 : }
729 :
730 34101473 : static inline int file_flag_has(const struct snapraid_file* file, unsigned mask)
731 : {
732 34101473 : return (file->flag & mask) == mask;
733 : }
734 :
735 5303517 : static inline void file_flag_set(struct snapraid_file* file, unsigned mask)
736 : {
737 5303517 : file->flag |= mask;
738 5303517 : }
739 :
740 18459 : static inline void file_flag_clear(struct snapraid_file* file, unsigned mask)
741 : {
742 18459 : file->flag &= ~mask;
743 18459 : }
744 :
745 : /**
746 : * Allocate a file.
747 : */
748 : struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical);
749 :
750 : /**
751 : * Duplicate a file.
752 : */
753 : struct snapraid_file* file_dup(struct snapraid_file* copy);
754 :
755 : /**
756 : * Deallocate a file.
757 : */
758 : void file_free(struct snapraid_file* file);
759 :
760 : /**
761 : * Rename a file.
762 : */
763 : void file_rename(struct snapraid_file* file, const char* sub);
764 :
765 : /**
766 : * Copy a file.
767 : */
768 : void file_copy(struct snapraid_file* src_file, struct snapraid_file* dest_file);
769 :
770 : /**
771 : * Return the block at the specified position.
772 : *
773 : * Note that the block size if a runtime value.
774 : */
775 54348022 : static inline struct snapraid_block* file_block(struct snapraid_file* file, size_t pos)
776 : {
777 54348022 : unsigned char* ptr = (unsigned char*)file->blockvec;
778 :
779 54348022 : return (struct snapraid_block*)(ptr + pos * block_sizeof());
780 : }
781 :
782 : /**
783 : * Return the name of the file, without the dir.
784 : */
785 : const char* file_name(const struct snapraid_file* file);
786 :
787 : /**
788 : * Check if the block is the last in the file.
789 : */
790 : int file_block_is_last(struct snapraid_file* file, block_off_t file_pos);
791 :
792 : /**
793 : * Get the size in bytes of the block.
794 : * If it's the last block of a file it could be less than block_size.
795 : */
796 : unsigned file_block_size(struct snapraid_file* file, block_off_t file_pos, unsigned block_size);
797 :
798 : /**
799 : * Compare a file with an inode.
800 : */
801 : int file_inode_compare_to_arg(const void* void_arg, const void* void_data);
802 :
803 : /**
804 : * Compare files by inode.
805 : */
806 : int file_inode_compare(const void* void_a, const void* void_b);
807 :
808 : /**
809 : * Compare files by path.
810 : */
811 : int file_path_compare(const void* void_a, const void* void_b);
812 :
813 : /**
814 : * Compare files by physical address.
815 : */
816 : int file_physical_compare(const void* void_a, const void* void_b);
817 :
818 : /**
819 : * Compute the hash of a file inode.
820 : */
821 5055426 : static inline tommy_uint32_t file_inode_hash(uint64_t inode)
822 : {
823 5055426 : return (tommy_uint32_t)tommy_inthash_u64(inode);
824 : }
825 :
826 : /**
827 : * Compare a file with a path.
828 : */
829 : int file_path_compare_to_arg(const void* void_arg, const void* void_data);
830 :
831 : /**
832 : * Compare a file with another file for name, stamp, and both.
833 : */
834 : int file_name_compare(const void* void_a, const void* void_b);
835 : int file_stamp_compare(const void* void_a, const void* void_b);
836 : int file_namestamp_compare(const void* void_a, const void* void_b);
837 : int file_pathstamp_compare(const void* void_a, const void* void_b);
838 :
839 : /**
840 : * Compute the hash of a file path.
841 : */
842 3891909 : static inline tommy_uint32_t file_path_hash(const char* sub)
843 : {
844 3891909 : return tommy_hash_u32(0, sub, strlen(sub));
845 : }
846 :
847 : /**
848 : * Compute the hash of a file stamp.
849 : */
850 6275848 : static inline tommy_uint32_t file_stamp_hash(data_off_t size, int64_t mtime_sec, int mtime_nsec)
851 : {
852 6275848 : return tommy_inthash_u32((tommy_uint32_t)size ^ tommy_inthash_u32(mtime_sec ^ tommy_inthash_u32(mtime_nsec)));
853 : }
854 :
855 : /**
856 : * Allocate a extent.
857 : */
858 : struct snapraid_extent* extent_alloc(block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos, block_off_t count);
859 :
860 : /**
861 : * Deallocate a extent.
862 : */
863 : void extent_free(struct snapraid_extent* extent);
864 :
865 : /**
866 : * Compare extent by parity position.
867 : */
868 : int extent_parity_compare(const void* void_a, const void* void_b);
869 :
870 : /**
871 : * Compare extent by file and file position.
872 : */
873 : int extent_file_compare(const void* void_a, const void* void_b);
874 :
875 209328 : static inline int link_flag_has(const struct snapraid_link* slink, unsigned mask)
876 : {
877 209328 : return (slink->flag & mask) == mask;
878 : }
879 :
880 39461 : static inline void link_flag_set(struct snapraid_link* slink, unsigned mask)
881 : {
882 39461 : slink->flag |= mask;
883 39461 : }
884 :
885 : static inline void link_flag_clear(struct snapraid_link* slink, unsigned mask)
886 : {
887 : slink->flag &= ~mask;
888 : }
889 :
890 6 : static inline void link_flag_let(struct snapraid_link* slink, unsigned flag, unsigned mask)
891 : {
892 6 : slink->flag &= ~mask;
893 6 : slink->flag |= flag & mask;
894 6 : }
895 :
896 91794 : static inline unsigned link_flag_get(struct snapraid_link* slink, unsigned mask)
897 : {
898 91794 : return slink->flag & mask;
899 : }
900 :
901 : /**
902 : * Allocate a link.
903 : */
904 : struct snapraid_link* link_alloc(const char* name, const char* slink, unsigned link_flag);
905 :
906 : /**
907 : * Deallocate a link.
908 : */
909 : void link_free(struct snapraid_link* slink);
910 :
911 : /**
912 : * Compare a link with a name.
913 : */
914 : int link_name_compare_to_arg(const void* void_arg, const void* void_data);
915 :
916 : /**
917 : * Compare links by path.
918 : */
919 : int link_alpha_compare(const void* void_a, const void* void_b);
920 :
921 : /**
922 : * Compute the hash of a link name.
923 : */
924 155706 : static inline tommy_uint32_t link_name_hash(const char* name)
925 : {
926 155706 : return tommy_hash_u32(0, name, strlen(name));
927 : }
928 :
929 940 : static inline int dir_flag_has(const struct snapraid_dir* dir, unsigned mask)
930 : {
931 940 : return (dir->flag & mask) == mask;
932 : }
933 :
934 278 : static inline void dir_flag_set(struct snapraid_dir* dir, unsigned mask)
935 : {
936 278 : dir->flag |= mask;
937 278 : }
938 :
939 : static inline void dir_flag_clear(struct snapraid_dir* dir, unsigned mask)
940 : {
941 : dir->flag &= ~mask;
942 : }
943 :
944 : /**
945 : * Allocate a dir.
946 : */
947 : struct snapraid_dir* dir_alloc(const char* name);
948 :
949 : /**
950 : * Deallocate a dir.
951 : */
952 : void dir_free(struct snapraid_dir* dir);
953 :
954 : /**
955 : * Compare a dir with a name.
956 : */
957 : int dir_name_compare(const void* void_arg, const void* void_data);
958 :
959 : /**
960 : * Compute the hash of a dir name.
961 : */
962 1059 : static inline tommy_uint32_t dir_name_hash(const char* name)
963 : {
964 1059 : return tommy_hash_u32(0, name, strlen(name));
965 : }
966 :
967 : /**
968 : * Allocate a disk.
969 : */
970 : struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev, const char* uuid, int skip);
971 :
972 : /**
973 : * Deallocate a disk.
974 : */
975 : void disk_free(struct snapraid_disk* disk);
976 :
977 : /**
978 : * Get the size of the disk in blocks.
979 : */
980 : block_off_t fs_size(struct snapraid_disk* disk);
981 :
982 : /**
983 : * Check if a disk is totally empty and can be discarded from the content file.
984 : * A disk is empty if it doesn't contain any file, symlink, hardlink or dir
985 : * and without any DELETED block.
986 : * The blockmax is used to limit the search of DELETED block up to blockmax.
987 : */
988 : int fs_is_empty(struct snapraid_disk* disk, block_off_t blockmax);
989 :
990 : /**
991 : * Check the file-system for errors.
992 : * Return 0 if it's OK.
993 : */
994 : int fs_check(struct snapraid_disk* disk);
995 :
996 : /**
997 : * Allocate a parity position for the specified file position.
998 : *
999 : * After this call you can use the par2file/par2block operations
1000 : * to query the relation.
1001 : *
1002 : * \note This function is NOT thread-safe as it uses the the disk cache. +
1003 : */
1004 : void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos);
1005 :
1006 : /**
1007 : * Deallocate the parity position.
1008 : *
1009 : * After this call the par2file/par2block operations
1010 : * won't find anymore the parity association.
1011 : *
1012 : * \note This function is NOT thread-safe as it uses the the disk cache.
1013 : */
1014 : void fs_deallocate(struct snapraid_disk* disk, block_off_t pos);
1015 :
1016 : /**
1017 : * Get the block from the file position.
1018 : */
1019 : struct snapraid_block* fs_file2block_get(struct snapraid_file* file, block_off_t file_pos);
1020 :
1021 : /**
1022 : * Get the file position from the parity position.
1023 : * Return 0 if no file is using it.
1024 : */
1025 : struct snapraid_file* fs_par2file_find(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos);
1026 :
1027 : /**
1028 : * Get the file position from the parity position.
1029 : */
1030 11721014 : static inline struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos)
1031 : {
1032 : struct snapraid_file* ret;
1033 :
1034 11721014 : ret = fs_par2file_find(disk, parity_pos, file_pos);
1035 11723751 : if (ret == 0) {
1036 : /* LCOV_EXCL_START */
1037 : log_fatal("Internal inconsistency when deresolving parity to file at position '%u' in disk '%s'\n", parity_pos, disk->name);
1038 : os_abort();
1039 : /* LCOV_EXCL_STOP */
1040 : }
1041 :
1042 11723751 : return ret;
1043 : }
1044 :
1045 : /**
1046 : * Get the parity position from the file position.
1047 : * Return POS_NULL if no parity is allocated.
1048 : */
1049 : block_off_t fs_file2par_find(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos);
1050 :
1051 : /**
1052 : * Get the parity position from the file position.
1053 : */
1054 4755439 : static inline block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos)
1055 : {
1056 : block_off_t ret;
1057 :
1058 4755439 : ret = fs_file2par_find(disk, file, file_pos);
1059 4755439 : if (ret == POS_NULL) {
1060 : /* LCOV_EXCL_START */
1061 : log_fatal("Internal inconsistency when resolving file '%s' at position '%u/%u' in disk '%s'\n", file->sub, file_pos, file->blockmax, disk->name);
1062 : os_abort();
1063 : /* LCOV_EXCL_STOP */
1064 : }
1065 :
1066 4755439 : return ret;
1067 : }
1068 :
1069 : /**
1070 : * Get the block from the parity position.
1071 : * Return BLOCK_NULL==0 if the block is over the end of the disk or not used.
1072 : */
1073 : struct snapraid_block* fs_par2block_find(struct snapraid_disk* disk, block_off_t parity_pos);
1074 :
1075 : /**
1076 : * Get the block from the parity position.
1077 : */
1078 15867 : static inline struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos)
1079 : {
1080 : struct snapraid_block* ret;
1081 :
1082 15867 : ret = fs_par2block_find(disk, parity_pos);
1083 15867 : if (ret == BLOCK_NULL) {
1084 : /* LCOV_EXCL_START */
1085 : log_fatal("Internal inconsistency when deresolving parity to block at position '%u' in disk '%s'\n", parity_pos, disk->name);
1086 : os_abort();
1087 : /* LCOV_EXCL_STOP */
1088 : }
1089 :
1090 15867 : return ret;
1091 : }
1092 :
1093 : /**
1094 : * Allocate a disk mapping.
1095 : * Uses uuid="" if not available.
1096 : */
1097 : struct snapraid_map* map_alloc(const char* name, unsigned position, block_off_t total_blocks, block_off_t free_blocks, const char* uuid);
1098 :
1099 : /**
1100 : * Deallocate a disk mapping.
1101 : */
1102 : void map_free(struct snapraid_map* map);
1103 :
1104 : /**
1105 : * Mask used to store additional information in the info bits.
1106 : *
1107 : * These bits reduce the granularity of the time in the memory representation.
1108 : */
1109 : #define INFO_MASK 0x7
1110 :
1111 : /**
1112 : * Make an info.
1113 : */
1114 194355 : static inline snapraid_info info_make(time_t last_access, int error, int rehash, int justsynced)
1115 : {
1116 : /* clear the lowest bits as reserved for other information */
1117 194355 : snapraid_info info = last_access & ~INFO_MASK;
1118 :
1119 194355 : if (error != 0)
1120 5273 : info |= 0x1;
1121 194355 : if (rehash != 0)
1122 14426 : info |= 0x2;
1123 194355 : if (justsynced != 0)
1124 142804 : info |= 0x4;
1125 194355 : return info;
1126 : }
1127 :
1128 : /**
1129 : * Extract the time information.
1130 : * This is the last time when the block was know to be correct.
1131 : * The "scrubbed" info tells if the time is referreing at the latest sync or scrub.
1132 : */
1133 883089 : static inline time_t info_get_time(snapraid_info info)
1134 : {
1135 883089 : return info & ~INFO_MASK;
1136 : }
1137 :
1138 : /**
1139 : * Extract the error information.
1140 : * Report if the block address had some problem.
1141 : */
1142 351938 : static inline int info_get_bad(snapraid_info info)
1143 : {
1144 351938 : return (info & 0x1) != 0;
1145 : }
1146 :
1147 : /**
1148 : * Extract the rehash information.
1149 : * Report if the block address is using the old hash and needs to be rehashed.
1150 : */
1151 2046765 : static inline int info_get_rehash(snapraid_info info)
1152 : {
1153 2046765 : return (info & 0x2) != 0;
1154 : }
1155 :
1156 : /**
1157 : * Extract the scrubbed information.
1158 : * Report if the block address was never scrubbed.
1159 : */
1160 68520 : static inline int info_get_justsynced(snapraid_info info)
1161 : {
1162 68520 : return (info & 0x4) != 0;
1163 : }
1164 :
1165 : /**
1166 : * Mark the block address as with error.
1167 : */
1168 1773 : static inline snapraid_info info_set_bad(snapraid_info info)
1169 : {
1170 1773 : return info | 0x1;
1171 : }
1172 :
1173 : /**
1174 : * Mark the block address as with rehash.
1175 : */
1176 9215 : static inline snapraid_info info_set_rehash(snapraid_info info)
1177 : {
1178 9215 : return info | 0x2;
1179 : }
1180 :
1181 : /**
1182 : * Set the info at the specified position.
1183 : * The position is allocated if not yet done.
1184 : */
1185 1698024 : static inline void info_set(tommy_arrayblkof* array, block_off_t pos, snapraid_info info)
1186 : {
1187 1698024 : tommy_arrayblkof_grow(array, pos + 1);
1188 :
1189 1698024 : memcpy(tommy_arrayblkof_ref(array, pos), &info, sizeof(snapraid_info));
1190 1698024 : }
1191 :
1192 : /**
1193 : * Get the info at the specified position.
1194 : * For not allocated position, 0 is returned.
1195 : */
1196 5098521 : static inline snapraid_info info_get(tommy_arrayblkof* array, block_off_t pos)
1197 : {
1198 : snapraid_info info;
1199 :
1200 5098521 : if (pos >= tommy_arrayblkof_size(array))
1201 262643 : return 0;
1202 :
1203 4835878 : memcpy(&info, tommy_arrayblkof_ref(array, pos), sizeof(snapraid_info));
1204 :
1205 4835878 : return info;
1206 : }
1207 :
1208 : /**
1209 : * Compare times
1210 : */
1211 : int time_compare(const void* void_a, const void* void_b);
1212 :
1213 : /****************************************************************************/
1214 : /* format */
1215 :
1216 : #define FMT_FILE 0 /**< Print only the file. */
1217 : #define FMT_DISK 1 /**< Print the disk name and the file. */
1218 : #define FMT_PATH 2 /**< Print the full path. */
1219 :
1220 : extern int FMT_MODE;
1221 :
1222 : /**
1223 : * Format a file path for poll reference
1224 : */
1225 : const char* fmt_poll(const struct snapraid_disk* disk, const char* str, char* buffer);
1226 :
1227 : /**
1228 : * Format a path name for terminal reference
1229 : */
1230 : const char* fmt_term(const struct snapraid_disk* disk, const char* str, char* buffer);
1231 :
1232 : #endif
1233 :
|
