Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #ifndef __ELEM_H
19 : #define __ELEM_H
20 :
21 : #include "util.h"
22 : #include "support.h"
23 : #include "tommyds/tommyhash.h"
24 : #include "tommyds/tommylist.h"
25 : #include "tommyds/tommytree.h"
26 : #include "tommyds/tommyhashdyn.h"
27 : #include "tommyds/tommyarray.h"
28 : #include "tommyds/tommyarrayblkof.h"
29 :
30 : /****************************************************************************/
31 : /* snapraid */
32 :
33 : /**
34 : * Number of measures of the operation progress.
35 : */
36 : #define PROGRESS_MAX 100
37 :
38 : /**
39 : * Max UUID length.
40 : */
41 : #define UUID_MAX 128
42 :
43 : /**
44 : * Invalid position.
45 : */
46 : #define POS_NULL ((block_off_t)-1)
47 :
48 : /**
49 : * Content file specification.
50 : */
51 : struct snapraid_content {
52 : char content[PATH_MAX]; /**< Path of the content file. */
53 : uint64_t device; /**< Device identifier. */
54 : void* context; /**< Context used for multithread operations. */
55 : tommy_node node; /**< Next node in the list. */
56 : };
57 :
58 : /**
59 : * Filter for paths.
60 : */
61 : struct snapraid_filter {
62 : char pattern[PATH_MAX]; /**< Filter pattern. */
63 : int is_disk; /**< If the pattern is a disk one. */
64 : int is_path; /**< If the pattern is only for the complete path. */
65 : int is_dir; /**< If the pattern is only for dir. */
66 : int direction; /**< If it's an inclusion (=1) or an exclusion (=-1). */
67 : tommy_node node; /**< Next node in the list. */
68 : };
69 :
70 : /**
71 : * Block pointer used to represent unused blocks.
72 : */
73 : #define BLOCK_NULL 0
74 :
75 : /**
76 : * This block is an empty one.
77 : * Note that an empty block is represent with ::BLOCK_NULL.
78 : */
79 : #define BLOCK_STATE_EMPTY 0
80 :
81 : /**
82 : * The block has both the hash and the parity computed.
83 : * This is the normal state of a saved block.
84 : *
85 : * The block hash field IS set.
86 : * The parity for this disk is updated.
87 : */
88 : #define BLOCK_STATE_BLK 1
89 :
90 : /**
91 : * The block is new and not yet hashed.
92 : * This happens when a new block overwrite a just removed block, or an empty space.
93 : *
94 : * The block hash field MAY be set and it represents the hash of the OLD data.
95 : * The hash may be also INVALID or ZERO.
96 : *
97 : * If the OLD block was empty, the hash is set to the special ZERO value.
98 : * If the OLD block was lost, the hash is set to the special INVALID value.
99 : *
100 : * The parity for this disk is not updated, but it contains the old data referenced by the hash.
101 : *
102 : * If the state is read from an incomplete sync, we don't really know if the hash is referring at the
103 : * data used to compute the parity, because the sync process was interrupted at an unknown point,
104 : * and the parity may or may not be updated.
105 : *
106 : * For this reason we clear all such hashes when reading the state from an incomplete sync before
107 : * starting a new sync, because sync is affected by such hashes, as sync updates the parity, only
108 : * if the new data read for CHG blocks has a mismatching hash.
109 : * Clearing is done setting the ::clear_past_hash flag before reading the state.
110 : * No clearing is done in other commands, as check and fix are instead able to work with unsynced
111 : * hashes, and scrub ignores CHG/DELETED blocks.
112 : */
113 : #define BLOCK_STATE_CHG 2
114 :
115 : /**
116 : * The block is new and hashed.
117 : * This happens when a new block overwrite a just removed block, or an empty space.
118 : *
119 : * Note that when the file copy heuristic is enabled, the REP blocks may be set
120 : * using this heuristic, meaning that the hash may be wrong.
121 : *
122 : * For this reason, when the ::force_nocopy flag is enabled in sync, we convert all the REP blocks
123 : * to CHG, invalidating the stored hash.
124 : * Clearing is done setting the ::clear_past_hash flag before reading the state.
125 : * No clearing is done in other commands, as they don't stop the process like in sync
126 : * when there is a false silent error.
127 : *
128 : * The block hash field IS set, and it represents the hash of the new data.
129 : * The parity for this disk is not updated.
130 : */
131 : #define BLOCK_STATE_REP 3
132 :
133 : /**
134 : * This block is a deleted one.
135 : * This happens when a file is deleted.
136 : *
137 : * The block hash field IS set, and it represents the hash of the previous data,
138 : * but only if it's different by all 0.
139 : * The parity for this disk is not updated, but it contains the old data referenced by the hash.
140 : *
141 : * If the state is read from an incomplete sync, we don't really know if the hash is referring at the
142 : * data used to compute the parity, because the sync process was interrupted at an unknown point,
143 : * and the parity may or may not be updated.
144 : *
145 : * A now the sync process is not affected by DELETED hash, so clearing won't be really needed,
146 : * but considering that we have to do it for CHG blocks, we do it also for DELETED ones,
147 : * clearing all the past hashes.
148 : * Clearing is done setting the ::clear_past_hash flag before reading the state.
149 : * No clearing is done in other commands, as check and fix are instead able to work with unsynced
150 : * hashes, and scrub ignores CHG/DELETED blocks.
151 : */
152 : #define BLOCK_STATE_DELETED 4
153 :
154 : /**
155 : * Block hash size.
156 : *
157 : * At max HASH_MAX.
158 : */
159 : extern int BLOCK_HASH_SIZE;
160 :
161 : /**
162 : * Block of a file.
163 : */
164 : struct snapraid_block {
165 : unsigned char state; /**< State of the block. */
166 :
167 : /**
168 : * Hash of the block.
169 : *
170 : * The effective stored size is BLOCK_HASH_SIZE.
171 : */
172 : unsigned char hash[HASH_MAX];
173 : };
174 :
175 : /**
176 : * If a file is present in the disk.
177 : * It's used only in scan to detect present and missing files.
178 : */
179 : #define FILE_IS_PRESENT 0x01
180 :
181 : /**
182 : * If it's an excluded file from the processing.
183 : * It's used in both check and fix to mark files to exclude from the processing.
184 : */
185 : #define FILE_IS_EXCLUDED 0x02
186 :
187 : /**
188 : * If a fix was attempted but it failed.
189 : * It's used only in fix to mark that some data is unrecoverable.
190 : */
191 : #define FILE_IS_DAMAGED 0x04
192 :
193 : /**
194 : * If a fix was done.
195 : * It's used only in fix to mark that some data was recovered.
196 : */
197 : #define FILE_IS_FIXED 0x08
198 :
199 : /**
200 : * If the file was originally missing, and it was created in the fix process.
201 : * It's used only in fix to mark files recovered from scratch,
202 : * meaning that they don't have any previous content.
203 : * This is important because it means that deleting them, you are not going
204 : * to lose something that cannot be recovered.
205 : * Note that excluded files won't ever get this flag.
206 : */
207 : #define FILE_IS_CREATED 0x10
208 :
209 : /**
210 : * If the file has completed its processing, meaning that it won't be opened anymore.
211 : * It's used only in fix to mark when we finish processing one file.
212 : * Note that excluded files won't ever get this flag.
213 : */
214 : #define FILE_IS_FINISHED 0x20
215 :
216 : /**
217 : * If the file hash was obtained from a file copy
218 : * identified by the same name, size and stamp.
219 : */
220 : #define FILE_IS_COPY 0x40
221 :
222 : /**
223 : * If the file was opened.
224 : * It's used in fix to detect if it's the first time a file is opened.
225 : */
226 : #define FILE_IS_OPENED 0x80
227 :
228 : /**
229 : * If the file is modified from the latest sync.
230 : * It's used in fix to store if the state of the file before being modified.
231 : */
232 : #define FILE_IS_UNSYNCED 0x100
233 :
234 : /**
235 : * If the file is without inode.
236 : * It could happen in file-system where inodes are not persistent,
237 : * or when restoring a full disk with "fix".
238 : * In such cases we have to clear any stored duplicate inode.
239 : * After the scan process completes, no file should have this flag set.
240 : */
241 : #define FILE_IS_WITHOUT_INODE 0x200
242 :
243 : /**
244 : * The file is deleted.
245 : * This happens when a file is deleted from the array,
246 : * but it's keep inside the parity until the next sync.
247 : *
248 : * During the file-system check we needs this information,
249 : * because deleted files may be present only partially.
250 : */
251 : #define FILE_IS_DELETED 0x400
252 :
253 : /**
254 : * The file is missing.
255 : * This happens in fix/check when a file is cannot be opened,
256 : * and marking it as such prevents to retry to open it again.
257 : */
258 : #define FILE_IS_MISSING 0x800
259 :
260 : #define FILE_IS_HARDLINK 0x1000 /**< If it's an hardlink. */
261 : #define FILE_IS_SYMLINK 0x2000 /**< If it's a file symlink. */
262 : #define FILE_IS_SYMDIR 0x4000 /**< If it's a dir symlink for Windows. Not yet supported. */
263 : #define FILE_IS_JUNCTION 0x8000 /**< If it's a junction for Windows. Not yet supported. */
264 : #define FILE_IS_LINK_MASK 0xF000 /**< Mask for link type. */
265 :
266 : /**
267 : * File.
268 : */
269 : struct snapraid_file {
270 : int64_t mtime_sec; /**< Modification time. */
271 : uint64_t inode; /**< Inode. */
272 : uint64_t physical; /**< Physical offset of the file. */
273 : data_off_t size; /**< Size of the file. */
274 : struct snapraid_block* blockvec; /**< All the blocks of the file. */
275 : int mtime_nsec; /**< Modification time nanoseconds. In the range 0 <= x < 1,000,000,000, or STAT_NSEC_INVALID if not present. */
276 : block_off_t blockmax; /**< Number of blocks. */
277 : unsigned flag; /**< FILE_IS_* flags. */
278 : char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
279 :
280 : /* nodes for data structures */
281 : tommy_node nodelist;
282 : tommy_hashdyn_node nodeset;
283 : tommy_hashdyn_node pathset;
284 : tommy_hashdyn_node stampset;
285 : };
286 :
287 : /**
288 : * Symbolic Link.
289 : */
290 : struct snapraid_link {
291 : unsigned flag; /**< FILE_IS_* flags. */
292 : char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
293 : char* linkto; /**< Link to. */
294 :
295 : /* nodes for data structures */
296 : tommy_node nodelist;
297 : tommy_hashdyn_node nodeset;
298 : };
299 :
300 : /**
301 : * Dir.
302 : */
303 : struct snapraid_dir {
304 : unsigned flag; /**< FILE_IS_* flags. */
305 : char* sub; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
306 :
307 : /* nodes for data structures */
308 : tommy_node nodelist;
309 : tommy_hashdyn_node nodeset;
310 : };
311 :
312 : /**
313 : * Chunk.
314 : *
315 : * A extent represents a fragment of a file mapped into the parity.
316 : */
317 : struct snapraid_extent {
318 : struct snapraid_file* file; /**< File containing this extent. */
319 : block_off_t parity_pos; /**< Parity position. */
320 : block_off_t file_pos; /**< Position in the file. */
321 : block_off_t count; /**< Number of sequential blocks in the file and parity. */
322 : tommy_tree_node parity_node; /**< Tree sorted by <parity_pos>. */
323 : tommy_tree_node file_node; /**< Tree sorter by <file,file_pos>. */
324 : };
325 :
326 : /**
327 : * Disk.
328 : */
329 : struct snapraid_disk {
330 : char name[PATH_MAX]; /**< Name of the disk. */
331 : char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */
332 : char smartctl[PATH_MAX]; /**< Custom command for smartctl. Empty means auto. */
333 : int smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for this device. */
334 : char uuid[UUID_MAX]; /**< UUID of the disk. */
335 :
336 : uint64_t device; /**< Device identifier. */
337 : block_off_t total_blocks; /**< Number of total blocks. */
338 : block_off_t free_blocks; /**< Number of free blocks at the last sync. */
339 :
340 : uint64_t tick; /**< Usage time. */
341 : uint64_t progress_tick[PROGRESS_MAX]; /**< Last ticks of progress. */
342 : unsigned cached_blocks; /**< Number of IO blocks cached. */
343 : struct snapraid_file* progress_file; /**< File in progress. */
344 :
345 : /**
346 : * First free searching block.
347 : * Note that it doesn't necessarily point at the first free block,
348 : * but it just tell you that no free block is present before this position.
349 : */
350 : block_off_t first_free_block;
351 :
352 : int has_volatile_inodes; /**< If the underline file-system has not persistent inodes. */
353 : int has_volatile_hardlinks; /**< If the underline file-system has not synchronized metadata for hardlink (NTFS). */
354 : int has_unreliable_physical; /**< If the physical offset of files has duplicates. */
355 : int has_different_uuid; /**< If the disk has a different UUID, meaning that it is not the same file-system. */
356 : int has_unsupported_uuid; /**< If the disk doesn't report UUID, meaning it's not supported. */
357 : int had_empty_uuid; /**< If the disk had an empty UUID, meaning that it's a new disk. */
358 : int mapping_idx; /**< Index in the mapping vector. Used only as buffer when writing the content file. */
359 : int skip_access; /**< If the disk is inaccessible and it should be skipped. */
360 :
361 : #if HAVE_THREAD
362 : /**
363 : * Mutex for protecting the filesystem structure.
364 : *
365 : * Specifically, this protects ::fs_parity, ::fs_file, and ::fs_last,
366 : * meaning that it protects only extents.
367 : *
368 : * Files, links and dirs are not protected as they are not expected to
369 : * change during multithread processing.
370 : */
371 : thread_mutex_t fs_mutex;
372 : int fs_mutex_enabled; /*< If the lock has to be used. */
373 :
374 : /**
375 : * Mutex for protecting the scan process.
376 : *
377 : * It's used during the scan process to protect the stampset to identity copy of files
378 : */
379 : thread_mutex_t stamp_mutex;
380 : #endif
381 :
382 : /**
383 : * Mapping of extents in the parity.
384 : * Sorted by <parity_pos> and by <file,file_pos>
385 : */
386 : tommy_tree fs_parity;
387 : tommy_tree fs_file;
388 :
389 : /**
390 : * Last extent we accessed.
391 : * It's used to optimize access of sequential blocks.
392 : */
393 : struct snapraid_extent* fs_last;
394 :
395 : /**
396 : * List of all the snapraid_file for the disk.
397 : */
398 : tommy_list filelist;
399 :
400 : /**
401 : * List of all the deleted file for the disk.
402 : *
403 : * These files are kept allocated, because the blocks are still referenced in
404 : * the ::blockarr.
405 : */
406 : tommy_list deletedlist;
407 :
408 : tommy_hashdyn inodeset; /**< Hashtable by inode of all the files. */
409 : tommy_hashdyn pathset; /**< Hashtable by path of all the files. */
410 : tommy_hashdyn stampset; /**< Hashtable by stamp (size and time) of all the files. */
411 : tommy_list linklist; /**< List of all the links. */
412 : tommy_hashdyn linkset; /**< Hashtable by name of all the links. */
413 : tommy_list dirlist; /**< List of all the empty dirs. */
414 : tommy_hashdyn dirset; /**< Hashtable by name of all the empty dirs. */
415 :
416 : /* nodes for data structures */
417 : tommy_node node;
418 : };
419 :
420 : /**
421 : * Disk mapping.
422 : */
423 : struct snapraid_map {
424 : char name[PATH_MAX]; /**< Name of the disk. */
425 : char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */
426 : block_off_t total_blocks; /**< Number of total blocks. */
427 : block_off_t free_blocks; /**< Number of free blocks at last 'sync'. */
428 : unsigned position; /**< Position of the disk in the parity. */
429 :
430 : /* nodes for data structures */
431 : tommy_node node;
432 : };
433 :
434 : /**
435 : * Max number of parity split.
436 : */
437 : #define SPLIT_MAX 8
438 :
439 : /**
440 : * Invalid parity size.
441 : *
442 : * This value is used to identify new parities,
443 : * like when you alter the configuration adding
444 : * a new parity level, creating it with 'fix'.
445 : * Given that 'fix' doesn't write the content file,
446 : * the new size will be written only at the next
447 : * 'sync'.
448 : */
449 : #define PARITY_SIZE_INVALID -1LL
450 :
451 : /**
452 : * Parity split.
453 : */
454 : struct snapraid_split {
455 : char path[PATH_MAX]; /**< Path of the parity file. */
456 : char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */
457 :
458 : /**
459 : * Size of the parity split.
460 : * Only the latest not zero size is allowed to grow.
461 : * If the value is unset, it's PARITY_SIZE_INVALID.
462 : */
463 : data_off_t size;
464 :
465 : uint64_t device; /**< Device identifier of the parity. */
466 : };
467 :
468 : /**
469 : * Parity.
470 : */
471 : struct snapraid_parity {
472 : struct snapraid_split split_map[SPLIT_MAX]; /**< Parity splits. */
473 : unsigned split_mac; /**< Number of parity splits. */
474 : char smartctl[PATH_MAX]; /**< Custom command for smartctl. Empty means auto. */
475 : int smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for this device. */
476 : block_off_t total_blocks; /**< Number of total blocks. */
477 : block_off_t free_blocks; /**< Number of free blocks at the last sync. */
478 : int is_excluded_by_filter; /**< If the parity is excluded by filters. */
479 : int skip_access; /**< If at least one of the parity disk is inaccessible and it should be skipped. */
480 : uint64_t tick; /**< Usage time. */
481 : uint64_t progress_tick[PROGRESS_MAX]; /**< Last cpu ticks of progress. */
482 : unsigned cached_blocks; /**< Number of IO blocks cached. */
483 : };
484 :
485 : /**
486 : * Info.
487 : */
488 : typedef uint64_t snapraid_info;
489 :
490 : /**
491 : * Allocate a content.
492 : */
493 : struct snapraid_content* content_alloc(const char* path, uint64_t dev);
494 :
495 : /**
496 : * Deallocate a content.
497 : */
498 : void content_free(struct snapraid_content* content);
499 :
500 : /**
501 : * Allocate a filter pattern for files and directories.
502 : */
503 : struct snapraid_filter* filter_alloc_file(int is_include, const char* pattern);
504 :
505 : /**
506 : * Allocate a filter pattern for disks.
507 : */
508 : struct snapraid_filter* filter_alloc_disk(int is_include, const char* pattern);
509 :
510 : /**
511 : * Deallocate an exclusion.
512 : */
513 : void filter_free(struct snapraid_filter* filter);
514 :
515 : /**
516 : * Filter type description.
517 : */
518 : const char* filter_type(struct snapraid_filter* filter, char* out, size_t out_size);
519 :
520 : /**
521 : * Filter hidden files.
522 : * Return !=0 if it matches and it should be excluded.
523 : */
524 3173346 : static inline int filter_hidden(int enable, struct dirent* dd)
525 : {
526 3173346 : if (enable && dirent_hidden(dd)) {
527 0 : return 1; /* filter out */
528 : }
529 :
530 3173346 : return 0;
531 : }
532 :
533 : /**
534 : * Filter a path using a list of filters.
535 : * For each element of the path all the filters are applied, until the first one that matches.
536 : * Return !=0 if it should be excluded.
537 : */
538 : int filter_path(tommy_list* filterlist, struct snapraid_filter** reason, const char* disk, const char* sub);
539 :
540 : /**
541 : * Filter a file/link/dir if missing.
542 : * This call imply a disk check for the file presence.
543 : * Return !=0 if the file is present and it should be excluded.
544 : */
545 : int filter_existence(int filter_missing, const char* dir, const char* sub);
546 :
547 : /**
548 : * Filter a file if bad.
549 : * Return !=0 if the file is correct and it should be excluded.
550 : */
551 : int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file);
552 :
553 : /**
554 : * Filter a dir using a list of filters.
555 : * For each element of the path all the filters are applied, until the first one that matches.
556 : * Thesesdir are always by included by default, to allow to apply rules at the contained files.
557 : * Return !=0 if should be excluded.
558 : */
559 : int filter_subdir(tommy_list* filterlist, struct snapraid_filter** reason, const char* disk, const char* sub);
560 :
561 : /**
562 : * Filter a dir using a list of filters.
563 : * For each element of the path all the filters are applied, until the first one that matches.
564 : * Return !=0 if should be excluded.
565 : */
566 : int filter_emptydir(tommy_list* filterlist, struct snapraid_filter** reason, const char* disk, const char* sub);
567 :
568 : /**
569 : * Filter a path if it's a content file.
570 : * Return !=0 if should be excluded.
571 : */
572 : int filter_content(tommy_list* contentlist, const char* path);
573 :
574 : /**
575 : * Check if the specified hash is invalid.
576 : *
577 : * An invalid hash is represented with all bytes at 0x00.
578 : *
579 : * If working with reduced hash lengths, this function always return 0.
580 : */
581 756575 : static inline int hash_is_invalid(const unsigned char* hash)
582 : {
583 : int i;
584 :
585 : /* if the hash is reduced, we cannot grant that it's a specific kind of hash */
586 756575 : if (BLOCK_HASH_SIZE != HASH_MAX)
587 0 : return 0;
588 :
589 1176326 : for (i = 0; i < BLOCK_HASH_SIZE; ++i)
590 1150262 : if (hash[i] != 0x00)
591 730511 : return 0;
592 :
593 26064 : return 1;
594 : }
595 :
596 : /**
597 : * Check if the specified hash represent the zero block.
598 : *
599 : * A zero hash is represented with all bytes at 0xFF.
600 : *
601 : * If working with reduced hash lengths, this function always return 0.
602 : */
603 758037 : static inline int hash_is_zero(const unsigned char* hash)
604 : {
605 : int i;
606 :
607 : /* if the hash is reduced, we cannot grant that it's a specific kind of hash */
608 758037 : if (BLOCK_HASH_SIZE != HASH_MAX)
609 0 : return 0;
610 :
611 992339 : for (i = 0; i < BLOCK_HASH_SIZE; ++i)
612 977870 : if (hash[i] != 0xFF)
613 743568 : return 0;
614 :
615 14469 : return 1;
616 : }
617 :
618 : /**
619 : * Check if the specified hash is unequivocally representing the data.
620 : *
621 : * If working with reduced hash lengths, this function always return 0.
622 : */
623 42003 : static inline int hash_is_unique(const unsigned char* hash)
624 : {
625 : /* if the hash is reduced, we cannot grant that it's a specific kind of hash */
626 42003 : if (BLOCK_HASH_SIZE != HASH_MAX)
627 0 : return 0;
628 :
629 42003 : return !hash_is_zero(hash) && !hash_is_invalid(hash);
630 : }
631 :
632 : /**
633 : * Set the hash to the special INVALID value.
634 : */
635 9244426 : static inline void hash_invalid_set(unsigned char* hash)
636 : {
637 9244426 : memset(hash, 0x00, BLOCK_HASH_SIZE);
638 9244426 : }
639 :
640 : /**
641 : * Set the hash to the special ZERO value.
642 : */
643 115543 : static inline void hash_zero_set(unsigned char* hash)
644 : {
645 115543 : memset(hash, 0xFF, BLOCK_HASH_SIZE);
646 115543 : }
647 :
648 : /**
649 : * Allocated space for block.
650 : */
651 55095695 : static inline size_t block_sizeof(void)
652 : {
653 55095695 : return 1 + BLOCK_HASH_SIZE;
654 : }
655 :
656 : /**
657 : * Get the state of the block.
658 : *
659 : * For this function, it's allowed to pass a NULL block
660 : * pointer than results in the BLOCK_STATE_EMPTY state.
661 : */
662 46798328 : static inline unsigned block_state_get(const struct snapraid_block* block)
663 : {
664 46798328 : if (block == BLOCK_NULL)
665 1514217 : return BLOCK_STATE_EMPTY;
666 :
667 45284111 : return block->state;
668 : }
669 :
670 : /**
671 : * Set the state of the block.
672 : */
673 19119835 : static inline void block_state_set(struct snapraid_block* block, unsigned state)
674 : {
675 19119835 : block->state = state;
676 19119835 : }
677 :
678 : /**
679 : * Check if the specified block has an updated hash.
680 : *
681 : * Note that EMPTY / CHG / DELETED return 0.
682 : */
683 2181182 : static inline int block_has_updated_hash(const struct snapraid_block* block)
684 : {
685 2181182 : unsigned state = block_state_get(block);
686 :
687 2181182 : return state == BLOCK_STATE_BLK || state == BLOCK_STATE_REP;
688 : }
689 :
690 : /**
691 : * Check if the specified block has a past hash,
692 : * i.e. the hash of the data that it's now overwritten or lost.
693 : *
694 : * Note that EMPTY / BLK / REP return 0.
695 : */
696 2542571 : static inline int block_has_past_hash(const struct snapraid_block* block)
697 : {
698 2542571 : unsigned state = block_state_get(block);
699 :
700 2542571 : return state == BLOCK_STATE_CHG || state == BLOCK_STATE_DELETED;
701 : }
702 :
703 : /**
704 : * Check if the specified block is part of a file.
705 : *
706 : * Note that EMPTY / DELETED return 0.
707 : */
708 11857533 : static inline int block_has_file(const struct snapraid_block* block)
709 : {
710 11857533 : unsigned state = block_state_get(block);
711 :
712 : return state == BLOCK_STATE_BLK
713 11857533 : || state == BLOCK_STATE_CHG || state == BLOCK_STATE_REP;
714 : }
715 :
716 : /**
717 : * Check if the block has an invalid parity than needs to be updated.
718 : *
719 : * Note that EMPTY / BLK return 0.
720 : */
721 9855272 : static inline int block_has_invalid_parity(const struct snapraid_block* block)
722 : {
723 9855272 : unsigned state = block_state_get(block);
724 :
725 : return state == BLOCK_STATE_DELETED
726 9855272 : || state == BLOCK_STATE_CHG || state == BLOCK_STATE_REP;
727 : }
728 :
729 : /**
730 : * Check if the block is part of a file with valid parity.
731 : *
732 : * Note that anything different than BLK return 0.
733 : */
734 220960 : static inline int block_has_file_and_valid_parity(const struct snapraid_block* block)
735 : {
736 220960 : unsigned state = block_state_get(block);
737 :
738 220960 : return state == BLOCK_STATE_BLK;
739 : }
740 :
741 33865472 : static inline int file_flag_has(const struct snapraid_file* file, unsigned mask)
742 : {
743 33865472 : return (file->flag & mask) == mask;
744 : }
745 :
746 5302860 : static inline void file_flag_set(struct snapraid_file* file, unsigned mask)
747 : {
748 5302860 : file->flag |= mask;
749 5302860 : }
750 :
751 11568 : static inline void file_flag_clear(struct snapraid_file* file, unsigned mask)
752 : {
753 11568 : file->flag &= ~mask;
754 11568 : }
755 :
756 : /**
757 : * Allocate a file.
758 : */
759 : struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical);
760 :
761 : /**
762 : * Duplicate a file.
763 : */
764 : struct snapraid_file* file_dup(struct snapraid_file* copy);
765 :
766 : /**
767 : * Deallocate a file.
768 : */
769 : void file_free(struct snapraid_file* file);
770 :
771 : /**
772 : * Rename a file.
773 : */
774 : void file_rename(struct snapraid_file* file, const char* sub);
775 :
776 : /**
777 : * Copy a file.
778 : */
779 : void file_copy(struct snapraid_file* src_file, struct snapraid_file* dest_file);
780 :
781 : /**
782 : * Return the block at the specified position.
783 : *
784 : * Note that the block size if a runtime value.
785 : */
786 51377857 : static inline struct snapraid_block* file_block(struct snapraid_file* file, size_t pos)
787 : {
788 51377857 : unsigned char* ptr = (unsigned char*)file->blockvec;
789 :
790 51377857 : return (struct snapraid_block*)(ptr + pos * block_sizeof());
791 : }
792 :
793 : /**
794 : * Return the name of the file, without the dir.
795 : */
796 : const char* file_name(const struct snapraid_file* file);
797 :
798 : /**
799 : * Check if the block is the last in the file.
800 : */
801 : int file_block_is_last(struct snapraid_file* file, block_off_t file_pos);
802 :
803 : /**
804 : * Get the size in bytes of the block.
805 : * If it's the last block of a file it could be less than block_size.
806 : */
807 : unsigned file_block_size(struct snapraid_file* file, block_off_t file_pos, unsigned block_size);
808 :
809 : /**
810 : * Compare a file with an inode.
811 : */
812 : int file_inode_compare_to_arg(const void* void_arg, const void* void_data);
813 :
814 : /**
815 : * Compare files by inode.
816 : */
817 : int file_inode_compare(const void* void_a, const void* void_b);
818 :
819 : /**
820 : * Compare files by path.
821 : */
822 : int file_path_compare(const void* void_a, const void* void_b);
823 :
824 : /**
825 : * Compare files by physical address.
826 : */
827 : int file_physical_compare(const void* void_a, const void* void_b);
828 :
829 : /**
830 : * Compute the hash of a file inode.
831 : */
832 5050478 : static inline tommy_uint32_t file_inode_hash(uint64_t inode)
833 : {
834 5050478 : return (tommy_uint32_t)tommy_inthash_u64(inode);
835 : }
836 :
837 : /**
838 : * Compare a file with a path.
839 : */
840 : int file_path_compare_to_arg(const void* void_arg, const void* void_data);
841 :
842 : /**
843 : * Compare a file with another file for name, stamp, and both.
844 : */
845 : int file_name_compare(const void* void_a, const void* void_b);
846 : int file_stamp_compare(const void* void_a, const void* void_b);
847 : int file_namestamp_compare(const void* void_a, const void* void_b);
848 : int file_pathstamp_compare(const void* void_a, const void* void_b);
849 :
850 : /**
851 : * Compute the hash of a file path.
852 : */
853 3890321 : static inline tommy_uint32_t file_path_hash(const char* sub)
854 : {
855 3890321 : return tommy_hash_u32(0, sub, strlen(sub));
856 : }
857 :
858 : /**
859 : * Compute the hash of a file stamp.
860 : */
861 6277957 : static inline tommy_uint32_t file_stamp_hash(data_off_t size, int64_t mtime_sec, int mtime_nsec)
862 : {
863 6277957 : return tommy_inthash_u32((tommy_uint32_t)size ^ tommy_inthash_u32(mtime_sec ^ tommy_inthash_u32(mtime_nsec)));
864 : }
865 :
866 : /**
867 : * Allocate a extent.
868 : */
869 : struct snapraid_extent* extent_alloc(block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos, block_off_t count);
870 :
871 : /**
872 : * Deallocate a extent.
873 : */
874 : void extent_free(struct snapraid_extent* extent);
875 :
876 : /**
877 : * Compare extent by parity position.
878 : */
879 : int extent_parity_compare(const void* void_a, const void* void_b);
880 :
881 : /**
882 : * Compare extent by file and file position.
883 : */
884 : int extent_file_compare(const void* void_a, const void* void_b);
885 :
886 209329 : static inline int link_flag_has(const struct snapraid_link* slink, unsigned mask)
887 : {
888 209329 : return (slink->flag & mask) == mask;
889 : }
890 :
891 39461 : static inline void link_flag_set(struct snapraid_link* slink, unsigned mask)
892 : {
893 39461 : slink->flag |= mask;
894 39461 : }
895 :
896 : static inline void link_flag_clear(struct snapraid_link* slink, unsigned mask)
897 : {
898 : slink->flag &= ~mask;
899 : }
900 :
901 6 : static inline void link_flag_let(struct snapraid_link* slink, unsigned flag, unsigned mask)
902 : {
903 6 : slink->flag &= ~mask;
904 6 : slink->flag |= flag & mask;
905 6 : }
906 :
907 90071 : static inline unsigned link_flag_get(struct snapraid_link* slink, unsigned mask)
908 : {
909 90071 : return slink->flag & mask;
910 : }
911 :
912 : /**
913 : * Allocate a link.
914 : */
915 : struct snapraid_link* link_alloc(const char* name, const char* slink, unsigned link_flag);
916 :
917 : /**
918 : * Deallocate a link.
919 : */
920 : void link_free(struct snapraid_link* slink);
921 :
922 : /**
923 : * Compare a link with a name.
924 : */
925 : int link_name_compare_to_arg(const void* void_arg, const void* void_data);
926 :
927 : /**
928 : * Compare links by path.
929 : */
930 : int link_alpha_compare(const void* void_a, const void* void_b);
931 :
932 : /**
933 : * Compute the hash of a link name.
934 : */
935 155917 : static inline tommy_uint32_t link_name_hash(const char* name)
936 : {
937 155917 : return tommy_hash_u32(0, name, strlen(name));
938 : }
939 :
940 940 : static inline int dir_flag_has(const struct snapraid_dir* dir, unsigned mask)
941 : {
942 940 : return (dir->flag & mask) == mask;
943 : }
944 :
945 278 : static inline void dir_flag_set(struct snapraid_dir* dir, unsigned mask)
946 : {
947 278 : dir->flag |= mask;
948 278 : }
949 :
950 : static inline void dir_flag_clear(struct snapraid_dir* dir, unsigned mask)
951 : {
952 : dir->flag &= ~mask;
953 : }
954 :
955 : /**
956 : * Allocate a dir.
957 : */
958 : struct snapraid_dir* dir_alloc(const char* name);
959 :
960 : /**
961 : * Deallocate a dir.
962 : */
963 : void dir_free(struct snapraid_dir* dir);
964 :
965 : /**
966 : * Compare a dir with a name.
967 : */
968 : int dir_name_compare(const void* void_arg, const void* void_data);
969 :
970 : /**
971 : * Compute the hash of a dir name.
972 : */
973 1059 : static inline tommy_uint32_t dir_name_hash(const char* name)
974 : {
975 1059 : return tommy_hash_u32(0, name, strlen(name));
976 : }
977 :
978 : /**
979 : * Allocate a disk.
980 : */
981 : struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev, const char* uuid, int skip_access);
982 :
983 : /**
984 : * Deallocate a disk.
985 : */
986 : void disk_free(struct snapraid_disk* disk);
987 :
988 : /**
989 : * Enable multithread support for the disk.
990 : */
991 : void disk_start_thread(struct snapraid_disk* disk);
992 :
993 : /**
994 : * Get the size of the disk in blocks.
995 : */
996 : block_off_t fs_size(struct snapraid_disk* disk);
997 :
998 : /**
999 : * Check if a disk is totally empty and can be discarded from the content file.
1000 : * A disk is empty if it doesn't contain any file, symlink, hardlink or dir
1001 : * and without any DELETED block.
1002 : * The blockmax is used to limit the search of DELETED block up to blockmax.
1003 : */
1004 : int fs_is_empty(struct snapraid_disk* disk, block_off_t blockmax);
1005 :
1006 : /**
1007 : * Check the file-system for errors.
1008 : * Return 0 if it's OK.
1009 : */
1010 : int fs_check(struct snapraid_disk* disk);
1011 :
1012 : /**
1013 : * Allocate a parity position for the specified file position.
1014 : *
1015 : * After this call you can use the par2file/par2block operations
1016 : * to query the relation.
1017 : *
1018 : * \note This function is NOT thread-safe as it uses the disk cache. +
1019 : */
1020 : void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos);
1021 :
1022 : /**
1023 : * Deallocate the parity position.
1024 : *
1025 : * After this call the par2file/par2block operations
1026 : * won't find anymore the parity association.
1027 : *
1028 : * \note This function is NOT thread-safe as it uses the disk cache.
1029 : */
1030 : void fs_deallocate(struct snapraid_disk* disk, block_off_t pos);
1031 :
1032 : /**
1033 : * Get the block from the file position.
1034 : */
1035 : struct snapraid_block* fs_file2block_get(struct snapraid_file* file, block_off_t file_pos);
1036 :
1037 : /**
1038 : * Get the file position from the parity position.
1039 : * Return 0 if no file is using it.
1040 : */
1041 : struct snapraid_file* fs_par2file_find(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos);
1042 :
1043 : /**
1044 : * Get the file position from the parity position.
1045 : */
1046 11510587 : static inline struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos)
1047 : {
1048 : struct snapraid_file* ret;
1049 :
1050 11510587 : ret = fs_par2file_find(disk, parity_pos, file_pos);
1051 11510587 : if (ret == 0) {
1052 : /* LCOV_EXCL_START */
1053 : log_fatal("Internal inconsistency: Deresolving parity to file at position '%u' in disk '%s'\n", parity_pos, disk->name);
1054 : os_abort();
1055 : /* LCOV_EXCL_STOP */
1056 : }
1057 :
1058 11510587 : return ret;
1059 : }
1060 :
1061 : /**
1062 : * Get the parity position from the file position.
1063 : * Return POS_NULL if no parity is allocated.
1064 : */
1065 : block_off_t fs_file2par_find(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos);
1066 :
1067 : /**
1068 : * Get the parity position from the file position.
1069 : */
1070 4629578 : static inline block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos)
1071 : {
1072 : block_off_t ret;
1073 :
1074 4629578 : ret = fs_file2par_find(disk, file, file_pos);
1075 4629578 : if (ret == POS_NULL) {
1076 : /* LCOV_EXCL_START */
1077 : log_fatal("Internal inconsistency: Resolving file '%s' at position '%u/%u' in disk '%s'\n", file->sub, file_pos, file->blockmax, disk->name);
1078 : os_abort();
1079 : /* LCOV_EXCL_STOP */
1080 : }
1081 :
1082 4629578 : return ret;
1083 : }
1084 :
1085 : /**
1086 : * Get the block from the parity position.
1087 : * Return BLOCK_NULL==0 if the block is over the end of the disk or not used.
1088 : */
1089 : struct snapraid_block* fs_par2block_find(struct snapraid_disk* disk, block_off_t parity_pos);
1090 :
1091 : /**
1092 : * Get the block from the parity position.
1093 : */
1094 15867 : static inline struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos)
1095 : {
1096 : struct snapraid_block* ret;
1097 :
1098 15867 : ret = fs_par2block_find(disk, parity_pos);
1099 15867 : if (ret == BLOCK_NULL) {
1100 : /* LCOV_EXCL_START */
1101 : log_fatal("Internal inconsistency: Deresolving parity to block at position '%u' in disk '%s'\n", parity_pos, disk->name);
1102 : os_abort();
1103 : /* LCOV_EXCL_STOP */
1104 : }
1105 :
1106 15867 : return ret;
1107 : }
1108 :
1109 : /**
1110 : * Allocate a disk mapping.
1111 : * Uses uuid="" if not available.
1112 : */
1113 : struct snapraid_map* map_alloc(const char* name, unsigned position, block_off_t total_blocks, block_off_t free_blocks, const char* uuid);
1114 :
1115 : /**
1116 : * Deallocate a disk mapping.
1117 : */
1118 : void map_free(struct snapraid_map* map);
1119 :
1120 : /**
1121 : * Mask used to store additional information in the info bits.
1122 : *
1123 : * These bits reduce the granularity of the time in the memory representation.
1124 : */
1125 : #define INFO_MASK ((snapraid_info)0x7)
1126 :
1127 : /**
1128 : * Make an info.
1129 : */
1130 177273 : static inline snapraid_info info_make(time_t last_access, int error, int rehash, int justsynced)
1131 : {
1132 : /* clear the lowest bits as reserved for other information */
1133 177273 : snapraid_info info = last_access & ~INFO_MASK;
1134 :
1135 177273 : if (error != 0)
1136 5304 : info |= 0x1;
1137 177273 : if (rehash != 0)
1138 14181 : info |= 0x2;
1139 177273 : if (justsynced != 0)
1140 131140 : info |= 0x4;
1141 177273 : return info;
1142 : }
1143 :
1144 : /**
1145 : * Extract the time information.
1146 : * This is the last time when the block was know to be correct.
1147 : * The "scrubbed" info tells if the time is referring at the latest sync or scrub.
1148 : */
1149 828753 : static inline time_t info_get_time(snapraid_info info)
1150 : {
1151 828753 : return info & ~INFO_MASK;
1152 : }
1153 :
1154 : /**
1155 : * Extract the error information.
1156 : * Report if the block address had some problem.
1157 : */
1158 279278 : static inline int info_get_bad(snapraid_info info)
1159 : {
1160 279278 : return (info & 0x1) != 0;
1161 : }
1162 :
1163 : /**
1164 : * Extract the rehash information.
1165 : * Report if the block address is using the old hash and needs to be rehashed.
1166 : */
1167 2018644 : static inline int info_get_rehash(snapraid_info info)
1168 : {
1169 2018644 : return (info & 0x2) != 0;
1170 : }
1171 :
1172 : /**
1173 : * Extract the scrubbed information.
1174 : * Report if the block address was never scrubbed.
1175 : */
1176 58782 : static inline int info_get_justsynced(snapraid_info info)
1177 : {
1178 58782 : return (info & 0x4) != 0;
1179 : }
1180 :
1181 : /**
1182 : * Mark the block address as with error.
1183 : */
1184 1772 : static inline snapraid_info info_set_bad(snapraid_info info)
1185 : {
1186 1772 : return info | 0x1;
1187 : }
1188 :
1189 : /**
1190 : * Mark the block address as with rehash.
1191 : */
1192 9215 : static inline snapraid_info info_set_rehash(snapraid_info info)
1193 : {
1194 9215 : return info | 0x2;
1195 : }
1196 :
1197 : /**
1198 : * Set the info at the specified position.
1199 : * The position is allocated if not yet done.
1200 : */
1201 1696798 : static inline void info_set(tommy_arrayblkof* array, block_off_t pos, snapraid_info info)
1202 : {
1203 1696798 : tommy_arrayblkof_grow(array, pos + 1);
1204 :
1205 1696798 : memcpy(tommy_arrayblkof_ref(array, pos), &info, sizeof(snapraid_info));
1206 1696798 : }
1207 :
1208 : /**
1209 : * Get the info at the specified position.
1210 : * For not allocated position, 0 is returned.
1211 : */
1212 3127326 : static inline snapraid_info info_get(tommy_arrayblkof* array, block_off_t pos)
1213 : {
1214 : snapraid_info info;
1215 :
1216 3127326 : if (pos >= tommy_arrayblkof_size(array))
1217 262643 : return 0;
1218 :
1219 2864683 : memcpy(&info, tommy_arrayblkof_ref(array, pos), sizeof(snapraid_info));
1220 :
1221 2864683 : return info;
1222 : }
1223 :
1224 : /**
1225 : * Compare times
1226 : */
1227 : int time_compare(const void* void_a, const void* void_b);
1228 :
1229 : /****************************************************************************/
1230 : /* format */
1231 :
1232 : #define FMT_FILE 0 /**< Print only the file. */
1233 : #define FMT_DISK 1 /**< Print the disk name and the file. */
1234 : #define FMT_PATH 2 /**< Print the full path. */
1235 :
1236 : extern int FMT_MODE;
1237 :
1238 : /**
1239 : * Format a file path for poll reference
1240 : */
1241 : const char* fmt_poll(const struct snapraid_disk* disk, const char* str, char* buffer);
1242 :
1243 : /**
1244 : * Format a path name for terminal reference
1245 : */
1246 : const char* fmt_term(const struct snapraid_disk* disk, const char* str, char* buffer);
1247 :
1248 : #endif
1249 :
|
