Line data Source code
1 : // SPDX-License-Identifier: GPL-3.0-or-later
2 : // Copyright (C) 2011 Andrea Mazzoleni
3 :
4 : #include "portable.h"
5 :
6 : #include "support.h"
7 : #include "elem.h"
8 : #include "state.h"
9 : #include "parity.h"
10 :
11 : struct snapraid_scan {
12 : struct snapraid_state* state; /**< State used. */
13 : struct snapraid_disk* disk; /**< Disk used. */
14 : thread_id_t thread; /**< Thread used for scanning the disk */
15 :
16 : int is_diff; /**< If it's a diff command or a scanning */
17 : int need_write; /**< If a state write is required */
18 :
19 : /**
20 : * Counters of changes.
21 : */
22 : unsigned count_equal; /**< Files equal. */
23 : unsigned count_move; /**< Files with a different name, but equal inode, size and timestamp in the same disk. */
24 : unsigned count_restore; /**< Files with equal name, size and timestamp, but different inode. */
25 : unsigned count_change; /**< Files with same name, but different size and/or timestamp. */
26 : unsigned count_copy; /**< Files new, with same name size and timestamp of a file in a different disk. */
27 : unsigned count_relocate; /**< Like copy, but with the original disappeared. */
28 : unsigned count_insert; /**< Files new. */
29 : unsigned count_remove; /**< Files removed. */
30 :
31 : tommy_list file_insert_list; /**< Files to insert. */
32 : tommy_list link_insert_list; /**< Links to insert. */
33 : tommy_list dir_insert_list; /**< Dirs to insert. */
34 : tommy_list local_filter_list; /**< Filter list specific for the disk. */
35 :
36 : /* nodes for data structures */
37 : tommy_node node;
38 : };
39 :
40 659 : static struct snapraid_scan* scan_alloc(struct snapraid_state* state, struct snapraid_disk* disk, int is_diff)
41 : {
42 : struct snapraid_scan* scan;
43 :
44 659 : scan = malloc_nofail(sizeof(struct snapraid_scan));
45 659 : scan->state = state;
46 659 : scan->disk = disk;
47 659 : scan->count_equal = 0;
48 659 : scan->count_move = 0;
49 659 : scan->count_copy = 0;
50 659 : scan->count_relocate = 0;
51 659 : scan->count_restore = 0;
52 659 : scan->count_change = 0;
53 659 : scan->count_remove = 0;
54 659 : scan->count_insert = 0;
55 659 : tommy_list_init(&scan->file_insert_list);
56 659 : tommy_list_init(&scan->link_insert_list);
57 659 : tommy_list_init(&scan->dir_insert_list);
58 659 : tommy_list_init(&scan->local_filter_list);
59 659 : scan->is_diff = is_diff;
60 659 : scan->need_write = 0;
61 :
62 : #if HAVE_THREAD
63 659 : thread_mutex_init(&disk->stamp_mutex);
64 : #endif
65 :
66 659 : return scan;
67 : }
68 :
69 653 : static void scan_free(struct snapraid_scan* scan)
70 : {
71 : #if HAVE_THREAD
72 653 : thread_mutex_destroy(&scan->disk->stamp_mutex);
73 : #endif
74 653 : tommy_list_foreach(&scan->local_filter_list, filter_free);
75 653 : free(scan);
76 653 : }
77 :
78 558309 : static void stamp_lock(struct snapraid_disk* disk)
79 : {
80 : #if HAVE_THREAD
81 558309 : thread_mutex_lock(&disk->stamp_mutex);
82 : #else
83 : (void)disk;
84 : #endif
85 558309 : }
86 :
87 558309 : static void stamp_unlock(struct snapraid_disk* disk)
88 : {
89 : #if HAVE_THREAD
90 558309 : thread_mutex_unlock(&disk->stamp_mutex);
91 : #else
92 : (void)disk;
93 : #endif
94 558309 : }
95 :
96 : /**
97 : * Remove the specified link from the data set.
98 : */
99 993 : static void scan_link_remove(struct snapraid_scan* scan, struct snapraid_link* slink)
100 : {
101 993 : struct snapraid_disk* disk = scan->disk;
102 :
103 : /* state changed */
104 993 : scan->need_write = 1;
105 :
106 : /* remove the file from the link containers */
107 993 : tommy_hashdyn_remove_existing(&disk->linkset, &slink->nodeset);
108 993 : tommy_list_remove_existing(&disk->linklist, &slink->nodelist);
109 :
110 : /* deallocate */
111 993 : link_free(slink);
112 993 : }
113 :
114 : /**
115 : * Insert the specified link in the data set.
116 : */
117 2566 : static void scan_link_insert(struct snapraid_scan* scan, struct snapraid_link* slink)
118 : {
119 2566 : struct snapraid_disk* disk = scan->disk;
120 :
121 : /* state changed */
122 2566 : scan->need_write = 1;
123 :
124 : /* insert the link in the link containers */
125 2566 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
126 2566 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
127 2566 : }
128 :
129 : /**
130 : * Process a symbolic link.
131 : */
132 40712 : static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub, const char* linkto, unsigned link_flag)
133 : {
134 40712 : struct snapraid_state* state = scan->state;
135 40712 : struct snapraid_disk* disk = scan->disk;
136 : struct snapraid_link* slink;
137 : char esc_buffer[ESC_MAX];
138 :
139 : /* check if the link already exists */
140 40712 : slink = tommy_hashdyn_search(&disk->linkset, link_name_compare_to_arg, sub, link_name_hash(sub));
141 40712 : if (slink) {
142 : /* check if multiple files have the same name */
143 38146 : if (link_flag_has(slink, FILE_IS_PRESENT)) {
144 : /* LCOV_EXCL_START */
145 : log_fatal(EINTERNAL, "Internal inconsistency for link '%s%s'\n", disk->dir, sub);
146 : os_abort();
147 : /* LCOV_EXCL_STOP */
148 : }
149 :
150 : /* mark as present */
151 38146 : link_flag_set(slink, FILE_IS_PRESENT);
152 :
153 : /* check if the link is not changed and it's of the same kind */
154 38146 : if (strcmp(slink->linkto, linkto) == 0 && link_flag == link_flag_get(slink, FILE_IS_LINK_MASK)) {
155 : /* it's equal */
156 38088 : ++scan->count_equal;
157 :
158 38088 : if (state->opt.gui_verbose) {
159 0 : log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
160 : }
161 : } else {
162 : /* it's an update */
163 :
164 : /* we have to save the linkto/type */
165 58 : scan->need_write = 1;
166 :
167 58 : ++scan->count_change;
168 :
169 58 : log_tag("scan:update:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
170 58 : if (is_diff) {
171 3 : msg_info("update %s\n", fmt_term(disk, slink->sub, esc_buffer));
172 : }
173 :
174 : /* update it */
175 58 : free(slink->linkto);
176 58 : slink->linkto = strdup_nofail(linkto);
177 58 : link_flag_let(slink, link_flag, FILE_IS_LINK_MASK);
178 : }
179 :
180 : /* nothing more to do */
181 38146 : return;
182 : } else {
183 : /* create the new link */
184 2566 : ++scan->count_insert;
185 :
186 2566 : log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
187 2566 : if (is_diff) {
188 599 : msg_info("add %s\n", fmt_term(disk, sub, esc_buffer));
189 : }
190 :
191 : /* and continue to insert it */
192 : }
193 :
194 : /* insert it */
195 2566 : slink = link_alloc(sub, linkto, link_flag);
196 :
197 : /* mark it as present */
198 2566 : link_flag_set(slink, FILE_IS_PRESENT);
199 :
200 : /* insert it in the delayed insert list */
201 2566 : tommy_list_insert_tail(&scan->link_insert_list, &slink->nodelist, slink);
202 : }
203 :
204 : /**
205 : * Insert the specified file in the parity.
206 : */
207 105934 : static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
208 : {
209 105934 : struct snapraid_state* state = scan->state;
210 105934 : struct snapraid_disk* disk = scan->disk;
211 : block_off_t i;
212 : block_off_t parity_pos;
213 :
214 : /* state changed */
215 105934 : scan->need_write = 1;
216 :
217 : /* allocate the blocks of the file */
218 105934 : parity_pos = disk->first_free_block;
219 360546 : for (i = 0; i < file->blockmax; ++i) {
220 : struct snapraid_block* block;
221 : struct snapraid_block* over_block;
222 : snapraid_info info;
223 :
224 : /* increment the position until the first really free block */
225 786215 : while (block_has_file(fs_par2block_find(disk, parity_pos)))
226 531603 : ++parity_pos;
227 :
228 : /* get block we are going to overwrite, if any */
229 254612 : over_block = fs_par2block_find(disk, parity_pos);
230 :
231 : /* deallocate it */
232 254612 : if (over_block != BLOCK_NULL)
233 93731 : fs_deallocate(disk, parity_pos);
234 :
235 : /* get block specific info */
236 254612 : info = info_get(&state->infoarr, parity_pos);
237 :
238 : /* get the new block we are going to write */
239 254612 : block = fs_file2block_get(file, i);
240 :
241 : /* if the file block already has an updated hash without rehash */
242 254612 : if (block_has_updated_hash(block) && !info_get_rehash(info)) {
243 : /* the only possible case is for REP blocks */
244 85548 : assert(block_state_get(block) == BLOCK_STATE_REP);
245 :
246 : /* convert to a REP block */
247 85548 : block_state_set(block, BLOCK_STATE_REP);
248 :
249 : /* and keep the hash as it's */
250 : } else {
251 : unsigned over_state;
252 :
253 : /* convert to a CHG block */
254 169064 : block_state_set(block, BLOCK_STATE_CHG);
255 :
256 : /* state of the block we are going to overwrite */
257 169064 : over_state = block_state_get(over_block);
258 :
259 : /* if the block is an empty one */
260 169064 : if (over_state == BLOCK_STATE_EMPTY) {
261 : /*
262 : * The block was empty and filled with zeros
263 : * set the hash to the special ZERO value
264 : */
265 120437 : hash_zero_set(block->hash);
266 : } else {
267 : /* otherwise it's a DELETED one */
268 48627 : assert(over_state == BLOCK_STATE_DELETED);
269 :
270 : /*
271 : * In this cases we don't know if the old state is still the one
272 : * stored inside the parity, because after an aborted sync, the parity
273 : * may be or may be not have been updated with the data that it's now
274 : * deleted.
275 : *
276 : * We anyway keep the hash as the new sync will not assume it correct,
277 : * but the check/fix may be using it.
278 : *
279 : * For example:
280 : * - One file is deleted
281 : * - Sync aborted after updating the parity to the new state,
282 : * but without saving the content file representing this new state.
283 : * - Another file is added again (exactly here)
284 : * with the hash of DELETED block not representing the real parity state
285 : */
286 :
287 : /* copy the past hash of the block */
288 48627 : memcpy(block->hash, over_block->hash, BLOCK_HASH_SIZE);
289 : }
290 : }
291 :
292 : /* store in the disk map, after invalidating all the other blocks */
293 254612 : fs_allocate(disk, parity_pos, file, i);
294 :
295 : /* set the new free position */
296 254612 : disk->first_free_block = parity_pos + 1;
297 : }
298 :
299 : /* insert in the list of contained files */
300 105934 : tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
301 105934 : }
302 :
303 : /**
304 : * Delete the specified file from the parity.
305 : *
306 : * Note that the parity remains allocated, but the blocks and the file are marked as DELETED.
307 : * The file is then inserted in the deleted set, and it should not be deallocated,
308 : * as the parity still references it.
309 : */
310 58857 : static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file)
311 : {
312 58857 : struct snapraid_disk* disk = scan->disk;
313 : block_off_t i;
314 :
315 : /* remove from the list of contained files */
316 58857 : tommy_list_remove_existing(&disk->filelist, &file->nodelist);
317 :
318 : /* state changed */
319 58857 : scan->need_write = 1;
320 :
321 : /*
322 : * Here we are supposed to adjust the ::first_free_block position
323 : * with the parity position we are deleting
324 : * but we also know that we do only delayed insert, after all the deletion,
325 : * so at this point ::first_free_block is always at 0, and we don't need to update it
326 : */
327 58857 : if (disk->first_free_block != 0) {
328 : /* LCOV_EXCL_START */
329 : log_fatal(EINTERNAL, "Internal inconsistency for first free position at '%u' deallocating file '%s'\n", disk->first_free_block, file->sub);
330 : os_abort();
331 : /* LCOV_EXCL_STOP */
332 : }
333 :
334 : /* free all the blocks of the file */
335 200683 : for (i = 0; i < file->blockmax; ++i) {
336 141826 : struct snapraid_block* block = fs_file2block_get(file, i);
337 : unsigned block_state;
338 :
339 : /*
340 : * In case we scan after an aborted sync,
341 : * we could get also intermediate states
342 : */
343 141826 : block_state = block_state_get(block);
344 141826 : switch (block_state) {
345 35354 : case BLOCK_STATE_BLK :
346 : /* we keep the hash making it an "old" hash, because the parity is still containing data for it */
347 35354 : break;
348 29813 : case BLOCK_STATE_CHG :
349 : /*
350 : * In this cases we don't know if the old state is still the one
351 : * stored inside the parity, because after an aborted sync, the parity
352 : * may be or may be not have been updated with the data that it's now
353 : * deleted.
354 : *
355 : * We anyway keep the hash as the new sync will not assume it correct,
356 : * but the check/fix may be using it.
357 : *
358 : * For example:
359 : * - One file is added
360 : * - Sync aborted after updating the parity to the new state,
361 : * but without saving the content file representing this new state.
362 : * - File is now deleted after the aborted sync
363 : * - Sync again, deleting the blocks (exactly here)
364 : * with the hash of CHG block not representing the real parity state
365 : */
366 29813 : break;
367 76659 : case BLOCK_STATE_REP :
368 : /* we just don't know the old hash, and then we set it to invalid */
369 76659 : hash_invalid_set(block->hash);
370 76659 : break;
371 0 : default :
372 : /* LCOV_EXCL_START */
373 : log_fatal(EINTERNAL, "Internal inconsistency in file '%s' deallocating block '%u:%u' state %u\n", file->sub, i, file->blockmax, block_state);
374 : os_abort();
375 : /* LCOV_EXCL_STOP */
376 : }
377 :
378 : /* set the block as deleted */
379 141826 : block_state_set(block, BLOCK_STATE_DELETED);
380 : }
381 :
382 : /* mark the file as deleted */
383 58857 : file_flag_set(file, FILE_IS_DELETED);
384 :
385 : /* insert it in the list of deleted blocks */
386 58857 : tommy_list_insert_tail(&disk->deletedlist, &file->nodelist, file);
387 58857 : }
388 :
389 105934 : static void scan_file_delayed_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
390 : {
391 105934 : struct snapraid_state* state = scan->state;
392 105934 : struct snapraid_disk* disk = scan->disk;
393 :
394 : /* if we sort for physical offsets we have to read them for new files */
395 105934 : if (state->opt.force_order == SORT_PHYSICAL
396 11432 : && file->physical == FILEPHY_UNREAD_OFFSET
397 : ) {
398 : char path_next[PATH_MAX];
399 :
400 11432 : pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, file->sub);
401 :
402 11432 : if (filephy(path_next, file->size, &file->physical) != 0) {
403 : /* LCOV_EXCL_START */
404 : log_fatal(errno, "Error in getting the physical offset of file '%s'. %s.\n", path_next, strerror(errno));
405 : exit(EXIT_FAILURE);
406 : /* LCOV_EXCL_STOP */
407 : }
408 : }
409 :
410 : /* insert in the delayed list */
411 105934 : tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file);
412 105934 : }
413 :
414 : /**
415 : * Check if a file is completely formed of blocks with invalid parity,
416 : * and no rehash is tagged, and if it has at least one block.
417 : */
418 1158929 : static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
419 : {
420 : block_off_t i;
421 :
422 : /* with no block, it never has an invalid parity */
423 1158929 : if (file->blockmax == 0)
424 1622 : return 0;
425 :
426 : /* check all blocks */
427 1216071 : for (i = 0; i < file->blockmax; ++i) {
428 : snapraid_info info;
429 1191589 : struct snapraid_block* block = fs_file2block_get(file, i);
430 : block_off_t parity_pos;
431 :
432 : /* exclude blocks with parity */
433 1191589 : if (!block_has_invalid_parity(block))
434 1132825 : return 0;
435 :
436 : /*
437 : * Get the parity position.
438 : *
439 : * Note that here we expect to always have mapped
440 : * parity, because kept files always have it.
441 : *
442 : * Anyway, checking for POS_NULL doesn't hurt.
443 : */
444 58764 : parity_pos = fs_file2par_find(disk, file, i);
445 :
446 : /* if it's not mapped, it cannot have rehash */
447 58764 : if (parity_pos != POS_NULL) {
448 : /* get block specific info */
449 58764 : info = info_get(&state->infoarr, parity_pos);
450 :
451 : /* if rehash fails */
452 58764 : if (info_get_rehash(info))
453 0 : return 0;
454 : }
455 : }
456 :
457 24482 : return 1;
458 : }
459 :
460 : /**
461 : * Check if a file is completely formed of blocks with an updated hash,
462 : * and no rehash is tagged, and if it has at least one block.
463 : */
464 23020 : static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
465 : {
466 : block_off_t i;
467 :
468 : /* with no block, it never has a hash */
469 23020 : if (file->blockmax == 0)
470 25 : return 0;
471 :
472 : /* check all blocks */
473 79548 : for (i = 0; i < file->blockmax; ++i) {
474 : snapraid_info info;
475 56559 : struct snapraid_block* block = fs_file2block_get(file, i);
476 : block_off_t parity_pos;
477 :
478 : /* exclude blocks without hash */
479 56559 : if (!block_has_updated_hash(block))
480 6 : return 0;
481 :
482 : /*
483 : * Get the parity position.
484 : *
485 : * Note that it's possible to have files
486 : * not mapped into the parity, even if they
487 : * have a valid hash.
488 : *
489 : * This happens for example, for 'copied' files
490 : * that have REP blocks, but not yet mapped.
491 : *
492 : * If there are multiple copies, it's also possible
493 : * that such files are used as 'source' to copy
494 : * hashes, and then to get them inside this function.
495 : */
496 56553 : parity_pos = fs_file2par_find(disk, file, i);
497 :
498 : /* if it's not mapped, it cannot have rehash */
499 56553 : if (parity_pos != POS_NULL) {
500 : /* get block specific info */
501 56545 : info = info_get(&state->infoarr, parity_pos);
502 :
503 : /* exclude blocks needing a rehash */
504 56545 : if (info_get_rehash(info))
505 0 : return 0;
506 : }
507 : }
508 :
509 22989 : return 1;
510 : }
511 :
512 : /**
513 : * Refresh the file info.
514 : *
515 : * This is needed by Windows as the normal way to list directories may report not
516 : * updated info. Only the GetFileInformationByHandle() func, called file-by-file,
517 : * really ensures to return synced info.
518 : *
519 : * If this happens, we read also the physical offset, to avoid to read it later.
520 : */
521 81452 : static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struct stat* st, uint64_t* physical)
522 : {
523 : #if HAVE_LSTAT_SYNC
524 : struct snapraid_state* state = scan->state;
525 : struct snapraid_disk* disk = scan->disk;
526 :
527 : /* if the st_sync is not set, ensure to get synced info */
528 : if (st->st_sync == 0) {
529 : char path_next[PATH_MAX];
530 : struct stat synced_st;
531 :
532 : pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, sub);
533 :
534 : /* if we sort for physical offsets we have to read them for new files */
535 : if (state->opt.force_order == SORT_PHYSICAL
536 : && *physical == FILEPHY_UNREAD_OFFSET
537 : ) {
538 : /* do nothing, leave the pointer to read the physical offset */
539 : } else {
540 : physical = 0; /* set the pointer to 0 to read nothing */
541 : }
542 :
543 : if (lstat_sync(path_next, &synced_st, physical) != 0) {
544 : /* LCOV_EXCL_START */
545 : log_fatal(errno, "Error in stat file '%s'. %s.\n", path_next, strerror(errno));
546 : exit(EXIT_FAILURE);
547 : /* LCOV_EXCL_STOP */
548 : }
549 :
550 : if (st->st_mtime != synced_st.st_mtime
551 : || st->st_mtimensec != synced_st.st_mtimensec
552 : ) {
553 : #ifndef _WIN32
554 : /*
555 : * In Windows having different metadata is expected with open files
556 : * because the metadata in the directory is updated only when the file
557 : * is closed.
558 : *
559 : * The same happens for hardlinks that duplicate metadata.
560 : * The link metadata is updated only when the link is opened.
561 : * This extends also to st_size and st_nlink.
562 : *
563 : * See also:
564 : * Why is the file size reported incorrectly for files that are still being written to?
565 : * http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx
566 : */
567 : log_fatal(ESOFT, "WARNING! Detected uncached time change from %" PRIu64 ".%09u to %" PRIu64 ".%09u for file '%s'\n",
568 : (uint64_t)st->st_mtime, (uint32_t)st->st_mtimensec, (uint64_t)synced_st.st_mtime, (uint32_t)synced_st.st_mtimensec, sub);
569 : log_fatal(ESOFT, "It's better if you run SnapRAID without other processes running.\n");
570 : #endif
571 : st->st_mtime = synced_st.st_mtime;
572 : st->st_mtimensec = synced_st.st_mtimensec;
573 : }
574 :
575 : if (st->st_size != synced_st.st_size) {
576 : #ifndef _WIN32
577 : log_fatal(ESOFT, "WARNING! Detected uncached size change from %" PRIu64 " to %" PRIu64 " for file '%s'\n",
578 : (uint64_t)st->st_size, (uint64_t)synced_st.st_size, sub);
579 : log_fatal(ESOFT, "It's better if you run SnapRAID without other processes running.\n");
580 : #endif
581 : st->st_size = synced_st.st_size;
582 : }
583 :
584 : if (st->st_nlink != synced_st.st_nlink) {
585 : #ifndef _WIN32
586 : log_fatal(ESOFT, "WARNING! Detected uncached nlink change from %u to %u for file '%s'\n",
587 : (uint32_t)st->st_nlink, (uint32_t)synced_st.st_nlink, sub);
588 : log_fatal(ESOFT, "It's better if you run SnapRAID without other processes running.\n");
589 : #endif
590 : st->st_nlink = synced_st.st_nlink;
591 : }
592 :
593 : if (st->st_ino != synced_st.st_ino) {
594 : log_fatal(ESOFT, "DANGER! Detected uncached inode change from %" PRIu64 " to %" PRIu64 " for file '%s'\n",
595 : (uint64_t)st->st_ino, (uint64_t)synced_st.st_ino, sub);
596 : log_fatal(ESOFT, "It's better if you run SnapRAID without other processes running.\n");
597 : /*
598 : * At this point, it's too late to change inode
599 : * and having inconsistent inodes may result to internal failures
600 : * so, it's better to abort
601 : */
602 : exit(EXIT_FAILURE);
603 : }
604 : }
605 : #else
606 : (void)scan;
607 : (void)sub;
608 : (void)st;
609 : (void)physical;
610 : #endif
611 81452 : }
612 :
613 : /**
614 : * Insert the file in the data set.
615 : */
616 105934 : static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
617 : {
618 105934 : struct snapraid_disk* disk = scan->disk;
619 :
620 : /* insert the file in the containers */
621 105934 : if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
622 105934 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
623 :
624 105934 : stamp_lock(disk);
625 105934 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
626 105934 : tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
627 105934 : stamp_unlock(disk);
628 :
629 : /* delayed allocation of the parity */
630 105934 : scan_file_delayed_allocate(scan, file);
631 105934 : }
632 :
633 : /**
634 : * Remove the file from the data set.
635 : *
636 : * File is then deleted.
637 : */
638 58857 : static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file, int keep_track)
639 : {
640 58857 : struct snapraid_disk* disk = scan->disk;
641 :
642 : /* remove the file from the containers */
643 58857 : if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
644 58827 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
645 58857 : tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
646 :
647 : /*
648 : * Keep track of the removed file (that won't be added later)
649 : *
650 : * This allows to record from where the parity was computed.
651 : */
652 58857 : if (keep_track) {
653 34375 : struct snapraid_dealloc* dealloc = dealloc_alloc(scan->state->block_size, file->sub, file->size, file->mtime_sec, file->mtime_nsec);
654 :
655 34375 : dealloc_import(dealloc, file);
656 :
657 : /* insert the dealloc in the dealloc containers */
658 34375 : tommy_list_insert_tail(&disk->dealloclist, &dealloc->nodelist, dealloc);
659 : }
660 :
661 58857 : stamp_lock(disk);
662 58857 : tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
663 58857 : stamp_unlock(disk);
664 :
665 : /* deallocate the file from the parity */
666 58857 : scan_file_deallocate(scan, file);
667 58857 : }
668 :
669 : /**
670 : * Keep the file as it's (or with only a name/inode modification).
671 : *
672 : * If the file is kept, nothing has to be done.
673 : *
674 : * But if a file contains only blocks with invalid parity, it's reallocated to ensure
675 : * to always minimize the space used in the parity.
676 : *
677 : * This could happen after a failed sync, when some other files are deleted,
678 : * and then new ones can be moved backward to fill the hole created.
679 : */
680 1158929 : static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* file)
681 : {
682 1158929 : struct snapraid_disk* disk = scan->disk;
683 :
684 : /* if the file is full invalid, schedule a reinsert at later stage */
685 1158929 : if (file_is_full_invalid_parity_and_stable(scan->state, disk, file)) {
686 24482 : struct snapraid_file* copy = file_dup(file);
687 :
688 : /* remove the file */
689 24482 : scan_file_remove(scan, file, 0);
690 :
691 : /* reinsert the copy in the delayed list */
692 24482 : scan_file_insert(scan, copy);
693 : }
694 1158929 : }
695 :
696 : /**
697 : * Process a file.
698 : */
699 1240603 : static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, struct stat* st, uint64_t physical)
700 : {
701 1240603 : struct snapraid_state* state = scan->state;
702 1240603 : struct snapraid_disk* disk = scan->disk;
703 : struct snapraid_file* file;
704 : tommy_node* i;
705 : int is_original_file_size_different_than_zero;
706 : int is_file_already_present;
707 : data_off_t file_already_present_size;
708 : int64_t file_already_present_mtime_sec;
709 : int file_already_present_mtime_nsec;
710 : int is_file_reported;
711 : char esc_buffer[ESC_MAX];
712 : char esc_buffer_alt[ESC_MAX];
713 :
714 : /*
715 : * If the disk has persistent inodes and UUID, try a search on the past inodes,
716 : * to detect moved files.
717 : *
718 : * For persistent inodes we mean inodes that keep their values when the file-system
719 : * is unmounted and remounted. This don't always happen.
720 : *
721 : * Cases found are:
722 : * - Linux FUSE with exFAT driver from https://code.google.com/p/exfat/.
723 : * Inodes are reassigned at every mount restarting from 1 and incrementing.
724 : * As worse, the exFAT support in FUSE doesn't use sub-second precision in timestamps
725 : * making inode collision more easy (exFAT by design supports 10ms precision).
726 : * - Linux VFAT kernel (3.2) driver. Inodes are fully reassigned at every mount.
727 : *
728 : * In such cases, to avoid possible random collisions, it's better to disable the moved
729 : * file recognition.
730 : *
731 : * For persistent UUID we mean that it has the same UUID as before.
732 : * Otherwise, if the UUID is changed, likely it's a new recreated file-system,
733 : * and then the inode have no meaning.
734 : *
735 : * Note that to disable the search by past inode, we do this implicitly
736 : * removing all the past inode before searching for files.
737 : * This ensures that no file is found with a past inode, but at the same time,
738 : * it allows to find new files with the same inode, to identify them as hardlinks.
739 : */
740 1240603 : int has_past_inodes = !disk->has_volatile_inodes && !disk->has_different_uuid && !disk->has_unsupported_uuid;
741 :
742 : /*
743 : * Always search with the new inode, in the all new inodes found until now,
744 : * with the eventual presence of also the past inodes
745 : */
746 1240603 : uint64_t inode = st->st_ino;
747 :
748 1240603 : file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
749 :
750 : /* identify moved files with past inodes and hardlinks with the new inodes */
751 1240603 : if (file) {
752 : /* check if the file is not changed */
753 1070742 : if (file->size == st->st_size
754 1056029 : && file->mtime_sec == st->st_mtime
755 1054197 : && (file->mtime_nsec == STAT_NSEC(st)
756 : /*
757 : * Always accept the stored value if it's STAT_NSEC_INVALID
758 : * it happens when upgrading from an old version of SnapRAID
759 : * not yet supporting the nanosecond field
760 : */
761 6 : || file->mtime_nsec == STAT_NSEC_INVALID
762 : )
763 : ) {
764 : /* check if multiple files have the same inode */
765 1054191 : if (file_flag_has(file, FILE_IS_PRESENT)) {
766 : /* if has_volatile_hardlinks is true, the nlink value is not reliable */
767 222 : if (!disk->has_volatile_hardlinks && st->st_nlink == 1) {
768 : /* LCOV_EXCL_START */
769 : log_fatal(EINTERNAL, "Internal inode '%" PRIu64 "' inconsistency for file '%s%s' already present\n", (uint64_t)st->st_ino, disk->dir, sub);
770 : os_abort();
771 : /* LCOV_EXCL_STOP */
772 : }
773 :
774 : /* it's a hardlink */
775 222 : scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
776 1159151 : return;
777 : }
778 :
779 : /* mark as present */
780 1053969 : file_flag_set(file, FILE_IS_PRESENT);
781 :
782 : /*
783 : * Update the nanoseconds mtime only if different
784 : * to avoid unneeded updates
785 : */
786 1053969 : if (file->mtime_nsec == STAT_NSEC_INVALID
787 0 : && STAT_NSEC(st) != file->mtime_nsec
788 : ) {
789 0 : file->mtime_nsec = STAT_NSEC(st);
790 :
791 : /* we have to save the new mtime */
792 0 : scan->need_write = 1;
793 : }
794 :
795 1053969 : if (strcmp(file->sub, sub) != 0) {
796 : /* if the path is different, it means a moved file with the same inode */
797 155 : ++scan->count_move;
798 :
799 155 : log_tag("scan:move:%s:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer), esc_tag(sub, esc_buffer_alt));
800 155 : if (is_diff) {
801 0 : msg_info("move %s -> %s\n", fmt_term(disk, file->sub, esc_buffer), fmt_term(disk, sub, esc_buffer_alt));
802 : }
803 :
804 : /* remove from the name set */
805 155 : tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
806 :
807 : /* save the new name */
808 155 : file_rename(file, sub);
809 :
810 : /* reinsert in the name set */
811 155 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
812 :
813 : /* we have to save the new name */
814 155 : scan->need_write = 1;
815 : } else {
816 : /* otherwise it's equal */
817 1053814 : ++scan->count_equal;
818 :
819 1053814 : if (state->opt.gui_verbose) {
820 0 : log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
821 : }
822 : }
823 :
824 : /* mark the file as kept */
825 1053969 : scan_file_keep(scan, file);
826 :
827 : /* nothing more to do */
828 1053969 : return;
829 : }
830 :
831 : /*
832 : * Here the file matches the inode, but not the other info
833 : *
834 : * It could be a modified file with the same name,
835 : * or a restored/copied file that get assigned a previously used inode,
836 : * or a file-system with not persistent inodes.
837 : *
838 : * In NTFS it could be also a hardlink, because in NTFS
839 : * hardlink don't share the same directory information,
840 : * like attribute and time.
841 : *
842 : * For example:
843 : * C:> echo A > A
844 : * C:> mklink /H B A
845 : * ...wait one minute
846 : * C:> echo AAAAAAAAAAAAAA > A
847 : * C:> dir
848 : * ...both time and size of A and B don't match!
849 : */
850 16551 : if (file_flag_has(file, FILE_IS_PRESENT)) {
851 : /* if has_volatile_hardlinks is true, the nlink value is not reliable */
852 0 : if (!disk->has_volatile_hardlinks && st->st_nlink == 1) {
853 : /* LCOV_EXCL_START */
854 : log_fatal(EINTERNAL, "Internal inode '%" PRIu64 "' inconsistency for files '%s%s' and '%s%s' with same inode but different attributes: size %" PRIu64 "?%" PRIu64 ", sec %" PRIu64 "?%" PRIu64 ", nsec %d?%d\n",
855 : file->inode, disk->dir, sub, disk->dir, file->sub,
856 : file->size, (uint64_t)st->st_size,
857 : file->mtime_sec, (uint64_t)st->st_mtime,
858 : file->mtime_nsec, STAT_NSEC(st));
859 : os_abort();
860 : /* LCOV_EXCL_STOP */
861 : }
862 :
863 : /* LCOV_EXCL_START */
864 : /* suppose it's hardlink with not synced metadata */
865 : scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
866 : return;
867 : /* LCOV_EXCL_STOP */
868 : }
869 :
870 : /*
871 : * Assume a previously used inode, it's the worst case
872 : * and we handle it removing the duplicate stored inode.
873 : * If the file is found by name later, it will have the inode restored,
874 : * otherwise, it will get removed
875 : */
876 :
877 : /* remove from the inode set */
878 16551 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
879 :
880 : /*
881 : * Clear the inode
882 : * this is not really needed for correct functionality
883 : * because we are going to set FILE_IS_WITHOUT_INODE
884 : * but it's easier for debugging to have invalid inodes set to 0
885 : */
886 16551 : file->inode = 0;
887 :
888 : /* mark as missing inode */
889 16551 : file_flag_set(file, FILE_IS_WITHOUT_INODE);
890 :
891 : /* go further to find it by name */
892 : }
893 :
894 : /* initialize for later overwrite */
895 186412 : is_file_reported = 0;
896 186412 : is_original_file_size_different_than_zero = 0;
897 :
898 : /* then try finding it by name */
899 186412 : file = tommy_hashdyn_search(&disk->pathset, file_path_compare_to_arg, sub, file_path_hash(sub));
900 :
901 : /* keep track if the file already exists */
902 186412 : is_file_already_present = file != 0;
903 :
904 186412 : if (is_file_already_present) {
905 : /* if the file is without an inode */
906 109661 : if (file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
907 : /* set it now */
908 16535 : file->inode = st->st_ino;
909 :
910 : /* insert in the set */
911 16535 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
912 :
913 : /* unmark as missing inode */
914 16535 : file_flag_clear(file, FILE_IS_WITHOUT_INODE);
915 : } else {
916 : /* here the inode has to be different, otherwise we would have found it before */
917 93126 : if (file->inode == st->st_ino) {
918 : /* LCOV_EXCL_START */
919 : log_fatal(EINTERNAL, "Internal inconsistency in inode '%" PRIu64 "' for files '%s%s' as unexpected matching\n", file->inode, disk->dir, sub);
920 : os_abort();
921 : /* LCOV_EXCL_STOP */
922 : }
923 : }
924 :
925 : /* for sure it cannot be already present */
926 109661 : if (file_flag_has(file, FILE_IS_PRESENT)) {
927 : /* LCOV_EXCL_START */
928 : log_fatal(EINTERNAL, "Internal inconsistency in path for file '%s%s' matching and already present\n", disk->dir, sub);
929 : os_abort();
930 : /* LCOV_EXCL_STOP */
931 : }
932 :
933 : /* check if the file is not changed */
934 109661 : if (file->size == st->st_size
935 107823 : && file->mtime_sec == st->st_mtime
936 104962 : && (file->mtime_nsec == STAT_NSEC(st)
937 : /*
938 : * Always accept the stored value if it's STAT_NSEC_INVALID
939 : * it happens when upgrading from an old version of SnapRAID
940 : * not yet supporting the nanosecond field
941 : */
942 2 : || file->mtime_nsec == STAT_NSEC_INVALID
943 : )
944 : ) {
945 : /* mark as present */
946 104960 : file_flag_set(file, FILE_IS_PRESENT);
947 :
948 : /*
949 : * Update the nano seconds mtime only if different
950 : * to avoid unneeded updates
951 : */
952 104960 : if (file->mtime_nsec == STAT_NSEC_INVALID
953 0 : && STAT_NSEC(st) != STAT_NSEC_INVALID
954 : ) {
955 0 : file->mtime_nsec = STAT_NSEC(st);
956 :
957 : /* we have to save the new mtime */
958 0 : scan->need_write = 1;
959 : }
960 :
961 : /* if when processing the disk we used the past inodes values */
962 104960 : if (has_past_inodes) {
963 : /*
964 : * If persistent inodes are supported, we are sure that the inode number
965 : * is now different, because otherwise the file would have been found
966 : * when searching by inode.
967 : * if the inode is different, it means a rewritten file with the same path
968 : * like when restoring a backup that restores also the timestamp
969 : */
970 104946 : ++scan->count_restore;
971 :
972 104946 : log_tag("scan:restore:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
973 104946 : if (is_diff) {
974 0 : msg_info("restore %s\n", fmt_term(disk, sub, esc_buffer));
975 : }
976 :
977 : /* remove from the inode set */
978 104946 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
979 :
980 : /* save the new inode */
981 104946 : file->inode = st->st_ino;
982 :
983 : /* reinsert in the inode set */
984 104946 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
985 :
986 : /* we have to save the new inode */
987 104946 : scan->need_write = 1;
988 : } else {
989 : /*
990 : * Otherwise it's the case of not persistent inode, where doesn't
991 : * matter if the inode is different or equal, because they have no
992 : * meaning, and then we don't even save them
993 : */
994 14 : ++scan->count_equal;
995 :
996 14 : if (state->opt.gui_verbose) {
997 0 : log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
998 : }
999 : }
1000 :
1001 : /* mark the file as kept */
1002 104960 : scan_file_keep(scan, file);
1003 :
1004 : /* nothing more to do */
1005 104960 : return;
1006 : }
1007 :
1008 : /* here if the file is changed but with the correct name */
1009 :
1010 : /* save the info for later printout */
1011 4701 : file_already_present_size = file->size;
1012 4701 : file_already_present_mtime_sec = file->mtime_sec;
1013 4701 : file_already_present_mtime_nsec = file->mtime_nsec;
1014 :
1015 : /* keep track if the original file was not of zero size */
1016 4701 : is_original_file_size_different_than_zero = file->size != 0;
1017 :
1018 : /* remove it, and continue to insert it again */
1019 4701 : scan_file_remove(scan, file, 1);
1020 :
1021 : /* and continue to insert it again */
1022 : } else {
1023 76751 : file_already_present_size = 0;
1024 76751 : file_already_present_mtime_sec = 0;
1025 76751 : file_already_present_mtime_nsec = 0;
1026 : }
1027 :
1028 : /*
1029 : * Refresh the info, to ensure that they are synced,
1030 : * note that we refresh only the info of the new or modified files
1031 : * because this is slow operation
1032 : */
1033 81452 : scan_file_refresh(scan, sub, st, &physical);
1034 :
1035 : #ifndef _WIN32
1036 : /*
1037 : * Do a safety check to ensure that the common ext4 case of zeroing
1038 : * the size of a file after a crash doesn't propagate to the backup
1039 : * this check is specific for Linux, so we disable it on Windows
1040 : */
1041 81452 : if (is_original_file_size_different_than_zero && st->st_size == 0) {
1042 0 : if (!state->opt.force_zero) {
1043 : /* LCOV_EXCL_START */
1044 : log_fatal(ESOFT, "The file '%s%s' has unexpected zero size!\n", disk->mount_point, sub);
1045 : log_fatal(ESOFT, "It's possible that after a kernel crash this file was lost,\n");
1046 : log_fatal(ESOFT, "and you can use 'snapraid fix -f /%s' to recover it.\n", fmt_poll(disk, sub, esc_buffer));
1047 : if (!is_diff) {
1048 : log_fatal(ESOFT, "If this an expected condition you can '%s' anyway using 'snapraid --force-zero %s'\n", state->command, state->command);
1049 : exit(EXIT_FAILURE);
1050 : }
1051 : /* LCOV_EXCL_STOP */
1052 : }
1053 : }
1054 : #else
1055 : /* avoid the unused warning in Windows */
1056 : (void)is_original_file_size_different_than_zero;
1057 : #endif
1058 :
1059 : /* insert it */
1060 81452 : file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime, STAT_NSEC(st), st->st_ino, physical);
1061 :
1062 : /* mark it as present */
1063 81452 : file_flag_set(file, FILE_IS_PRESENT);
1064 :
1065 : /*
1066 : * If copy detection is enabled
1067 : * note that the copy detection is tried also for updated files
1068 : * this makes sense because it may happen to have two different copies
1069 : * of the same file, and we move the right one over the wrong one
1070 : * in such case we have a "copy" over an "update"
1071 : */
1072 81452 : if (!state->opt.force_nocopy) {
1073 81452 : tommy_uint32_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec);
1074 :
1075 : /* search for a file with the same name and stamp in all the disks */
1076 428992 : for (i = state->disklist; i != 0; i = i->next) {
1077 370529 : struct snapraid_disk* other_disk = i->data;
1078 : struct snapraid_file* other_file;
1079 :
1080 370529 : stamp_lock(other_disk);
1081 : /*
1082 : * If the nanosecond part of the time stamp is valid, search
1083 : * for name and stamp, otherwise for path and stamp
1084 : */
1085 370529 : if (file->mtime_nsec != 0 && file->mtime_nsec != STAT_NSEC_INVALID)
1086 370421 : other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash);
1087 : else
1088 108 : other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
1089 370529 : stamp_unlock(other_disk);
1090 :
1091 : /* if found, and it's a fully hashed file */
1092 370529 : if (other_file && file_is_full_hashed_and_stable(scan->state, other_disk, other_file)) {
1093 : char path_other[PATH_MAX];
1094 : struct stat other_st;
1095 :
1096 : /*
1097 : * Protect the write as multiple threads may write the same FILE_IS_RELOCATED bit.
1098 : *
1099 : * The bit is always written as 1 and never read, so protection is likely unnecessary
1100 : * but still valuable to avoid data race reports from checker tools
1101 : */
1102 22989 : stamp_lock(other_disk);
1103 22989 : file_flag_set(other_file, FILE_IS_RELOCATED);
1104 22989 : stamp_unlock(other_disk);
1105 :
1106 : /* assume that the file is a copy, and reuse the hash */
1107 22989 : file_copy(other_file, file);
1108 :
1109 : /* check if other file still exists */
1110 22989 : pathprint(path_other, sizeof(path_other), "%s%s", other_disk->dir, other_file->sub);
1111 22989 : if (lstat(path_other, &other_st) == 0) {
1112 15975 : ++scan->count_copy;
1113 :
1114 15975 : log_tag("scan:copy:%s:%s:%s:%s\n", other_disk->name, esc_tag(other_file->sub, esc_buffer), disk->name, esc_tag(file->sub, esc_buffer_alt));
1115 15975 : if (is_diff) {
1116 0 : msg_info("copy %s -> %s\n", fmt_term(other_disk, other_file->sub, esc_buffer), fmt_term(disk, file->sub, esc_buffer_alt));
1117 : }
1118 : } else {
1119 7014 : ++scan->count_relocate;
1120 :
1121 7014 : log_tag("scan:relocate:%s:%s:%s:%s\n", other_disk->name, esc_tag(other_file->sub, esc_buffer), disk->name, esc_tag(file->sub, esc_buffer_alt));
1122 7014 : if (is_diff) {
1123 31 : msg_info("relocate %s -> %s\n", fmt_term(other_disk, other_file->sub, esc_buffer), fmt_term(disk, file->sub, esc_buffer_alt));
1124 : }
1125 : }
1126 :
1127 : /* mark it as reported */
1128 22989 : is_file_reported = 1;
1129 :
1130 : /* no need to continue the search */
1131 22989 : break;
1132 : }
1133 : }
1134 : }
1135 :
1136 : /*
1137 : * If not yet reported, do it now
1138 : * we postpone this to avoid to print two times the copied files
1139 : */
1140 81452 : if (!is_file_reported) {
1141 58463 : if (is_file_already_present) {
1142 4701 : ++scan->count_change;
1143 :
1144 4701 : log_tag("scan:update:%s:%s: %" PRIu64 " %" PRIu64 ".%d -> %" PRIu64 " %" PRIu64 ".%d\n", disk->name, esc_tag(sub, esc_buffer),
1145 : file_already_present_size, file_already_present_mtime_sec, file_already_present_mtime_nsec,
1146 : file->size, file->mtime_sec, file->mtime_nsec
1147 : );
1148 :
1149 4701 : if (is_diff) {
1150 118 : msg_info("update %s\n", fmt_term(disk, sub, esc_buffer));
1151 : }
1152 : } else {
1153 53762 : ++scan->count_insert;
1154 :
1155 53762 : log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
1156 53762 : if (is_diff) {
1157 17054 : msg_info("add %s\n", fmt_term(disk, sub, esc_buffer));
1158 : }
1159 : }
1160 : }
1161 :
1162 : /* insert the file in the delayed list */
1163 81452 : scan_file_insert(scan, file);
1164 : }
1165 :
1166 : /**
1167 : * Remove the specified dir from the data set.
1168 : */
1169 2 : static void scan_emptydir_remove(struct snapraid_scan* scan, struct snapraid_dir* dir)
1170 : {
1171 2 : struct snapraid_disk* disk = scan->disk;
1172 :
1173 : /* state changed */
1174 2 : scan->need_write = 1;
1175 :
1176 : /* remove the file from the dir containers */
1177 2 : tommy_hashdyn_remove_existing(&disk->dirset, &dir->nodeset);
1178 2 : tommy_list_remove_existing(&disk->dirlist, &dir->nodelist);
1179 :
1180 : /* deallocate */
1181 2 : dir_free(dir);
1182 2 : }
1183 :
1184 : /**
1185 : * Insert the specified dir in the data set.
1186 : */
1187 10 : static void scan_emptydir_insert(struct snapraid_scan* scan, struct snapraid_dir* dir)
1188 : {
1189 10 : struct snapraid_disk* disk = scan->disk;
1190 :
1191 : /* state changed */
1192 10 : scan->need_write = 1;
1193 :
1194 : /* insert the dir in the dir containers */
1195 10 : tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
1196 10 : tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
1197 10 : }
1198 :
1199 : /**
1200 : * Process a dir.
1201 : */
1202 295 : static void scan_emptydir(struct snapraid_scan* scan, const char* sub)
1203 : {
1204 295 : struct snapraid_disk* disk = scan->disk;
1205 : struct snapraid_dir* dir;
1206 :
1207 : /* check if the dir already exists */
1208 295 : dir = tommy_hashdyn_search(&disk->dirset, dir_name_compare, sub, dir_name_hash(sub));
1209 295 : if (dir) {
1210 : /* check if multiple files have the same name */
1211 285 : if (dir_flag_has(dir, FILE_IS_PRESENT)) {
1212 : /* LCOV_EXCL_START */
1213 : log_fatal(EINTERNAL, "Internal inconsistency for dir '%s%s'\n", disk->dir, sub);
1214 : os_abort();
1215 : /* LCOV_EXCL_STOP */
1216 : }
1217 :
1218 : /* mark as present */
1219 285 : dir_flag_set(dir, FILE_IS_PRESENT);
1220 :
1221 : /* nothing more to do */
1222 285 : return;
1223 : } else {
1224 : /* and continue to insert it */
1225 : }
1226 :
1227 : /* insert it */
1228 10 : dir = dir_alloc(sub);
1229 :
1230 : /* mark it as present */
1231 10 : dir_flag_set(dir, FILE_IS_PRESENT);
1232 :
1233 : /* insert it in the delayed insert list */
1234 10 : tommy_list_insert_tail(&scan->dir_insert_list, &dir->nodelist, dir);
1235 : }
1236 :
1237 : struct dirent_sorted {
1238 : /* node for data structures */
1239 : tommy_node node;
1240 :
1241 : #if HAVE_STRUCT_DIRENT_D_INO
1242 : uint64_t d_ino; /**< Inode number. */
1243 : #endif
1244 : #if HAVE_STRUCT_DIRENT_D_TYPE
1245 : uint32_t d_type; /**< File type. */
1246 : #endif
1247 : #if HAVE_STRUCT_DIRENT_D_STAT
1248 : struct stat d_stat; /**< Stat result. */
1249 : #endif
1250 : char d_name[]; /**< Variable length name. It must be the last field. */
1251 : };
1252 :
1253 : #if HAVE_STRUCT_DIRENT_D_INO
1254 103636 : static int dd_ino_compare(const void* void_a, const void* void_b)
1255 : {
1256 103636 : const struct dirent_sorted* a = void_a;
1257 103636 : const struct dirent_sorted* b = void_b;
1258 :
1259 103636 : if (a->d_ino < b->d_ino)
1260 49708 : return -1;
1261 53928 : if (a->d_ino > b->d_ino)
1262 53928 : return 1;
1263 :
1264 0 : return 0;
1265 : }
1266 : #endif
1267 :
1268 12822062 : static int dd_name_compare(const void* void_a, const void* void_b)
1269 : {
1270 12822062 : const struct dirent_sorted* a = void_a;
1271 12822062 : const struct dirent_sorted* b = void_b;
1272 :
1273 12822062 : return strcmp(a->d_name, b->d_name);
1274 : }
1275 :
1276 : /**
1277 : * Return the stat info of a dir entry.
1278 : */
1279 : #if HAVE_STRUCT_DIRENT_D_STAT
1280 : #define DSTAT(file, dd, buf) dstat(dd)
1281 : struct stat* dstat(struct dirent_sorted* dd)
1282 : {
1283 : return &dd->d_stat;
1284 : }
1285 : #else
1286 : #define DSTAT(file, dd, buf) dstat(file, buf)
1287 1242087 : struct stat* dstat(const char* file, struct stat* st)
1288 : {
1289 1242087 : if (lstat(file, st) != 0) {
1290 : /* LCOV_EXCL_START */
1291 : log_fatal(errno, "Error in stat file/directory '%s'. %s.\n", file, strerror(errno));
1292 : exit(EXIT_FAILURE);
1293 : /* LCOV_EXCL_STOP */
1294 : }
1295 1242087 : return st;
1296 : }
1297 : #endif
1298 :
1299 : /**
1300 : * Process a directory.
1301 : * Return != 0 if at least one file or link is processed.
1302 : */
1303 2143 : static int scan_sub(struct snapraid_scan* scan, int level, int is_diff, char* path_next, char* sub_next, char* tmp)
1304 : {
1305 2143 : struct snapraid_state* state = scan->state;
1306 2143 : struct snapraid_disk* disk = scan->disk;
1307 2143 : int processed = 0;
1308 : DIR* d;
1309 : tommy_list list;
1310 : tommy_node* node;
1311 : size_t path_len;
1312 : size_t sub_len;
1313 :
1314 2143 : path_len = strlen(path_next);
1315 2143 : sub_len = strlen(sub_next);
1316 :
1317 2143 : tommy_list_init(&list);
1318 :
1319 2143 : d = opendir(path_next);
1320 2143 : if (!d) {
1321 : /* LCOV_EXCL_START */
1322 : log_fatal(errno, "Error opening directory '%s'. %s.\n", path_next, strerror(errno));
1323 : if (level == 0)
1324 : log_fatal(errno, "If this is the disk mount point, remember to create it manually\n");
1325 : else
1326 : log_fatal(errno, "If it's a permission problem, you can exclude it in the config file with:\n\texclude /%s\n", sub_next);
1327 : exit(EXIT_FAILURE);
1328 : /* LCOV_EXCL_STOP */
1329 : }
1330 :
1331 : /* read the full directory */
1332 1287027 : while (1) {
1333 : struct dirent_sorted* entry;
1334 : const char* name;
1335 : struct dirent* dd;
1336 : size_t name_len;
1337 :
1338 : /*
1339 : * Clear errno to differentiate the end of the stream and an error condition
1340 : *
1341 : * From the Linux readdir() manpage:
1342 : * "If the end of the directory stream is reached, NULL is returned and errno is not changed.
1343 : * If an error occurs, NULL is returned and errno is set appropriately."
1344 : */
1345 1289170 : errno = 0;
1346 1289170 : dd = readdir(d);
1347 1289170 : if (dd == 0 && errno != 0) {
1348 : /* LCOV_EXCL_START */
1349 : /* restore removing additions */
1350 : path_next[path_len] = 0;
1351 : sub_next[sub_len] = 0;
1352 : log_fatal(errno, "Error reading directory '%s'. %s.\n", path_next, strerror(errno));
1353 : log_fatal(errno, "You can exclude it in the config file with:\n\texclude /%s\n", sub_next);
1354 : exit(EXIT_FAILURE);
1355 : /* LCOV_EXCL_STOP */
1356 : }
1357 1289170 : if (dd == 0) {
1358 2143 : break; /* finished */
1359 : }
1360 :
1361 : /* skip "." and ".." files */
1362 1287027 : name = dd->d_name;
1363 1287027 : if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
1364 4286 : continue;
1365 :
1366 1282741 : pathcatl(path_next, path_len, PATH_MAX, name);
1367 :
1368 : /* check for not supported file names */
1369 1282741 : if (name[0] == 0) {
1370 : /* LCOV_EXCL_START */
1371 : log_fatal(ESOFT, "Unsupported name '%s' in file '%s'.\n", name, path_next);
1372 : exit(EXIT_FAILURE);
1373 : /* LCOV_EXCL_STOP */
1374 : }
1375 :
1376 : /* exclude hidden files even before calling lstat() */
1377 1282741 : if (filter_hidden(state->filter_hidden, dd) != 0) {
1378 0 : msg_verbose("Excluding hidden '%s'\n", path_next);
1379 0 : continue;
1380 : }
1381 :
1382 : /* exclude content files even before calling lstat() */
1383 1282741 : if (filter_content(&state->contentlist, path_next) != 0) {
1384 0 : msg_verbose("Excluding content '%s'\n", path_next);
1385 0 : continue;
1386 : }
1387 :
1388 1282741 : name_len = strlen(dd->d_name);
1389 1282741 : entry = malloc_nofail(sizeof(struct dirent_sorted) + name_len + 1);
1390 :
1391 : /* copy the dir entry */
1392 : #if HAVE_STRUCT_DIRENT_D_INO
1393 1282741 : entry->d_ino = dd->d_ino;
1394 : #endif
1395 : #if HAVE_STRUCT_DIRENT_D_TYPE
1396 1282741 : entry->d_type = dd->d_type;
1397 : #endif
1398 : #if HAVE_STRUCT_DIRENT_D_STAT
1399 : /* convert dirent to lstat result */
1400 : dirent_lstat(dd, &entry->d_stat);
1401 :
1402 : /* note that at this point the st_mode may be 0 */
1403 : #endif
1404 1282741 : memcpy(entry->d_name, dd->d_name, name_len + 1);
1405 :
1406 : /* insert in the list */
1407 1282741 : tommy_list_insert_tail(&list, &entry->node, entry);
1408 :
1409 : /* process ignore files */
1410 1282741 : if (strcmp(".snapraidignore", dd->d_name) == 0)
1411 66 : state_load_ignore_file(&scan->local_filter_list, path_next, sub_next);
1412 : }
1413 :
1414 2143 : if (closedir(d) != 0) {
1415 : /* LCOV_EXCL_START */
1416 : /* restore removing additions */
1417 : path_next[path_len] = 0;
1418 : log_fatal(errno, "Error closing directory '%s'. %s.\n", path_next, strerror(errno));
1419 : exit(EXIT_FAILURE);
1420 : /* LCOV_EXCL_STOP */
1421 : }
1422 :
1423 2143 : if (state->opt.force_order == SORT_ALPHA) {
1424 : /*
1425 : * If requested sort alphabetically
1426 : * this is mainly done for testing to ensure to always
1427 : * process in the same way in different platforms
1428 : */
1429 2107 : tommy_list_sort(&list, dd_name_compare);
1430 : }
1431 : #if HAVE_STRUCT_DIRENT_D_INO
1432 36 : else if (!disk->has_volatile_inodes) {
1433 : /*
1434 : * If inodes are persistent
1435 : * sort the list of dir entries by inodes
1436 : */
1437 36 : tommy_list_sort(&list, dd_ino_compare);
1438 : }
1439 : /* otherwise just keep the insertion order */
1440 : #endif
1441 :
1442 : /* process the sorted dir entries */
1443 2143 : node = list;
1444 1284884 : while (node != 0) {
1445 1282741 : struct snapraid_filter* reason = 0;
1446 1282741 : struct dirent_sorted* dd = node->data;
1447 1282741 : const char* name = dd->d_name;
1448 : struct stat* st;
1449 : int type;
1450 : #if !HAVE_STRUCT_DIRENT_D_STAT
1451 : struct stat st_buf;
1452 : #endif
1453 :
1454 1282741 : pathcatl(path_next, path_len, PATH_MAX, name);
1455 1282741 : pathcatl(sub_next, sub_len, PATH_MAX, name);
1456 :
1457 : /* start with an unknown type */
1458 1282741 : type = -1;
1459 1282741 : st = 0;
1460 :
1461 : /* if dirent has the type, use it */
1462 : #if HAVE_STRUCT_DIRENT_D_TYPE
1463 1282741 : switch (dd->d_type) {
1464 0 : case DT_UNKNOWN : break;
1465 1240735 : case DT_REG : type = 0; break;
1466 40492 : case DT_LNK : type = 1; break;
1467 1514 : case DT_DIR : type = 2; break;
1468 0 : default : type = 3; break;
1469 : }
1470 : #endif
1471 :
1472 : /* if type is still unknown */
1473 1282741 : if (type < 0) {
1474 : /* get the type from stat */
1475 0 : st = DSTAT(path_next, dd, &st_buf);
1476 :
1477 : #if HAVE_STRUCT_DIRENT_D_STAT
1478 : /*
1479 : * If the st_mode field is missing, takes care to fill it using normal lstat()
1480 : * at now this can happen only in Windows (with HAVE_STRUCT_DIRENT_D_STAT defined),
1481 : * because we use a directory reading method that doesn't read info about ReparsePoint.
1482 : * Note that here we cannot call here lstat_sync(), because we don't know what kind
1483 : * of file is it, and lstat_sync() doesn't always work
1484 : */
1485 : if (st->st_mode == 0) {
1486 : if (lstat(path_next, st) != 0) {
1487 : /* LCOV_EXCL_START */
1488 : log_fatal(errno, "Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno));
1489 : exit(EXIT_FAILURE);
1490 : /* LCOV_EXCL_STOP */
1491 : }
1492 : }
1493 : #endif
1494 :
1495 0 : if (S_ISREG(st->st_mode))
1496 0 : type = 0;
1497 0 : else if (S_ISLNK(st->st_mode))
1498 0 : type = 1;
1499 0 : else if (S_ISDIR(st->st_mode))
1500 0 : type = 2;
1501 : else
1502 0 : type = 3;
1503 : }
1504 :
1505 1282741 : if (type == 0) { /* REG */
1506 1240735 : if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0
1507 1240735 : && filter_path(&scan->local_filter_list, &reason, disk->name, sub_next) == 0) {
1508 :
1509 : /* late stat, if not yet called */
1510 1240603 : if (!st)
1511 1240603 : st = DSTAT(path_next, dd, &st_buf);
1512 :
1513 : #if HAVE_LSTAT_SYNC
1514 : /*
1515 : * If the st_ino field is missing, takes care to fill it using the extended lstat()
1516 : * this can happen only in Windows
1517 : */
1518 : if (st->st_ino == 0 || st->st_nlink == 0) {
1519 : if (lstat_sync(path_next, st, 0) != 0) {
1520 : /* LCOV_EXCL_START */
1521 : log_fatal(errno, "Error in stat file '%s'. %s.\n", path_next, strerror(errno));
1522 : exit(EXIT_FAILURE);
1523 : /* LCOV_EXCL_STOP */
1524 : }
1525 : }
1526 : #endif
1527 :
1528 1240603 : scan_file(scan, is_diff, sub_next, st, FILEPHY_UNREAD_OFFSET);
1529 1240603 : processed = 1;
1530 : } else {
1531 132 : msg_verbose("Excluding file '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
1532 : }
1533 42006 : } else if (type == 1) { /* LNK */
1534 40492 : if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0
1535 40492 : && filter_path(&scan->local_filter_list, &reason, disk->name, sub_next) == 0) {
1536 : int ret;
1537 :
1538 40490 : ret = readlink(path_next, tmp, PATH_MAX);
1539 40490 : if (ret >= PATH_MAX) {
1540 : /* LCOV_EXCL_START */
1541 : log_fatal(EINTERNAL, "Error in readlink file '%s'. Symlink too long.\n", path_next);
1542 : exit(EXIT_FAILURE);
1543 : /* LCOV_EXCL_STOP */
1544 : }
1545 40490 : if (ret < 0) {
1546 : /* LCOV_EXCL_START */
1547 : log_fatal(errno, "Error in readlink file '%s'. %s.\n", path_next, strerror(errno));
1548 : exit(EXIT_FAILURE);
1549 : /* LCOV_EXCL_STOP */
1550 : }
1551 40490 : if (ret == 0)
1552 0 : log_fatal(ESOFT, "WARNING! Empty symbolic link '%s'.\n", path_next);
1553 :
1554 : /* readlink doesn't put the final 0 */
1555 40490 : tmp[ret] = 0;
1556 :
1557 : /* process as a symbolic link */
1558 40490 : scan_link(scan, is_diff, sub_next, tmp, FILE_IS_SYMLINK);
1559 40490 : processed = 1;
1560 : } else {
1561 2 : msg_verbose("Excluding link '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
1562 : }
1563 1514 : } else if (type == 2) { /* DIR */
1564 1514 : if (filter_subdir(&state->filterlist, &reason, disk->name, sub_next) == 0
1565 1484 : && filter_subdir(&scan->local_filter_list, &reason, disk->name, sub_next) == 0) {
1566 : #ifndef _WIN32
1567 : /* late stat, if not yet called */
1568 1484 : if (!st)
1569 1484 : st = DSTAT(path_next, dd, &st_buf);
1570 :
1571 : /*
1572 : * In Unix don't follow mount points in different devices
1573 : * in Windows we are already skipping them reporting them as special files
1574 : */
1575 1484 : if ((uint64_t)st->st_dev != disk->dir_device) {
1576 0 : log_fatal(ESOFT, "WARNING! Ignoring mount point '%s' because it appears to be in a different device\n", path_next);
1577 : } else
1578 : #endif
1579 : {
1580 : /* recurse */
1581 1484 : pathslash(path_next, PATH_MAX);
1582 1484 : pathslash(sub_next, PATH_MAX);
1583 1484 : if (scan_sub(scan, level + 1, is_diff, path_next, sub_next, tmp) == 0) {
1584 : /* restore removing additions */
1585 295 : pathcatl(sub_next, sub_len, PATH_MAX, name);
1586 : /* scan the directory as empty dir */
1587 295 : scan_emptydir(scan, sub_next);
1588 : }
1589 : /* or we processed something internally, or we have added the empty dir */
1590 1484 : processed = 1;
1591 : }
1592 : } else {
1593 30 : msg_verbose("Excluding directory '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
1594 : }
1595 : } else {
1596 0 : if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0
1597 0 : && filter_path(&scan->local_filter_list, &reason, disk->name, sub_next) == 0) {
1598 : /* late stat, if not yet called */
1599 0 : if (!st)
1600 0 : st = DSTAT(path_next, dd, &st_buf);
1601 :
1602 0 : log_fatal(ESOFT, "WARNING! Ignoring special '%s' file '%s'\n", stat_desc(st), path_next);
1603 : } else {
1604 0 : msg_verbose("Excluding special file '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
1605 : }
1606 : }
1607 :
1608 : /* next entry */
1609 1282741 : node = node->next;
1610 :
1611 : /* free the present one */
1612 1282741 : free(dd);
1613 : }
1614 :
1615 2143 : return processed;
1616 : }
1617 :
1618 : /**
1619 : * Process a directory.
1620 : * Return != 0 if at least one file or link is processed.
1621 : */
1622 659 : static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const char* dir, const char* sub)
1623 : {
1624 : /* working buffers used by scan_sub() */
1625 : char path_next[PATH_MAX];
1626 : char sub_next[PATH_MAX];
1627 : char tmp[PATH_MAX];
1628 :
1629 659 : pathcpy(path_next, sizeof(path_next), dir);
1630 659 : pathcpy(sub_next, sizeof(sub_next), sub);
1631 :
1632 659 : return scan_sub(scan, level, is_diff, path_next, sub_next, tmp);
1633 : }
1634 :
1635 659 : static void* scan_disk(void* arg)
1636 : {
1637 659 : struct snapraid_scan* scan = arg;
1638 659 : struct snapraid_disk* disk = scan->disk;
1639 : int ret;
1640 : int has_persistent_inodes;
1641 : int has_syncronized_hardlinks;
1642 : uint64_t start;
1643 :
1644 : /* check if the disk supports persistent inodes */
1645 659 : ret = fsinfo(disk->dir, &has_persistent_inodes, &has_syncronized_hardlinks, 0, 0, 0, 0, 0, 0);
1646 659 : if (ret < 0) {
1647 : /* LCOV_EXCL_START */
1648 : log_fatal(errno, "Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
1649 : exit(EXIT_FAILURE);
1650 : /* LCOV_EXCL_STOP */
1651 : }
1652 659 : if (!has_persistent_inodes) {
1653 0 : disk->has_volatile_inodes = 1;
1654 : }
1655 659 : if (!has_syncronized_hardlinks) {
1656 0 : disk->has_volatile_hardlinks = 1;
1657 : }
1658 :
1659 : /* if inodes or UUID are not persistent/changed/unsupported */
1660 659 : if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) {
1661 : /*
1662 : * Remove all the inodes from the inode collection
1663 : * if they are not persistent, all of them could be changed now
1664 : * and we don't want to find false matching ones
1665 : * See scan_file() for more details
1666 : */
1667 97 : tommy_node* node = disk->filelist;
1668 111 : while (node) {
1669 14 : struct snapraid_file* file = node->data;
1670 :
1671 14 : node = node->next;
1672 :
1673 : /* remove from the inode set */
1674 14 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
1675 :
1676 : /* clear the inode */
1677 14 : file->inode = 0;
1678 :
1679 : /* mark as missing inode */
1680 14 : file_flag_set(file, FILE_IS_WITHOUT_INODE);
1681 : }
1682 : }
1683 :
1684 659 : start = os_tick_ms();
1685 :
1686 659 : scan_dir(scan, 0, scan->is_diff, disk->dir, "");
1687 :
1688 659 : if (!scan->is_diff)
1689 611 : msg_progress("Scanned %s in %" PRIu64 " seconds\n", disk->name, (os_tick_ms() - start) / 1000);
1690 :
1691 659 : return 0;
1692 : }
1693 :
1694 115 : static int state_diffscan(struct snapraid_state* state, int is_diff)
1695 : {
1696 : tommy_node* i;
1697 : tommy_node* j;
1698 : tommy_list scanlist;
1699 : int done;
1700 : msg_ptr* msg;
1701 : struct snapraid_scan total;
1702 : int no_difference;
1703 : char esc_buffer[ESC_MAX];
1704 :
1705 115 : tommy_list_init(&scanlist);
1706 :
1707 115 : if (is_diff)
1708 8 : msg_progress("Comparing...\n");
1709 : else
1710 107 : msg_progress("Scanning...\n");
1711 :
1712 115 : log_tag("list:scan_begin\n");
1713 :
1714 : /* allocate all the scan data */
1715 774 : for (i = state->disklist; i != 0; i = i->next) {
1716 659 : struct snapraid_disk* disk = i->data;
1717 : struct snapraid_scan* scan;
1718 :
1719 659 : scan = scan_alloc(state, disk, is_diff);
1720 :
1721 659 : tommy_list_insert_tail(&scanlist, &scan->node, scan);
1722 : }
1723 :
1724 : /* first scan all the directory and find new and deleted files */
1725 774 : for (i = scanlist; i != 0; i = i->next) {
1726 659 : struct snapraid_scan* scan = i->data;
1727 : #if HAVE_THREAD
1728 659 : if (state->opt.skip_multi_scan)
1729 0 : scan_disk(scan);
1730 : else
1731 659 : thread_create(&scan->thread, scan_disk, scan);
1732 : #else
1733 : scan_disk(scan);
1734 : #endif
1735 : }
1736 :
1737 : #if HAVE_THREAD
1738 : /* wait for all threads to terminate */
1739 774 : for (i = scanlist; i != 0; i = i->next) {
1740 659 : struct snapraid_scan* scan = i->data;
1741 : void* retval;
1742 :
1743 : /* wait for thread termination */
1744 659 : if (!state->opt.skip_multi_scan)
1745 659 : thread_join(scan->thread, &retval);
1746 : }
1747 : #endif
1748 :
1749 : /*
1750 : * We split the search in two phases because to detect files
1751 : * moved from one disk to another we have to start deletion
1752 : * only when all disks have all the new files found
1753 : */
1754 :
1755 : /* now process all the new and deleted files */
1756 774 : for (i = scanlist; i != 0; i = i->next) {
1757 659 : struct snapraid_scan* scan = i->data;
1758 659 : struct snapraid_disk* disk = scan->disk;
1759 : tommy_node* node;
1760 : unsigned phy_dup;
1761 : uint64_t phy_last;
1762 : struct snapraid_file* phy_file_last;
1763 :
1764 : /* check for removed files */
1765 659 : node = disk->filelist;
1766 1164780 : while (node) {
1767 1164121 : struct snapraid_file* file = node->data;
1768 :
1769 : /* next node */
1770 1164121 : node = node->next;
1771 :
1772 : /* remove if not present */
1773 1164121 : if (!file_flag_has(file, FILE_IS_PRESENT)) {
1774 29674 : if (!file_flag_has(file, FILE_IS_RELOCATED)) {
1775 22660 : ++scan->count_remove;
1776 :
1777 22660 : log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
1778 22660 : if (is_diff) {
1779 66 : msg_info("remove %s\n", fmt_term(disk, file->sub, esc_buffer));
1780 : }
1781 : }
1782 :
1783 29674 : scan_file_remove(scan, file, 1);
1784 : }
1785 : }
1786 :
1787 : /* check for removed links */
1788 659 : node = disk->linklist;
1789 39798 : while (node) {
1790 39139 : struct snapraid_link* slink = node->data;
1791 :
1792 : /* next node */
1793 39139 : node = node->next;
1794 :
1795 : /* remove if not present */
1796 39139 : if (!link_flag_has(slink, FILE_IS_PRESENT)) {
1797 993 : ++scan->count_remove;
1798 :
1799 993 : log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
1800 993 : if (is_diff) {
1801 10 : msg_info("remove %s\n", fmt_term(disk, slink->sub, esc_buffer));
1802 : }
1803 :
1804 993 : scan_link_remove(scan, slink);
1805 : }
1806 : }
1807 :
1808 : /* check for removed dirs */
1809 659 : node = disk->dirlist;
1810 946 : while (node) {
1811 287 : struct snapraid_dir* dir = node->data;
1812 :
1813 : /* next node */
1814 287 : node = node->next;
1815 :
1816 : /* remove if not present */
1817 287 : if (!dir_flag_has(dir, FILE_IS_PRESENT)) {
1818 2 : scan_emptydir_remove(scan, dir);
1819 : }
1820 : }
1821 :
1822 : /*
1823 : * Sort the files before inserting them
1824 : * we use a stable sort to ensure that if the reported physical offset/inode
1825 : * are always 0, we keep at least the directory order
1826 : */
1827 659 : switch (state->opt.force_order) {
1828 12 : case SORT_PHYSICAL :
1829 12 : tommy_list_sort(&scan->file_insert_list, file_physical_compare);
1830 12 : break;
1831 0 : case SORT_INODE :
1832 0 : tommy_list_sort(&scan->file_insert_list, file_inode_compare);
1833 0 : break;
1834 647 : case SORT_ALPHA :
1835 647 : tommy_list_sort(&scan->file_insert_list, file_path_compare);
1836 647 : break;
1837 0 : case SORT_DIR :
1838 : /* already in order */
1839 0 : break;
1840 : }
1841 :
1842 : /*
1843 : * Insert all the new files, we insert them only after the deletion
1844 : * to reuse the just freed space
1845 : * also check if the physical offset reported are fakes or not
1846 : */
1847 659 : node = scan->file_insert_list;
1848 659 : phy_dup = 0;
1849 659 : phy_last = FILEPHY_UNREAD_OFFSET;
1850 659 : phy_file_last = 0;
1851 106593 : while (node) {
1852 105934 : struct snapraid_file* file = node->data;
1853 :
1854 : /* if the file is not empty, count duplicate physical offsets */
1855 105934 : if (state->opt.force_order == SORT_PHYSICAL && file->size != 0) {
1856 11418 : if (phy_file_last != 0 && file->physical == phy_last
1857 : /* files without offset are expected to have duplicates */
1858 0 : && phy_last != FILEPHY_WITHOUT_OFFSET
1859 : ) {
1860 : /*
1861 : * If verbose, print the list of duplicates real offsets
1862 : * other cases are for offsets not supported, so we don't need to report them file by file
1863 : */
1864 0 : if (phy_last >= FILEPHY_REAL_OFFSET) {
1865 0 : log_fatal(ESOFT, "WARNING! Files '%s%s' and '%s%s' share the same physical offset %" PRId64 ".\n", disk->mount_point, phy_file_last->sub, disk->mount_point, file->sub, phy_last);
1866 : }
1867 0 : ++phy_dup;
1868 : }
1869 11418 : phy_file_last = file;
1870 11418 : phy_last = file->physical;
1871 : }
1872 :
1873 : /* next node */
1874 105934 : node = node->next;
1875 :
1876 : /* insert in the parity */
1877 105934 : scan_file_allocate(scan, file);
1878 : }
1879 :
1880 : /*
1881 : * Mark the disk without reliable physical offset if it has duplicates
1882 : * here it should never happen because we already sorted out hardlinks
1883 : */
1884 659 : if (state->opt.force_order == SORT_PHYSICAL && phy_dup > 0) {
1885 0 : disk->has_unreliable_physical = 1;
1886 : }
1887 :
1888 : /* insert all the new links */
1889 659 : node = scan->link_insert_list;
1890 3225 : while (node) {
1891 2566 : struct snapraid_link* slink = node->data;
1892 :
1893 : /* next node */
1894 2566 : node = node->next;
1895 :
1896 : /* insert it */
1897 2566 : scan_link_insert(scan, slink);
1898 : }
1899 :
1900 : /* insert all the new dirs */
1901 659 : node = scan->dir_insert_list;
1902 669 : while (node) {
1903 10 : struct snapraid_dir* dir = node->data;
1904 :
1905 : /* next node */
1906 10 : node = node->next;
1907 :
1908 : /* insert it */
1909 10 : scan_emptydir_insert(scan, dir);
1910 : }
1911 : }
1912 :
1913 : /* propagate the state change (after all the scan operations are called) */
1914 774 : for (i = scanlist; i != 0; i = i->next) {
1915 659 : struct snapraid_scan* scan = i->data;
1916 659 : if (scan->need_write) {
1917 233 : state->need_write = 1;
1918 : }
1919 : }
1920 :
1921 : /* check for disks where all the previously existing files where removed */
1922 115 : if (!state->opt.force_empty) {
1923 109 : int all_missing = 0;
1924 109 : int all_rewritten = 0;
1925 109 : done = 0;
1926 732 : for (i = state->disklist, j = scanlist; i != 0; i = i->next, j = j->next) {
1927 623 : struct snapraid_disk* disk = i->data;
1928 623 : struct snapraid_scan* scan = j->data;
1929 :
1930 623 : if (scan->count_equal == 0
1931 77 : && scan->count_move == 0
1932 77 : && scan->count_restore == 0
1933 77 : && (scan->count_remove != 0 || scan->count_change != 0)
1934 : ) {
1935 1 : if (!done) {
1936 1 : done = 1;
1937 1 : log_fatal(ESOFT, "WARNING! All the files previously present in disk '%s' at dir '%s'", disk->name, disk->mount_point);
1938 : } else {
1939 0 : log_fatal(ESOFT, ", disk '%s' at dir '%s'", disk->name, disk->mount_point);
1940 : }
1941 :
1942 : /* detect the special condition of all files missing */
1943 1 : if (scan->count_change == 0)
1944 1 : all_missing = 1;
1945 :
1946 : /* detect the special condition of all files rewritten */
1947 1 : if (scan->count_remove == 0)
1948 0 : all_rewritten = 1;
1949 : }
1950 : }
1951 109 : if (done) {
1952 1 : log_fatal(ESOFT, "\nare now missing or have been rewritten!\n");
1953 1 : if (all_rewritten) {
1954 0 : log_fatal(ESOFT, "This could occur when restoring a disk from a backup\n");
1955 0 : log_fatal(ESOFT, "program that is not setting correctly the timestamps.\n");
1956 : }
1957 1 : if (all_missing) {
1958 1 : log_fatal(ESOFT, "This could occur when some disks are not mounted\n");
1959 1 : log_fatal(ESOFT, "in the expected directory.\n");
1960 : }
1961 1 : if (!is_diff) {
1962 1 : log_fatal(ESOFT, "If you want to '%s' anyway, use 'snapraid --force-empty %s'.\n", state->command, state->command);
1963 1 : exit(EXIT_FAILURE);
1964 : }
1965 : }
1966 : }
1967 :
1968 : /* check for disks without the physical offset support */
1969 114 : if (state->opt.force_order == SORT_PHYSICAL) {
1970 2 : done = 0;
1971 14 : for (i = state->disklist; i != 0; i = i->next) {
1972 12 : struct snapraid_disk* disk = i->data;
1973 :
1974 12 : if (disk->has_unreliable_physical) {
1975 0 : if (!done) {
1976 0 : done = 1;
1977 0 : log_fatal(ESOFT, "WARNING! Physical offsets not supported for disk '%s'", disk->name);
1978 : } else {
1979 0 : log_fatal(ESOFT, ", '%s'", disk->name);
1980 : }
1981 : }
1982 : }
1983 2 : if (done) {
1984 0 : log_fatal(ESOFT, ". The order of files won't be optimal.\n");
1985 : }
1986 : }
1987 :
1988 : /* check for disks without persistent inodes */
1989 114 : done = 0;
1990 767 : for (i = state->disklist; i != 0; i = i->next) {
1991 653 : struct snapraid_disk* disk = i->data;
1992 :
1993 653 : if (disk->has_volatile_inodes) {
1994 0 : if (!done) {
1995 0 : done = 1;
1996 0 : log_fatal(ESOFT, "WARNING! Inodes are not persistent for disks: '%s'", disk->name);
1997 : } else {
1998 0 : log_fatal(ESOFT, ", '%s'", disk->name);
1999 : }
2000 : }
2001 : }
2002 114 : if (done) {
2003 0 : log_fatal(ESOFT, ". Inodes are not used to detect move operations.\n");
2004 : }
2005 :
2006 : /* check for disks with changed UUID */
2007 114 : done = 0;
2008 767 : for (i = state->disklist; i != 0; i = i->next) {
2009 653 : struct snapraid_disk* disk = i->data;
2010 :
2011 : /*
2012 : * Don't print the message if the UUID changed because before
2013 : * it was no set.
2014 : * This is the normal condition for an empty disk because it
2015 : * isn't stored
2016 : */
2017 653 : if (disk->has_different_uuid && !disk->had_empty_uuid) {
2018 4 : if (!done) {
2019 2 : done = 1;
2020 2 : log_fatal(ESOFT, "WARNING! UUID is changed for disks: '%s'", disk->name);
2021 : } else {
2022 2 : log_fatal(ESOFT, ", '%s'", disk->name);
2023 : }
2024 : }
2025 : }
2026 114 : if (done) {
2027 2 : log_fatal(ESOFT, ". Inodes are not used to detect move operations.\n");
2028 : }
2029 :
2030 : /* check for disks with unsupported UUID */
2031 114 : done = 0;
2032 767 : for (i = state->disklist; i != 0; i = i->next) {
2033 653 : struct snapraid_disk* disk = i->data;
2034 :
2035 653 : if (disk->has_unsupported_uuid) {
2036 0 : if (!done) {
2037 0 : done = 1;
2038 0 : log_fatal(ESOFT, "WARNING! UUID is unsupported for disks: '%s'", disk->name);
2039 : } else {
2040 0 : log_fatal(ESOFT, ", '%s'", disk->name);
2041 : }
2042 : }
2043 : }
2044 114 : if (done) {
2045 0 : log_fatal(ESOFT, ". Not using inodes to detect move operations.\n");
2046 : #if defined(_linux) && !HAVE_BLKID
2047 : log_fatal(ESOFT, "The 'blkid' library is not linked in SnapRAID!\n");
2048 : log_fatal(ESOFT, "Try rebuilding it after installing the libblkid-dev or libblkid-devel package.\n");
2049 : #endif
2050 : }
2051 :
2052 114 : total.count_equal = 0;
2053 114 : total.count_move = 0;
2054 114 : total.count_copy = 0;
2055 114 : total.count_relocate = 0;
2056 114 : total.count_restore = 0;
2057 114 : total.count_change = 0;
2058 114 : total.count_remove = 0;
2059 114 : total.count_insert = 0;
2060 :
2061 767 : for (i = scanlist; i != 0; i = i->next) {
2062 653 : struct snapraid_scan* scan = i->data;
2063 653 : total.count_equal += scan->count_equal;
2064 653 : total.count_move += scan->count_move;
2065 653 : total.count_copy += scan->count_copy;
2066 653 : total.count_relocate += scan->count_relocate;
2067 653 : total.count_restore += scan->count_restore;
2068 653 : total.count_change += scan->count_change;
2069 653 : total.count_remove += scan->count_remove;
2070 653 : total.count_insert += scan->count_insert;
2071 : }
2072 :
2073 114 : if (is_diff) {
2074 8 : msg_status("\n");
2075 8 : msg = msg_status;
2076 : } else {
2077 106 : msg = msg_verbose;
2078 : }
2079 :
2080 114 : msg("%8u equal\n", total.count_equal);
2081 114 : msg("%8u added\n", total.count_insert);
2082 114 : msg("%8u removed\n", total.count_remove);
2083 114 : msg("%8u updated\n", total.count_change);
2084 114 : msg("%8u moved\n", total.count_move);
2085 114 : msg("%8u copied\n", total.count_copy);
2086 114 : msg("%8u relocated\n", total.count_relocate);
2087 114 : msg("%8u restored\n", total.count_restore);
2088 :
2089 114 : log_tag("summary:equal:%u\n", total.count_equal);
2090 114 : log_tag("summary:added:%u\n", total.count_insert);
2091 114 : log_tag("summary:removed:%u\n", total.count_remove);
2092 114 : log_tag("summary:updated:%u\n", total.count_change);
2093 114 : log_tag("summary:moved:%u\n", total.count_move);
2094 114 : log_tag("summary:copied:%u\n", total.count_copy);
2095 114 : log_tag("summary:relocated:%u\n", total.count_relocate);
2096 114 : log_tag("summary:restored:%u\n", total.count_restore);
2097 114 : log_tag("list:scan_end\n");
2098 :
2099 : /* save in the state */
2100 114 : state->removed_files = total.count_remove;
2101 114 : state->updated_files = total.count_change;
2102 :
2103 113 : no_difference = !total.count_move && !total.count_copy && !total.count_relocate && !total.count_restore
2104 227 : && !total.count_change && !total.count_remove && !total.count_insert;
2105 :
2106 114 : if (is_diff) {
2107 8 : if (!no_difference) {
2108 6 : msg_status("There are differences!\n");
2109 : } else {
2110 2 : msg_status("No differences\n");
2111 : }
2112 8 : if (state->unsynced_blocks != 0)
2113 1 : log_error(EUSER, "The last sync was interrupted. Run it again!\n");
2114 :
2115 8 : if (state->unsynced_blocks != 0) {
2116 1 : log_tag("summary:exit:unsynced\n");
2117 7 : } else if (!no_difference) {
2118 6 : log_tag("summary:exit:diff\n");
2119 : } else {
2120 1 : log_tag("summary:exit:equal\n");
2121 : }
2122 : }
2123 :
2124 114 : log_flush();
2125 :
2126 114 : tommy_list_foreach(&scanlist, (tommy_foreach_func*)scan_free);
2127 :
2128 : /* check the file-system on all disks */
2129 114 : state_fscheck(state, "after scan");
2130 :
2131 114 : if (is_diff) {
2132 : /* check for file difference */
2133 8 : if (!no_difference)
2134 6 : return 1;
2135 :
2136 : /* check also for incomplete "sync" */
2137 2 : if (state->unsynced_blocks != 0)
2138 1 : return 1;
2139 : }
2140 :
2141 107 : return 0;
2142 : }
2143 :
2144 8 : int state_diff(struct snapraid_state* state)
2145 : {
2146 8 : return state_diffscan(state, 1);
2147 : }
2148 :
2149 107 : void state_scan(struct snapraid_state* state)
2150 : {
2151 107 : (void)state_diffscan(state, 0); /* ignore return value */
2152 106 : }
2153 :
|
