Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "portable.h"
19 :
20 : #include "support.h"
21 : #include "elem.h"
22 : #include "state.h"
23 : #include "parity.h"
24 :
25 : struct snapraid_scan {
26 : struct snapraid_state* state; /**< State used. */
27 : struct snapraid_disk* disk; /**< Disk used. */
28 : thread_id_t thread; /**< Thread used for scanning the disk */
29 :
30 : int is_diff; /**< If it's a diff command or a scanning */
31 : int need_write; /**< If a state write is required */
32 :
33 : #if HAVE_THREAD
34 : /**
35 : * Mutex for protecting the disk stampset table
36 : */
37 : thread_mutex_t mutex;
38 : #endif
39 :
40 : /**
41 : * Counters of changes.
42 : */
43 : unsigned count_equal; /**< Files equal. */
44 : unsigned count_move; /**< Files with a different name, but equal inode, size and timestamp in the same disk. */
45 : unsigned count_restore; /**< Files with equal name, size and timestamp, but different inode. */
46 : unsigned count_change; /**< Files with same name, but different size and/or timestamp. */
47 : unsigned count_copy; /**< Files new, with same name size and timestamp of a file in a different disk. */
48 : unsigned count_insert; /**< Files new. */
49 : unsigned count_remove; /**< Files removed. */
50 :
51 : tommy_list file_insert_list; /**< Files to insert. */
52 : tommy_list link_insert_list; /**< Links to insert. */
53 : tommy_list dir_insert_list; /**< Dirs to insert. */
54 :
55 : /* nodes for data structures */
56 : tommy_node node;
57 : };
58 :
59 575 : static struct snapraid_scan* scan_alloc(struct snapraid_state* state, struct snapraid_disk* disk, int is_diff)
60 : {
61 : struct snapraid_scan* scan;
62 :
63 575 : scan = malloc_nofail(sizeof(struct snapraid_scan));
64 575 : scan->state = state;
65 575 : scan->disk = disk;
66 575 : scan->count_equal = 0;
67 575 : scan->count_move = 0;
68 575 : scan->count_copy = 0;
69 575 : scan->count_restore = 0;
70 575 : scan->count_change = 0;
71 575 : scan->count_remove = 0;
72 575 : scan->count_insert = 0;
73 575 : tommy_list_init(&scan->file_insert_list);
74 575 : tommy_list_init(&scan->link_insert_list);
75 575 : tommy_list_init(&scan->dir_insert_list);
76 575 : scan->is_diff = is_diff;
77 575 : scan->need_write = 0;
78 :
79 : #if HAVE_THREAD
80 575 : thread_mutex_init(&disk->stamp_mutex);
81 : #endif
82 :
83 575 : return scan;
84 : }
85 :
86 569 : static void scan_free(struct snapraid_scan* scan)
87 : {
88 : #if HAVE_THREAD
89 569 : thread_mutex_destroy(&scan->disk->stamp_mutex);
90 : #endif
91 :
92 569 : free(scan);
93 569 : }
94 :
95 512477 : static void stamp_lock(struct snapraid_disk* disk)
96 : {
97 : #if HAVE_THREAD
98 512477 : thread_mutex_lock(&disk->stamp_mutex);
99 : #else
100 : (void)disk;
101 : #endif
102 512477 : }
103 :
104 512477 : static void stamp_unlock(struct snapraid_disk* disk)
105 : {
106 : #if HAVE_THREAD
107 512477 : thread_mutex_unlock(&disk->stamp_mutex);
108 : #else
109 : (void)disk;
110 : #endif
111 512477 : }
112 :
113 : /**
114 : * Remove the specified link from the data set.
115 : */
116 956 : static void scan_link_remove(struct snapraid_scan* scan, struct snapraid_link* slink)
117 : {
118 956 : struct snapraid_disk* disk = scan->disk;
119 :
120 : /* state changed */
121 956 : scan->need_write = 1;
122 :
123 : /* remove the file from the link containers */
124 956 : tommy_hashdyn_remove_existing(&disk->linkset, &slink->nodeset);
125 956 : tommy_list_remove_existing(&disk->linklist, &slink->nodelist);
126 :
127 : /* deallocate */
128 956 : link_free(slink);
129 956 : }
130 :
131 : /**
132 : * Insert the specified link in the data set.
133 : */
134 2457 : static void scan_link_insert(struct snapraid_scan* scan, struct snapraid_link* slink)
135 : {
136 2457 : struct snapraid_disk* disk = scan->disk;
137 :
138 : /* state changed */
139 2457 : scan->need_write = 1;
140 :
141 : /* insert the link in the link containers */
142 2457 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
143 2457 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
144 2457 : }
145 :
146 : /**
147 : * Process a symbolic link.
148 : */
149 36184 : static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub, const char* linkto, unsigned link_flag)
150 : {
151 36184 : struct snapraid_state* state = scan->state;
152 36184 : struct snapraid_disk* disk = scan->disk;
153 : struct snapraid_link* slink;
154 : char esc_buffer[ESC_MAX];
155 :
156 : /* check if the link already exists */
157 36184 : slink = tommy_hashdyn_search(&disk->linkset, link_name_compare_to_arg, sub, link_name_hash(sub));
158 36184 : if (slink) {
159 : /* check if multiple files have the same name */
160 33727 : if (link_flag_has(slink, FILE_IS_PRESENT)) {
161 : /* LCOV_EXCL_START */
162 : log_fatal("Internal inconsistency for link '%s%s'\n", disk->dir, sub);
163 : os_abort();
164 : /* LCOV_EXCL_STOP */
165 : }
166 :
167 : /* mark as present */
168 33727 : link_flag_set(slink, FILE_IS_PRESENT);
169 :
170 : /* check if the link is not changed and it's of the same kind */
171 33727 : if (strcmp(slink->linkto, linkto) == 0 && link_flag == link_flag_get(slink, FILE_IS_LINK_MASK)) {
172 : /* it's equal */
173 33721 : ++scan->count_equal;
174 :
175 33721 : if (state->opt.gui) {
176 829 : log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
177 : }
178 : } else {
179 : /* it's an update */
180 :
181 : /* we have to save the linkto/type */
182 6 : scan->need_write = 1;
183 :
184 6 : ++scan->count_change;
185 :
186 6 : log_tag("scan:update:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
187 6 : if (is_diff) {
188 3 : msg_info("update %s\n", fmt_term(disk, slink->sub, esc_buffer));
189 : }
190 :
191 : /* update it */
192 6 : free(slink->linkto);
193 6 : slink->linkto = strdup_nofail(linkto);
194 6 : link_flag_let(slink, link_flag, FILE_IS_LINK_MASK);
195 : }
196 :
197 : /* nothing more to do */
198 33727 : return;
199 : } else {
200 : /* create the new link */
201 2457 : ++scan->count_insert;
202 :
203 2457 : log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
204 2457 : if (is_diff) {
205 599 : msg_info("add %s\n", fmt_term(disk, sub, esc_buffer));
206 : }
207 :
208 : /* and continue to insert it */
209 : }
210 :
211 : /* insert it */
212 2457 : slink = link_alloc(sub, linkto, link_flag);
213 :
214 : /* mark it as present */
215 2457 : link_flag_set(slink, FILE_IS_PRESENT);
216 :
217 : /* insert it in the delayed insert list */
218 2457 : tommy_list_insert_tail(&scan->link_insert_list, &slink->nodelist, slink);
219 : }
220 :
221 : /**
222 : * Insert the specified file in the parity.
223 : */
224 100097 : static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
225 : {
226 100097 : struct snapraid_state* state = scan->state;
227 100097 : struct snapraid_disk* disk = scan->disk;
228 : block_off_t i;
229 : block_off_t parity_pos;
230 :
231 : /* state changed */
232 100097 : scan->need_write = 1;
233 :
234 : /* allocate the blocks of the file */
235 100097 : parity_pos = disk->first_free_block;
236 346887 : for (i = 0; i < file->blockmax; ++i) {
237 : struct snapraid_block* block;
238 : struct snapraid_block* over_block;
239 : snapraid_info info;
240 :
241 : /* increment the position until the first really free block */
242 761974 : while (block_has_file(fs_par2block_find(disk, parity_pos)))
243 515184 : ++parity_pos;
244 :
245 : /* get block we are going to overwrite, if any */
246 246790 : over_block = fs_par2block_find(disk, parity_pos);
247 :
248 : /* deallocate it */
249 246790 : if (over_block != BLOCK_NULL)
250 90876 : fs_deallocate(disk, parity_pos);
251 :
252 : /* get block specific info */
253 246790 : info = info_get(&state->infoarr, parity_pos);
254 :
255 : /* get the new block we are going to write */
256 246790 : block = fs_file2block_get(file, i);
257 :
258 : /* if the file block already has an updated hash without rehash */
259 246790 : if (block_has_updated_hash(block) && !info_get_rehash(info)) {
260 : /* the only possible case is for REP blocks */
261 86215 : assert(block_state_get(block) == BLOCK_STATE_REP);
262 :
263 : /* convert to a REP block */
264 86215 : block_state_set(block, BLOCK_STATE_REP);
265 :
266 : /* and keep the hash as it's */
267 : } else {
268 : unsigned over_state;
269 :
270 : /* convert to a CHG block */
271 160575 : block_state_set(block, BLOCK_STATE_CHG);
272 :
273 : /* state of the block we are going to overwrite */
274 160575 : over_state = block_state_get(over_block);
275 :
276 : /* if the block is an empty one */
277 160575 : if (over_state == BLOCK_STATE_EMPTY) {
278 : /* the block was empty and filled with zeros */
279 : /* set the hash to the special ZERO value */
280 115543 : hash_zero_set(block->hash);
281 : } else {
282 : /* otherwise it's a DELETED one */
283 45032 : assert(over_state == BLOCK_STATE_DELETED);
284 :
285 : /* copy the past hash of the block */
286 45032 : memcpy(block->hash, over_block->hash, BLOCK_HASH_SIZE);
287 :
288 : /* if we have not already cleared the past hash */
289 45032 : if (!state->clear_past_hash) {
290 : /* in this case we don't know if the old state is still the one */
291 : /* stored inside the parity, because after an aborted sync, the parity */
292 : /* may be or may be not have been updated with the new data */
293 : /* Then we reset the hash to a bogus value */
294 : /* For example: */
295 : /* - One file is deleted */
296 : /* - Sync aborted after, updating the parity to the new state, */
297 : /* but without saving the content file representing this new state. */
298 : /* - Another file is added again (exactly here) */
299 : /* with the hash of DELETED block not representing the real parity state */
300 421 : hash_invalid_set(block->hash);
301 : }
302 : }
303 : }
304 :
305 : /* store in the disk map, after invalidating all the other blocks */
306 246790 : fs_allocate(disk, parity_pos, file, i);
307 :
308 : /* set the new free position */
309 246790 : disk->first_free_block = parity_pos + 1;
310 : }
311 :
312 : /* insert in the list of contained files */
313 100097 : tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
314 100097 : }
315 :
316 : /**
317 : * Delete the specified file from the parity.
318 : *
319 : * Note that the parity remains allocated, but the blocks and the file are marked as DELETED.
320 : * The file is then inserted in the deleted set, and it should not be deallocated,
321 : * as the parity still references it.
322 : */
323 55231 : static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file)
324 : {
325 55231 : struct snapraid_state* state = scan->state;
326 55231 : struct snapraid_disk* disk = scan->disk;
327 : block_off_t i;
328 :
329 : /* remove from the list of contained files */
330 55231 : tommy_list_remove_existing(&disk->filelist, &file->nodelist);
331 :
332 : /* state changed */
333 55231 : scan->need_write = 1;
334 :
335 : /* here we are supposed to adjust the ::first_free_block position */
336 : /* with the parity position we are deleting */
337 : /* but we also know that we do only delayed insert, after all the deletion, */
338 : /* so at this point ::first_free_block is always at 0, and we don't need to update it */
339 55231 : if (disk->first_free_block != 0) {
340 : /* LCOV_EXCL_START */
341 : log_fatal("Internal inconsistency for first free position at '%u' deallocating file '%s'\n", disk->first_free_block, file->sub);
342 : os_abort();
343 : /* LCOV_EXCL_STOP */
344 : }
345 :
346 : /* free all the blocks of the file */
347 191406 : for (i = 0; i < file->blockmax; ++i) {
348 136175 : struct snapraid_block* block = fs_file2block_get(file, i);
349 : unsigned block_state;
350 :
351 : /* in case we scan after an aborted sync, */
352 : /* we could get also intermediate states */
353 136175 : block_state = block_state_get(block);
354 136175 : switch (block_state) {
355 31179 : case BLOCK_STATE_BLK :
356 : /* we keep the hash making it an "old" hash, because the parity is still containing data for it */
357 31179 : break;
358 28040 : case BLOCK_STATE_CHG :
359 : /* if we have not already cleared the past hash */
360 28040 : if (!state->clear_past_hash) {
361 : /* in these cases we don't know if the old state is still the one */
362 : /* stored inside the parity, because after an aborted sync, the parity */
363 : /* may be or may be not have been updated with the data that it's now */
364 : /* deleted. Then we reset the hash to a bogus value. */
365 : /* For example: */
366 : /* - One file is added */
367 : /* - Sync aborted after updating the parity to the new state, */
368 : /* but without saving the content file representing this new state. */
369 : /* - File is now deleted after the aborted sync */
370 : /* - Sync again, deleting the blocks (exactly here) */
371 : /* with the hash of CHG block not representing the real parity state */
372 40 : hash_invalid_set(block->hash);
373 : }
374 28040 : break;
375 76956 : case BLOCK_STATE_REP :
376 : /* we just don't know the old hash, and then we set it to invalid */
377 76956 : hash_invalid_set(block->hash);
378 76956 : break;
379 0 : default :
380 : /* LCOV_EXCL_START */
381 : log_fatal("Internal inconsistency in file '%s' deallocating block '%u:%u' state %u\n", file->sub, i, file->blockmax, block_state);
382 : os_abort();
383 : /* LCOV_EXCL_STOP */
384 : }
385 :
386 : /* set the block as deleted */
387 136175 : block_state_set(block, BLOCK_STATE_DELETED);
388 : }
389 :
390 : /* mark the file as deleted */
391 55231 : file_flag_set(file, FILE_IS_DELETED);
392 :
393 : /* insert it in the list of deleted blocks */
394 55231 : tommy_list_insert_tail(&disk->deletedlist, &file->nodelist, file);
395 55231 : }
396 :
397 100097 : static void scan_file_delayed_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
398 : {
399 100097 : struct snapraid_state* state = scan->state;
400 100097 : struct snapraid_disk* disk = scan->disk;
401 :
402 : /* if we sort for physical offsets we have to read them for new files */
403 100097 : if (state->opt.force_order == SORT_PHYSICAL
404 11432 : && file->physical == FILEPHY_UNREAD_OFFSET
405 : ) {
406 : char path_next[PATH_MAX];
407 :
408 11432 : pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, file->sub);
409 :
410 11432 : if (filephy(path_next, file->size, &file->physical) != 0) {
411 : /* LCOV_EXCL_START */
412 : log_fatal("Error in getting the physical offset of file '%s'. %s.\n", path_next, strerror(errno));
413 : exit(EXIT_FAILURE);
414 : /* LCOV_EXCL_STOP */
415 : }
416 : }
417 :
418 : /* insert in the delayed list */
419 100097 : tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file);
420 100097 : }
421 :
422 : /**
423 : * Check if a file is completely formed of blocks with invalid parity,
424 : * and no rehash is tagged, and if it has at least one block.
425 : */
426 1027333 : static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
427 : {
428 : block_off_t i;
429 :
430 : /* with no block, it never has an invalid parity */
431 1027333 : if (file->blockmax == 0)
432 1438 : return 0;
433 :
434 : /* check all blocks */
435 1081862 : for (i = 0; i < file->blockmax; ++i) {
436 : snapraid_info info;
437 1059176 : struct snapraid_block* block = fs_file2block_get(file, i);
438 : block_off_t parity_pos;
439 :
440 : /* exclude blocks with parity */
441 1059176 : if (!block_has_invalid_parity(block))
442 1003209 : return 0;
443 :
444 : /*
445 : * Get the parity position.
446 : *
447 : * Note that here we expect to always have mapped
448 : * parity, because kept files always have it.
449 : *
450 : * Anyway, checking for POS_NULL doesn't hurt.
451 : */
452 55967 : parity_pos = fs_file2par_find(disk, file, i);
453 :
454 : /* if it's not mapped, it cannot have rehash */
455 55967 : if (parity_pos != POS_NULL) {
456 : /* get block specific info */
457 55967 : info = info_get(&state->infoarr, parity_pos);
458 :
459 : /* if rehash fails */
460 55967 : if (info_get_rehash(info))
461 0 : return 0;
462 : }
463 : }
464 :
465 22686 : return 1;
466 : }
467 :
468 : /**
469 : * Check if a file is completely formed of blocks with an updated hash,
470 : * and no rehash is tagged, and if it has at least one block.
471 : */
472 23669 : static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
473 : {
474 : block_off_t i;
475 :
476 : /* with no block, it never has a hash */
477 23669 : if (file->blockmax == 0)
478 27 : return 0;
479 :
480 : /* check all blocks */
481 81886 : for (i = 0; i < file->blockmax; ++i) {
482 : snapraid_info info;
483 58249 : struct snapraid_block* block = fs_file2block_get(file, i);
484 : block_off_t parity_pos;
485 :
486 : /* exclude blocks without hash */
487 58249 : if (!block_has_updated_hash(block))
488 5 : return 0;
489 :
490 : /*
491 : * Get the parity position.
492 : *
493 : * Note that it's possible to have files
494 : * not mapped into the parity, even if they
495 : * have a valid hash.
496 : *
497 : * This happens for example, for 'copied' files
498 : * that have REP blocks, but not yet mapped.
499 : *
500 : * If there are multiple copies, it's also possible
501 : * that such files are used as 'source' to copy
502 : * hashes, and then to get them inside this function.
503 : */
504 58244 : parity_pos = fs_file2par_find(disk, file, i);
505 :
506 : /* if it's not mapped, it cannot have rehash */
507 58244 : if (parity_pos != POS_NULL) {
508 : /* get block specific info */
509 58236 : info = info_get(&state->infoarr, parity_pos);
510 :
511 : /* exclude blocks needing a rehash */
512 58236 : if (info_get_rehash(info))
513 0 : return 0;
514 : }
515 : }
516 :
517 23637 : return 1;
518 : }
519 :
520 : /**
521 : * Refresh the file info.
522 : *
523 : * This is needed by Windows as the normal way to list directories may report not
524 : * updated info. Only the GetFileInformationByHandle() func, called file-by-file,
525 : * really ensures to return synced info.
526 : *
527 : * If this happens, we read also the physical offset, to avoid to read it later.
528 : */
529 77411 : static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struct stat* st, uint64_t* physical)
530 : {
531 : #if HAVE_LSTAT_SYNC
532 : struct snapraid_state* state = scan->state;
533 : struct snapraid_disk* disk = scan->disk;
534 :
535 : /* if the st_sync is not set, ensure to get synced info */
536 : if (st->st_sync == 0) {
537 : char path_next[PATH_MAX];
538 : struct stat synced_st;
539 :
540 : pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, sub);
541 :
542 : /* if we sort for physical offsets we have to read them for new files */
543 : if (state->opt.force_order == SORT_PHYSICAL
544 : && *physical == FILEPHY_UNREAD_OFFSET
545 : ) {
546 : /* do nothing, leave the pointer to read the physical offset */
547 : } else {
548 : physical = 0; /* set the pointer to 0 to read nothing */
549 : }
550 :
551 : if (lstat_sync(path_next, &synced_st, physical) != 0) {
552 : /* LCOV_EXCL_START */
553 : log_fatal("Error in stat file '%s'. %s.\n", path_next, strerror(errno));
554 : exit(EXIT_FAILURE);
555 : /* LCOV_EXCL_STOP */
556 : }
557 :
558 : if (st->st_mtime != synced_st.st_mtime
559 : || st->st_mtimensec != synced_st.st_mtimensec
560 : ) {
561 : #ifndef _WIN32
562 : /*
563 : * In Windows having different metadata is expected with open files
564 : * because the metadata in the directory is updated only when the file
565 : * is closed.
566 : *
567 : * The same happens for hardlinks that duplicate metadata.
568 : * The link metadata is updated only when the link is opened.
569 : * This extends also to st_size and st_nlink.
570 : *
571 : * See also:
572 : * Why is the file size reported incorrectly for files that are still being written to?
573 : * http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx
574 : */
575 : log_fatal("WARNING! Detected uncached time change from %" PRIu64 ".%09u to %" PRIu64 ".%09u for file '%s'\n",
576 : (uint64_t)st->st_mtime, (uint32_t)st->st_mtimensec, (uint64_t)synced_st.st_mtime, (uint32_t)synced_st.st_mtimensec, sub);
577 : log_fatal("It's better if you run SnapRAID without other processes running.\n");
578 : #endif
579 : st->st_mtime = synced_st.st_mtime;
580 : st->st_mtimensec = synced_st.st_mtimensec;
581 : }
582 :
583 : if (st->st_size != synced_st.st_size) {
584 : #ifndef _WIN32
585 : log_fatal("WARNING! Detected uncached size change from %" PRIu64 " to %" PRIu64 " for file '%s'\n",
586 : (uint64_t)st->st_size, (uint64_t)synced_st.st_size, sub);
587 : log_fatal("It's better if you run SnapRAID without other processes running.\n");
588 : #endif
589 : st->st_size = synced_st.st_size;
590 : }
591 :
592 : if (st->st_nlink != synced_st.st_nlink) {
593 : #ifndef _WIN32
594 : log_fatal("WARNING! Detected uncached nlink change from %u to %u for file '%s'\n",
595 : (uint32_t)st->st_nlink, (uint32_t)synced_st.st_nlink, sub);
596 : log_fatal("It's better if you run SnapRAID without other processes running.\n");
597 : #endif
598 : st->st_nlink = synced_st.st_nlink;
599 : }
600 :
601 : if (st->st_ino != synced_st.st_ino) {
602 : log_fatal("DANGER! Detected uncached inode change from %" PRIu64 " to %" PRIu64 " for file '%s'\n",
603 : (uint64_t)st->st_ino, (uint64_t)synced_st.st_ino, sub);
604 : log_fatal("It's better if you run SnapRAID without other processes running.\n");
605 : /* at this point, it's too late to change inode */
606 : /* and having inconsistent inodes may result to internal failures */
607 : /* so, it's better to abort */
608 : exit(EXIT_FAILURE);
609 : }
610 : }
611 : #else
612 : (void)scan;
613 : (void)sub;
614 : (void)st;
615 : (void)physical;
616 : #endif
617 77411 : }
618 :
619 : /**
620 : * Insert the file in the data set.
621 : */
622 100097 : static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
623 : {
624 100097 : struct snapraid_disk* disk = scan->disk;
625 :
626 : /* insert the file in the containers */
627 100097 : if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
628 100097 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
629 :
630 100097 : stamp_lock(disk);
631 100097 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
632 100097 : tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
633 100097 : stamp_unlock(disk);
634 :
635 : /* delayed allocation of the parity */
636 100097 : scan_file_delayed_allocate(scan, file);
637 100097 : }
638 :
639 : /**
640 : * Remove the file from the data set.
641 : *
642 : * File is then deleted.
643 : */
644 55231 : static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
645 : {
646 55231 : struct snapraid_disk* disk = scan->disk;
647 :
648 : /* remove the file from the containers */
649 55231 : if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
650 55174 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
651 55231 : tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
652 :
653 55231 : stamp_lock(disk);
654 55231 : tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
655 55231 : stamp_unlock(disk);
656 :
657 : /* deallocate the file from the parity */
658 55231 : scan_file_deallocate(scan, file);
659 55231 : }
660 :
661 : /**
662 : * Keep the file as it's (or with only a name/inode modification).
663 : *
664 : * If the file is kept, nothing has to be done.
665 : *
666 : * But if a file contains only blocks with invalid parity, it's reallocated to ensure
667 : * to always minimize the space used in the parity.
668 : *
669 : * This could happen after a failed sync, when some other files are deleted,
670 : * and then new ones can be moved backward to fill the hole created.
671 : */
672 1027333 : static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* file)
673 : {
674 1027333 : struct snapraid_disk* disk = scan->disk;
675 :
676 : /* if the file is full invalid, schedule a reinsert at later stage */
677 1027333 : if (file_is_full_invalid_parity_and_stable(scan->state, disk, file)) {
678 22686 : struct snapraid_file* copy = file_dup(file);
679 :
680 : /* remove the file */
681 22686 : scan_file_remove(scan, file);
682 :
683 : /* reinsert the copy in the delayed list */
684 22686 : scan_file_insert(scan, copy);
685 : }
686 1027333 : }
687 :
688 : /**
689 : * Process a file.
690 : */
691 1104939 : static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, struct stat* st, uint64_t physical)
692 : {
693 1104939 : struct snapraid_state* state = scan->state;
694 1104939 : struct snapraid_disk* disk = scan->disk;
695 : struct snapraid_file* file;
696 : tommy_node* i;
697 : int is_original_file_size_different_than_zero;
698 : int is_file_already_present;
699 : data_off_t file_already_present_size;
700 : int64_t file_already_present_mtime_sec;
701 : int file_already_present_mtime_nsec;
702 : int is_file_reported;
703 : char esc_buffer[ESC_MAX];
704 : char esc_buffer_alt[ESC_MAX];
705 :
706 : /*
707 : * If the disk has persistent inodes and UUID, try a search on the past inodes,
708 : * to detect moved files.
709 : *
710 : * For persistent inodes we mean inodes that keep their values when the file-system
711 : * is unmounted and remounted. This don't always happen.
712 : *
713 : * Cases found are:
714 : * - Linux FUSE with exFAT driver from https://code.google.com/p/exfat/.
715 : * Inodes are reassigned at every mount restarting from 1 and incrementing.
716 : * As worse, the exFAT support in FUSE doesn't use sub-second precision in timestamps
717 : * making inode collision more easy (exFAT by design supports 10ms precision).
718 : * - Linux VFAT kernel (3.2) driver. Inodes are fully reassigned at every mount.
719 : *
720 : * In such cases, to avoid possible random collisions, it's better to disable the moved
721 : * file recognition.
722 : *
723 : * For persistent UUID we mean that it has the same UUID as before.
724 : * Otherwise, if the UUID is changed, likely it's a new recreated file-system,
725 : * and then the inode have no meaning.
726 : *
727 : * Note that to disable the search by past inode, we do this implicitly
728 : * removing all the past inode before searching for files.
729 : * This ensures that no file is found with a past inode, but at the same time,
730 : * it allows to find new files with the same inode, to identify them as hardlinks.
731 : */
732 1104939 : int has_past_inodes = !disk->has_volatile_inodes && !disk->has_different_uuid && !disk->has_unsupported_uuid;
733 :
734 : /* always search with the new inode, in the all new inodes found until now, */
735 : /* with the eventual presence of also the past inodes */
736 1104939 : uint64_t inode = st->st_ino;
737 :
738 1104939 : file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
739 :
740 : /* identify moved files with past inodes and hardlinks with the new inodes */
741 1104939 : if (file) {
742 : /* check if the file is not changed */
743 944187 : if (file->size == st->st_size
744 933852 : && file->mtime_sec == st->st_mtime
745 932579 : && (file->mtime_nsec == STAT_NSEC(st)
746 : /* always accept the stored value if it's STAT_NSEC_INVALID */
747 : /* it happens when upgrading from an old version of SnapRAID */
748 : /* not yet supporting the nanosecond field */
749 3 : || file->mtime_nsec == STAT_NSEC_INVALID
750 : )
751 : ) {
752 : /* check if multiple files have the same inode */
753 932576 : if (file_flag_has(file, FILE_IS_PRESENT)) {
754 : /* if has_volatile_hardlinks is true, the nlink value is not reliable */
755 195 : if (!disk->has_volatile_hardlinks && st->st_nlink == 1) {
756 : /* LCOV_EXCL_START */
757 : log_fatal("Internal inode '%" PRIu64 "' inconsistency for file '%s%s' already present\n", (uint64_t)st->st_ino, disk->dir, sub);
758 : os_abort();
759 : /* LCOV_EXCL_STOP */
760 : }
761 :
762 : /* it's a hardlink */
763 195 : scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
764 1027528 : return;
765 : }
766 :
767 : /* mark as present */
768 932381 : file_flag_set(file, FILE_IS_PRESENT);
769 :
770 : /* update the nanoseconds mtime only if different */
771 : /* to avoid unneeded updates */
772 932381 : if (file->mtime_nsec == STAT_NSEC_INVALID
773 0 : && STAT_NSEC(st) != file->mtime_nsec
774 : ) {
775 0 : file->mtime_nsec = STAT_NSEC(st);
776 :
777 : /* we have to save the new mtime */
778 0 : scan->need_write = 1;
779 : }
780 :
781 932381 : if (strcmp(file->sub, sub) != 0) {
782 : /* if the path is different, it means a moved file with the same inode */
783 155 : ++scan->count_move;
784 :
785 155 : log_tag("scan:move:%s:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer), esc_tag(sub, esc_buffer_alt));
786 155 : if (is_diff) {
787 0 : msg_info("move %s -> %s\n", fmt_term(disk, file->sub, esc_buffer), fmt_term(disk, sub, esc_buffer_alt));
788 : }
789 :
790 : /* remove from the name set */
791 155 : tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
792 :
793 : /* save the new name */
794 155 : file_rename(file, sub);
795 :
796 : /* reinsert in the name set */
797 155 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
798 :
799 : /* we have to save the new name */
800 155 : scan->need_write = 1;
801 : } else {
802 : /* otherwise it's equal */
803 932226 : ++scan->count_equal;
804 :
805 932226 : if (state->opt.gui) {
806 22611 : log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
807 : }
808 : }
809 :
810 : /* mark the file as kept */
811 932381 : scan_file_keep(scan, file);
812 :
813 : /* nothing more to do */
814 932381 : return;
815 : }
816 :
817 : /*
818 : * Here the file matches the inode, but not the other info
819 : *
820 : * It could be a modified file with the same name,
821 : * or a restored/copied file that get assigned a previously used inode,
822 : * or a file-system with not persistent inodes.
823 : *
824 : * In NTFS it could be also a hardlink, because in NTFS
825 : * hardlink don't share the same directory information,
826 : * like attribute and time.
827 : *
828 : * For example:
829 : * C:> echo A > A
830 : * C:> mklink /H B A
831 : * ...wait one minute
832 : * C:> echo AAAAAAAAAAAAAA > A
833 : * C:> dir
834 : * ...both time and size of A and B don't match!
835 : */
836 11611 : if (file_flag_has(file, FILE_IS_PRESENT)) {
837 : /* if has_volatile_hardlinks is true, the nlink value is not reliable */
838 0 : if (!disk->has_volatile_hardlinks && st->st_nlink == 1) {
839 : /* LCOV_EXCL_START */
840 : log_fatal("Internal inode '%" PRIu64 "' inconsistency for files '%s%s' and '%s%s' with same inode but different attributes: size %" PRIu64 "?%" PRIu64 ", sec %" PRIu64 "?%" PRIu64 ", nsec %d?%d\n",
841 : file->inode, disk->dir, sub, disk->dir, file->sub,
842 : file->size, (uint64_t)st->st_size,
843 : file->mtime_sec, (uint64_t)st->st_mtime,
844 : file->mtime_nsec, STAT_NSEC(st));
845 : os_abort();
846 : /* LCOV_EXCL_STOP */
847 : }
848 :
849 : /* LCOV_EXCL_START */
850 : /* suppose it's hardlink with not synced metadata */
851 : scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
852 : return;
853 : /* LCOV_EXCL_STOP */
854 : }
855 :
856 : /* assume a previously used inode, it's the worst case */
857 : /* and we handle it removing the duplicate stored inode. */
858 : /* If the file is found by name later, it will have the inode restored, */
859 : /* otherwise, it will get removed */
860 :
861 : /* remove from the inode set */
862 11611 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
863 :
864 : /* clear the inode */
865 : /* this is not really needed for correct functionality */
866 : /* because we are going to set FILE_IS_WITHOUT_INODE */
867 : /* but it's easier for debugging to have invalid inodes set to 0 */
868 11611 : file->inode = 0;
869 :
870 : /* mark as missing inode */
871 11611 : file_flag_set(file, FILE_IS_WITHOUT_INODE);
872 :
873 : /* go further to find it by name */
874 : }
875 :
876 : /* initialize for later overwrite */
877 172363 : is_file_reported = 0;
878 172363 : is_original_file_size_different_than_zero = 0;
879 :
880 : /* then try finding it by name */
881 172363 : file = tommy_hashdyn_search(&disk->pathset, file_path_compare_to_arg, sub, file_path_hash(sub));
882 :
883 : /* keep track if the file already exists */
884 172363 : is_file_already_present = file != 0;
885 :
886 172363 : if (is_file_already_present) {
887 : /* if the file is without an inode */
888 97849 : if (file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
889 : /* set it now */
890 11568 : file->inode = st->st_ino;
891 :
892 : /* insert in the set */
893 11568 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
894 :
895 : /* unmark as missing inode */
896 11568 : file_flag_clear(file, FILE_IS_WITHOUT_INODE);
897 : } else {
898 : /* here the inode has to be different, otherwise we would have found it before */
899 86281 : if (file->inode == st->st_ino) {
900 : /* LCOV_EXCL_START */
901 : log_fatal("Internal inconsistency in inode '%" PRIu64 "' for files '%s%s' as unexpected matching\n", file->inode, disk->dir, sub);
902 : os_abort();
903 : /* LCOV_EXCL_STOP */
904 : }
905 : }
906 :
907 : /* for sure it cannot be already present */
908 97849 : if (file_flag_has(file, FILE_IS_PRESENT)) {
909 : /* LCOV_EXCL_START */
910 : log_fatal("Internal inconsistency in path for file '%s%s' matching and already present\n", disk->dir, sub);
911 : os_abort();
912 : /* LCOV_EXCL_STOP */
913 : }
914 :
915 : /* check if the file is not changed */
916 97849 : if (file->size == st->st_size
917 97253 : && file->mtime_sec == st->st_mtime
918 94954 : && (file->mtime_nsec == STAT_NSEC(st)
919 : /* always accept the stored value if it's STAT_NSEC_INVALID */
920 : /* it happens when upgrading from an old version of SnapRAID */
921 : /* not yet supporting the nanosecond field */
922 2 : || file->mtime_nsec == STAT_NSEC_INVALID
923 : )
924 : ) {
925 : /* mark as present */
926 94952 : file_flag_set(file, FILE_IS_PRESENT);
927 :
928 : /* update the nano seconds mtime only if different */
929 : /* to avoid unneeded updates */
930 94952 : if (file->mtime_nsec == STAT_NSEC_INVALID
931 0 : && STAT_NSEC(st) != STAT_NSEC_INVALID
932 : ) {
933 0 : file->mtime_nsec = STAT_NSEC(st);
934 :
935 : /* we have to save the new mtime */
936 0 : scan->need_write = 1;
937 : }
938 :
939 : /* if when processing the disk we used the past inodes values */
940 94952 : if (has_past_inodes) {
941 : /* if persistent inodes are supported, we are sure that the inode number */
942 : /* is now different, because otherwise the file would have been found */
943 : /* when searching by inode. */
944 : /* if the inode is different, it means a rewritten file with the same path */
945 : /* like when restoring a backup that restores also the timestamp */
946 94938 : ++scan->count_restore;
947 :
948 94938 : log_tag("scan:restore:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
949 94938 : if (is_diff) {
950 0 : msg_info("restore %s\n", fmt_term(disk, sub, esc_buffer));
951 : }
952 :
953 : /* remove from the inode set */
954 94938 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
955 :
956 : /* save the new inode */
957 94938 : file->inode = st->st_ino;
958 :
959 : /* reinsert in the inode set */
960 94938 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
961 :
962 : /* we have to save the new inode */
963 94938 : scan->need_write = 1;
964 : } else {
965 : /* otherwise it's the case of not persistent inode, where doesn't */
966 : /* matter if the inode is different or equal, because they have no */
967 : /* meaning, and then we don't even save them */
968 14 : ++scan->count_equal;
969 :
970 14 : if (state->opt.gui) {
971 0 : log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
972 : }
973 : }
974 :
975 : /* mark the file as kept */
976 94952 : scan_file_keep(scan, file);
977 :
978 : /* nothing more to do */
979 94952 : return;
980 : }
981 :
982 : /* here if the file is changed but with the correct name */
983 :
984 : /* save the info for later printout */
985 2897 : file_already_present_size = file->size;
986 2897 : file_already_present_mtime_sec = file->mtime_sec;
987 2897 : file_already_present_mtime_nsec = file->mtime_nsec;
988 :
989 : /* keep track if the original file was not of zero size */
990 2897 : is_original_file_size_different_than_zero = file->size != 0;
991 :
992 : /* remove it, and continue to insert it again */
993 2897 : scan_file_remove(scan, file);
994 :
995 : /* and continue to insert it again */
996 : } else {
997 74514 : file_already_present_size = 0;
998 74514 : file_already_present_mtime_sec = 0;
999 74514 : file_already_present_mtime_nsec = 0;
1000 : }
1001 :
1002 : /* refresh the info, to ensure that they are synced, */
1003 : /* note that we refresh only the info of the new or modified files */
1004 : /* because this is slow operation */
1005 77411 : scan_file_refresh(scan, sub, st, &physical);
1006 :
1007 : #ifndef _WIN32
1008 : /* do a safety check to ensure that the common ext4 case of zeroing */
1009 : /* the size of a file after a crash doesn't propagate to the backup */
1010 : /* this check is specific for Linux, so we disable it on Windows */
1011 77411 : if (is_original_file_size_different_than_zero && st->st_size == 0) {
1012 0 : if (!state->opt.force_zero) {
1013 : /* LCOV_EXCL_START */
1014 : log_fatal("The file '%s%s' has unexpected zero size!\n", disk->dir, sub);
1015 : log_fatal("It's possible that after a kernel crash this file was lost,\n");
1016 : log_fatal("and you can use 'snapraid fix -f /%s' to recover it.\n", fmt_poll(disk, sub, esc_buffer));
1017 : if (!is_diff) {
1018 : log_fatal("If this an expected condition you can '%s' anyway using 'snapraid --force-zero %s'\n", state->command, state->command);
1019 : exit(EXIT_FAILURE);
1020 : }
1021 : /* LCOV_EXCL_STOP */
1022 : }
1023 : }
1024 : #else
1025 : /* avoid the unused warning in Windows */
1026 : (void)is_original_file_size_different_than_zero;
1027 : #endif
1028 :
1029 : /* insert it */
1030 77411 : file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime, STAT_NSEC(st), st->st_ino, physical);
1031 :
1032 : /* mark it as present */
1033 77411 : file_flag_set(file, FILE_IS_PRESENT);
1034 :
1035 : /* if copy detection is enabled */
1036 : /* note that the copy detection is tried also for updated files */
1037 : /* this makes sense because it may happen to have two different copies */
1038 : /* of the same file, and we move the right one over the wrong one */
1039 : /* in such case we have a "copy" over an "update" */
1040 77411 : if (!state->opt.force_nocopy) {
1041 77411 : tommy_uint32_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec);
1042 :
1043 : /* search for a file with the same name and stamp in all the disks */
1044 410923 : for (i = state->disklist; i != 0; i = i->next) {
1045 357149 : struct snapraid_disk* other_disk = i->data;
1046 : struct snapraid_file* other_file;
1047 :
1048 357149 : stamp_lock(other_disk);
1049 : /* if the nanosecond part of the time stamp is valid, search */
1050 : /* for name and stamp, otherwise for path and stamp */
1051 357149 : if (file->mtime_nsec != 0 && file->mtime_nsec != STAT_NSEC_INVALID)
1052 357041 : other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash);
1053 : else
1054 108 : other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
1055 357149 : stamp_unlock(other_disk);
1056 :
1057 : /* if found, and it's a fully hashed file */
1058 357149 : if (other_file && file_is_full_hashed_and_stable(scan->state, other_disk, other_file)) {
1059 : /* assume that the file is a copy, and reuse the hash */
1060 23637 : file_copy(other_file, file);
1061 :
1062 : /* revert old counter and use the copy one */
1063 23637 : ++scan->count_copy;
1064 :
1065 23637 : log_tag("scan:copy:%s:%s:%s:%s\n", other_disk->name, esc_tag(other_file->sub, esc_buffer), disk->name, esc_tag(file->sub, esc_buffer_alt));
1066 23637 : if (is_diff) {
1067 31 : msg_info("copy %s -> %s\n", fmt_term(other_disk, other_file->sub, esc_buffer), fmt_term(disk, file->sub, esc_buffer_alt));
1068 : }
1069 :
1070 : /* mark it as reported */
1071 23637 : is_file_reported = 1;
1072 :
1073 : /* no need to continue the search */
1074 23637 : break;
1075 : }
1076 : }
1077 : }
1078 :
1079 : /* if not yet reported, do it now */
1080 : /* we postpone this to avoid to print two times the copied files */
1081 77411 : if (!is_file_reported) {
1082 53774 : if (is_file_already_present) {
1083 2897 : ++scan->count_change;
1084 :
1085 2897 : log_tag("scan:update:%s:%s: %" PRIu64 " %" PRIu64 ".%d -> %" PRIu64 " %" PRIu64 ".%d\n", disk->name, esc_tag(sub, esc_buffer),
1086 : file_already_present_size, file_already_present_mtime_sec, file_already_present_mtime_nsec,
1087 : file->size, file->mtime_sec, file->mtime_nsec
1088 : );
1089 :
1090 2897 : if (is_diff) {
1091 118 : msg_info("update %s\n", fmt_term(disk, sub, esc_buffer));
1092 : }
1093 : } else {
1094 50877 : ++scan->count_insert;
1095 :
1096 50877 : log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
1097 50877 : if (is_diff) {
1098 17054 : msg_info("add %s\n", fmt_term(disk, sub, esc_buffer));
1099 : }
1100 : }
1101 : }
1102 :
1103 : /* insert the file in the delayed list */
1104 77411 : scan_file_insert(scan, file);
1105 : }
1106 :
1107 : /**
1108 : * Remove the specified dir from the data set.
1109 : */
1110 2 : static void scan_emptydir_remove(struct snapraid_scan* scan, struct snapraid_dir* dir)
1111 : {
1112 2 : struct snapraid_disk* disk = scan->disk;
1113 :
1114 : /* state changed */
1115 2 : scan->need_write = 1;
1116 :
1117 : /* remove the file from the dir containers */
1118 2 : tommy_hashdyn_remove_existing(&disk->dirset, &dir->nodeset);
1119 2 : tommy_list_remove_existing(&disk->dirlist, &dir->nodelist);
1120 :
1121 : /* deallocate */
1122 2 : dir_free(dir);
1123 2 : }
1124 :
1125 : /**
1126 : * Insert the specified dir in the data set.
1127 : */
1128 10 : static void scan_emptydir_insert(struct snapraid_scan* scan, struct snapraid_dir* dir)
1129 : {
1130 10 : struct snapraid_disk* disk = scan->disk;
1131 :
1132 : /* state changed */
1133 10 : scan->need_write = 1;
1134 :
1135 : /* insert the dir in the dir containers */
1136 10 : tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
1137 10 : tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
1138 10 : }
1139 :
1140 : /**
1141 : * Process a dir.
1142 : */
1143 259 : static void scan_emptydir(struct snapraid_scan* scan, const char* sub)
1144 : {
1145 259 : struct snapraid_disk* disk = scan->disk;
1146 : struct snapraid_dir* dir;
1147 :
1148 : /* check if the dir already exists */
1149 259 : dir = tommy_hashdyn_search(&disk->dirset, dir_name_compare, sub, dir_name_hash(sub));
1150 259 : if (dir) {
1151 : /* check if multiple files have the same name */
1152 249 : if (dir_flag_has(dir, FILE_IS_PRESENT)) {
1153 : /* LCOV_EXCL_START */
1154 : log_fatal("Internal inconsistency for dir '%s%s'\n", disk->dir, sub);
1155 : os_abort();
1156 : /* LCOV_EXCL_STOP */
1157 : }
1158 :
1159 : /* mark as present */
1160 249 : dir_flag_set(dir, FILE_IS_PRESENT);
1161 :
1162 : /* nothing more to do */
1163 249 : return;
1164 : } else {
1165 : /* and continue to insert it */
1166 : }
1167 :
1168 : /* insert it */
1169 10 : dir = dir_alloc(sub);
1170 :
1171 : /* mark it as present */
1172 10 : dir_flag_set(dir, FILE_IS_PRESENT);
1173 :
1174 : /* insert it in the delayed insert list */
1175 10 : tommy_list_insert_tail(&scan->dir_insert_list, &dir->nodelist, dir);
1176 : }
1177 :
1178 : struct dirent_sorted {
1179 : /* node for data structures */
1180 : tommy_node node;
1181 :
1182 : #if HAVE_STRUCT_DIRENT_D_INO
1183 : uint64_t d_ino; /**< Inode number. */
1184 : #endif
1185 : #if HAVE_STRUCT_DIRENT_D_TYPE
1186 : uint32_t d_type; /**< File type. */
1187 : #endif
1188 : #if HAVE_STRUCT_DIRENT_D_STAT
1189 : struct stat d_stat; /**< Stat result. */
1190 : #endif
1191 : char d_name[]; /**< Variable length name. It must be the last field. */
1192 : };
1193 :
1194 : #if HAVE_STRUCT_DIRENT_D_INO
1195 103638 : static int dd_ino_compare(const void* void_a, const void* void_b)
1196 : {
1197 103638 : const struct dirent_sorted* a = void_a;
1198 103638 : const struct dirent_sorted* b = void_b;
1199 :
1200 103638 : if (a->d_ino < b->d_ino)
1201 49706 : return -1;
1202 53932 : if (a->d_ino > b->d_ino)
1203 53932 : return 1;
1204 :
1205 0 : return 0;
1206 : }
1207 : #endif
1208 :
1209 11528720 : static int dd_name_compare(const void* void_a, const void* void_b)
1210 : {
1211 11528720 : const struct dirent_sorted* a = void_a;
1212 11528720 : const struct dirent_sorted* b = void_b;
1213 :
1214 11528720 : return strcmp(a->d_name, b->d_name);
1215 : }
1216 :
1217 : /**
1218 : * Return the stat info of a dir entry.
1219 : */
1220 : #if HAVE_STRUCT_DIRENT_D_STAT
1221 : #define DSTAT(file, dd, buf) dstat(dd)
1222 : struct stat* dstat(struct dirent_sorted* dd)
1223 : {
1224 : return &dd->d_stat;
1225 : }
1226 : #else
1227 : #define DSTAT(file, dd, buf) dstat(file, buf)
1228 1106210 : struct stat* dstat(const char* file, struct stat* st)
1229 : {
1230 1106210 : if (lstat(file, st) != 0) {
1231 : /* LCOV_EXCL_START */
1232 : log_fatal("Error in stat file/directory '%s'. %s.\n", file, strerror(errno));
1233 : exit(EXIT_FAILURE);
1234 : /* LCOV_EXCL_STOP */
1235 : }
1236 1106210 : return st;
1237 : }
1238 : #endif
1239 :
1240 : /**
1241 : * Process a directory.
1242 : * Return != 0 if at least one file or link is processed.
1243 : */
1244 1846 : static int scan_sub(struct snapraid_scan* scan, int level, int is_diff, char* path_next, char* sub_next, char* tmp)
1245 : {
1246 1846 : struct snapraid_state* state = scan->state;
1247 1846 : struct snapraid_disk* disk = scan->disk;
1248 1846 : int processed = 0;
1249 : DIR* d;
1250 : tommy_list list;
1251 : tommy_node* node;
1252 : size_t path_len;
1253 : size_t sub_len;
1254 :
1255 1846 : path_len = strlen(path_next);
1256 1846 : sub_len = strlen(sub_next);
1257 :
1258 1846 : tommy_list_init(&list);
1259 :
1260 1846 : d = opendir(path_next);
1261 1846 : if (!d) {
1262 : /* LCOV_EXCL_START */
1263 : log_fatal("Error opening directory '%s'. %s.\n", path_next, strerror(errno));
1264 : if (level == 0)
1265 : log_fatal("If this is the disk mount point, remember to create it manually\n");
1266 : else
1267 : log_fatal("If it's a permission problem, you can exclude it in the config file with:\n\texclude /%s\n", sub_next);
1268 : exit(EXIT_FAILURE);
1269 : /* LCOV_EXCL_STOP */
1270 : }
1271 :
1272 : /* read the full directory */
1273 1145891 : while (1) {
1274 : struct dirent_sorted* entry;
1275 : const char* name;
1276 : struct dirent* dd;
1277 : size_t name_len;
1278 :
1279 : /*
1280 : * Clear errno to differentiate the end of the stream and an error condition
1281 : *
1282 : * From the Linux readdir() manpage:
1283 : * "If the end of the directory stream is reached, NULL is returned and errno is not changed.
1284 : * If an error occurs, NULL is returned and errno is set appropriately."
1285 : */
1286 1147737 : errno = 0;
1287 1147737 : dd = readdir(d);
1288 1147737 : if (dd == 0 && errno != 0) {
1289 : /* LCOV_EXCL_START */
1290 : /* restore removing additions */
1291 : path_next[path_len] = 0;
1292 : sub_next[sub_len] = 0;
1293 : log_fatal("Error reading directory '%s'. %s.\n", path_next, strerror(errno));
1294 : log_fatal("You can exclude it in the config file with:\n\texclude /%s\n", sub_next);
1295 : exit(EXIT_FAILURE);
1296 : /* LCOV_EXCL_STOP */
1297 : }
1298 1147737 : if (dd == 0) {
1299 1846 : break; /* finished */
1300 : }
1301 :
1302 : /* skip "." and ".." files */
1303 1145891 : name = dd->d_name;
1304 1145891 : if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
1305 3692 : continue;
1306 :
1307 1142199 : pathcatl(path_next, path_len, PATH_MAX, name);
1308 :
1309 : /* check for not supported file names */
1310 1142199 : if (name[0] == 0) {
1311 : /* LCOV_EXCL_START */
1312 : log_fatal("Unsupported name '%s' in file '%s'.\n", name, path_next);
1313 : exit(EXIT_FAILURE);
1314 : /* LCOV_EXCL_STOP */
1315 : }
1316 :
1317 : /* exclude hidden files even before calling lstat() */
1318 1142199 : if (filter_hidden(state->filter_hidden, dd) != 0) {
1319 0 : msg_verbose("Excluding hidden '%s'\n", path_next);
1320 0 : continue;
1321 : }
1322 :
1323 : /* exclude content files even before calling lstat() */
1324 1142199 : if (filter_content(&state->contentlist, path_next) != 0) {
1325 0 : msg_verbose("Excluding content '%s'\n", path_next);
1326 0 : continue;
1327 : }
1328 :
1329 1142199 : name_len = strlen(dd->d_name);
1330 1142199 : entry = malloc_nofail(sizeof(struct dirent_sorted) + name_len + 1);
1331 :
1332 : /* copy the dir entry */
1333 : #if HAVE_STRUCT_DIRENT_D_INO
1334 1142199 : entry->d_ino = dd->d_ino;
1335 : #endif
1336 : #if HAVE_STRUCT_DIRENT_D_TYPE
1337 1142199 : entry->d_type = dd->d_type;
1338 : #endif
1339 : #if HAVE_STRUCT_DIRENT_D_STAT
1340 : /* convert dirent to lstat result */
1341 : dirent_lstat(dd, &entry->d_stat);
1342 :
1343 : /* note that at this point the st_mode may be 0 */
1344 : #endif
1345 1142199 : memcpy(entry->d_name, dd->d_name, name_len + 1);
1346 :
1347 : /* insert in the list */
1348 1142199 : tommy_list_insert_tail(&list, &entry->node, entry);
1349 : }
1350 :
1351 1846 : if (closedir(d) != 0) {
1352 : /* LCOV_EXCL_START */
1353 : /* restore removing additions */
1354 : path_next[path_len] = 0;
1355 : log_fatal("Error closing directory '%s'. %s.\n", path_next, strerror(errno));
1356 : exit(EXIT_FAILURE);
1357 : /* LCOV_EXCL_STOP */
1358 : }
1359 :
1360 1846 : if (state->opt.force_order == SORT_ALPHA) {
1361 : /* if requested sort alphabetically */
1362 : /* this is mainly done for testing to ensure to always */
1363 : /* process in the same way in different platforms */
1364 1810 : tommy_list_sort(&list, dd_name_compare);
1365 : }
1366 : #if HAVE_STRUCT_DIRENT_D_INO
1367 36 : else if (!disk->has_volatile_inodes) {
1368 : /* if inodes are persistent */
1369 : /* sort the list of dir entries by inodes */
1370 36 : tommy_list_sort(&list, dd_ino_compare);
1371 : }
1372 : /* otherwise just keep the insertion order */
1373 : #endif
1374 :
1375 : /* process the sorted dir entries */
1376 1846 : node = list;
1377 1144045 : while (node != 0) {
1378 1142199 : struct snapraid_filter* reason = 0;
1379 1142199 : struct dirent_sorted* dd = node->data;
1380 1142199 : const char* name = dd->d_name;
1381 : struct stat* st;
1382 : int type;
1383 : #if !HAVE_STRUCT_DIRENT_D_STAT
1384 : struct stat st_buf;
1385 : #endif
1386 :
1387 1142199 : pathcatl(path_next, path_len, PATH_MAX, name);
1388 1142199 : pathcatl(sub_next, sub_len, PATH_MAX, name);
1389 :
1390 : /* start with an unknown type */
1391 1142199 : type = -1;
1392 1142199 : st = 0;
1393 :
1394 : /* if dirent has the type, use it */
1395 : #if HAVE_STRUCT_DIRENT_D_TYPE
1396 1142199 : switch (dd->d_type) {
1397 0 : case DT_UNKNOWN : break;
1398 1104939 : case DT_REG : type = 0; break;
1399 35989 : case DT_LNK : type = 1; break;
1400 1271 : case DT_DIR : type = 2; break;
1401 0 : default : type = 3; break;
1402 : }
1403 : #endif
1404 :
1405 : /* if type is still unknown */
1406 1142199 : if (type < 0) {
1407 : /* get the type from stat */
1408 0 : st = DSTAT(path_next, dd, &st_buf);
1409 :
1410 : #if HAVE_STRUCT_DIRENT_D_STAT
1411 : /* if the st_mode field is missing, takes care to fill it using normal lstat() */
1412 : /* at now this can happen only in Windows (with HAVE_STRUCT_DIRENT_D_STAT defined), */
1413 : /* because we use a directory reading method that doesn't read info about ReparsePoint. */
1414 : /* Note that here we cannot call here lstat_sync(), because we don't know what kind */
1415 : /* of file is it, and lstat_sync() doesn't always work */
1416 : if (st->st_mode == 0) {
1417 : if (lstat(path_next, st) != 0) {
1418 : /* LCOV_EXCL_START */
1419 : log_fatal("Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno));
1420 : exit(EXIT_FAILURE);
1421 : /* LCOV_EXCL_STOP */
1422 : }
1423 : }
1424 : #endif
1425 :
1426 0 : if (S_ISREG(st->st_mode))
1427 0 : type = 0;
1428 0 : else if (S_ISLNK(st->st_mode))
1429 0 : type = 1;
1430 0 : else if (S_ISDIR(st->st_mode))
1431 0 : type = 2;
1432 : else
1433 0 : type = 3;
1434 : }
1435 :
1436 1142199 : if (type == 0) { /* REG */
1437 1104939 : if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
1438 :
1439 : /* late stat, if not yet called */
1440 1104939 : if (!st)
1441 1104939 : st = DSTAT(path_next, dd, &st_buf);
1442 :
1443 : #if HAVE_LSTAT_SYNC
1444 : /* if the st_ino field is missing, takes care to fill it using the extended lstat() */
1445 : /* this can happen only in Windows */
1446 : if (st->st_ino == 0 || st->st_nlink == 0) {
1447 : if (lstat_sync(path_next, st, 0) != 0) {
1448 : /* LCOV_EXCL_START */
1449 : log_fatal("Error in stat file '%s'. %s.\n", path_next, strerror(errno));
1450 : exit(EXIT_FAILURE);
1451 : /* LCOV_EXCL_STOP */
1452 : }
1453 : }
1454 : #endif
1455 :
1456 1104939 : scan_file(scan, is_diff, sub_next, st, FILEPHY_UNREAD_OFFSET);
1457 1104939 : processed = 1;
1458 : } else {
1459 0 : msg_verbose("Excluding file '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
1460 : }
1461 37260 : } else if (type == 1) { /* LNK */
1462 35989 : if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
1463 : int ret;
1464 :
1465 35989 : ret = readlink(path_next, tmp, PATH_MAX);
1466 35989 : if (ret >= PATH_MAX) {
1467 : /* LCOV_EXCL_START */
1468 : log_fatal("Error in readlink file '%s'. Symlink too long.\n", path_next);
1469 : exit(EXIT_FAILURE);
1470 : /* LCOV_EXCL_STOP */
1471 : }
1472 35989 : if (ret < 0) {
1473 : /* LCOV_EXCL_START */
1474 : log_fatal("Error in readlink file '%s'. %s.\n", path_next, strerror(errno));
1475 : exit(EXIT_FAILURE);
1476 : /* LCOV_EXCL_STOP */
1477 : }
1478 35989 : if (ret == 0)
1479 0 : log_fatal("WARNING! Empty symbolic link '%s'.\n", path_next);
1480 :
1481 : /* readlink doesn't put the final 0 */
1482 35989 : tmp[ret] = 0;
1483 :
1484 : /* process as a symbolic link */
1485 35989 : scan_link(scan, is_diff, sub_next, tmp, FILE_IS_SYMLINK);
1486 35989 : processed = 1;
1487 : } else {
1488 0 : msg_verbose("Excluding link '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
1489 : }
1490 1271 : } else if (type == 2) { /* DIR */
1491 1271 : if (filter_subdir(&state->filterlist, &reason, disk->name, sub_next) == 0) {
1492 : #ifndef _WIN32
1493 : /* late stat, if not yet called */
1494 1271 : if (!st)
1495 1271 : st = DSTAT(path_next, dd, &st_buf);
1496 :
1497 : /* in Unix don't follow mount points in different devices */
1498 : /* in Windows we are already skipping them reporting them as special files */
1499 1271 : if ((uint64_t)st->st_dev != disk->device) {
1500 0 : log_fatal("WARNING! Ignoring mount point '%s' because it appears to be in a different device\n", path_next);
1501 : } else
1502 : #endif
1503 : {
1504 : /* recurse */
1505 1271 : pathslash(path_next, PATH_MAX);
1506 1271 : pathslash(sub_next, PATH_MAX);
1507 1271 : if (scan_sub(scan, level + 1, is_diff, path_next, sub_next, tmp) == 0) {
1508 : /* restore removing additions */
1509 259 : pathcatl(sub_next, sub_len, PATH_MAX, name);
1510 : /* scan the directory as empty dir */
1511 259 : scan_emptydir(scan, sub_next);
1512 : }
1513 : /* or we processed something internally, or we have added the empty dir */
1514 1271 : processed = 1;
1515 : }
1516 : } else {
1517 0 : msg_verbose("Excluding directory '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
1518 : }
1519 : } else {
1520 0 : if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
1521 : /* late stat, if not yet called */
1522 0 : if (!st)
1523 0 : st = DSTAT(path_next, dd, &st_buf);
1524 :
1525 0 : log_fatal("WARNING! Ignoring special '%s' file '%s'\n", stat_desc(st), path_next);
1526 : } else {
1527 0 : msg_verbose("Excluding special file '%s' for rule '%s'\n", path_next, filter_type(reason, tmp, PATH_MAX));
1528 : }
1529 : }
1530 :
1531 : /* next entry */
1532 1142199 : node = node->next;
1533 :
1534 : /* free the present one */
1535 1142199 : free(dd);
1536 : }
1537 :
1538 1846 : return processed;
1539 : }
1540 :
1541 : /**
1542 : * Process a directory.
1543 : * Return != 0 if at least one file or link is processed.
1544 : */
1545 575 : static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const char* dir, const char* sub)
1546 : {
1547 : /* working buffers used by scan_sub() */
1548 : char path_next[PATH_MAX];
1549 : char sub_next[PATH_MAX];
1550 : char tmp[PATH_MAX];
1551 :
1552 575 : pathcpy(path_next, sizeof(path_next), dir);
1553 575 : pathcpy(sub_next, sizeof(sub_next), sub);
1554 :
1555 575 : return scan_sub(scan, level, is_diff, path_next, sub_next, tmp);
1556 : }
1557 :
1558 575 : static void* scan_disk(void* arg)
1559 : {
1560 575 : struct snapraid_scan* scan = arg;
1561 575 : struct snapraid_disk* disk = scan->disk;
1562 : int ret;
1563 : int has_persistent_inodes;
1564 : int has_syncronized_hardlinks;
1565 : uint64_t start;
1566 :
1567 : /* check if the disk supports persistent inodes */
1568 575 : ret = fsinfo(disk->dir, &has_persistent_inodes, &has_syncronized_hardlinks, 0, 0);
1569 575 : if (ret < 0) {
1570 : /* LCOV_EXCL_START */
1571 : log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
1572 : exit(EXIT_FAILURE);
1573 : /* LCOV_EXCL_STOP */
1574 : }
1575 575 : if (!has_persistent_inodes) {
1576 0 : disk->has_volatile_inodes = 1;
1577 : }
1578 575 : if (!has_syncronized_hardlinks) {
1579 0 : disk->has_volatile_hardlinks = 1;
1580 : }
1581 :
1582 : /* if inodes or UUID are not persistent/changed/unsupported */
1583 575 : if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) {
1584 : /* remove all the inodes from the inode collection */
1585 : /* if they are not persistent, all of them could be changed now */
1586 : /* and we don't want to find false matching ones */
1587 : /* see scan_file() for more details */
1588 94 : tommy_node* node = disk->filelist;
1589 108 : while (node) {
1590 14 : struct snapraid_file* file = node->data;
1591 :
1592 14 : node = node->next;
1593 :
1594 : /* remove from the inode set */
1595 14 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
1596 :
1597 : /* clear the inode */
1598 14 : file->inode = 0;
1599 :
1600 : /* mark as missing inode */
1601 14 : file_flag_set(file, FILE_IS_WITHOUT_INODE);
1602 : }
1603 : }
1604 :
1605 575 : start = tick_ms();
1606 :
1607 575 : scan_dir(scan, 0, scan->is_diff, disk->dir, "");
1608 :
1609 575 : if (!scan->is_diff)
1610 527 : msg_progress("Scanned %s in %" PRIu64 " seconds\n", disk->name, (tick_ms() - start) / 1000);
1611 :
1612 575 : return 0;
1613 : }
1614 :
1615 96 : static int state_diffscan(struct snapraid_state* state, int is_diff)
1616 : {
1617 : tommy_node* i;
1618 : tommy_node* j;
1619 : tommy_list scanlist;
1620 : int done;
1621 : fptr* msg;
1622 : struct snapraid_scan total;
1623 : int no_difference;
1624 : char esc_buffer[ESC_MAX];
1625 :
1626 96 : tommy_list_init(&scanlist);
1627 :
1628 96 : if (is_diff)
1629 8 : msg_progress("Comparing...\n");
1630 : else
1631 88 : msg_progress("Scanning...\n");
1632 :
1633 : /* allocate all the scan data */
1634 671 : for (i = state->disklist; i != 0; i = i->next) {
1635 575 : struct snapraid_disk* disk = i->data;
1636 : struct snapraid_scan* scan;
1637 :
1638 575 : scan = scan_alloc(state, disk, is_diff);
1639 :
1640 575 : tommy_list_insert_tail(&scanlist, &scan->node, scan);
1641 : }
1642 :
1643 : /* first scan all the directory and find new and deleted files */
1644 671 : for (i = scanlist; i != 0; i = i->next) {
1645 575 : struct snapraid_scan* scan = i->data;
1646 : #if HAVE_THREAD
1647 575 : if (state->opt.skip_multi_scan)
1648 0 : scan_disk(scan);
1649 : else
1650 575 : thread_create(&scan->thread, scan_disk, scan);
1651 : #else
1652 : scan_disk(scan);
1653 : #endif
1654 : }
1655 :
1656 : #if HAVE_THREAD
1657 : /* wait for all threads to terminate */
1658 671 : for (i = scanlist; i != 0; i = i->next) {
1659 575 : struct snapraid_scan* scan = i->data;
1660 : void* retval;
1661 :
1662 : /* wait for thread termination */
1663 575 : if (!state->opt.skip_multi_scan)
1664 575 : thread_join(scan->thread, &retval);
1665 : }
1666 : #endif
1667 :
1668 : /* we split the search in two phases because to detect files */
1669 : /* moved from one disk to another we have to start deletion */
1670 : /* only when all disks have all the new files found */
1671 :
1672 : /* now process all the new and deleted files */
1673 671 : for (i = scanlist; i != 0; i = i->next) {
1674 575 : struct snapraid_scan* scan = i->data;
1675 575 : struct snapraid_disk* disk = scan->disk;
1676 : tommy_node* node;
1677 : unsigned phy_dup;
1678 : uint64_t phy_last;
1679 : struct snapraid_file* phy_file_last;
1680 :
1681 : /* check for removed files */
1682 575 : node = disk->filelist;
1683 1034870 : while (node) {
1684 1034295 : struct snapraid_file* file = node->data;
1685 :
1686 : /* next node */
1687 1034295 : node = node->next;
1688 :
1689 : /* remove if not present */
1690 1034295 : if (!file_flag_has(file, FILE_IS_PRESENT)) {
1691 29648 : ++scan->count_remove;
1692 :
1693 29648 : log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
1694 29648 : if (is_diff) {
1695 97 : msg_info("remove %s\n", fmt_term(disk, file->sub, esc_buffer));
1696 : }
1697 :
1698 29648 : scan_file_remove(scan, file);
1699 : }
1700 : }
1701 :
1702 : /* check for removed links */
1703 575 : node = disk->linklist;
1704 35258 : while (node) {
1705 34683 : struct snapraid_link* slink = node->data;
1706 :
1707 : /* next node */
1708 34683 : node = node->next;
1709 :
1710 : /* remove if not present */
1711 34683 : if (!link_flag_has(slink, FILE_IS_PRESENT)) {
1712 956 : ++scan->count_remove;
1713 :
1714 956 : log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
1715 956 : if (is_diff) {
1716 10 : msg_info("remove %s\n", fmt_term(disk, slink->sub, esc_buffer));
1717 : }
1718 :
1719 956 : scan_link_remove(scan, slink);
1720 : }
1721 : }
1722 :
1723 : /* check for removed dirs */
1724 575 : node = disk->dirlist;
1725 826 : while (node) {
1726 251 : struct snapraid_dir* dir = node->data;
1727 :
1728 : /* next node */
1729 251 : node = node->next;
1730 :
1731 : /* remove if not present */
1732 251 : if (!dir_flag_has(dir, FILE_IS_PRESENT)) {
1733 2 : scan_emptydir_remove(scan, dir);
1734 : }
1735 : }
1736 :
1737 : /* sort the files before inserting them */
1738 : /* we use a stable sort to ensure that if the reported physical offset/inode */
1739 : /* are always 0, we keep at least the directory order */
1740 575 : switch (state->opt.force_order) {
1741 12 : case SORT_PHYSICAL :
1742 12 : tommy_list_sort(&scan->file_insert_list, file_physical_compare);
1743 12 : break;
1744 0 : case SORT_INODE :
1745 0 : tommy_list_sort(&scan->file_insert_list, file_inode_compare);
1746 0 : break;
1747 563 : case SORT_ALPHA :
1748 563 : tommy_list_sort(&scan->file_insert_list, file_path_compare);
1749 563 : break;
1750 0 : case SORT_DIR :
1751 : /* already in order */
1752 0 : break;
1753 : }
1754 :
1755 : /* insert all the new files, we insert them only after the deletion */
1756 : /* to reuse the just freed space */
1757 : /* also check if the physical offset reported are fakes or not */
1758 575 : node = scan->file_insert_list;
1759 575 : phy_dup = 0;
1760 575 : phy_last = FILEPHY_UNREAD_OFFSET;
1761 575 : phy_file_last = 0;
1762 100672 : while (node) {
1763 100097 : struct snapraid_file* file = node->data;
1764 :
1765 : /* if the file is not empty, count duplicate physical offsets */
1766 100097 : if (state->opt.force_order == SORT_PHYSICAL && file->size != 0) {
1767 11418 : if (phy_file_last != 0 && file->physical == phy_last
1768 : /* files without offset are expected to have duplicates */
1769 0 : && phy_last != FILEPHY_WITHOUT_OFFSET
1770 : ) {
1771 : /* if verbose, print the list of duplicates real offsets */
1772 : /* other cases are for offsets not supported, so we don't need to report them file by file */
1773 0 : if (phy_last >= FILEPHY_REAL_OFFSET) {
1774 0 : log_fatal("WARNING! Files '%s%s' and '%s%s' share the same physical offset %" PRId64 ".\n", disk->dir, phy_file_last->sub, disk->dir, file->sub, phy_last);
1775 : }
1776 0 : ++phy_dup;
1777 : }
1778 11418 : phy_file_last = file;
1779 11418 : phy_last = file->physical;
1780 : }
1781 :
1782 : /* next node */
1783 100097 : node = node->next;
1784 :
1785 : /* insert in the parity */
1786 100097 : scan_file_allocate(scan, file);
1787 : }
1788 :
1789 : /* mark the disk without reliable physical offset if it has duplicates */
1790 : /* here it should never happen because we already sorted out hardlinks */
1791 575 : if (state->opt.force_order == SORT_PHYSICAL && phy_dup > 0) {
1792 0 : disk->has_unreliable_physical = 1;
1793 : }
1794 :
1795 : /* insert all the new links */
1796 575 : node = scan->link_insert_list;
1797 3032 : while (node) {
1798 2457 : struct snapraid_link* slink = node->data;
1799 :
1800 : /* next node */
1801 2457 : node = node->next;
1802 :
1803 : /* insert it */
1804 2457 : scan_link_insert(scan, slink);
1805 : }
1806 :
1807 : /* insert all the new dirs */
1808 575 : node = scan->dir_insert_list;
1809 585 : while (node) {
1810 10 : struct snapraid_dir* dir = node->data;
1811 :
1812 : /* next node */
1813 10 : node = node->next;
1814 :
1815 : /* insert it */
1816 10 : scan_emptydir_insert(scan, dir);
1817 : }
1818 : }
1819 :
1820 : /* propagate the state change (after all the scan operations are called) */
1821 671 : for (i = scanlist; i != 0; i = i->next) {
1822 575 : struct snapraid_scan* scan = i->data;
1823 575 : if (scan->need_write) {
1824 207 : state->need_write = 1;
1825 : }
1826 : }
1827 :
1828 : /* check for disks where all the previously existing files where removed */
1829 96 : if (!state->opt.force_empty) {
1830 90 : int all_missing = 0;
1831 90 : int all_rewritten = 0;
1832 90 : done = 0;
1833 629 : for (i = state->disklist, j = scanlist; i != 0; i = i->next, j = j->next) {
1834 539 : struct snapraid_disk* disk = i->data;
1835 539 : struct snapraid_scan* scan = j->data;
1836 :
1837 539 : if (scan->count_equal == 0
1838 74 : && scan->count_move == 0
1839 74 : && scan->count_restore == 0
1840 74 : && (scan->count_remove != 0 || scan->count_change != 0)
1841 : ) {
1842 1 : if (!done) {
1843 1 : done = 1;
1844 1 : log_fatal("WARNING! All the files previously present in disk '%s' at dir '%s'", disk->name, disk->dir);
1845 : } else {
1846 0 : log_fatal(", disk '%s' at dir '%s'", disk->name, disk->dir);
1847 : }
1848 :
1849 : /* detect the special condition of all files missing */
1850 1 : if (scan->count_change == 0)
1851 1 : all_missing = 1;
1852 :
1853 : /* detect the special condition of all files rewritten */
1854 1 : if (scan->count_remove == 0)
1855 0 : all_rewritten = 1;
1856 : }
1857 : }
1858 90 : if (done) {
1859 1 : log_fatal("\nare now missing or have been rewritten!\n");
1860 1 : if (all_rewritten) {
1861 0 : log_fatal("This could occur when restoring a disk from a backup\n");
1862 0 : log_fatal("program that is not setting correctly the timestamps.\n");
1863 : }
1864 1 : if (all_missing) {
1865 1 : log_fatal("This could occur when some disks are not mounted\n");
1866 1 : log_fatal("in the expected directory.\n");
1867 : }
1868 1 : if (!is_diff) {
1869 1 : log_fatal("If you want to '%s' anyway, use 'snapraid --force-empty %s'.\n", state->command, state->command);
1870 1 : exit(EXIT_FAILURE);
1871 : }
1872 : }
1873 : }
1874 :
1875 : /* check for disks without the physical offset support */
1876 95 : if (state->opt.force_order == SORT_PHYSICAL) {
1877 2 : done = 0;
1878 14 : for (i = state->disklist; i != 0; i = i->next) {
1879 12 : struct snapraid_disk* disk = i->data;
1880 :
1881 12 : if (disk->has_unreliable_physical) {
1882 0 : if (!done) {
1883 0 : done = 1;
1884 0 : log_fatal("WARNING! Physical offsets not supported for disk '%s'", disk->name);
1885 : } else {
1886 0 : log_fatal(", '%s'", disk->name);
1887 : }
1888 : }
1889 : }
1890 2 : if (done) {
1891 0 : log_fatal(". The order of files won't be optimal.\n");
1892 : }
1893 : }
1894 :
1895 : /* check for disks without persistent inodes */
1896 95 : done = 0;
1897 664 : for (i = state->disklist; i != 0; i = i->next) {
1898 569 : struct snapraid_disk* disk = i->data;
1899 :
1900 569 : if (disk->has_volatile_inodes) {
1901 0 : if (!done) {
1902 0 : done = 1;
1903 0 : log_fatal("WARNING! Inodes are not persistent for disks: '%s'", disk->name);
1904 : } else {
1905 0 : log_fatal(", '%s'", disk->name);
1906 : }
1907 : }
1908 : }
1909 95 : if (done) {
1910 0 : log_fatal(". Inodes are not used to detect move operations.\n");
1911 : }
1912 :
1913 : /* check for disks with changed UUID */
1914 95 : done = 0;
1915 664 : for (i = state->disklist; i != 0; i = i->next) {
1916 569 : struct snapraid_disk* disk = i->data;
1917 :
1918 : /* don't print the message if the UUID changed because before */
1919 : /* it was no set. */
1920 : /* this is the normal condition for an empty disk because it */
1921 : /* isn't stored */
1922 569 : if (disk->has_different_uuid && !disk->had_empty_uuid) {
1923 4 : if (!done) {
1924 2 : done = 1;
1925 2 : log_fatal("WARNING! UUID is changed for disks: '%s'", disk->name);
1926 : } else {
1927 2 : log_fatal(", '%s'", disk->name);
1928 : }
1929 : }
1930 : }
1931 95 : if (done) {
1932 2 : log_fatal(". Inodes are not used to detect move operations.\n");
1933 : }
1934 :
1935 : /* check for disks with unsupported UUID */
1936 95 : done = 0;
1937 664 : for (i = state->disklist; i != 0; i = i->next) {
1938 569 : struct snapraid_disk* disk = i->data;
1939 :
1940 569 : if (disk->has_unsupported_uuid) {
1941 0 : if (!done) {
1942 0 : done = 1;
1943 0 : log_fatal("WARNING! UUID is unsupported for disks: '%s'", disk->name);
1944 : } else {
1945 0 : log_fatal(", '%s'", disk->name);
1946 : }
1947 : }
1948 : }
1949 95 : if (done) {
1950 0 : log_fatal(". Not using inodes to detect move operations.\n");
1951 : #if defined(_linux) && !HAVE_BLKID
1952 : log_fatal("The 'blkid' library is not linked in SnapRAID!\n");
1953 : log_fatal("Try rebuilding it after installing the libblkid-dev or libblkid-devel package.\n");
1954 : #endif
1955 : }
1956 :
1957 95 : total.count_equal = 0;
1958 95 : total.count_move = 0;
1959 95 : total.count_copy = 0;
1960 95 : total.count_restore = 0;
1961 95 : total.count_change = 0;
1962 95 : total.count_remove = 0;
1963 95 : total.count_insert = 0;
1964 :
1965 664 : for (i = scanlist; i != 0; i = i->next) {
1966 569 : struct snapraid_scan* scan = i->data;
1967 569 : total.count_equal += scan->count_equal;
1968 569 : total.count_move += scan->count_move;
1969 569 : total.count_copy += scan->count_copy;
1970 569 : total.count_restore += scan->count_restore;
1971 569 : total.count_change += scan->count_change;
1972 569 : total.count_remove += scan->count_remove;
1973 569 : total.count_insert += scan->count_insert;
1974 : }
1975 :
1976 95 : if (is_diff) {
1977 8 : msg_status("\n");
1978 8 : msg = msg_status;
1979 : } else {
1980 87 : msg = msg_verbose;
1981 : }
1982 :
1983 95 : msg("%8u equal\n", total.count_equal);
1984 95 : msg("%8u added\n", total.count_insert);
1985 95 : msg("%8u removed\n", total.count_remove);
1986 95 : msg("%8u updated\n", total.count_change);
1987 95 : msg("%8u moved\n", total.count_move);
1988 95 : msg("%8u copied\n", total.count_copy);
1989 95 : msg("%8u restored\n", total.count_restore);
1990 :
1991 95 : log_tag("summary:equal:%u\n", total.count_equal);
1992 95 : log_tag("summary:added:%u\n", total.count_insert);
1993 95 : log_tag("summary:removed:%u\n", total.count_remove);
1994 95 : log_tag("summary:updated:%u\n", total.count_change);
1995 95 : log_tag("summary:moved:%u\n", total.count_move);
1996 95 : log_tag("summary:copied:%u\n", total.count_copy);
1997 95 : log_tag("summary:restored:%u\n", total.count_restore);
1998 :
1999 94 : no_difference = !total.count_move && !total.count_copy && !total.count_restore
2000 189 : && !total.count_change && !total.count_remove && !total.count_insert;
2001 :
2002 95 : if (is_diff) {
2003 8 : if (no_difference) {
2004 2 : msg_status("No differences\n");
2005 : } else {
2006 6 : msg_status("There are differences!\n");
2007 : }
2008 : }
2009 :
2010 95 : if (no_difference) {
2011 21 : log_tag("summary:exit:equal\n");
2012 : } else {
2013 74 : log_tag("summary:exit:diff\n");
2014 : }
2015 95 : log_flush();
2016 :
2017 95 : tommy_list_foreach(&scanlist, (tommy_foreach_func*)scan_free);
2018 :
2019 : /* check the file-system on all disks */
2020 95 : state_fscheck(state, "after scan");
2021 :
2022 95 : if (is_diff) {
2023 : /* check for file difference */
2024 8 : if (!no_difference)
2025 6 : return 1;
2026 :
2027 : /* check also for incomplete "sync" */
2028 2 : if (parity_is_invalid(state))
2029 1 : return 1;
2030 : }
2031 :
2032 88 : return 0;
2033 : }
2034 :
2035 8 : int state_diff(struct snapraid_state* state)
2036 : {
2037 8 : return state_diffscan(state, 1);
2038 : }
2039 :
2040 88 : void state_scan(struct snapraid_state* state)
2041 : {
2042 88 : (void)state_diffscan(state, 0); /* ignore return value */
2043 87 : }
2044 :
|
