Line data Source code
1 : /*
2 : * Copyright (C) 2011 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 3 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * You should have received a copy of the GNU General Public License
15 : * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 : */
17 :
18 : #include "portable.h"
19 :
20 : #include "support.h"
21 : #include "elem.h"
22 : #include "state.h"
23 : #include "parity.h"
24 :
25 : struct snapraid_scan {
26 : struct snapraid_state* state; /**< State used. */
27 : struct snapraid_disk* disk; /**< Disk used. */
28 :
29 : /**
30 : * Counters of changes.
31 : */
32 : unsigned count_equal; /**< Files equal. */
33 : unsigned count_move; /**< Files with a different name, but equal inode, size and timestamp in the same disk. */
34 : unsigned count_restore; /**< Files with equal name, size and timestamp, but different inode. */
35 : unsigned count_change; /**< Files with same name, but different size and/or timestamp. */
36 : unsigned count_copy; /**< Files new, with same name size and timestamp of a file in a different disk. */
37 : unsigned count_insert; /**< Files new. */
38 : unsigned count_remove; /**< Files removed. */
39 :
40 : tommy_list file_insert_list; /**< Files to insert. */
41 : tommy_list link_insert_list; /**< Links to insert. */
42 : tommy_list dir_insert_list; /**< Dirs to insert. */
43 :
44 : /* nodes for data structures */
45 : tommy_node node;
46 : };
47 :
48 : /**
49 : * Remove the specified link from the data set.
50 : */
51 956 : static void scan_link_remove(struct snapraid_scan* scan, struct snapraid_link* slink)
52 : {
53 956 : struct snapraid_state* state = scan->state;
54 956 : struct snapraid_disk* disk = scan->disk;
55 :
56 : /* state changed */
57 956 : state->need_write = 1;
58 :
59 : /* remove the file from the link containers */
60 956 : tommy_hashdyn_remove_existing(&disk->linkset, &slink->nodeset);
61 956 : tommy_list_remove_existing(&disk->linklist, &slink->nodelist);
62 :
63 : /* deallocate */
64 956 : link_free(slink);
65 956 : }
66 :
67 : /**
68 : * Insert the specified link in the data set.
69 : */
70 2457 : static void scan_link_insert(struct snapraid_scan* scan, struct snapraid_link* slink)
71 : {
72 2457 : struct snapraid_state* state = scan->state;
73 2457 : struct snapraid_disk* disk = scan->disk;
74 :
75 : /* state changed */
76 2457 : state->need_write = 1;
77 :
78 : /* insert the link in the link containers */
79 2457 : tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
80 2457 : tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
81 2457 : }
82 :
83 : /**
84 : * Process a symbolic link.
85 : */
86 36184 : static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub, const char* linkto, unsigned link_flag)
87 : {
88 36184 : struct snapraid_state* state = scan->state;
89 36184 : struct snapraid_disk* disk = scan->disk;
90 : struct snapraid_link* slink;
91 : char esc_buffer[ESC_MAX];
92 :
93 : /* check if the link already exists */
94 36184 : slink = tommy_hashdyn_search(&disk->linkset, link_name_compare_to_arg, sub, link_name_hash(sub));
95 36184 : if (slink) {
96 : /* check if multiple files have the same name */
97 33727 : if (link_flag_has(slink, FILE_IS_PRESENT)) {
98 : /* LCOV_EXCL_START */
99 : log_fatal("Internal inconsistency for link '%s%s'\n", disk->dir, sub);
100 : os_abort();
101 : /* LCOV_EXCL_STOP */
102 : }
103 :
104 : /* mark as present */
105 33727 : link_flag_set(slink, FILE_IS_PRESENT);
106 :
107 : /* check if the link is not changed and it's of the same kind */
108 33727 : if (strcmp(slink->linkto, linkto) == 0 && link_flag == link_flag_get(slink, FILE_IS_LINK_MASK)) {
109 : /* it's equal */
110 33721 : ++scan->count_equal;
111 :
112 67442 : if (state->opt.gui) {
113 829 : log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
114 : }
115 : } else {
116 : /* it's an update */
117 :
118 : /* we have to save the linkto/type */
119 6 : state->need_write = 1;
120 :
121 6 : ++scan->count_change;
122 :
123 6 : log_tag("scan:update:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
124 6 : if (is_diff) {
125 3 : printf("update %s\n", fmt_term(disk, slink->sub, esc_buffer));
126 : }
127 :
128 : /* update it */
129 6 : free(slink->linkto);
130 6 : slink->linkto = strdup_nofail(linkto);
131 6 : link_flag_let(slink, link_flag, FILE_IS_LINK_MASK);
132 : }
133 :
134 : /* nothing more to do */
135 33727 : return;
136 : } else {
137 : /* create the new link */
138 2457 : ++scan->count_insert;
139 :
140 2457 : log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
141 2457 : if (is_diff) {
142 599 : printf("add %s\n", fmt_term(disk, sub, esc_buffer));
143 : }
144 :
145 : /* and continue to insert it */
146 : }
147 :
148 : /* insert it */
149 2457 : slink = link_alloc(sub, linkto, link_flag);
150 :
151 : /* mark it as present */
152 2457 : link_flag_set(slink, FILE_IS_PRESENT);
153 :
154 : /* insert it in the delayed insert list */
155 2457 : tommy_list_insert_tail(&scan->link_insert_list, &slink->nodelist, slink);
156 : }
157 :
158 : /**
159 : * Insert the specified file in the parity.
160 : */
161 101861 : static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
162 : {
163 101861 : struct snapraid_state* state = scan->state;
164 101861 : struct snapraid_disk* disk = scan->disk;
165 : block_off_t i;
166 : block_off_t parity_pos;
167 :
168 : /* state changed */
169 101861 : state->need_write = 1;
170 :
171 : /* allocate the blocks of the file */
172 101861 : parity_pos = disk->first_free_block;
173 353042 : for (i = 0; i < file->blockmax; ++i) {
174 : struct snapraid_block* block;
175 : struct snapraid_block* over_block;
176 : snapraid_info info;
177 :
178 : /* increment the position until the first really free block */
179 1022727 : while (block_has_file(fs_par2block_find(disk, parity_pos)))
180 520365 : ++parity_pos;
181 :
182 : /* get block we are going to overwrite, if any */
183 251181 : over_block = fs_par2block_find(disk, parity_pos);
184 :
185 : /* deallocate it */
186 251181 : if (over_block != BLOCK_NULL)
187 95268 : fs_deallocate(disk, parity_pos);
188 :
189 : /* get block specific info */
190 251181 : info = info_get(&state->infoarr, parity_pos);
191 :
192 : /* get the new block we are going to write */
193 251181 : block = fs_file2block_get(file, i);
194 :
195 : /* if the file block already has an updated hash without rehash */
196 251181 : if (block_has_updated_hash(block) && !info_get_rehash(info)) {
197 : /* the only possible case is for REP blocks */
198 86215 : assert(block_state_get(block) == BLOCK_STATE_REP);
199 :
200 : /* convert to a REP block */
201 86215 : block_state_set(block, BLOCK_STATE_REP);
202 :
203 : /* and keep the hash as it's */
204 : } else {
205 : unsigned over_state;
206 :
207 : /* convert to a CHG block */
208 164966 : block_state_set(block, BLOCK_STATE_CHG);
209 :
210 : /* state of the block we are going to overwrite */
211 164966 : over_state = block_state_get(over_block);
212 :
213 : /* if the block is an empty one */
214 164966 : if (over_state == BLOCK_STATE_EMPTY) {
215 : /* the block was empty and filled with zeros */
216 : /* set the hash to the special ZERO value */
217 115543 : hash_zero_set(block->hash);
218 : } else {
219 : /* otherwise it's a DELETED one */
220 49423 : assert(over_state == BLOCK_STATE_DELETED);
221 :
222 : /* copy the past hash of the block */
223 49423 : memcpy(block->hash, over_block->hash, BLOCK_HASH_SIZE);
224 :
225 : /* if we have not already cleared the past hash */
226 49423 : if (!state->clear_past_hash) {
227 : /* in this case we don't know if the old state is still the one */
228 : /* stored inside the parity, because after an aborted sync, the parity */
229 : /* may be or may be not have been updated with the new data */
230 : /* Then we reset the hash to a bogus value */
231 : /* For example: */
232 : /* - One file is deleted */
233 : /* - Sync aborted after, updating the parity to the new state, */
234 : /* but without saving the content file representing this new state. */
235 : /* - Another file is added again (exactly here) */
236 : /* with the hash of DELETED block not representing the real parity state */
237 421 : hash_invalid_set(block->hash);
238 : }
239 : }
240 : }
241 :
242 : /* store in the disk map, after invalidating all the other blocks */
243 251181 : fs_allocate(disk, parity_pos, file, i);
244 :
245 : /* set the new free position */
246 251181 : disk->first_free_block = parity_pos + 1;
247 : }
248 :
249 : /* insert in the list of contained files */
250 101861 : tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
251 101861 : }
252 :
253 : /**
254 : * Delete the specified file from the parity.
255 : *
256 : * Note that the parity remains allocated, but the blocks and the file are marked as DELETED.
257 : * The file is then inserted in the deleted set, and it should not be deallocated,
258 : * as the parity still references it.
259 : */
260 56995 : static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file)
261 : {
262 56995 : struct snapraid_state* state = scan->state;
263 56995 : struct snapraid_disk* disk = scan->disk;
264 : block_off_t i;
265 :
266 : /* remove from the list of contained files */
267 56995 : tommy_list_remove_existing(&disk->filelist, &file->nodelist);
268 :
269 : /* state changed */
270 56995 : state->need_write = 1;
271 :
272 : /* here we are supposed to adjust the ::first_free_block position */
273 : /* with the parity position we are deleting */
274 : /* but we also know that we do only delayed insert, after all the deletion, */
275 : /* so at this point ::first_free_block is always at 0, and we don't need to update it */
276 56995 : if (disk->first_free_block != 0) {
277 : /* LCOV_EXCL_START */
278 : log_fatal("Internal inconsistency for first free position at '%u' deallocating file '%s'\n", disk->first_free_block, file->sub);
279 : os_abort();
280 : /* LCOV_EXCL_STOP */
281 : }
282 :
283 : /* free all the blocks of the file */
284 197561 : for (i = 0; i < file->blockmax; ++i) {
285 140566 : struct snapraid_block* block = fs_file2block_get(file, i);
286 : unsigned block_state;
287 :
288 : /* in case we scan after an aborted sync, */
289 : /* we could get also intermediate states */
290 140566 : block_state = block_state_get(block);
291 140566 : switch (block_state) {
292 : case BLOCK_STATE_BLK :
293 : /* we keep the hash making it an "old" hash, because the parity is still containing data for it */
294 35569 : break;
295 : case BLOCK_STATE_CHG :
296 : /* if we have not already cleared the past hash */
297 28040 : if (!state->clear_past_hash) {
298 : /* in these cases we don't know if the old state is still the one */
299 : /* stored inside the parity, because after an aborted sync, the parity */
300 : /* may be or may be not have been updated with the data that it's now */
301 : /* deleted. Then we reset the hash to a bogus value. */
302 : /* For example: */
303 : /* - One file is added */
304 : /* - Sync aborted after updating the parity to the new state, */
305 : /* but without saving the content file representing this new state. */
306 : /* - File is now deleted after the aborted sync */
307 : /* - Sync again, deleting the blocks (exactly here) */
308 : /* with the hash of CHG block not representing the real parity state */
309 40 : hash_invalid_set(block->hash);
310 : }
311 28040 : break;
312 : case BLOCK_STATE_REP :
313 : /* we just don't know the old hash, and then we set it to invalid */
314 76957 : hash_invalid_set(block->hash);
315 76957 : break;
316 : default :
317 : /* LCOV_EXCL_START */
318 : log_fatal("Internal inconsistency in file '%s' deallocating block '%u:%u' state %u\n", file->sub, i, file->blockmax, block_state);
319 : os_abort();
320 : /* LCOV_EXCL_STOP */
321 : }
322 :
323 : /* set the block as deleted */
324 140566 : block_state_set(block, BLOCK_STATE_DELETED);
325 : }
326 :
327 : /* mark the file as deleted */
328 56995 : file_flag_set(file, FILE_IS_DELETED);
329 :
330 : /* insert it in the list of deleted blocks */
331 56995 : tommy_list_insert_tail(&disk->deletedlist, &file->nodelist, file);
332 56995 : }
333 :
334 101861 : static void scan_file_delayed_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
335 : {
336 101861 : struct snapraid_state* state = scan->state;
337 101861 : struct snapraid_disk* disk = scan->disk;
338 :
339 : /* if we sort for physical offsets we have to read them for new files */
340 101861 : if (state->opt.force_order == SORT_PHYSICAL
341 11432 : && file->physical == FILEPHY_UNREAD_OFFSET
342 : ) {
343 : char path_next[PATH_MAX];
344 :
345 11432 : pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, file->sub);
346 :
347 11432 : if (filephy(path_next, file->size, &file->physical) != 0) {
348 : /* LCOV_EXCL_START */
349 : log_fatal("Error in getting the physical offset of file '%s'. %s.\n", path_next, strerror(errno));
350 : exit(EXIT_FAILURE);
351 : /* LCOV_EXCL_STOP */
352 : }
353 : }
354 :
355 : /* insert in the delayed list */
356 101861 : tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file);
357 101861 : }
358 :
359 : /**
360 : * Check if a file is completely formed of blocks with invalid parity,
361 : * and no rehash is tagged, and if it has at least one block.
362 : */
363 1025570 : static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
364 : {
365 : block_off_t i;
366 :
367 : /* with no block, it never has an invalid parity */
368 1025570 : if (file->blockmax == 0)
369 1437 : return 0;
370 :
371 : /* check all blocks */
372 1080101 : for (i = 0; i < file->blockmax; ++i) {
373 : snapraid_info info;
374 1057414 : struct snapraid_block* block = fs_file2block_get(file, i);
375 : block_off_t parity_pos;
376 :
377 : /* exclude blocks with parity */
378 1057414 : if (!block_has_invalid_parity(block))
379 1001446 : return 0;
380 :
381 : /*
382 : * Get the parity position.
383 : *
384 : * Note that here we expect to always have mapped
385 : * parity, because kept files always have it.
386 : *
387 : * Anyway, checking for POS_NULL doesn't hurt.
388 : */
389 55968 : parity_pos = fs_file2par_find(disk, file, i);
390 :
391 : /* if it's not mapped, it cannot have rehash */
392 55968 : if (parity_pos != POS_NULL) {
393 : /* get block specific info */
394 55968 : info = info_get(&state->infoarr, parity_pos);
395 :
396 : /* if rehash fails */
397 55968 : if (info_get_rehash(info))
398 0 : return 0;
399 : }
400 : }
401 :
402 22687 : return 1;
403 : }
404 :
405 : /**
406 : * Check if a file is completely formed of blocks with an updated hash,
407 : * and no rehash is tagged, and if it has at least one block.
408 : */
409 23670 : static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
410 : {
411 : block_off_t i;
412 :
413 : /* with no block, it never has a hash */
414 23670 : if (file->blockmax == 0)
415 27 : return 0;
416 :
417 : /* check all blocks */
418 81886 : for (i = 0; i < file->blockmax; ++i) {
419 : snapraid_info info;
420 58250 : struct snapraid_block* block = fs_file2block_get(file, i);
421 : block_off_t parity_pos;
422 :
423 : /* exclude blocks without hash */
424 58250 : if (!block_has_updated_hash(block))
425 7 : return 0;
426 :
427 : /*
428 : * Get the parity position.
429 : *
430 : * Note that it's possible to have files
431 : * not mapped into the parity, even if they
432 : * have a valid hash.
433 : *
434 : * This happens for example, for 'copied' files
435 : * that have REP blocks, but not yet mapped.
436 : *
437 : * If there are multiple copies, it's also possible
438 : * that such files are used as 'source' to copy
439 : * hashes, and then to get them inside this function.
440 : */
441 58243 : parity_pos = fs_file2par_find(disk, file, i);
442 :
443 : /* if it's not mapped, it cannot have rehash */
444 58243 : if (parity_pos != POS_NULL) {
445 : /* get block specific info */
446 58235 : info = info_get(&state->infoarr, parity_pos);
447 :
448 : /* exclude blocks needing a rehash */
449 58235 : if (info_get_rehash(info))
450 0 : return 0;
451 : }
452 : }
453 :
454 23636 : return 1;
455 : }
456 :
457 : /**
458 : * Refresh the file info.
459 : *
460 : * This is needed by Windows as the normal way to list directories may report not
461 : * updated info. Only the GetFileInformationByHandle() func, called file-by-file,
462 : * really ensures to return synced info.
463 : *
464 : * If this happens, we read also the physical offset, to avoid to read it later.
465 : */
466 79174 : static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struct stat* st, uint64_t* physical)
467 : {
468 : #if HAVE_LSTAT_SYNC
469 : struct snapraid_state* state = scan->state;
470 : struct snapraid_disk* disk = scan->disk;
471 :
472 : /* if the st_sync is not set, ensure to get synced info */
473 : if (st->st_sync == 0) {
474 : char path_next[PATH_MAX];
475 : struct stat synced_st;
476 :
477 : pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, sub);
478 :
479 : /* if we sort for physical offsets we have to read them for new files */
480 : if (state->opt.force_order == SORT_PHYSICAL
481 : && *physical == FILEPHY_UNREAD_OFFSET
482 : ) {
483 : /* do nothing, leave the pointer to read the physical offset */
484 : } else {
485 : physical = 0; /* set the pointer to 0 to read nothing */
486 : }
487 :
488 : if (lstat_sync(path_next, &synced_st, physical) != 0) {
489 : /* LCOV_EXCL_START */
490 : log_fatal("Error in stat file '%s'. %s.\n", path_next, strerror(errno));
491 : exit(EXIT_FAILURE);
492 : /* LCOV_EXCL_STOP */
493 : }
494 :
495 : if (st->st_mtime != synced_st.st_mtime
496 : || st->st_mtimensec != synced_st.st_mtimensec
497 : ) {
498 : #ifndef _WIN32
499 : /*
500 : * In Windows having different metadata is expected with open files
501 : * because the metadata in the directory is updated only when the file
502 : * is closed.
503 : *
504 : * The same happens for hardlinks that duplicate metatada.
505 : * The link metadata is updated only when the link is opened.
506 : * This extends also to st_size and st_nlink.
507 : *
508 : * See also:
509 : * Why is the file size reported incorrectly for files that are still being written to?
510 : * http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx
511 : */
512 : log_fatal("WARNING! Detected uncached time change from %" PRIu64 ".%09u to %" PRIu64 ".%09u for file '%s'\n",
513 : (uint64_t)st->st_mtime, (uint32_t)st->st_mtimensec, (uint64_t)synced_st.st_mtime, (uint32_t)synced_st.st_mtimensec, sub);
514 : log_fatal("It's better if you run SnapRAID without other processes running.\n");
515 : #endif
516 : st->st_mtime = synced_st.st_mtime;
517 : st->st_mtimensec = synced_st.st_mtimensec;
518 : }
519 :
520 : if (st->st_size != synced_st.st_size) {
521 : #ifndef _WIN32
522 : log_fatal("WARNING! Detected uncached size change from %" PRIu64 " to %" PRIu64 " for file '%s'\n",
523 : (uint64_t)st->st_size, (uint64_t)synced_st.st_size, sub);
524 : log_fatal("It's better if you run SnapRAID without other processes running.\n");
525 : #endif
526 : st->st_size = synced_st.st_size;
527 : }
528 :
529 : if (st->st_nlink != synced_st.st_nlink) {
530 : #ifndef _WIN32
531 : log_fatal("WARNING! Detected uncached nlink change from %u to %u for file '%s'\n",
532 : (uint32_t)st->st_nlink, (uint32_t)synced_st.st_nlink, sub);
533 : log_fatal("It's better if you run SnapRAID without other processes running.\n");
534 : #endif
535 : st->st_nlink = synced_st.st_nlink;
536 : }
537 :
538 : if (st->st_ino != synced_st.st_ino) {
539 : log_fatal("DANGER! Detected uncached inode change from %" PRIu64 " to %" PRIu64 " for file '%s'\n",
540 : (uint64_t)st->st_ino, (uint64_t)synced_st.st_ino, sub);
541 : log_fatal("It's better if you run SnapRAID without other processes running.\n");
542 : /* at this point, it's too late to change inode */
543 : /* and having inconsistent inodes may result to internal failures */
544 : /* so, it's better to abort */
545 : exit(EXIT_FAILURE);
546 : }
547 : }
548 : #else
549 : (void)scan;
550 : (void)sub;
551 : (void)st;
552 : (void)physical;
553 : #endif
554 79174 : }
555 :
556 : /**
557 : * Insert the file in the data set.
558 : */
559 101861 : static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
560 : {
561 101861 : struct snapraid_disk* disk = scan->disk;
562 :
563 : /* insert the file in the containers */
564 101861 : if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
565 101861 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
566 101861 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
567 101861 : tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
568 :
569 : /* delayed allocation of the parity */
570 101861 : scan_file_delayed_allocate(scan, file);
571 101861 : }
572 :
573 : /**
574 : * Remove the file from the data set.
575 : *
576 : * File is then deleted.
577 : */
578 56995 : static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
579 : {
580 56995 : struct snapraid_disk* disk = scan->disk;
581 :
582 : /* remove the file from the containers */
583 56995 : if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
584 56950 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
585 56995 : tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
586 56995 : tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
587 :
588 : /* deallocate the file from the parity */
589 56995 : scan_file_deallocate(scan, file);
590 56995 : }
591 :
592 : /**
593 : * Keep the file as it's (or with only a name/inode modification).
594 : *
595 : * If the file is kept, nothing has to be done.
596 : *
597 : * But if a file contains only blocks with invalid parity, it's reallocated to ensure
598 : * to always minimize the space used in the parity.
599 : *
600 : * This could happen after a failed sync, when some other files are deleted,
601 : * and then new ones can be moved backward to fill the hole created.
602 : */
603 1025570 : static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* file)
604 : {
605 1025570 : struct snapraid_disk* disk = scan->disk;
606 :
607 : /* if the file is full invalid, schedule a reinsert at later stage */
608 1025570 : if (file_is_full_invalid_parity_and_stable(scan->state, disk, file)) {
609 22687 : struct snapraid_file* copy = file_dup(file);
610 :
611 : /* remove the file */
612 22687 : scan_file_remove(scan, file);
613 :
614 : /* reinsert the copy in the delayed list */
615 22687 : scan_file_insert(scan, copy);
616 : }
617 1025570 : }
618 :
619 : /**
620 : * Process a file.
621 : */
622 1104939 : static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, struct stat* st, uint64_t physical)
623 : {
624 1104939 : struct snapraid_state* state = scan->state;
625 1104939 : struct snapraid_disk* disk = scan->disk;
626 : struct snapraid_file* file;
627 : tommy_node* i;
628 : int is_original_file_size_different_than_zero;
629 : int is_file_already_present;
630 : data_off_t file_already_present_size;
631 : int64_t file_already_present_mtime_sec;
632 : int file_already_present_mtime_nsec;
633 : int is_file_reported;
634 : char esc_buffer[ESC_MAX];
635 : char esc_buffer_alt[ESC_MAX];
636 :
637 : /*
638 : * If the disk has persistent inodes and UUID, try a search on the past inodes,
639 : * to detect moved files.
640 : *
641 : * For persistent inodes we mean inodes that keep their values when the file-system
642 : * is unmounted and remounted. This don't always happen.
643 : *
644 : * Cases found are:
645 : * - Linux FUSE with exFAT driver from https://code.google.com/p/exfat/.
646 : * Inodes are reassigned at every mount restarting from 1 and incrementing.
647 : * As worse, the exFAT support in FUSE doesn't use sub-second precision in timestamps
648 : * making inode collision more easy (exFAT by design supports 10ms precision).
649 : * - Linux VFAT kernel (3.2) driver. Inodes are fully reassigned at every mount.
650 : *
651 : * In such cases, to avoid possible random collisions, it's better to disable the moved
652 : * file recognition.
653 : *
654 : * For persistent UUID we mean that it has the same UUID as before.
655 : * Otherwise, if the UUID is changed, likely it's a new recreated file-system,
656 : * and then the inode have no meaning.
657 : *
658 : * Note that to disable the search by past inode, we do this implicitly
659 : * removing all the past inode before searching for files.
660 : * This ensures that no file is found with a past inode, but at the same time,
661 : * it allows to find new files with the same inode, to identify them as hardlinks.
662 : */
663 1104939 : int has_past_inodes = !disk->has_volatile_inodes && !disk->has_different_uuid && !disk->has_unsupported_uuid;
664 :
665 : /* always search with the new inode, in the all new inodes found until now, */
666 : /* with the eventual presence of also the past inodes */
667 1104939 : uint64_t inode = st->st_ino;
668 :
669 1104939 : file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
670 :
671 : /* identify moved files with past inodes and hardlinks with the new inodes */
672 1104939 : if (file) {
673 : /* check if the file is not changed */
674 945298 : if (file->size == st->st_size
675 929856 : && file->mtime_sec == st->st_mtime
676 926818 : && (file->mtime_nsec == STAT_NSEC(st)
677 : /* always accept the stored value if it's STAT_NSEC_INVALID */
678 : /* it happens when upgrading from an old version of SnapRAID */
679 : /* not yet supporting the nanosecond field */
680 10 : || file->mtime_nsec == STAT_NSEC_INVALID
681 : )
682 : ) {
683 : /* check if multiple files have the same inode */
684 926808 : if (file_flag_has(file, FILE_IS_PRESENT)) {
685 : /* if has_volatile_hardlinks is true, the nlink value is not reliable */
686 195 : if (!disk->has_volatile_hardlinks && st->st_nlink == 1) {
687 : /* LCOV_EXCL_START */
688 : log_fatal("Internal inode '%" PRIu64 "' inconsistency for file '%s%s' already present\n", (uint64_t)st->st_ino, disk->dir, sub);
689 : os_abort();
690 : /* LCOV_EXCL_STOP */
691 : }
692 :
693 : /* it's a hardlink */
694 195 : scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
695 195 : return;
696 : }
697 :
698 : /* mark as present */
699 926613 : file_flag_set(file, FILE_IS_PRESENT);
700 :
701 : /* update the nanoseconds mtime only if different */
702 : /* to avoid unneeded updates */
703 926613 : if (file->mtime_nsec == STAT_NSEC_INVALID
704 0 : && STAT_NSEC(st) != file->mtime_nsec
705 : ) {
706 0 : file->mtime_nsec = STAT_NSEC(st);
707 :
708 : /* we have to save the new mtime */
709 0 : state->need_write = 1;
710 : }
711 :
712 926613 : if (strcmp(file->sub, sub) != 0) {
713 : /* if the path is different, it means a moved file with the same inode */
714 155 : ++scan->count_move;
715 :
716 155 : log_tag("scan:move:%s:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer), esc_tag(sub, esc_buffer_alt));
717 155 : if (is_diff) {
718 0 : printf("move %s -> %s\n", fmt_term(disk, file->sub, esc_buffer), fmt_term(disk, sub, esc_buffer_alt));
719 : }
720 :
721 : /* remove from the name set */
722 155 : tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
723 :
724 : /* save the new name */
725 155 : file_rename(file, sub);
726 :
727 : /* reinsert in the name set */
728 155 : tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
729 :
730 : /* we have to save the new name */
731 155 : state->need_write = 1;
732 : } else {
733 : /* otherwise it's equal */
734 926458 : ++scan->count_equal;
735 :
736 926458 : if (state->opt.gui) {
737 22611 : log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
738 : }
739 : }
740 :
741 : /* mark the file as kept */
742 926613 : scan_file_keep(scan, file);
743 :
744 : /* nothing more to do */
745 926613 : return;
746 : }
747 :
748 : /*
749 : * Here the file matches the inode, but not the other info
750 : *
751 : * It could be a modified file with the same name,
752 : * or a restored/copied file that get assigned a previously used inode,
753 : * or a file-system with not persistent inodes.
754 : *
755 : * In NTFS it could be also a hardlink, because in NTFS
756 : * hardlink don't share the same directory information,
757 : * like attribute and time.
758 : *
759 : * For example:
760 : * C:> echo A > A
761 : * C:> mklink /H B A
762 : * ...wait one minute
763 : * C:> echo AAAAAAAAAAAAAA > A
764 : * C:> dir
765 : * ...both time and size of A and B don't match!
766 : */
767 18490 : if (file_flag_has(file, FILE_IS_PRESENT)) {
768 : /* if has_volatile_hardlinks is true, the nlink value is not reliable */
769 0 : if (!disk->has_volatile_hardlinks && st->st_nlink == 1) {
770 : /* LCOV_EXCL_START */
771 : log_fatal("Internal inode '%" PRIu64 "' inconsistency for files '%s%s' and '%s%s' with same inode but different attributes: size %" PRIu64 "?%" PRIu64 ", sec %" PRIu64 "?%" PRIu64 ", nsec %d?%d\n",
772 : file->inode, disk->dir, sub, disk->dir, file->sub,
773 : file->size, (uint64_t)st->st_size,
774 : file->mtime_sec, (uint64_t)st->st_mtime,
775 : file->mtime_nsec, STAT_NSEC(st));
776 : os_abort();
777 : /* LCOV_EXCL_STOP */
778 : }
779 :
780 : /* LCOV_EXCL_START */
781 : /* suppose it's hardlink with not synced metadata */
782 : scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
783 : return;
784 : /* LCOV_EXCL_STOP */
785 : }
786 :
787 : /* assume a previously used inode, it's the worst case */
788 : /* and we handle it removing the duplicate stored inode. */
789 : /* If the file is found by name later, it will have the inode restored, */
790 : /* otherwise, it will get removed */
791 :
792 : /* remove from the inode set */
793 18490 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
794 :
795 : /* clear the inode */
796 : /* this is not really needed for correct functionality */
797 : /* because we are going to set FILE_IS_WITHOUT_INODE */
798 : /* but it's easier for debugging to have invalid inodes set to 0 */
799 18490 : file->inode = 0;
800 :
801 : /* mark as missing inode */
802 18490 : file_flag_set(file, FILE_IS_WITHOUT_INODE);
803 :
804 : /* go further to find it by name */
805 : }
806 :
807 : /* initialize for later overwrite */
808 178131 : is_file_reported = 0;
809 178131 : is_original_file_size_different_than_zero = 0;
810 :
811 : /* then try finding it by name */
812 178131 : file = tommy_hashdyn_search(&disk->pathset, file_path_compare_to_arg, sub, file_path_hash(sub));
813 :
814 : /* keep track if the file already exists */
815 178131 : is_file_already_present = file != 0;
816 :
817 178131 : if (is_file_already_present) {
818 : /* if the file is without an inode */
819 103618 : if (file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
820 : /* set it now */
821 18459 : file->inode = st->st_ino;
822 :
823 : /* insert in the set */
824 18459 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
825 :
826 : /* unmark as missing inode */
827 18459 : file_flag_clear(file, FILE_IS_WITHOUT_INODE);
828 : } else {
829 : /* here the inode has to be different, otherwise we would have found it before */
830 85159 : if (file->inode == st->st_ino) {
831 : /* LCOV_EXCL_START */
832 : log_fatal("Internal inconsistency in inode '%" PRIu64 "' for files '%s%s' as unexpected matching\n", file->inode, disk->dir, sub);
833 : os_abort();
834 : /* LCOV_EXCL_STOP */
835 : }
836 : }
837 :
838 : /* for sure it cannot be already present */
839 103618 : if (file_flag_has(file, FILE_IS_PRESENT)) {
840 : /* LCOV_EXCL_START */
841 : log_fatal("Internal inconsistency in path for file '%s%s' matching and already present\n", disk->dir, sub);
842 : os_abort();
843 : /* LCOV_EXCL_STOP */
844 : }
845 :
846 : /* check if the file is not changed */
847 103618 : if (file->size == st->st_size
848 103022 : && file->mtime_sec == st->st_mtime
849 98959 : && (file->mtime_nsec == STAT_NSEC(st)
850 : /* always accept the stored value if it's STAT_NSEC_INVALID */
851 : /* it happens when upgrading from an old version of SnapRAID */
852 : /* not yet supporting the nanosecond field */
853 2 : || file->mtime_nsec == STAT_NSEC_INVALID
854 : )
855 : ) {
856 : /* mark as present */
857 98957 : file_flag_set(file, FILE_IS_PRESENT);
858 :
859 : /* update the nano seconds mtime only if different */
860 : /* to avoid unneeded updates */
861 98957 : if (file->mtime_nsec == STAT_NSEC_INVALID
862 0 : && STAT_NSEC(st) != STAT_NSEC_INVALID
863 : ) {
864 0 : file->mtime_nsec = STAT_NSEC(st);
865 :
866 : /* we have to save the new mtime */
867 0 : state->need_write = 1;
868 : }
869 :
870 : /* if when processing the disk we used the past inodes values */
871 98957 : if (has_past_inodes) {
872 : /* if persistent inodes are supported, we are sure that the inode number */
873 : /* is now different, because otherwise the file would have been found */
874 : /* when searching by inode. */
875 : /* if the inode is different, it means a rewritten file with the same path */
876 : /* like when restoring a backup that restores also the timestamp */
877 98943 : ++scan->count_restore;
878 :
879 98943 : log_tag("scan:restore:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
880 98943 : if (is_diff) {
881 0 : printf("restore %s\n", fmt_term(disk, sub, esc_buffer));
882 : }
883 :
884 : /* remove from the inode set */
885 98943 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
886 :
887 : /* save the new inode */
888 98943 : file->inode = st->st_ino;
889 :
890 : /* reinsert in the inode set */
891 98943 : tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
892 :
893 : /* we have to save the new inode */
894 98943 : state->need_write = 1;
895 : } else {
896 : /* otherwise it's the case of not persistent inode, where doesn't */
897 : /* matter if the inode is different or equal, because they have no */
898 : /* meaning, and then we don't even save them */
899 14 : ++scan->count_equal;
900 :
901 14 : if (state->opt.gui) {
902 0 : log_tag("scan:equal:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
903 : }
904 : }
905 :
906 : /* mark the file as kept */
907 98957 : scan_file_keep(scan, file);
908 :
909 : /* nothing more to do */
910 98957 : return;
911 : }
912 :
913 : /* here if the file is changed but with the correct name */
914 :
915 : /* save the info for later printout */
916 4661 : file_already_present_size = file->size;
917 4661 : file_already_present_mtime_sec = file->mtime_sec;
918 4661 : file_already_present_mtime_nsec = file->mtime_nsec;
919 :
920 : /* keep track if the original file was not of zero size */
921 4661 : is_original_file_size_different_than_zero = file->size != 0;
922 :
923 : /* remove it, and continue to insert it again */
924 4661 : scan_file_remove(scan, file);
925 :
926 : /* and continue to insert it again */
927 : } else {
928 74513 : file_already_present_size = 0;
929 74513 : file_already_present_mtime_sec = 0;
930 74513 : file_already_present_mtime_nsec = 0;
931 : }
932 :
933 : /* refresh the info, to ensure that they are synced, */
934 : /* note that we refresh only the info of the new or modified files */
935 : /* because this is slow operation */
936 79174 : scan_file_refresh(scan, sub, st, &physical);
937 :
938 : #ifndef _WIN32
939 : /* do a safety check to ensure that the common ext4 case of zeroing */
940 : /* the size of a file after a crash doesn't propagate to the backup */
941 : /* this check is specific for Linux, so we disable it on Windows */
942 79174 : if (is_original_file_size_different_than_zero && st->st_size == 0) {
943 0 : if (!state->opt.force_zero) {
944 : /* LCOV_EXCL_START */
945 : log_fatal("The file '%s%s' has unexpected zero size!\n", disk->dir, sub);
946 : log_fatal("It's possible that after a kernel crash this file was lost,\n");
947 : log_fatal("and you can use 'snapraid fix -f /%s' to recover it.\n", fmt_poll(disk, sub, esc_buffer));
948 : if (!is_diff) {
949 : log_fatal("If this an expected condition you can '%s' anyway using 'snapraid --force-zero %s'\n", state->command, state->command);
950 : exit(EXIT_FAILURE);
951 : }
952 : /* LCOV_EXCL_STOP */
953 : }
954 : }
955 : #else
956 : /* avoid the unused warning in Windows */
957 : (void)is_original_file_size_different_than_zero;
958 : #endif
959 :
960 : /* insert it */
961 79174 : file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime, STAT_NSEC(st), st->st_ino, physical);
962 :
963 : /* mark it as present */
964 79174 : file_flag_set(file, FILE_IS_PRESENT);
965 :
966 : /* if copy detection is enabled */
967 : /* note that the copy detection is tried also for updated files */
968 : /* this makes sense because it may happen to have two different copies */
969 : /* of the same file, and we move the right one over the wrong one */
970 : /* in such case we have a "copy" over an "update" */
971 79174 : if (!state->opt.force_nocopy) {
972 79174 : tommy_uint32_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec);
973 :
974 : /* search for a file with the same name and stamp in all the disks */
975 423270 : for (i = state->disklist; i != 0; i = i->next) {
976 367732 : struct snapraid_disk* other_disk = i->data;
977 : struct snapraid_file* other_file;
978 :
979 : /* if the nanosecond part of the time stamp is valid, search */
980 : /* for name and stamp, otherwise for path and stamp */
981 367732 : if (file->mtime_nsec != 0 && file->mtime_nsec != STAT_NSEC_INVALID)
982 367624 : other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash);
983 : else
984 108 : other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
985 :
986 : /* if found, and it's a fully hashed file */
987 367732 : if (other_file && file_is_full_hashed_and_stable(scan->state, other_disk, other_file)) {
988 : /* assume that the file is a copy, and reuse the hash */
989 23636 : file_copy(other_file, file);
990 :
991 : /* revert old counter and use the copy one */
992 23636 : ++scan->count_copy;
993 :
994 23636 : log_tag("scan:copy:%s:%s:%s:%s\n", other_disk->name, esc_tag(other_file->sub, esc_buffer), disk->name, esc_tag(file->sub, esc_buffer_alt));
995 23636 : if (is_diff) {
996 31 : printf("copy %s -> %s\n", fmt_term(other_disk, other_file->sub, esc_buffer), fmt_term(disk, file->sub, esc_buffer_alt));
997 : }
998 :
999 : /* mark it as reported */
1000 23636 : is_file_reported = 1;
1001 :
1002 : /* no need to continue the search */
1003 23636 : break;
1004 : }
1005 : }
1006 : }
1007 :
1008 : /* if not yet reported, do it now */
1009 : /* we postpone this to avoid to print two times the copied files */
1010 79174 : if (!is_file_reported) {
1011 55538 : if (is_file_already_present) {
1012 4661 : ++scan->count_change;
1013 :
1014 4661 : log_tag("scan:update:%s:%s: %" PRIu64 " %" PRIu64 ".%d -> %" PRIu64 " %" PRIu64 ".%d\n", disk->name, esc_tag(sub, esc_buffer),
1015 : file_already_present_size, file_already_present_mtime_sec, file_already_present_mtime_nsec,
1016 : file->size, file->mtime_sec, file->mtime_nsec
1017 : );
1018 :
1019 4661 : if (is_diff) {
1020 118 : printf("update %s\n", fmt_term(disk, sub, esc_buffer));
1021 : }
1022 : } else {
1023 50877 : ++scan->count_insert;
1024 :
1025 50877 : log_tag("scan:add:%s:%s\n", disk->name, esc_tag(sub, esc_buffer));
1026 50877 : if (is_diff) {
1027 17054 : printf("add %s\n", fmt_term(disk, sub, esc_buffer));
1028 : }
1029 : }
1030 : }
1031 :
1032 : /* insert the file in the delayed list */
1033 79174 : scan_file_insert(scan, file);
1034 : }
1035 :
1036 : /**
1037 : * Remove the specified dir from the data set.
1038 : */
1039 2 : static void scan_emptydir_remove(struct snapraid_scan* scan, struct snapraid_dir* dir)
1040 : {
1041 2 : struct snapraid_state* state = scan->state;
1042 2 : struct snapraid_disk* disk = scan->disk;
1043 :
1044 : /* state changed */
1045 2 : state->need_write = 1;
1046 :
1047 : /* remove the file from the dir containers */
1048 2 : tommy_hashdyn_remove_existing(&disk->dirset, &dir->nodeset);
1049 2 : tommy_list_remove_existing(&disk->dirlist, &dir->nodelist);
1050 :
1051 : /* deallocate */
1052 2 : dir_free(dir);
1053 2 : }
1054 :
1055 : /**
1056 : * Insert the specified dir in the data set.
1057 : */
1058 10 : static void scan_emptydir_insert(struct snapraid_scan* scan, struct snapraid_dir* dir)
1059 : {
1060 10 : struct snapraid_state* state = scan->state;
1061 10 : struct snapraid_disk* disk = scan->disk;
1062 :
1063 : /* state changed */
1064 10 : state->need_write = 1;
1065 :
1066 : /* insert the dir in the dir containers */
1067 10 : tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
1068 10 : tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
1069 10 : }
1070 :
1071 : /**
1072 : * Process a dir.
1073 : */
1074 259 : static void scan_emptydir(struct snapraid_scan* scan, const char* sub)
1075 : {
1076 259 : struct snapraid_disk* disk = scan->disk;
1077 : struct snapraid_dir* dir;
1078 :
1079 : /* check if the dir already exists */
1080 259 : dir = tommy_hashdyn_search(&disk->dirset, dir_name_compare, sub, dir_name_hash(sub));
1081 259 : if (dir) {
1082 : /* check if multiple files have the same name */
1083 249 : if (dir_flag_has(dir, FILE_IS_PRESENT)) {
1084 : /* LCOV_EXCL_START */
1085 : log_fatal("Internal inconsistency for dir '%s%s'\n", disk->dir, sub);
1086 : os_abort();
1087 : /* LCOV_EXCL_STOP */
1088 : }
1089 :
1090 : /* mark as present */
1091 249 : dir_flag_set(dir, FILE_IS_PRESENT);
1092 :
1093 : /* nothing more to do */
1094 249 : return;
1095 : } else {
1096 : /* and continue to insert it */
1097 : }
1098 :
1099 : /* insert it */
1100 10 : dir = dir_alloc(sub);
1101 :
1102 : /* mark it as present */
1103 10 : dir_flag_set(dir, FILE_IS_PRESENT);
1104 :
1105 : /* insert it in the delayed insert list */
1106 10 : tommy_list_insert_tail(&scan->dir_insert_list, &dir->nodelist, dir);
1107 : }
1108 :
1109 : struct dirent_sorted {
1110 : /* node for data structures */
1111 : tommy_node node;
1112 :
1113 : #if HAVE_STRUCT_DIRENT_D_INO
1114 : uint64_t d_ino; /**< Inode number. */
1115 : #endif
1116 : #if HAVE_STRUCT_DIRENT_D_TYPE
1117 : uint32_t d_type; /**< File type. */
1118 : #endif
1119 : #if HAVE_STRUCT_DIRENT_D_STAT
1120 : struct stat d_stat; /**< Stat result. */
1121 : #endif
1122 : char d_name[]; /**< Variable length name. It must be the last field. */
1123 : };
1124 :
1125 : #if HAVE_STRUCT_DIRENT_D_INO
1126 103550 : static int dd_ino_compare(const void* void_a, const void* void_b)
1127 : {
1128 103550 : const struct dirent_sorted* a = void_a;
1129 103550 : const struct dirent_sorted* b = void_b;
1130 :
1131 103550 : if (a->d_ino < b->d_ino)
1132 49454 : return -1;
1133 54096 : if (a->d_ino > b->d_ino)
1134 54096 : return 1;
1135 :
1136 0 : return 0;
1137 : }
1138 : #endif
1139 :
1140 11524282 : static int dd_name_compare(const void* void_a, const void* void_b)
1141 : {
1142 11524282 : const struct dirent_sorted* a = void_a;
1143 11524282 : const struct dirent_sorted* b = void_b;
1144 :
1145 11524282 : return strcmp(a->d_name, b->d_name);
1146 : }
1147 :
1148 : /**
1149 : * Return the stat info of a dir entry.
1150 : */
1151 : #if HAVE_STRUCT_DIRENT_D_STAT
1152 : #define DSTAT(file, dd, buf) dstat(dd)
1153 : struct stat* dstat(struct dirent_sorted* dd)
1154 : {
1155 : return &dd->d_stat;
1156 : }
1157 : #else
1158 : #define DSTAT(file, dd, buf) dstat(file, buf)
1159 1106210 : struct stat* dstat(const char* file, struct stat* st)
1160 : {
1161 1106210 : if (lstat(file, st) != 0) {
1162 : /* LCOV_EXCL_START */
1163 : log_fatal("Error in stat file/directory '%s'. %s.\n", file, strerror(errno));
1164 : exit(EXIT_FAILURE);
1165 : /* LCOV_EXCL_STOP */
1166 : }
1167 1106210 : return st;
1168 : }
1169 : #endif
1170 :
1171 : /**
1172 : * Process a directory.
1173 : * Return != 0 if at least one file or link is processed.
1174 : */
1175 1846 : static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const char* dir, const char* sub)
1176 : {
1177 1846 : struct snapraid_state* state = scan->state;
1178 1846 : struct snapraid_disk* disk = scan->disk;
1179 1846 : int processed = 0;
1180 : DIR* d;
1181 : tommy_list list;
1182 : tommy_node* node;
1183 :
1184 1846 : tommy_list_init(&list);
1185 :
1186 1846 : d = opendir(dir);
1187 1846 : if (!d) {
1188 : /* LCOV_EXCL_START */
1189 : log_fatal("Error opening directory '%s'. %s.\n", dir, strerror(errno));
1190 : if (level == 0)
1191 : log_fatal("If this is the disk mount point, remember to create it manually\n");
1192 : else
1193 : log_fatal("If it's a permission problem, you can exclude it in the config file with:\n\texclude /%s\n", sub);
1194 : exit(EXIT_FAILURE);
1195 : /* LCOV_EXCL_STOP */
1196 : }
1197 :
1198 : /* read the full directory */
1199 : while (1) {
1200 : char path_next[PATH_MAX];
1201 : char sub_next[PATH_MAX];
1202 : struct dirent_sorted* entry;
1203 : const char* name;
1204 : struct dirent* dd;
1205 : size_t name_len;
1206 :
1207 : /*
1208 : * Clear errno to differentiate the end of the stream and an error condition
1209 : *
1210 : * From the Linux readdir() manpage:
1211 : * "If the end of the directory stream is reached, NULL is returned and errno is not changed.
1212 : * If an error occurs, NULL is returned and errno is set appropriately."
1213 : */
1214 1147737 : errno = 0;
1215 1147737 : dd = readdir(d);
1216 1147737 : if (dd == 0 && errno != 0) {
1217 : /* LCOV_EXCL_START */
1218 : log_fatal("Error reading directory '%s'. %s.\n", dir, strerror(errno));
1219 : log_fatal("You can exclude it in the config file with:\n\texclude /%s\n", sub);
1220 : exit(EXIT_FAILURE);
1221 : /* LCOV_EXCL_STOP */
1222 : }
1223 1147737 : if (dd == 0) {
1224 1846 : break; /* finished */
1225 : }
1226 :
1227 : /* skip "." and ".." files */
1228 1145891 : name = dd->d_name;
1229 1145891 : if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
1230 7384 : continue;
1231 :
1232 1142199 : pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
1233 1142199 : pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
1234 :
1235 : /* check for not supported file names */
1236 1142199 : if (name[0] == 0) {
1237 : /* LCOV_EXCL_START */
1238 : log_fatal("Unsupported name '%s' in file '%s'.\n", name, path_next);
1239 : exit(EXIT_FAILURE);
1240 : /* LCOV_EXCL_STOP */
1241 : }
1242 :
1243 : /* exclude hidden files even before calling lstat() */
1244 1142199 : if (filter_hidden(state->filter_hidden, dd) != 0) {
1245 0 : msg_verbose("Excluding hidden '%s'\n", path_next);
1246 0 : continue;
1247 : }
1248 :
1249 : /* exclude content files even before calling lstat() */
1250 1142199 : if (filter_content(&state->contentlist, path_next) != 0) {
1251 0 : msg_verbose("Excluding content '%s'\n", path_next);
1252 0 : continue;
1253 : }
1254 :
1255 1142199 : name_len = strlen(dd->d_name);
1256 1142199 : entry = malloc_nofail(sizeof(struct dirent_sorted) + name_len + 1);
1257 :
1258 : /* copy the dir entry */
1259 : #if HAVE_STRUCT_DIRENT_D_INO
1260 1142199 : entry->d_ino = dd->d_ino;
1261 : #endif
1262 : #if HAVE_STRUCT_DIRENT_D_TYPE
1263 1142199 : entry->d_type = dd->d_type;
1264 : #endif
1265 : #if HAVE_STRUCT_DIRENT_D_STAT
1266 : /* convert dirent to lstat result */
1267 : dirent_lstat(dd, &entry->d_stat);
1268 :
1269 : /* note that at this point the st_mode may be 0 */
1270 : #endif
1271 1142199 : memcpy(entry->d_name, dd->d_name, name_len + 1);
1272 :
1273 : /* insert in the list */
1274 1142199 : tommy_list_insert_tail(&list, &entry->node, entry);
1275 1145891 : }
1276 :
1277 1846 : if (closedir(d) != 0) {
1278 : /* LCOV_EXCL_START */
1279 : log_fatal("Error closing directory '%s'. %s.\n", dir, strerror(errno));
1280 : exit(EXIT_FAILURE);
1281 : /* LCOV_EXCL_STOP */
1282 : }
1283 :
1284 1846 : if (state->opt.force_order == SORT_ALPHA) {
1285 : /* if requested sort alphabetically */
1286 : /* this is mainly done for testing to ensure to always */
1287 : /* process in the same way in different platforms */
1288 1810 : tommy_list_sort(&list, dd_name_compare);
1289 : }
1290 : #if HAVE_STRUCT_DIRENT_D_INO
1291 36 : else if (!disk->has_volatile_inodes) {
1292 : /* if inodes are persistent */
1293 : /* sort the list of dir entries by inodes */
1294 36 : tommy_list_sort(&list, dd_ino_compare);
1295 : }
1296 : /* otherwise just keep the insertion order */
1297 : #endif
1298 :
1299 : /* process the sorted dir entries */
1300 1846 : node = list;
1301 1145891 : while (node != 0) {
1302 : char path_next[PATH_MAX];
1303 : char sub_next[PATH_MAX];
1304 : char out[PATH_MAX];
1305 1142199 : struct snapraid_filter* reason = 0;
1306 1142199 : struct dirent_sorted* dd = node->data;
1307 1142199 : const char* name = dd->d_name;
1308 : struct stat* st;
1309 : int type;
1310 : #if !HAVE_STRUCT_DIRENT_D_STAT
1311 : struct stat st_buf;
1312 : #endif
1313 :
1314 1142199 : pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
1315 1142199 : pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
1316 :
1317 : /* start with an unknown type */
1318 1142199 : type = -1;
1319 1142199 : st = 0;
1320 :
1321 : /* if dirent has the type, use it */
1322 : #if HAVE_STRUCT_DIRENT_D_TYPE
1323 1142199 : switch (dd->d_type) {
1324 0 : case DT_UNKNOWN : break;
1325 1104939 : case DT_REG : type = 0; break;
1326 35989 : case DT_LNK : type = 1; break;
1327 1271 : case DT_DIR : type = 2; break;
1328 0 : default : type = 3; break;
1329 : }
1330 : #endif
1331 :
1332 : /* if type is still unknown */
1333 1142199 : if (type < 0) {
1334 : /* get the type from stat */
1335 0 : st = DSTAT(path_next, dd, &st_buf);
1336 :
1337 : #if HAVE_STRUCT_DIRENT_D_STAT
1338 : /* if the st_mode field is missing, takes care to fill it using normal lstat() */
1339 : /* at now this can happen only in Windows (with HAVE_STRUCT_DIRENT_D_STAT defined), */
1340 : /* because we use a directory reading method that doesn't read info about ReparsePoint. */
1341 : /* Note that here we cannot call here lstat_sync(), because we don't know what kind */
1342 : /* of file is it, and lstat_sync() doesn't always work */
1343 : if (st->st_mode == 0) {
1344 : if (lstat(path_next, st) != 0) {
1345 : /* LCOV_EXCL_START */
1346 : log_fatal("Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno));
1347 : exit(EXIT_FAILURE);
1348 : /* LCOV_EXCL_STOP */
1349 : }
1350 : }
1351 : #endif
1352 :
1353 0 : if (S_ISREG(st->st_mode))
1354 0 : type = 0;
1355 0 : else if (S_ISLNK(st->st_mode))
1356 0 : type = 1;
1357 0 : else if (S_ISDIR(st->st_mode))
1358 0 : type = 2;
1359 : else
1360 0 : type = 3;
1361 : }
1362 :
1363 1142199 : if (type == 0) { /* REG */
1364 1104939 : if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
1365 :
1366 : /* late stat, if not yet called */
1367 1104939 : if (!st)
1368 1104939 : st = DSTAT(path_next, dd, &st_buf);
1369 :
1370 : #if HAVE_LSTAT_SYNC
1371 : /* if the st_ino field is missing, takes care to fill it using the extended lstat() */
1372 : /* this can happen only in Windows */
1373 : if (st->st_ino == 0 || st->st_nlink == 0) {
1374 : if (lstat_sync(path_next, st, 0) != 0) {
1375 : /* LCOV_EXCL_START */
1376 : log_fatal("Error in stat file '%s'. %s.\n", path_next, strerror(errno));
1377 : exit(EXIT_FAILURE);
1378 : /* LCOV_EXCL_STOP */
1379 : }
1380 : }
1381 : #endif
1382 :
1383 1104939 : scan_file(scan, is_diff, sub_next, st, FILEPHY_UNREAD_OFFSET);
1384 1104939 : processed = 1;
1385 : } else {
1386 0 : msg_verbose("Excluding file '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
1387 : }
1388 37260 : } else if (type == 1) { /* LNK */
1389 35989 : if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
1390 : char subnew[PATH_MAX];
1391 : int ret;
1392 :
1393 35989 : ret = readlink(path_next, subnew, sizeof(subnew));
1394 35989 : if (ret >= PATH_MAX) {
1395 : /* LCOV_EXCL_START */
1396 : log_fatal("Error in readlink file '%s'. Symlink too long.\n", path_next);
1397 : exit(EXIT_FAILURE);
1398 : /* LCOV_EXCL_STOP */
1399 : }
1400 35989 : if (ret < 0) {
1401 : /* LCOV_EXCL_START */
1402 : log_fatal("Error in readlink file '%s'. %s.\n", path_next, strerror(errno));
1403 : exit(EXIT_FAILURE);
1404 : /* LCOV_EXCL_STOP */
1405 : }
1406 35989 : if (ret == 0)
1407 0 : log_fatal("WARNING! Empty symbolic link '%s'.\n", path_next);
1408 :
1409 : /* readlink doesn't put the final 0 */
1410 35989 : subnew[ret] = 0;
1411 :
1412 : /* process as a symbolic link */
1413 35989 : scan_link(scan, is_diff, sub_next, subnew, FILE_IS_SYMLINK);
1414 35989 : processed = 1;
1415 : } else {
1416 0 : msg_verbose("Excluding link '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
1417 : }
1418 1271 : } else if (type == 2) { /* DIR */
1419 1271 : if (filter_subdir(&state->filterlist, &reason, disk->name, sub_next) == 0) {
1420 : #ifndef _WIN32
1421 : /* late stat, if not yet called */
1422 1271 : if (!st)
1423 1271 : st = DSTAT(path_next, dd, &st_buf);
1424 :
1425 : /* in Unix don't follow mount points in different devices */
1426 : /* in Windows we are already skipping them reporting them as special files */
1427 1271 : if ((uint64_t)st->st_dev != disk->device) {
1428 0 : log_fatal("WARNING! Ignoring mount point '%s' because it appears to be in a different device\n", path_next);
1429 : } else
1430 : #endif
1431 : {
1432 : char sub_dir[PATH_MAX];
1433 :
1434 : /* recurse */
1435 1271 : pathslash(path_next, sizeof(path_next));
1436 1271 : pathcpy(sub_dir, sizeof(sub_dir), sub_next);
1437 1271 : pathslash(sub_dir, sizeof(sub_dir));
1438 1271 : if (scan_dir(scan, level + 1, is_diff, path_next, sub_dir) == 0) {
1439 : /* scan the directory as empty dir */
1440 259 : scan_emptydir(scan, sub_next);
1441 : }
1442 : /* or we processed something internally, or we have added the empty dir */
1443 1271 : processed = 1;
1444 : }
1445 : } else {
1446 0 : msg_verbose("Excluding directory '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
1447 : }
1448 : } else {
1449 0 : if (filter_path(&state->filterlist, &reason, disk->name, sub_next) == 0) {
1450 : /* late stat, if not yet called */
1451 0 : if (!st)
1452 0 : st = DSTAT(path_next, dd, &st_buf);
1453 :
1454 0 : log_fatal("WARNING! Ignoring special '%s' file '%s'\n", stat_desc(st), path_next);
1455 : } else {
1456 0 : msg_verbose("Excluding special file '%s' for rule '%s'\n", path_next, filter_type(reason, out, sizeof(out)));
1457 : }
1458 : }
1459 :
1460 : /* next entry */
1461 1142199 : node = node->next;
1462 :
1463 : /* free the present one */
1464 1142199 : free(dd);
1465 : }
1466 :
1467 1846 : return processed;
1468 : }
1469 :
1470 96 : static int state_diffscan(struct snapraid_state* state, int is_diff)
1471 : {
1472 : tommy_node* i;
1473 : tommy_node* j;
1474 : tommy_list scanlist;
1475 : int done;
1476 : fptr* msg;
1477 : struct snapraid_scan total;
1478 : int no_difference;
1479 : char esc_buffer[ESC_MAX];
1480 :
1481 96 : tommy_list_init(&scanlist);
1482 :
1483 96 : if (is_diff)
1484 8 : msg_progress("Comparing...\n");
1485 :
1486 : /* first scan all the directory and find new and deleted files */
1487 671 : for (i = state->disklist; i != 0; i = i->next) {
1488 575 : struct snapraid_disk* disk = i->data;
1489 : struct snapraid_scan* scan;
1490 : tommy_node* node;
1491 : int ret;
1492 : int has_persistent_inodes;
1493 : int has_syncronized_hardlinks;
1494 :
1495 575 : scan = malloc_nofail(sizeof(struct snapraid_scan));
1496 575 : scan->state = state;
1497 575 : scan->disk = disk;
1498 575 : scan->count_equal = 0;
1499 575 : scan->count_move = 0;
1500 575 : scan->count_copy = 0;
1501 575 : scan->count_restore = 0;
1502 575 : scan->count_change = 0;
1503 575 : scan->count_remove = 0;
1504 575 : scan->count_insert = 0;
1505 575 : tommy_list_init(&scan->file_insert_list);
1506 575 : tommy_list_init(&scan->link_insert_list);
1507 575 : tommy_list_init(&scan->dir_insert_list);
1508 :
1509 575 : tommy_list_insert_tail(&scanlist, &scan->node, scan);
1510 :
1511 575 : if (!is_diff)
1512 527 : msg_progress("Scanning disk %s...\n", disk->name);
1513 :
1514 : /* check if the disk supports persistent inodes */
1515 575 : ret = fsinfo(disk->dir, &has_persistent_inodes, &has_syncronized_hardlinks, 0, 0);
1516 575 : if (ret < 0) {
1517 : /* LCOV_EXCL_START */
1518 : log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
1519 : exit(EXIT_FAILURE);
1520 : /* LCOV_EXCL_STOP */
1521 : }
1522 575 : if (!has_persistent_inodes) {
1523 0 : disk->has_volatile_inodes = 1;
1524 : }
1525 575 : if (!has_syncronized_hardlinks) {
1526 0 : disk->has_volatile_hardlinks = 1;
1527 : }
1528 :
1529 : /* if inodes or UUID are not persistent/changed/unsupported */
1530 575 : if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) {
1531 : /* remove all the inodes from the inode collection */
1532 : /* if they are not persistent, all of them could be changed now */
1533 : /* and we don't want to find false matching ones */
1534 : /* see scan_file() for more details */
1535 94 : node = disk->filelist;
1536 202 : while (node) {
1537 14 : struct snapraid_file* file = node->data;
1538 :
1539 14 : node = node->next;
1540 :
1541 : /* remove from the inode set */
1542 14 : tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
1543 :
1544 : /* clear the inode */
1545 14 : file->inode = 0;
1546 :
1547 : /* mark as missing inode */
1548 14 : file_flag_set(file, FILE_IS_WITHOUT_INODE);
1549 : }
1550 : }
1551 :
1552 575 : scan_dir(scan, 0, is_diff, disk->dir, "");
1553 : }
1554 :
1555 : /* we split the search in two phases because to detect files */
1556 : /* moved from one disk to another we have to start deletion */
1557 : /* only when all disks have all the new files found */
1558 :
1559 : /* now process all the new and deleted files */
1560 671 : for (i = scanlist; i != 0; i = i->next) {
1561 575 : struct snapraid_scan* scan = i->data;
1562 575 : struct snapraid_disk* disk = scan->disk;
1563 : tommy_node* node;
1564 : unsigned phy_count;
1565 : unsigned phy_dup;
1566 : uint64_t phy_last;
1567 : struct snapraid_file* phy_file_last;
1568 :
1569 : /* check for removed files */
1570 575 : node = disk->filelist;
1571 1033680 : while (node) {
1572 1032530 : struct snapraid_file* file = node->data;
1573 :
1574 : /* next node */
1575 1032530 : node = node->next;
1576 :
1577 : /* remove if not present */
1578 1032530 : if (!file_flag_has(file, FILE_IS_PRESENT)) {
1579 29647 : ++scan->count_remove;
1580 :
1581 29647 : log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(file->sub, esc_buffer));
1582 29647 : if (is_diff) {
1583 97 : printf("remove %s\n", fmt_term(disk, file->sub, esc_buffer));
1584 : }
1585 :
1586 29647 : scan_file_remove(scan, file);
1587 : }
1588 : }
1589 :
1590 : /* check for removed links */
1591 575 : node = disk->linklist;
1592 35833 : while (node) {
1593 34683 : struct snapraid_link* slink = node->data;
1594 :
1595 : /* next node */
1596 34683 : node = node->next;
1597 :
1598 : /* remove if not present */
1599 34683 : if (!link_flag_has(slink, FILE_IS_PRESENT)) {
1600 956 : ++scan->count_remove;
1601 :
1602 956 : log_tag("scan:remove:%s:%s\n", disk->name, esc_tag(slink->sub, esc_buffer));
1603 956 : if (is_diff) {
1604 10 : printf("remove %s\n", fmt_term(disk, slink->sub, esc_buffer));
1605 : }
1606 :
1607 956 : scan_link_remove(scan, slink);
1608 : }
1609 : }
1610 :
1611 : /* check for removed dirs */
1612 575 : node = disk->dirlist;
1613 1401 : while (node) {
1614 251 : struct snapraid_dir* dir = node->data;
1615 :
1616 : /* next node */
1617 251 : node = node->next;
1618 :
1619 : /* remove if not present */
1620 251 : if (!dir_flag_has(dir, FILE_IS_PRESENT)) {
1621 2 : scan_emptydir_remove(scan, dir);
1622 : }
1623 : }
1624 :
1625 : /* sort the files before inserting them */
1626 : /* we use a stable sort to ensure that if the reported physical offset/inode */
1627 : /* are always 0, we keep at least the directory order */
1628 575 : switch (state->opt.force_order) {
1629 : case SORT_PHYSICAL :
1630 12 : tommy_list_sort(&scan->file_insert_list, file_physical_compare);
1631 12 : break;
1632 : case SORT_INODE :
1633 0 : tommy_list_sort(&scan->file_insert_list, file_inode_compare);
1634 0 : break;
1635 : case SORT_ALPHA :
1636 563 : tommy_list_sort(&scan->file_insert_list, file_path_compare);
1637 563 : break;
1638 : case SORT_DIR :
1639 : /* already in order */
1640 0 : break;
1641 : }
1642 :
1643 : /* insert all the new files, we insert them only after the deletion */
1644 : /* to reuse the just freed space */
1645 : /* also check if the physical offset reported are fakes or not */
1646 575 : node = scan->file_insert_list;
1647 575 : phy_count = 0;
1648 575 : phy_dup = 0;
1649 575 : phy_last = FILEPHY_UNREAD_OFFSET;
1650 575 : phy_file_last = 0;
1651 103011 : while (node) {
1652 101861 : struct snapraid_file* file = node->data;
1653 :
1654 : /* if the file is not empty, count duplicate physical offsets */
1655 101861 : if (state->opt.force_order == SORT_PHYSICAL && file->size != 0) {
1656 11418 : if (phy_file_last != 0 && file->physical == phy_last
1657 : /* files without offset are expected to have duplicates */
1658 0 : && phy_last != FILEPHY_WITHOUT_OFFSET
1659 : ) {
1660 : /* if verbose, print the list of duplicates real offsets */
1661 : /* other cases are for offsets not supported, so we don't need to report them file by file */
1662 0 : if (phy_last >= FILEPHY_REAL_OFFSET) {
1663 0 : log_fatal("WARNING! Files '%s%s' and '%s%s' have the same physical offset %" PRId64 ".\n", disk->dir, phy_file_last->sub, disk->dir, file->sub, phy_last);
1664 : }
1665 0 : ++phy_dup;
1666 : }
1667 11418 : phy_file_last = file;
1668 11418 : phy_last = file->physical;
1669 11418 : ++phy_count;
1670 : }
1671 :
1672 : /* next node */
1673 101861 : node = node->next;
1674 :
1675 : /* insert in the parity */
1676 101861 : scan_file_allocate(scan, file);
1677 : }
1678 :
1679 : /* mark the disk without reliable physical offset if it has duplicates */
1680 : /* here it should never happen because we already sorted out hardlinks */
1681 575 : if (state->opt.force_order == SORT_PHYSICAL && phy_dup > 0) {
1682 0 : disk->has_unreliable_physical = 1;
1683 : }
1684 :
1685 : /* insert all the new links */
1686 575 : node = scan->link_insert_list;
1687 3607 : while (node) {
1688 2457 : struct snapraid_link* slink = node->data;
1689 :
1690 : /* next node */
1691 2457 : node = node->next;
1692 :
1693 : /* insert it */
1694 2457 : scan_link_insert(scan, slink);
1695 : }
1696 :
1697 : /* insert all the new dirs */
1698 575 : node = scan->dir_insert_list;
1699 1160 : while (node) {
1700 10 : struct snapraid_dir* dir = node->data;
1701 :
1702 : /* next node */
1703 10 : node = node->next;
1704 :
1705 : /* insert it */
1706 10 : scan_emptydir_insert(scan, dir);
1707 : }
1708 : }
1709 :
1710 : /* check for disks where all the previously existing files where removed */
1711 96 : if (!state->opt.force_empty) {
1712 90 : int all_missing = 0;
1713 90 : int all_rewritten = 0;
1714 90 : done = 0;
1715 629 : for (i = state->disklist, j = scanlist; i != 0; i = i->next, j = j->next) {
1716 539 : struct snapraid_disk* disk = i->data;
1717 539 : struct snapraid_scan* scan = j->data;
1718 :
1719 539 : if (scan->count_equal == 0
1720 74 : && scan->count_move == 0
1721 74 : && scan->count_restore == 0
1722 74 : && (scan->count_remove != 0 || scan->count_change != 0)
1723 : ) {
1724 1 : if (!done) {
1725 1 : done = 1;
1726 1 : log_fatal("WARNING! All the files previously present in disk '%s' at dir '%s'", disk->name, disk->dir);
1727 : } else {
1728 0 : log_fatal(", disk '%s' at dir '%s'", disk->name, disk->dir);
1729 : }
1730 :
1731 : /* detect the special condition of all files missing */
1732 1 : if (scan->count_change == 0)
1733 1 : all_missing = 1;
1734 :
1735 : /* detect the special condition of all files rewritten */
1736 1 : if (scan->count_remove == 0)
1737 0 : all_rewritten = 1;
1738 : }
1739 : }
1740 90 : if (done) {
1741 1 : log_fatal("\nare now missing or rewritten!\n");
1742 1 : if (all_rewritten) {
1743 0 : log_fatal("This could happen when restoring a disk with a backup\n");
1744 0 : log_fatal("program that is not setting correctly the timestamps.\n");
1745 : }
1746 1 : if (all_missing) {
1747 1 : log_fatal("This could happen when some disks are not mounted\n");
1748 1 : log_fatal("in the expected directory.\n");
1749 : }
1750 1 : if (!is_diff) {
1751 1 : log_fatal("If you want to '%s' anyway, use 'snapraid --force-empty %s'.\n", state->command, state->command);
1752 1 : exit(EXIT_FAILURE);
1753 : }
1754 : }
1755 : }
1756 :
1757 : /* check for disks without the physical offset support */
1758 95 : if (state->opt.force_order == SORT_PHYSICAL) {
1759 2 : done = 0;
1760 14 : for (i = state->disklist; i != 0; i = i->next) {
1761 12 : struct snapraid_disk* disk = i->data;
1762 :
1763 12 : if (disk->has_unreliable_physical) {
1764 0 : if (!done) {
1765 0 : done = 1;
1766 0 : log_fatal("WARNING! Physical offsets not supported for disk '%s'", disk->name);
1767 : } else {
1768 0 : log_fatal(", '%s'", disk->name);
1769 : }
1770 : }
1771 : }
1772 2 : if (done) {
1773 0 : log_fatal(". Files order won't be optimal.\n");
1774 : }
1775 : }
1776 :
1777 : /* check for disks without persistent inodes */
1778 95 : done = 0;
1779 664 : for (i = state->disklist; i != 0; i = i->next) {
1780 569 : struct snapraid_disk* disk = i->data;
1781 :
1782 569 : if (disk->has_volatile_inodes) {
1783 0 : if (!done) {
1784 0 : done = 1;
1785 0 : log_fatal("WARNING! Inodes are not persistent for disks: '%s'", disk->name);
1786 : } else {
1787 0 : log_fatal(", '%s'", disk->name);
1788 : }
1789 : }
1790 : }
1791 95 : if (done) {
1792 0 : log_fatal(". Not using inodes to detect move operations.\n");
1793 : }
1794 :
1795 : /* check for disks with changed UUID */
1796 95 : done = 0;
1797 664 : for (i = state->disklist; i != 0; i = i->next) {
1798 569 : struct snapraid_disk* disk = i->data;
1799 :
1800 : /* don't print the message if the UUID changed because before */
1801 : /* it was no set. */
1802 : /* this is the normal condition for an empty disk because it */
1803 : /* isn't stored */
1804 569 : if (disk->has_different_uuid && !disk->had_empty_uuid) {
1805 4 : if (!done) {
1806 2 : done = 1;
1807 2 : log_fatal("WARNING! UUID is changed for disks: '%s'", disk->name);
1808 : } else {
1809 2 : log_fatal(", '%s'", disk->name);
1810 : }
1811 : }
1812 : }
1813 95 : if (done) {
1814 2 : log_fatal(". Not using inodes to detect move operations.\n");
1815 : }
1816 :
1817 : /* check for disks with unsupported UUID */
1818 95 : done = 0;
1819 664 : for (i = state->disklist; i != 0; i = i->next) {
1820 569 : struct snapraid_disk* disk = i->data;
1821 :
1822 569 : if (disk->has_unsupported_uuid) {
1823 0 : if (!done) {
1824 0 : done = 1;
1825 0 : log_fatal("WARNING! UUID is unsupported for disks: '%s'", disk->name);
1826 : } else {
1827 0 : log_fatal(", '%s'", disk->name);
1828 : }
1829 : }
1830 : }
1831 95 : if (done) {
1832 0 : log_fatal(". Not using inodes to detect move operations.\n");
1833 : #if defined(_linux) && !HAVE_BLKID
1834 : log_fatal("The 'blkid' library is not linked in SnapRAID!\n");
1835 : log_fatal("Try rebuilding it after installing the libblkid-dev or libblkid-devel package.\n");
1836 : #endif
1837 : }
1838 :
1839 95 : total.count_equal = 0;
1840 95 : total.count_move = 0;
1841 95 : total.count_copy = 0;
1842 95 : total.count_restore = 0;
1843 95 : total.count_change = 0;
1844 95 : total.count_remove = 0;
1845 95 : total.count_insert = 0;
1846 :
1847 664 : for (i = scanlist; i != 0; i = i->next) {
1848 569 : struct snapraid_scan* scan = i->data;
1849 569 : total.count_equal += scan->count_equal;
1850 569 : total.count_move += scan->count_move;
1851 569 : total.count_copy += scan->count_copy;
1852 569 : total.count_restore += scan->count_restore;
1853 569 : total.count_change += scan->count_change;
1854 569 : total.count_remove += scan->count_remove;
1855 569 : total.count_insert += scan->count_insert;
1856 : }
1857 :
1858 95 : if (is_diff) {
1859 8 : msg_status("\n");
1860 8 : msg = msg_status;
1861 : } else {
1862 87 : msg = msg_verbose;
1863 : }
1864 :
1865 95 : msg("%8u equal\n", total.count_equal);
1866 95 : msg("%8u added\n", total.count_insert);
1867 95 : msg("%8u removed\n", total.count_remove);
1868 95 : msg("%8u updated\n", total.count_change);
1869 95 : msg("%8u moved\n", total.count_move);
1870 95 : msg("%8u copied\n", total.count_copy);
1871 95 : msg("%8u restored\n", total.count_restore);
1872 :
1873 95 : log_tag("summary:equal:%u\n", total.count_equal);
1874 95 : log_tag("summary:added:%u\n", total.count_insert);
1875 95 : log_tag("summary:removed:%u\n", total.count_remove);
1876 95 : log_tag("summary:updated:%u\n", total.count_change);
1877 95 : log_tag("summary:moved:%u\n", total.count_move);
1878 95 : log_tag("summary:copied:%u\n", total.count_copy);
1879 95 : log_tag("summary:restored:%u\n", total.count_restore);
1880 :
1881 284 : no_difference = !total.count_move && !total.count_copy && !total.count_restore
1882 159 : && !total.count_change && !total.count_remove && !total.count_insert;
1883 :
1884 95 : if (is_diff) {
1885 8 : if (no_difference) {
1886 2 : msg_status("No differences\n");
1887 : } else {
1888 6 : msg_status("There are differences!\n");
1889 : }
1890 : }
1891 :
1892 95 : if (no_difference) {
1893 20 : log_tag("summary:exit:equal\n");
1894 : } else {
1895 75 : log_tag("summary:exit:diff\n");
1896 : }
1897 95 : log_flush();
1898 :
1899 95 : tommy_list_foreach(&scanlist, (tommy_foreach_func*)free);
1900 :
1901 : /* check the file-system on all disks */
1902 95 : state_fscheck(state, "after scan");
1903 :
1904 95 : if (is_diff) {
1905 : /* check for file difference */
1906 8 : if (!no_difference)
1907 6 : return 1;
1908 :
1909 : /* check also for incomplete "sync" */
1910 2 : if (parity_is_invalid(state))
1911 1 : return 1;
1912 : }
1913 :
1914 88 : return 0;
1915 : }
1916 :
1917 8 : int state_diff(struct snapraid_state* state)
1918 : {
1919 8 : return state_diffscan(state, 1);
1920 : }
1921 :
1922 88 : void state_scan(struct snapraid_state* state)
1923 : {
1924 88 : (void)state_diffscan(state, 0); /* ignore return value */
1925 87 : }
1926 :
|
