proptest/test_runner/failure_persistence/
file.rs

1//-
2// Copyright 2017, 2018, 2019 The proptest developers
3//
4// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
5// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7// option. This file may not be copied, modified, or distributed
8// except according to those terms.
9
10use core::any::Any;
11use core::fmt::Debug;
12use std::borrow::{Cow, ToOwned};
13use std::boxed::Box;
14use std::env;
15use std::fs;
16use std::io::{self, BufRead, Write};
17use std::path::{Path, PathBuf};
18use std::string::{String, ToString};
19use std::sync::RwLock;
20use std::vec::Vec;
21
22use self::FileFailurePersistence::*;
23use crate::test_runner::failure_persistence::{
24    FailurePersistence, PersistedSeed,
25};
26
27/// Describes how failing test cases are persisted.
28///
29/// Note that file names in this enum are `&str` rather than `&Path` since
30/// constant functions are not yet in Rust stable as of 2017-12-16.
31///
32/// In all cases, if a derived path references a directory which does not yet
33/// exist, proptest will attempt to create all necessary parent directories.
34#[derive(Clone, Copy, Debug, PartialEq)]
35pub enum FileFailurePersistence {
36    /// Completely disables persistence of failing test cases.
37    ///
38    /// This is semantically equivalent to `Direct("/dev/null")` on Unix and
39    /// `Direct("NUL")` on Windows (though it is internally handled by simply
40    /// not doing any I/O).
41    Off,
42    /// The path of the source file under test is traversed up the directory tree
43    /// until a directory containing a file named `lib.rs` or `main.rs` is found.
44    /// A sibling to that directory with the name given by the string in this
45    /// configuration is created, and a file with the same name and path relative
46    /// to the source directory, but with the extension changed to `.txt`, is used.
47    ///
48    /// For example, given a source path of
49    /// `/home/jsmith/code/project/src/foo/bar.rs` and a configuration of
50    /// `SourceParallel("proptest-regressions")` (the default), assuming the
51    /// `src` directory has a `lib.rs` or `main.rs`, the resulting file would
52    /// be `/home/jsmith/code/project/proptest-regressions/foo/bar.txt`.
53    ///
54    /// If no `lib.rs` or `main.rs` can be found, a warning is printed and this
55    /// behaves like `WithSource`.
56    ///
57    /// If no source file has been configured, a warning is printed and this
58    /// behaves like `Off`.
59    SourceParallel(&'static str),
60    /// Failures are persisted in a file with the same path as the source file
61    /// under test, but the extension is changed to the string given in this
62    /// configuration.
63    ///
64    /// For example, given a source path of
65    /// `/home/jsmith/code/project/src/foo/bar.rs` and a configuration of
66    /// `WithSource("regressions")`, the resulting path would be
67    /// `/home/jsmith/code/project/src/foo/bar.regressions`.
68    WithSource(&'static str),
69    /// The string given in this option is directly used as a file path without
70    /// any further processing.
71    Direct(&'static str),
72    #[doc(hidden)]
73    #[allow(missing_docs)]
74    _NonExhaustive,
75}
76
77impl Default for FileFailurePersistence {
78    fn default() -> Self {
79        SourceParallel("proptest-regressions")
80    }
81}
82
83impl FailurePersistence for FileFailurePersistence {
84    fn load_persisted_failures2(
85        &self,
86        source_file: Option<&'static str>,
87    ) -> Vec<PersistedSeed> {
88        let p = self.resolve(
89            source_file
90                .and_then(|s| absolutize_source_file(Path::new(s)))
91                .as_ref()
92                .map(|cow| &**cow),
93        );
94
95        let path: Option<&PathBuf> = p.as_ref();
96        let result: io::Result<Vec<PersistedSeed>> = path.map_or_else(
97            || Ok(vec![]),
98            |path| {
99                // .ok() instead of .unwrap() so we don't propagate panics here
100                let _lock = PERSISTENCE_LOCK.read().ok();
101                io::BufReader::new(fs::File::open(path)?)
102                    .lines()
103                    .enumerate()
104                    .filter_map(|(lineno, line)| match line {
105                        Err(err) => Some(Err(err)),
106                        Ok(line) => parse_seed_line(line, path, lineno).map(Ok),
107                    })
108                    .collect()
109            },
110        );
111
112        unwrap_or!(result, err => {
113            if io::ErrorKind::NotFound != err.kind() {
114                eprintln!(
115                    "proptest: failed to open {}: {}",
116                    &path.map(|x| &**x)
117                        .unwrap_or_else(|| Path::new("??"))
118                        .display(),
119                    err
120                );
121            }
122            vec![]
123        })
124    }
125
126    fn save_persisted_failure2(
127        &mut self,
128        source_file: Option<&'static str>,
129        seed: PersistedSeed,
130        shrunken_value: &dyn Debug,
131    ) {
132        let path = self.resolve(source_file.map(Path::new));
133        if let Some(path) = path {
134            // .ok() instead of .unwrap() so we don't propagate panics here
135            let _lock = PERSISTENCE_LOCK.write().ok();
136            let is_new = !path.is_file();
137
138            let mut to_write = Vec::<u8>::new();
139            if is_new {
140                write_header(&mut to_write)
141                    .expect("proptest: couldn't write header.");
142            }
143
144            write_seed_line(&mut to_write, &seed, shrunken_value)
145                .expect("proptest: couldn't write seed line.");
146
147            if let Err(e) = write_seed_data_to_file(&path, &to_write) {
148                eprintln!(
149                    "proptest: failed to append to {}: {}",
150                    path.display(),
151                    e
152                );
153            } else {
154                eprintln!(
155                    "proptest: Saving this and future failures in {}\n\
156                     proptest: If this test was run on a CI system, you may \
157                     wish to add the following line to your copy of the file.{}\n\
158                     {}",
159                    path.display(),
160                    if is_new { " (You may need to create it.)" } else { "" },
161                    seed);
162            }
163        }
164    }
165
166    fn box_clone(&self) -> Box<dyn FailurePersistence> {
167        Box::new(*self)
168    }
169
170    fn eq(&self, other: &dyn FailurePersistence) -> bool {
171        other
172            .as_any()
173            .downcast_ref::<Self>()
174            .map_or(false, |x| x == self)
175    }
176
177    fn as_any(&self) -> &dyn Any {
178        self
179    }
180}
181
182/// Ensure that the source file to use for resolving the location of the persisted
183/// failing cases file is absolute.
184///
185/// The source location can only be used if it is absolute. If `source` is
186/// not an absolute path, an attempt will be made to determine the absolute
187/// path based on the current working directory and its parents. If no
188/// absolute path can be determined, a warning will be printed and proptest
189/// will continue as if this function had never been called.
190///
191/// See [`FileFailurePersistence`](enum.FileFailurePersistence.html) for details on
192/// how this value is used once it is made absolute.
193///
194/// This is normally called automatically by the `proptest!` macro, which
195/// passes `file!()`.
196///
197fn absolutize_source_file<'a>(source: &'a Path) -> Option<Cow<'a, Path>> {
198    absolutize_source_file_with_cwd(env::current_dir, source)
199}
200
201fn absolutize_source_file_with_cwd<'a>(
202    getcwd: impl FnOnce() -> io::Result<PathBuf>,
203    source: &'a Path,
204) -> Option<Cow<'a, Path>> {
205    if source.is_absolute() {
206        // On Unix, `file!()` is absolute. In these cases, we can use
207        // that path directly.
208        Some(Cow::Borrowed(source))
209    } else {
210        // On Windows, `file!()` is relative to the crate root, but the
211        // test is not generally run with the crate root as the working
212        // directory, so the path is not directly usable. However, the
213        // working directory is almost always a subdirectory of the crate
214        // root, so pop directories off until pushing the source onto the
215        // directory results in a path that refers to an existing file.
216        // Once we find such a path, we can use that.
217        //
218        // If we can't figure out an absolute path, print a warning and act
219        // as if no source had been given.
220        match getcwd() {
221            Ok(mut cwd) => loop {
222                let joined = cwd.join(source);
223                if joined.is_file() {
224                    break Some(Cow::Owned(joined));
225                }
226
227                if !cwd.pop() {
228                    eprintln!(
229                        "proptest: Failed to find absolute path of \
230                         source file '{:?}'. Ensure the test is \
231                         being run from somewhere within the crate \
232                         directory hierarchy.",
233                        source
234                    );
235                    break None;
236                }
237            },
238
239            Err(e) => {
240                eprintln!(
241                    "proptest: Failed to determine current \
242                     directory, so the relative source path \
243                     '{:?}' cannot be resolved: {}",
244                    source, e
245                );
246                None
247            }
248        }
249    }
250}
251
252fn parse_seed_line(
253    mut line: String,
254    path: &Path,
255    lineno: usize,
256) -> Option<PersistedSeed> {
257    // Remove anything after and including '#':
258    if let Some(comment_start) = line.find('#') {
259        line.truncate(comment_start);
260    }
261
262    if line.len() > 0 {
263        let ret = line.parse::<PersistedSeed>().ok();
264        if !ret.is_some() {
265            eprintln!(
266                "proptest: {}:{}: unparsable line, ignoring",
267                path.display(),
268                lineno + 1
269            );
270        }
271        return ret;
272    }
273
274    None
275}
276
277fn write_seed_line(
278    buf: &mut Vec<u8>,
279    seed: &PersistedSeed,
280    shrunken_value: &dyn Debug,
281) -> io::Result<()> {
282    // Write the seed itself
283    write!(buf, "{}", seed.to_string())?;
284
285    // Write out comment:
286    let debug_start = buf.len();
287    write!(buf, " # shrinks to {:?}", shrunken_value)?;
288
289    // Ensure there are no newlines in the debug output
290    for byte in &mut buf[debug_start..] {
291        if b'\n' == *byte || b'\r' == *byte {
292            *byte = b' ';
293        }
294    }
295
296    buf.push(b'\n');
297
298    Ok(())
299}
300
301fn write_header(buf: &mut Vec<u8>) -> io::Result<()> {
302    writeln!(
303        buf,
304        "\
305# Seeds for failure cases proptest has generated in the past. It is
306# automatically read and these particular cases re-run before any
307# novel cases are generated.
308#
309# It is recommended to check this file in to source control so that
310# everyone who runs the test benefits from these saved cases."
311    )
312}
313
314fn write_seed_data_to_file(dst: &Path, data: &[u8]) -> io::Result<()> {
315    if let Some(parent) = dst.parent() {
316        fs::create_dir_all(parent)?;
317    }
318
319    let mut options = fs::OpenOptions::new();
320    options.append(true).create(true);
321    let mut out = options.open(dst)?;
322    out.write_all(data)?;
323
324    Ok(())
325}
326
327impl FileFailurePersistence {
328    /// Given the nominal source path, determine the location of the failure
329    /// persistence file, if any.
330    pub(super) fn resolve(&self, source: Option<&Path>) -> Option<PathBuf> {
331        let source = source.and_then(absolutize_source_file);
332
333        match *self {
334            Off => None,
335
336            SourceParallel(sibling) => match source {
337                Some(source_path) => {
338                    let mut dir = Cow::into_owned(source_path.clone());
339                    let mut found = false;
340                    while dir.pop() {
341                        if dir.join("lib.rs").is_file()
342                            || dir.join("main.rs").is_file()
343                        {
344                            found = true;
345                            break;
346                        }
347                    }
348
349                    if !found {
350                        eprintln!(
351                            "proptest: FileFailurePersistence::SourceParallel set, \
352                             but failed to find lib.rs or main.rs"
353                        );
354                        WithSource(sibling).resolve(Some(&*source_path))
355                    } else {
356                        let suffix = source_path
357                            .strip_prefix(&dir)
358                            .expect("parent of source is not a prefix of it?")
359                            .to_owned();
360                        let mut result = dir;
361                        // If we've somehow reached the root, or someone gave
362                        // us a relative path that we've exhausted, just accept
363                        // creating a subdirectory instead.
364                        let _ = result.pop();
365                        result.push(sibling);
366                        result.push(&suffix);
367                        result.set_extension("txt");
368                        Some(result)
369                    }
370                }
371                None => {
372                    eprintln!(
373                        "proptest: FileFailurePersistence::SourceParallel set, \
374                         but no source file known"
375                    );
376                    None
377                }
378            },
379
380            WithSource(extension) => match source {
381                Some(source_path) => {
382                    let mut result = Cow::into_owned(source_path);
383                    result.set_extension(extension);
384                    Some(result)
385                }
386
387                None => {
388                    eprintln!(
389                        "proptest: FileFailurePersistence::WithSource set, \
390                         but no source file known"
391                    );
392                    None
393                }
394            },
395
396            Direct(path) => Some(Path::new(path).to_owned()),
397
398            _NonExhaustive => {
399                panic!("FailurePersistence set to _NonExhaustive")
400            }
401        }
402    }
403}
404
405lazy_static! {
406    /// Used to guard access to the persistence file(s) so that a single
407    /// process will not step on its own toes.
408    ///
409    /// We don't have much protecting us should two separate process try to
410    /// write to the same file at once (depending on how atomic append mode is
411    /// on the OS), but this should be extremely rare.
412    static ref PERSISTENCE_LOCK: RwLock<()> = RwLock::new(());
413}
414
415#[cfg(test)]
416mod tests {
417    use super::*;
418
419    struct TestPaths {
420        crate_root: &'static Path,
421        src_file: PathBuf,
422        subdir_file: PathBuf,
423        misplaced_file: PathBuf,
424    }
425
426    lazy_static! {
427        static ref TEST_PATHS: TestPaths = {
428            let crate_root = Path::new(env!("CARGO_MANIFEST_DIR"));
429            let lib_root = crate_root.join("src");
430            let src_subdir = lib_root.join("strategy");
431            let src_file = lib_root.join("foo.rs");
432            let subdir_file = src_subdir.join("foo.rs");
433            let misplaced_file = crate_root.join("foo.rs");
434            TestPaths {
435                crate_root,
436                src_file,
437                subdir_file,
438                misplaced_file,
439            }
440        };
441    }
442
443    #[test]
444    fn persistence_file_location_resolved_correctly() {
445        // If off, there is never a file
446        assert_eq!(None, Off.resolve(None));
447        assert_eq!(None, Off.resolve(Some(&TEST_PATHS.subdir_file)));
448
449        // For direct, we don't care about the source file, and instead always
450        // use whatever is in the config.
451        assert_eq!(
452            Some(Path::new("bar.txt").to_owned()),
453            Direct("bar.txt").resolve(None)
454        );
455        assert_eq!(
456            Some(Path::new("bar.txt").to_owned()),
457            Direct("bar.txt").resolve(Some(&TEST_PATHS.subdir_file))
458        );
459
460        // For WithSource, only the extension changes, but we get nothing if no
461        // source file was configured.
462        // Accounting for the way absolute paths work on Windows would be more
463        // complex, so for now don't test that case.
464        #[cfg(unix)]
465        fn absolute_path_case() {
466            assert_eq!(
467                Some(Path::new("/foo/bar.ext").to_owned()),
468                WithSource("ext").resolve(Some(Path::new("/foo/bar.rs")))
469            );
470        }
471        #[cfg(not(unix))]
472        fn absolute_path_case() {}
473        absolute_path_case();
474        assert_eq!(None, WithSource("ext").resolve(None));
475
476        // For SourceParallel, we make a sibling directory tree and change the
477        // extensions to .txt ...
478        assert_eq!(
479            Some(TEST_PATHS.crate_root.join("sib").join("foo.txt")),
480            SourceParallel("sib").resolve(Some(&TEST_PATHS.src_file))
481        );
482        assert_eq!(
483            Some(
484                TEST_PATHS
485                    .crate_root
486                    .join("sib")
487                    .join("strategy")
488                    .join("foo.txt")
489            ),
490            SourceParallel("sib").resolve(Some(&TEST_PATHS.subdir_file))
491        );
492        // ... but if we can't find lib.rs / main.rs, give up and set the
493        // extension instead ...
494        assert_eq!(
495            Some(TEST_PATHS.crate_root.join("foo.sib")),
496            SourceParallel("sib").resolve(Some(&TEST_PATHS.misplaced_file))
497        );
498        // ... and if no source is configured, we do nothing
499        assert_eq!(None, SourceParallel("ext").resolve(None));
500    }
501
502    #[test]
503    fn relative_source_files_absolutified() {
504        const TEST_RUNNER_PATH: &[&str] = &["src", "test_runner", "mod.rs"];
505        lazy_static! {
506            static ref TEST_RUNNER_RELATIVE: PathBuf =
507                TEST_RUNNER_PATH.iter().collect();
508        }
509        const CARGO_DIR: &str = env!("CARGO_MANIFEST_DIR");
510
511        let expected = ::std::iter::once(CARGO_DIR)
512            .chain(TEST_RUNNER_PATH.iter().map(|s| *s))
513            .collect::<PathBuf>();
514
515        // Running from crate root
516        assert_eq!(
517            &*expected,
518            absolutize_source_file_with_cwd(
519                || Ok(Path::new(CARGO_DIR).to_owned()),
520                &TEST_RUNNER_RELATIVE
521            )
522            .unwrap()
523        );
524
525        // Running from test subdirectory
526        assert_eq!(
527            &*expected,
528            absolutize_source_file_with_cwd(
529                || Ok(Path::new(CARGO_DIR).join("target")),
530                &TEST_RUNNER_RELATIVE
531            )
532            .unwrap()
533        );
534    }
535}
proptest/test_runner/failure_persistence/file.rs

proptest/test_runner/failure_persistence/
file.rs
