diff --git a/Makefile.in b/Makefile.in index c41e6b170..3f236786b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -562,25 +562,16 @@ nasm.spec: nasm.spec.in nasm.spec.sed version.sed perlbreq.si splint: splint -weak *.c -.PHONY: test -test: $(PROGS) - cd $(srcdir)/test && \ - $(RUNPERL) performtest.pl --nasm=../nasm$(X) *.asm - -golden: $(PROGS) - cd $(srcdir)/test && \ - $(RUNPERL) performtest.pl --golden --nasm=../nasm$(X) *.asm - # # Travis tests # -travis: $(PROGS) +travis test: $(PROGS) $(MAKE) -f travis.mk all clean-travis travis-clean: $(MAKE) -f travis.mk clean -update-travis travis-update: +update-travis travis-update golden: $(MAKE) -f travis.mk update # diff --git a/test/Makefile.in b/test/Makefile.in index fee7dde69..36aeed6fd 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -118,15 +118,6 @@ cryptography-primitivestest: all: -golden: performtest.pl $(TESTS) - $(PERL) performtest.pl --golden --nasm='$(NASM)' $(TESTS) - -test: performtest.pl $(NASM) $(TESTS) - $(PERL) performtest.pl --nasm='$(NASM)' $(TESTS) - -diff: performtest.pl $(NASM) $(TESTS) - $(PERL) performtest.pl --diff --nasm='$(NASM)' $(TESTS) - clean: $(RM_F) *.com *.o *.o64 *.aout *.obj *.win32 *.win64 *.exe *.lst *.bin $(RM_F) *.bin *.bin16 *.bin32 *.bin64 diff --git a/test/bisect.sh b/test/bisect.sh deleted file mode 100755 index 98bdb37d2..000000000 --- a/test/bisect.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -# Usage: - -# Make a test and a golden file, read ./performtest.pl --help - -# cd nasm -# cp -r test somewhere (copy test dir out of the tree) -# git bisect start HEAD nasm-2.07 (where HEAD is bad and nasm-2.07 is good) -# git bisect run somewhere/test/bisect.sh br2148476 (what you want to test) - -# Done - - -# Slow but sure -./autogen.sh -./configure -make - -NASMDIR=$(pwd) -cd $(dirname "$0") -./performtest.pl "--nasm=$NASMDIR/nasm" "$1.asm" --verbose diff --git a/test/performtest.pl b/test/performtest.pl deleted file mode 100755 index 2426848fb..000000000 --- a/test/performtest.pl +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/perl -#Perform tests on nasm - -use strict; -use warnings; - -use Getopt::Long qw(GetOptions); -use Pod::Usage qw(pod2usage); - -use File::Basename qw(fileparse); -use File::Compare qw(compare compare_text); -use File::Copy qw(move); -use File::Path qw(mkpath rmtree); - -#sub debugprint { print (pop() . "\n"); } - sub debugprint { } - -my $globalresult = 0; - -#Process one testfile -sub perform { - my ($clean, $diff, $golden, $nasm, $quiet, $testpath) = @_; - my ($stdoutfile, $stderrfile) = ("stdout", "stderr"); - - my ($testname, $ignoredpath, $ignoredsuffix) = fileparse($testpath, ".asm"); - debugprint $testname; - - my $outputdir = $golden ? "golden" : "testresults"; - - mkdir "$outputdir" unless -d "$outputdir"; - - if ($clean) { - rmtree "$outputdir/$testname"; - return; - } - - if(-d "$outputdir/$testname") { - rmtree "$outputdir/$testname"; - } - - open(TESTFILE, '<', $testpath) or (warn "Can't open $testpath\n", return); - TEST: - while() { - #See if there is a test case - last unless /Testname=(.*);\s*Arguments=(.*);\s*Files=([^;]*)(?:;\s*Validate=(.*))?/; - my ($subname, $arguments, $files, $validate) = ($1, $2, $3, $4); - chomp $files; - debugprint("$subname | $arguments | $files"); - - #Call nasm with this test case - system("$nasm $arguments $testpath > $stdoutfile 2> $stderrfile"); - debugprint("$nasm $arguments $testpath > $stdoutfile 2> $stderrfile ----> $?"); - - if($validate) { - if(system("$validate >> $stdoutfile 2>> $stderrfile") != 0) { - print "Test $testname/$subname validation failed\n"; - $globalresult = 1; - } - } - - #Move the output to the test dir - mkpath("$outputdir/$testname/$subname"); - foreach(split / /,$files) { - if (-f $_) { - move($_, "$outputdir/$testname/$subname/$_") or die $! - } - } - unlink ("$stdoutfile", "$stderrfile"); #Just to be sure - - if($golden) { - print "Test $testname/$subname created.\n" unless $quiet; - } else { - #Compare them with the golden files - my $result = 0; - my @failedfiles = (); - foreach(split / /, $files) { - if(-f "$outputdir/$testname/$subname/$_") { - my $temp; - if($_ eq $stdoutfile or $_ eq $stderrfile) { - #Compare stdout and stderr in text mode so line ending changes won't matter - $temp = compare_text("$outputdir/$testname/$subname/$_", "golden/$testname/$subname/$_", - sub { my ($a, $b) = @_; - $a =~ s/\r//g; - $b =~ s/\r//g; - $a ne $b; } ); - } else { - $temp = compare("$outputdir/$testname/$subname/$_", "golden/$testname/$subname/$_"); - } - - if($temp == 1) { - #different - $result = 1; - $globalresult = 1; - push @failedfiles, $_; - } elsif($temp == -1) { - #error - print "Can't compare at $testname/$subname file $_\n"; - next TEST; - } - } elsif (-f "golden/$testname/$subname/$_") { - #File exists in golden but not in output - $result = 1; - $globalresult = 1; - push @failedfiles, $_; - } - } - - if($result == 0) { - print "Test $testname/$subname succeeded.\n" unless $quiet; - } elsif ($result == 1) { - print "Test $testname/$subname failed on @failedfiles.\n"; - if($diff) { - for(@failedfiles) { - if($_ eq $stdoutfile or $_ eq $stderrfile) { - system "diff -u golden/$testname/$subname/$_ $outputdir/$testname/$subname/$_"; - print "\n"; - } - } - } - } else { - die "Impossible result"; - } - } - } - close(TESTFILE); -} - -my $nasm; -my $clean = 0; -my $diff = 0; -my $golden = 0; -my $help = 0; -my $verbose = 0; - -GetOptions('clean' => \$clean, - 'diff'=> \$diff, - 'golden' => \$golden, - 'help' => \$help, - 'verbose' => \$verbose, - 'nasm=s' => \$nasm - ) or pod2usage(); - -pod2usage() if $help; -die "Please specify either --nasm or --clean. Use --help for help.\n" -unless $nasm or $clean; -die "Please specify the test files, e.g. *.asm\n" unless @ARGV; - -unless (!defined $nasm or -x $nasm) { - warn "Warning: $nasm may not be executable. Expect problems.\n\n"; - sleep 5; -} - -perform($clean, $diff, $golden, $nasm, ! $verbose, $_) foreach @ARGV; -exit $globalresult; - -__END__ - -=head1 NAME - -performtest.pl - NASM regression tester based on golden files - -=head1 SYNOPSIS - -performtest.pl [options] [testfile.asm ...] - -Runs NASM on the specified test files and compare the results -with "golden" output files. - - Options: - --clean Clean up test results (or golden files with --golden) - --diff Execute diff when stdout or stderr don't match - --golden Create golden files - --help Get this help - --nasm=file Specify the file name for the NASM executable, e.g. ../nasm - --verbose Get more output - - If --clean is not specified, --nasm is required. - - testfile.asm ...: - One or more files that NASM should be tested with, - often *.asm in the test directory. - It should contain one or more option lines at the start, - in the following format: - -;Testname=; Arguments=; Files= - - If no such lines are found at the start, the file is skipped. - testname should ideally describe the arguments, eg. unoptimized for -O0. - arguments can be an optimization level (-O), an output format (-f), - an output file specifier (-o) etc. - The output files should be a space separated list of files that will - be checked for regressions. This should often be the output file - and the special files stdout and stderr. - -Any mismatch could be a regression, -but it doesn't have to be. COFF files have a timestamp which -makes this method useless. ELF files have a comment section -with the current version of NASM, so they will change each version number. - -=cut