mirror of
https://github.com/netwide-assembler/nasm
synced 2026-08-26 16:23:04 -04:00
test: remove test/performtest.pl and redirect {test,golden} targets
Remove the long-since-unmaintained test/performtest.pl script and redirect the "test" and "golden" Makefile targets to their respective travis targets. The test/ directory still has two important functions: 1. Running ad hoc tests manually. The test/ directory has infrastructure for running a quick test manually in a large number of configurations. This is highly useful during development. 2. It contains the infrastructure for running regression tests on external applications, far too big to include into NASM itself. Reported-by: Ross Burton <ross@burtonini.com> Signed-off-by: H. Peter Anvin (Intel) <hpa@zytor.com>
This commit is contained in:
H. Peter Anvin (Intel)
parent
c0b64a692b
commit
75024b8109
4 changed files with 2 additions and 242 deletions
13
Makefile.in
13
Makefile.in
|
|
@ -562,25 +562,16 @@ nasm.spec: nasm.spec.in nasm.spec.sed version.sed perlbreq.si
|
|||
splint:
|
||||
splint -weak *.c
|
||||
|
||||
.PHONY: test
|
||||
test: $(PROGS)
|
||||
cd $(srcdir)/test && \
|
||||
$(RUNPERL) performtest.pl --nasm=../nasm$(X) *.asm
|
||||
|
||||
golden: $(PROGS)
|
||||
cd $(srcdir)/test && \
|
||||
$(RUNPERL) performtest.pl --golden --nasm=../nasm$(X) *.asm
|
||||
|
||||
#
|
||||
# Travis tests
|
||||
#
|
||||
travis: $(PROGS)
|
||||
travis test: $(PROGS)
|
||||
$(MAKE) -f travis.mk all
|
||||
|
||||
clean-travis travis-clean:
|
||||
$(MAKE) -f travis.mk clean
|
||||
|
||||
update-travis travis-update:
|
||||
update-travis travis-update golden:
|
||||
$(MAKE) -f travis.mk update
|
||||
|
||||
#
|
||||
|
|
|
|||
|
|
@ -118,15 +118,6 @@ cryptography-primitivestest:
|
|||
|
||||
all:
|
||||
|
||||
golden: performtest.pl $(TESTS)
|
||||
$(PERL) performtest.pl --golden --nasm='$(NASM)' $(TESTS)
|
||||
|
||||
test: performtest.pl $(NASM) $(TESTS)
|
||||
$(PERL) performtest.pl --nasm='$(NASM)' $(TESTS)
|
||||
|
||||
diff: performtest.pl $(NASM) $(TESTS)
|
||||
$(PERL) performtest.pl --diff --nasm='$(NASM)' $(TESTS)
|
||||
|
||||
clean:
|
||||
$(RM_F) *.com *.o *.o64 *.aout *.obj *.win32 *.win64 *.exe *.lst *.bin
|
||||
$(RM_F) *.bin *.bin16 *.bin32 *.bin64
|
||||
|
|
|
|||
|
|
@ -1,22 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
# Usage:
|
||||
|
||||
# Make a test and a golden file, read ./performtest.pl --help
|
||||
|
||||
# cd nasm
|
||||
# cp -r test somewhere (copy test dir out of the tree)
|
||||
# git bisect start HEAD nasm-2.07 (where HEAD is bad and nasm-2.07 is good)
|
||||
# git bisect run somewhere/test/bisect.sh br2148476 (what you want to test)
|
||||
|
||||
# Done
|
||||
|
||||
|
||||
# Slow but sure
|
||||
./autogen.sh
|
||||
./configure
|
||||
make
|
||||
|
||||
NASMDIR=$(pwd)
|
||||
cd $(dirname "$0")
|
||||
./performtest.pl "--nasm=$NASMDIR/nasm" "$1.asm" --verbose
|
||||
|
|
@ -1,200 +0,0 @@
|
|||
#!/usr/bin/perl
|
||||
#Perform tests on nasm
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
|
||||
use Getopt::Long qw(GetOptions);
|
||||
use Pod::Usage qw(pod2usage);
|
||||
|
||||
use File::Basename qw(fileparse);
|
||||
use File::Compare qw(compare compare_text);
|
||||
use File::Copy qw(move);
|
||||
use File::Path qw(mkpath rmtree);
|
||||
|
||||
#sub debugprint { print (pop() . "\n"); }
|
||||
sub debugprint { }
|
||||
|
||||
my $globalresult = 0;
|
||||
|
||||
#Process one testfile
|
||||
sub perform {
|
||||
my ($clean, $diff, $golden, $nasm, $quiet, $testpath) = @_;
|
||||
my ($stdoutfile, $stderrfile) = ("stdout", "stderr");
|
||||
|
||||
my ($testname, $ignoredpath, $ignoredsuffix) = fileparse($testpath, ".asm");
|
||||
debugprint $testname;
|
||||
|
||||
my $outputdir = $golden ? "golden" : "testresults";
|
||||
|
||||
mkdir "$outputdir" unless -d "$outputdir";
|
||||
|
||||
if ($clean) {
|
||||
rmtree "$outputdir/$testname";
|
||||
return;
|
||||
}
|
||||
|
||||
if(-d "$outputdir/$testname") {
|
||||
rmtree "$outputdir/$testname";
|
||||
}
|
||||
|
||||
open(TESTFILE, '<', $testpath) or (warn "Can't open $testpath\n", return);
|
||||
TEST:
|
||||
while(<TESTFILE>) {
|
||||
#See if there is a test case
|
||||
last unless /Testname=(.*);\s*Arguments=(.*);\s*Files=([^;]*)(?:;\s*Validate=(.*))?/;
|
||||
my ($subname, $arguments, $files, $validate) = ($1, $2, $3, $4);
|
||||
chomp $files;
|
||||
debugprint("$subname | $arguments | $files");
|
||||
|
||||
#Call nasm with this test case
|
||||
system("$nasm $arguments $testpath > $stdoutfile 2> $stderrfile");
|
||||
debugprint("$nasm $arguments $testpath > $stdoutfile 2> $stderrfile ----> $?");
|
||||
|
||||
if($validate) {
|
||||
if(system("$validate >> $stdoutfile 2>> $stderrfile") != 0) {
|
||||
print "Test $testname/$subname validation failed\n";
|
||||
$globalresult = 1;
|
||||
}
|
||||
}
|
||||
|
||||
#Move the output to the test dir
|
||||
mkpath("$outputdir/$testname/$subname");
|
||||
foreach(split / /,$files) {
|
||||
if (-f $_) {
|
||||
move($_, "$outputdir/$testname/$subname/$_") or die $!
|
||||
}
|
||||
}
|
||||
unlink ("$stdoutfile", "$stderrfile"); #Just to be sure
|
||||
|
||||
if($golden) {
|
||||
print "Test $testname/$subname created.\n" unless $quiet;
|
||||
} else {
|
||||
#Compare them with the golden files
|
||||
my $result = 0;
|
||||
my @failedfiles = ();
|
||||
foreach(split / /, $files) {
|
||||
if(-f "$outputdir/$testname/$subname/$_") {
|
||||
my $temp;
|
||||
if($_ eq $stdoutfile or $_ eq $stderrfile) {
|
||||
#Compare stdout and stderr in text mode so line ending changes won't matter
|
||||
$temp = compare_text("$outputdir/$testname/$subname/$_", "golden/$testname/$subname/$_",
|
||||
sub { my ($a, $b) = @_;
|
||||
$a =~ s/\r//g;
|
||||
$b =~ s/\r//g;
|
||||
$a ne $b; } );
|
||||
} else {
|
||||
$temp = compare("$outputdir/$testname/$subname/$_", "golden/$testname/$subname/$_");
|
||||
}
|
||||
|
||||
if($temp == 1) {
|
||||
#different
|
||||
$result = 1;
|
||||
$globalresult = 1;
|
||||
push @failedfiles, $_;
|
||||
} elsif($temp == -1) {
|
||||
#error
|
||||
print "Can't compare at $testname/$subname file $_\n";
|
||||
next TEST;
|
||||
}
|
||||
} elsif (-f "golden/$testname/$subname/$_") {
|
||||
#File exists in golden but not in output
|
||||
$result = 1;
|
||||
$globalresult = 1;
|
||||
push @failedfiles, $_;
|
||||
}
|
||||
}
|
||||
|
||||
if($result == 0) {
|
||||
print "Test $testname/$subname succeeded.\n" unless $quiet;
|
||||
} elsif ($result == 1) {
|
||||
print "Test $testname/$subname failed on @failedfiles.\n";
|
||||
if($diff) {
|
||||
for(@failedfiles) {
|
||||
if($_ eq $stdoutfile or $_ eq $stderrfile) {
|
||||
system "diff -u golden/$testname/$subname/$_ $outputdir/$testname/$subname/$_";
|
||||
print "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
die "Impossible result";
|
||||
}
|
||||
}
|
||||
}
|
||||
close(TESTFILE);
|
||||
}
|
||||
|
||||
my $nasm;
|
||||
my $clean = 0;
|
||||
my $diff = 0;
|
||||
my $golden = 0;
|
||||
my $help = 0;
|
||||
my $verbose = 0;
|
||||
|
||||
GetOptions('clean' => \$clean,
|
||||
'diff'=> \$diff,
|
||||
'golden' => \$golden,
|
||||
'help' => \$help,
|
||||
'verbose' => \$verbose,
|
||||
'nasm=s' => \$nasm
|
||||
) or pod2usage();
|
||||
|
||||
pod2usage() if $help;
|
||||
die "Please specify either --nasm or --clean. Use --help for help.\n"
|
||||
unless $nasm or $clean;
|
||||
die "Please specify the test files, e.g. *.asm\n" unless @ARGV;
|
||||
|
||||
unless (!defined $nasm or -x $nasm) {
|
||||
warn "Warning: $nasm may not be executable. Expect problems.\n\n";
|
||||
sleep 5;
|
||||
}
|
||||
|
||||
perform($clean, $diff, $golden, $nasm, ! $verbose, $_) foreach @ARGV;
|
||||
exit $globalresult;
|
||||
|
||||
__END__
|
||||
|
||||
=head1 NAME
|
||||
|
||||
performtest.pl - NASM regression tester based on golden files
|
||||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
performtest.pl [options] [testfile.asm ...]
|
||||
|
||||
Runs NASM on the specified test files and compare the results
|
||||
with "golden" output files.
|
||||
|
||||
Options:
|
||||
--clean Clean up test results (or golden files with --golden)
|
||||
--diff Execute diff when stdout or stderr don't match
|
||||
--golden Create golden files
|
||||
--help Get this help
|
||||
--nasm=file Specify the file name for the NASM executable, e.g. ../nasm
|
||||
--verbose Get more output
|
||||
|
||||
If --clean is not specified, --nasm is required.
|
||||
|
||||
testfile.asm ...:
|
||||
One or more files that NASM should be tested with,
|
||||
often *.asm in the test directory.
|
||||
It should contain one or more option lines at the start,
|
||||
in the following format:
|
||||
|
||||
;Testname=<testname>; Arguments=<arguments to nasm>; Files=<output files>
|
||||
|
||||
If no such lines are found at the start, the file is skipped.
|
||||
testname should ideally describe the arguments, eg. unoptimized for -O0.
|
||||
arguments can be an optimization level (-O), an output format (-f),
|
||||
an output file specifier (-o) etc.
|
||||
The output files should be a space separated list of files that will
|
||||
be checked for regressions. This should often be the output file
|
||||
and the special files stdout and stderr.
|
||||
|
||||
Any mismatch could be a regression,
|
||||
but it doesn't have to be. COFF files have a timestamp which
|
||||
makes this method useless. ELF files have a comment section
|
||||
with the current version of NASM, so they will change each version number.
|
||||
|
||||
=cut
|
||||
Loading…
Add table
Add a link
Reference in a new issue