Support filtering wordlists by regex in cdbmake-wordlist

Add a new option, -x or --exclude, that excludes words from the
resulting CDB database by regular expression.  This option may
be given repeatedly to filter out multiple regular expressions.

diff --git a/tests/tools/cdbmake-wordlist-t b/tests/tools/cdbmake-wordlist-t
index e22587dd1fa1fdad90fa65e9b5e16c925d500b56..ca4b53df9a5e142625e8a4f3e96f49969056b5eb 100755 (executable)
--- a/tests/tools/cdbmake-wordlist-t
+++ b/tests/tools/cdbmake-wordlist-t
@@ -17,7 +17,7 @@ if ! command -v cdb >/dev/null 2>&1 ; then
 fi
 
 # Output the test plan.
-plan 14
+plan 18
 
 # Create a temporary directory and wordlist and ensure it's writable.
 tmpdir=`test_tmpdir`
@@ -69,6 +69,16 @@ ok_program 'Database still contains bitterbane' 0 '1' \
 ok_program 'Database does not contain happenstance' 100 '' \
     cdb -q "$tmpdir/wordlist.cdb" happenstance
 
+# Regenerate the database, filtering out words starting with b or ending in d.
+ok_program 'Database generation with no b passwords' 0 '' \
+    "$cdbmake" -x '\Ab' -x '.*d' "$tmpdir/wordlist"
+ok_program 'Database does not contain bitterbane' 100 '' \
+    cdb -q "$tmpdir/wordlist.cdb" bitterbane
+ok_program 'Database still contains happenstance' 0 '1' \
+    cdb -q "$tmpdir/wordlist.cdb" happenstance
+ok_program 'Database does not contain password' 100 '' \
+    cdb -q "$tmpdir/wordlist.cdb" password
+
 # Clean up.
 rm -f "$tmpdir/wordlist.cdb"
 rm -f "$tmpdir/wordlist"

diff --git a/tools/cdbmake-wordlist b/tools/cdbmake-wordlist
index c0ca0c4f2559b0fb68c015d8c1bd0bfefb7f9995..8cc86c57522551c788f195f2b8e6fed836629df5 100755 (executable)
--- a/tools/cdbmake-wordlist
+++ b/tools/cdbmake-wordlist
@@ -42,13 +42,14 @@ my $fullpath = $0;
 local $0 = basename($0);
 
 # Parse the argument list.
-my ($ascii, $max_length, $min_length, $manual);
+my ($ascii, @exclude, $max_length, $min_length, $manual);
 Getopt::Long::config('bundling', 'no_ignore_case');
 GetOptions(
     'ascii|a'        => \$ascii,
     'max-length|L=i' => \$max_length,
     'min-length|l=i' => \$min_length,
-    'manual|man|m'   => \$manual
+    'manual|man|m'   => \$manual,
+    'exclude|x=s'    => \@exclude,
 );
 if ($manual) {
     print_fh(\*STDOUT, "Feeding myself to perldoc, please wait...\n");
@@ -72,7 +73,7 @@ open(my $in, '<', $input)
   or die "$0: cannot open input file $input: $!\n";
 open(my $out, '>', $output)
   or die "$0: cannot create output file $output: $!\n";
-while (defined(my $word = <$in>)) {
+WORD: while (defined(my $word = <$in>)) {
     chomp($word);
     my $length = length($word);
     next if (defined($min_length) && $length < $min_length);
@@ -81,6 +82,9 @@ while (defined(my $word = <$in>)) {
         next if $word =~ m{ [^[:ascii:]] }xms;
         next if $word =~ m{ [[:cntrl:]] }xms;
     }
+    for my $pattern (@exclude) {
+        next WORD if $word =~ m{ $pattern }xms;
+    }
     print_fh($out, "+$length,1:$word->1\n");
 }
 print_fh($out, "\n");
@@ -109,7 +113,7 @@ cdbmake-wordlist - Create a cdb database from a wordlist
 =head1 SYNOPSIS
 
 B<cdbmake-wordlist> [B<-am>] [B<-l> I<min-length>] [B<-L> I<max-length>]
-    I<wordlist>
+    [B<-x> I<exclude> ...] I<wordlist>
 
 =head1 DESCRIPTION
 
@@ -164,6 +168,13 @@ The default is not to filter out any words for minimum length.
 Print out this documentation (which is done simply by feeding the script to
 C<perldoc -t>).
 
+=item B<-x> I<exclude>, B<--exclude>=I<exclude>
+
+Filter all words matching the regular expression I<exclude> from the
+resulting cdb database.  This regular expression will be matched against
+each line of the source wordlist after the trailing newline is removed.
+This option may be given repeatedly to add multiple exclusion regexes.
+
 =back
 
 =head1 AUTHOR