#!/usr/bin/perl -w
# Extract all female names from www.20000-names.com
  
use strict;
use LWP::Simple;
use HTML::TreeBuilder;
use Encode;

my $filename = "female_names.txt";
my @number_of_pages_per_letter = ( # arrays are zero-indexed
  7,3,3,2,3,2, # A, B, C, D, E, F
  2,2,2,3,3,3,5, # G, H, I, J, K, L, M
  2,1,2,1,3,4,2, # N, O, P, Q, R, S, T
  1,1,1,1,1,1 # U, V, W, X, Y, Z
);
my %names_and_descriptions; # hash, indexed by name

open(OUTFILE, ">$filename");
for (my $letter = 1; $letter <= 26; $letter++) {
  process_letter($letter, $number_of_pages_per_letter[$letter-1]);
}
for my $name ( sort keys %names_and_descriptions ) {
  print OUTFILE "$name\t$names_and_descriptions{$name}\n"; # tab-separated (since descriptions can include commas
}
print "Results written to $filename.\n";

# -------------------------------------
# End of main code. Subroutines follow.
# -------------------------------------

sub process_letter{
  my $letter = shift;
  my $numberofpages = shift;
  my $httpaddress;
  for (my $count = 1; $count <= $numberofpages; $count++) {
    $httpaddress = "http://www.20000-names.com/female_"
      . chr(ord("a") - 1 + $letter)
      . "_names"
      . ( ($count > 1) ? ("_" . chr(ord("0") + $count)) : "")
      . ".htm";
    extract_info_from($httpaddress);
  }
}

sub extract_info_from{
  my $httpaddress = shift;
  print "Extracting names from: $httpaddress\n";
  my $html = get($httpaddress)
    or die "Couldn't fetch page: $httpaddress\n";
  # decode any UTF; see http://ripary.com/utf8.html
  # $html = decode_my_html($html);
  # $html = Encode::decode("utf8", $html);
  # $html = Encode::encode("iso-8859-1", $html);
  my $root = HTML::TreeBuilder->new_from_content($html);
  extract_names_from_HTML($root);
  $root->delete;
}

sub extract_names_from_HTML{
  my $root = shift;
  my $temp;
  
  # List names in web page
  # They begin with <li><p align="left"><b><font size="2" color="#9393FF">NAME</font></b>
  # Searching by color finds the name. That node's parent's parent gives name and description.

  foreach my $node ($root->find_by_attribute("color", "#9393FF")) {
    $names_and_descriptions{ $node->as_text } = $node->parent->parent->as_text; # syntax: $hash{ $key } = $value;
  }
}
