#!/usr/bin/perl -w
# Extract all male names from www.20000-names.com
  
use strict;
use LWP::Simple;
use HTML::TreeBuilder;
use Encode;

my $filename = "male_names.txt";
my @number_of_pages_per_letter = ( # arrays are zero-indexed
  8,3,3,3,3,3, # A, B, C, D, E, F
  3,3,2,3,3,3,4, # G, H, I, J, K, L, M
  2,2,3,1,3,4,3, # N, O, P, Q, R, S, T
  1,2,1,1,1,1 # U, V, W, X, Y, Z
);
my %names_and_descriptions; # hash, indexed by name

open(OUTFILE, ">$filename");
# extract_info_from("http://www.20000-names.com/male_a_names_3.htm");
for (my $letter = 1; $letter <= 26; $letter++) {
  process_letter($letter, $number_of_pages_per_letter[$letter-1]);
}
for my $name ( sort keys %names_and_descriptions ) {
  print OUTFILE "$name\t$names_and_descriptions{$name}\n"; # tab-separated (since descriptions can include commas
}
print "Results written to $filename.\n";

# -------------------------------------
# End of main code. Subroutines follow.
# -------------------------------------

sub process_letter{
  my $letter = shift;
  my $numberofpages = shift;
  my $httpaddress;
  for (my $count = 1; $count <= $numberofpages; $count++) {
    $httpaddress = "http://www.20000-names.com/male_"
      . chr(ord("a") - 1 + $letter)
      . "_names"
      . ( ($count > 1) ? ("_" . chr(ord("0") + $count)) : "")
      . ".htm";
    extract_info_from($httpaddress);
  }
}

sub extract_info_from{
  my $httpaddress = shift;
  print "Extracting names from: $httpaddress\n";
  my $html = get($httpaddress)
    or die "Couldn't fetch page: $httpaddress\n";
  # decode any UTF; see http://ripary.com/utf8.html
  # $html = decode_my_html($html);
  # $html = Encode::decode("utf8", $html);
  # $html = Encode::encode("iso-8859-1", $html);
  my $root = HTML::TreeBuilder->new_from_content($html);
  extract_names_from_HTML($root);
  $root->delete;
}

sub extract_images_from_HTML{
  my $root = shift;
  # List images in web page
  print "IMAGES:\n";
  my %images;
  foreach my $node ($root->find_by_tag_name('img')) {
    $images{ $node->attr('src') }++;
  }
  foreach my $pic (sort keys %images) {
    print "$pic\n";
  }
  print "END OF IMAGES.\n";
}

sub extract_names_from_HTML{
  my $root = shift;
  my $temp;
  
  # List names in web page
  # They begin with <li><p align="left"><b><font size="2" color="#9393FF">NAME</font></b>
  # Searching by color finds the name. That node's parent's parent gives name and description.

  foreach my $node ($root->find_by_attribute("color", "#9393FF")) {
    $names_and_descriptions{ $node->as_text } = $node->parent->parent->as_text; # syntax: $hash{ $key } = $value;
  }
}

sub decode_my_html{
  my $html = shift;
  if ($html =~ /<?xml[^>]+encoding[\s\x0d\x0a]*=[\s\x0d\x0a]*['"]utf-?8/i ||
    $html =~ /<meta[^>]+charset[\s\x0d\x0a]*=[\s\x0d\x0a]*utf-?8/i) {
    $html = decode('utf8', $html);
  }
  $html =~ s/(.)/asciiize($1)/eg; # Process by char
  return($html);
}

sub asciiize {
    return $_[0] if (ord($_[0]) < 128);     # ASCII
    return sprintf('&#x%04X;', ord($_[0])); # Non-ASCII
}
