Difference between revisions of "ResultScraping"

From timswiki
Jump to navigation Jump to search
 
(6 intermediate revisions by the same user not shown)
Line 1: Line 1:
An introduction to scraping results from ipscan web sessions.
+
An example perl script to scrape results from [https://github.com/timsgit/ipscan IPscan] web sessions.
 
 
<analytics uacct="UA-27654202-1" ></analytics>
 
  
 
==Results Scraping==
 
==Results Scraping==
Line 16: Line 14:
 
    
 
    
 
   {
 
   {
        local $/=undef;
+
      local $/=undef;
        open FILE, "ipv6.html" or die "Couldn't open ipv6 file: $!";
+
      open FILE, "ipv6.html" or die "Couldn't open ipv6 file: $!";
        $scan = <FILE>;
+
      $scan = <FILE>;
        close FILE;
+
      close FILE;
 
   }
 
   }
 
   my $te = new HTML::TableExtract();
 
   my $te = new HTML::TableExtract();
 
   $te->parse($scan);
 
   $te->parse($scan);
 
    
 
    
   # my $pingtable = $te->table(0,0);
+
   my $pingtable = $te->table(0,0);
   my $porttable = $te->table(0,1);
+
   my $udpporttable = $te->table(0,1);
 +
  my $tcpporttable = $te->table(0,2);
 +
 
 +
  # Ping result
 +
  foreach my $row ($pingtable->rows)
 +
  {
 +
      foreach my $ele ( @$row )
 +
      {
 +
              # <TD TITLE="IPv6 ping">ICMPv6 ECHO REQUEST returned : </TD><TD style="background-color:yellow">ECHO REPLY</TD>
 +
              if (defined $ele)
 +
              {
 +
                      if ($ele =~ m/ECHO(.*)REPLY$/)
 +
                      {
 +
                              printf("PING response was %sREPLY\n",$1);
 +
                      }
 +
              }
 +
      }
 +
  }
 
    
 
    
 
   # Currently prints out the OPEN ports
 
   # Currently prints out the OPEN ports
   foreach my $row ($porttable->rows)
+
   foreach my $row ($udpporttable->rows)
 +
  {
 +
      foreach my $ele ( @$row )
 +
      {
 +
              # Port 7 = RFSD
 +
              if (defined $ele)
 +
              {
 +
                      if ($ele =~ m/Port\s+(\d+)\s+=\s+([A-Z]+)/)
 +
                      {
 +
                              my $port = $1;
 +
                              my $state = $2;
 +
                              if ($state =~ m/OPEN/)
 +
                              {
 +
                                        printf("UDP PORT %5d was %s\n",$port,$state);
 +
                              }
 +
                      }
 +
              }
 +
      }
 +
  }
 +
 
 +
  foreach my $row ($tcpporttable->rows)
 
   {
 
   {
        foreach my $ele ( @$row )
+
      foreach my $ele ( @$row )
        {
+
      {
                # Port 7 = RFSD
+
              # Port 7 = RFSD
                if (defined $ele)
+
              if (defined $ele)
                {
+
              {
                        if ($ele =~ m/Port\s+(\d+)\s+=\s+([A-Z]+)/)
+
                      if ($ele =~ m/Port\s+(\d+)\s+=\s+([A-Z]+)/)
                        {
+
                      {
                                my $port = $1;
+
                              my $port = $1;
                                my $state = $2;
+
                              my $state = $2;
                                if ($state =~ m/OPEN/)
+
                              if ($state =~ m/OPEN/)
                                {
+
                              {
                                        printf("PORT %5d was %s\n",$port,$state);
+
                                        printf("TCP PORT %5d was %s\n",$port,$state);
                                }
+
                              }
                        }
+
                      }
                }
+
              }
        }
+
      }
 
   }
 
   }
 +
 
    
 
    
  
Line 52: Line 88:
  
 
   $ ./scrape.pl
 
   $ ./scrape.pl
   PORT    22 was OPEN
+
   PING response was REPLY
   PORT   25 was OPEN
+
   UDP PORT   123 was UDPOPEN
   PORT    80 was OPEN
+
   TCP PORT    80 was OPEN
   PORT  443 was OPEN
+
   TCP PORT  443 was OPEN
   $  
+
   $
 
+
 
This website publishes a [http://ipv6.chappell-family.com/html/privacy_policy.html Privacy Policy.] Continued use of this website implies your consent to the storage of data outlined in the policy.
+
This website publishes a Privacy Policy (link at the bottom of every page). Continued use of this website implies your consent to the use of data outlined in the policy.
  
 
----
 
----
 
 
<adsense>1</adsense>
 

Latest revision as of 23:39, 4 January 2019

An example perl script to scrape results from IPscan web sessions.

Results Scraping

Here is a simple Perl script (scrape.pl) which dumps out the open ports:


 #!/usr/bin/perl
 use strict;
 use warnings;
 use HTML::TableExtract;
 
 my $scan = "";
 
 {
      local $/=undef;
      open FILE, "ipv6.html" or die "Couldn't open ipv6 file: $!";
      $scan = <FILE>;
      close FILE;
 }
 my $te = new HTML::TableExtract();
 $te->parse($scan);
 
 my $pingtable = $te->table(0,0);
 my $udpporttable = $te->table(0,1);
 my $tcpporttable = $te->table(0,2);
 
 # Ping result
 foreach my $row ($pingtable->rows)
 {
      foreach my $ele ( @$row )
      {

# ICMPv6 ECHO REQUEST returned : ECHO REPLY

              if (defined $ele)
              {
                      if ($ele =~ m/ECHO(.*)REPLY$/)
                      {
                             printf("PING response was %sREPLY\n",$1);
                      }
              }
      }
 }
 
 # Currently prints out the OPEN ports
 foreach my $row ($udpporttable->rows)
 {
      foreach my $ele ( @$row )
      {
              # Port 7 = RFSD
              if (defined $ele)
              {
                      if ($ele =~ m/Port\s+(\d+)\s+=\s+([A-Z]+)/)
                      {
                              my $port = $1;
                              my $state = $2;
                              if ($state =~ m/OPEN/)
                              {
                                       printf("UDP PORT %5d was %s\n",$port,$state);
                              }
                      }
              }
      }
 }
 
 foreach my $row ($tcpporttable->rows)
 {
      foreach my $ele ( @$row )
      {
              # Port 7 = RFSD
              if (defined $ele)
              {
                      if ($ele =~ m/Port\s+(\d+)\s+=\s+([A-Z]+)/)
                      {
                              my $port = $1;
                              my $state = $2;
                              if ($state =~ m/OPEN/)
                              {
                                       printf("TCP PORT %5d was %s\n",$port,$state);
                              }
                      }
              }
      }
 }


Typical Scraped results

 $ ./scrape.pl
 PING response was  REPLY
 UDP PORT   123 was UDPOPEN
 TCP PORT    80 was OPEN
 TCP PORT   443 was OPEN
 $
 

This website publishes a Privacy Policy (link at the bottom of every page). Continued use of this website implies your consent to the use of data outlined in the policy.