ports-mgmt/portscout: Make GitHub ports work, fix bugs

A number of people reported no updates being detected for ports using
USE_GITHUB=yes, even after originally adding the GitHub site handler in
r401037 [1]. Investigation revealed that the FindNewestFile subroutine
and the vercompare() method assumed (or are designed so) that responses
returned from site handlers will be in a normalised version format. For
site handlers that return 'versions', this works well. For the github
handler, in the fallback use of the API for fetching repository tags, it
does not.

Additionally it turns out, portscout currently only uses/stores a
normalized version ('ver') in its database, in its general design
attempt to be a generic version comparison tool

In particular, portscout does not reference or store PORTVERSION
or DISTVERSION{FULL}, so we have nothing 'canonical' to compare
the responses from Github (tags) against.

This change special-cases Github in the FindNewestFile subroutine, which
was obtained via Portroach [2] (OpenBSD's portscout fork).

Extending this, we also now only match version-esque looking strings
from the tag, in an attempt to normalise, because they come in many
forms, including {foo-}X.Y.Z{-bar}, foo_X_Y_Z, among others.

While I'm here,

* Fix copypasta of $github_client_id, when $github_client_secret was
  intended
* Add code to use authenticated requests for Github project tags in the
  fallback (to /releases) case.
* Add and update some more debug messages to help diagnosis of future
  issues

Special thank you's to:

 * matthew, allanjude, mandree, des, Brendan Molloy for your regex, perl
   help and moral support.

[1] http://svnweb.freebsd.org/changeset/ports/401037
[2] https://github.com/jasperla/portroach

Differential Revision:	D4780

3 files changed, 178 insertions, 4 deletions

diff --git a/ports-mgmt/portscout/Makefile b/ports-mgmt/portscout/Makefile
index 96d9444c09db..de30f917c811 100644
--- a/ports-mgmt/portscout/Makefile
+++ b/ports-mgmt/portscout/Makefile
@@ -3,7 +3,7 @@
 
 PORTNAME=	portscout
 PORTVERSION=	0.8.1
-PORTREVISION=	4
+PORTREVISION=	5
 CATEGORIES=	ports-mgmt
 MASTER_SITES=	http://mirror.inerd.com/FreeBSD/distfiles/${PORTNAME}/ \
 		http://www.atarininja.org/~wxs/distfiles/ \
diff --git a/ports-mgmt/portscout/files/patch-Portscout_SiteHandler_GitHub.pm b/ports-mgmt/portscout/files/patch-Portscout_SiteHandler_GitHub.pm
index 7d7f9ea87504..57eed2ba7093 100644
--- a/ports-mgmt/portscout/files/patch-Portscout_SiteHandler_GitHub.pm
+++ b/ports-mgmt/portscout/files/patch-Portscout_SiteHandler_GitHub.pm
@@ -1,4 +1,4 @@
---- Portscout/SiteHandler/GitHub.pm.orig	2015-10-25 05:00:48 UTC
+--- Portscout/SiteHandler/GitHub.pm.orig	2016-01-04 10:46:49 UTC
 +++ Portscout/SiteHandler/GitHub.pm
 @@ -97,7 +97,9 @@ sub GetFiles
  	my ($url, $port, $files) = @_;
@@ -18,9 +18,21 @@
 -
 +		# Add GitHub Client ID & Secret if they are set in settings
 +		# https://developer.github.com/v3/#authentication
-+		if ($settings{github_client_id} && $settings{github_client_id}) {
++		if ($settings{github_client_id} && $settings{github_client_secret}) {
 +			$query = $query . "?client_id=$settings{github_client_id}&client_secret=$settings{github_client_secret}";
 +		}
  		_debug("GET $query");
  		$ua = LWP::UserAgent->new;
  		$ua->agent(USER_AGENT);
+@@ -120,6 +126,11 @@ sub GetFiles
+ 			_debug('GET failed: ' . $response->status_line);
+ 			# Project didn't do any releases, so let's try tags instead.
+ 			$query = 'https://api.github.com/repos/' . $projname . '/tags';
++			# Add GitHub Client ID & Secret if they are set in settings
++			# https://developer.github.com/v3/#authentication
++			if ($settings{github_client_id} && $settings{github_client_secret}) {
++				$query = $query . "?client_id=$settings{github_client_id}&client_secret=$settings{github_client_secret}";
++			}
+ 			_debug("GET $query");
+ 			$ua = LWP::UserAgent->new;
+ 			$ua->agent(USER_AGENT);
diff --git a/ports-mgmt/portscout/files/patch-portscout.pl b/ports-mgmt/portscout/files/patch-portscout.pl
index a3295f6b2e50..66b69118075f 100644
--- a/ports-mgmt/portscout/files/patch-portscout.pl
+++ b/ports-mgmt/portscout/files/patch-portscout.pl
@@ -1,4 +1,4 @@
---- portscout.pl.orig	2015-10-25 05:00:48 UTC
+--- portscout.pl.orig	2016-01-04 10:46:49 UTC
 +++ portscout.pl
 @@ -463,7 +463,7 @@ sub VersionCheck
  
@@ -53,3 +53,165 @@
  						}
  
  						last if ($new_found);
+@@ -867,7 +876,10 @@ sub FindNewestFile
+ 
+ 	foreach my $file (@$files)
+ 	{
+-		my $poss_path;
++		my ($poss_path, $github);
++
++		print "FindNewest: Checking $file ... against port DISTFILES. \n"
++			if ($settings{debug});
+ 
+ 		if ($file =~ /^(.*)\/(.*?)$/) {
+ 			# Files from SiteHandlers can come with paths
+@@ -888,6 +900,9 @@ sub FindNewestFile
+ 
+ 			my $skip = 0;
+ 
++			print "FindNewest: Checking DISTFILE ... $distfile (ver: $v, sufx: $s)\n"
++				if ($settings{debug});
++
+ 			if ($poss_path) {
+ 				# Do a full-URL comparison for $old_found
+ 				# if we're dealing with paths too.
+@@ -908,14 +923,20 @@ sub FindNewestFile
+ 			} else {
+ 				if ($file eq $distfile) {
+ 					$old_found = 1;
++					print "FindNewest: File matches existing port DISTFILE. Old found. \n"
++						if ($settings{debug});
+ 					next;
+ 				}
+ 			}
+ 
++			$github = 1 if ($site->clone =~ /^https?:\/\/([^\/.]+\.)?github\.com\/(.*?)\/tar.gz/);
++
+ 			# Skip beta versions if requested
+ 
+ 			if ($port->{skipbeta}) {
+ 				if (isbeta($file) && !isbeta($distfile)) {
++					print "FindNewest: File is beta, skipbeta is defined. Skipping ...\n"
++						if ($settings{debug});
+ 					next;
+ 				}
+ 			}
+@@ -940,31 +961,64 @@ sub FindNewestFile
+ 
+ 			# Possible candidate - extract version
+ 
+-			if ($file =~ /^($distfile)$/ && $2)
++			#warn "distfile = $distfile 2 = $2";
++			if (($file =~ /^($distfile)$/ && $2) or $github)
+ 			{
+-				my $version = $2;
+-				my $new_v = lc $version;
++				my ($version, $new_v, $matchver);
+ 
+-				# Catch a few missed cases
++				unless ($github) {
++					$version = $2;
++					$new_v = lc $version;
+ 
+-				$new_v =~ s/(?:$ext_regex)$//;
++					# Catch a few missed cases
++					$new_v =~ s/(?:$ext_regex)$//;
+ 
+-				# Version is much longer than original - skip it
++					# Version is much longer than original - skip it
++					next if (length $new_v > (12 + length $old_v));
+ 
+-				next if (length $new_v > (12 + length $old_v));
++					# New version is in date format (or contains a date-like
++					# string) - old one is not. Probably best to ignore.
+ 
+-				# New version is in date format (or contains a date-like
+-				# string) - old one is not. Probably best to ignore.
++					next if (
++						$new_v =~ /$date_regex/i &&
++						$old_v !~ /$date_regex/i
++					);
+ 
+-				next if (
+-					$new_v =~ /$date_regex/i &&
+-					$old_v !~ /$date_regex/i
+-				);
++					# Skip a few strange version format change cases
++					# (formatted -> "just a number")
++					next if ($new_v !~ /\./ && $old_v =~ /\./);
++				} else {
++					# Github is "special" since the actual URI we get back from the
++					# handler isn't the same as what is actually being retrieved.
++					# So fall back on comparing tags instead.
+ 
+-				# Skip a few strange version format change cases
+-				# (formatted -> "just a number")
++					$new_v = $file;
++					$version = lc $new_v;
+ 
+-				next if ($new_v !~ /\./ && $old_v =~ /\./);
++					# Only match version-esque looking strings from the tag
++					# because portscout currently only uses/stores a normalized
++					# 'ver', not PORTVERSION or DISTVERSION{FULL} so we have nothing
++					# canonical to compare agains. Currently the match is for digits
++					# with any non-digit separators, without non-digit prefixes and
++					# suffixes. This is likely the best we can get for now, without
++					# introducing false positives.
++
++					($matchver) = ($version =~ m/((?:\d+.)*\d+)/);
++
++					print "FindNewestFile::Github: Input: $file Matchver: $matchver \n"
++						if ($settings{debug});
++					
++					# Replace non-digit separators of digits with dots.
++
++					$matchver =~ s/[^\d+]/\./g;
++
++					if (defined($matchver)) {
++						$new_v = $matchver;
++					}
++
++					print "FindNewestFile::Github: Matchver: $matchver Normalized: $new_v \n" 
++						if ($settings{debug});
++				}
+ 
+ 				# Skip any specific versions if requested
+ 
+@@ -973,6 +1027,8 @@ sub FindNewestFile
+ 
+ 					foreach (split (/\s+/, $port->{skipversions})) {
+ 						if ($new_v eq $_) {
++							print "FindNewest: skipversions is defined. Matched $_. Skipping...\n"
++								if ($settings{debug});
+ 							$skip = 1;
+ 							last;
+ 						}
+@@ -981,7 +1037,7 @@ sub FindNewestFile
+ 					next if ($skip);
+ 				}
+ 
+-				unless ($settings{sillystrings_enable}) {
++				unless ($settings{sillystrings_enable} or $github) {
+ 					if ($new_v =~ /[-_.]([A-Za-z]+[A-Za-z_-]{2,})$/) {
+ 						my $str = $1;
+ 						next if (
+@@ -1026,12 +1082,19 @@ sub FindNewestFile
+ 
+ 				# Test our new version string
+ 
++				print "FindNewest: Comparing Old ($old_v) vs New ($new_v) version strings... \n"
++					if ($settings{debug});
++
+ 				if ($new_v eq $old_v)
+ 				{
++					print "FindNewest: Old ($old_v) == New ($new_v). Old found.\n"
++						if ($settings{debug});
+ 					$old_found = 1;
+ 				}
+ 				elsif (vercompare($new_v, $old_v))
+ 				{
++					print "FindNewest: Old ($old_v) < New ($new_v). New found.\n"
++						if ($settings{debug});
+ 					$new_found = 1;
+ 
+ 					# Keep going until we find the newest version