Files
Saxonite/SaxoniteCoreWeb/Base/Procs/SaxGolfUsgaDataLookup.cs
T
2024-06-19 17:19:26 +01:00

1071 lines
44 KiB
C#

using DocumentFormat.OpenXml.InkML;
using HtmlAgilityPack;
using MailKit;
using Microsoft.Net.Http.Headers;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Sarand.Net.Standard.Base.App;
using Sarand.Net.Standard.Base.Bus;
using Sarand.Net.Standard.Base.Core;
using Sarand.Net.Standard.Base.Dbas;
using Sarand.Net.Standard.Base.Helpers;
using Sarand.Net.Standard.Base.Net;
using Sarand.Net.Standard.Base.Utils;
using Saxonite.Net.Standard.Base.App;
using Saxonite.Net.Standard.Base.Bus.Reference;
using Saxonite.Net.Standard.Base.Mgr.Reference;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using static Saxonite.Net.Core.Web.Base.Procs.SaxGolfUsgaDataLookup;
namespace Saxonite.Net.Core.Web.Base.Procs
{
//======================================================================
// USGA course rating & slope database: https://ncrdb.usga.org/
// https://ncrdb.usga.org/courseTeeInfo.aspx?CourseID=14758
// Ref. https://zylalabs.com/blog/enhance-your-game-with-a-free-golf-api
public partial class SaxGolfUsgaDataLookup : SaxProcess_Web
{
//-------------------------------------------------------------------
// Definitions.
public enum Task
{
CleanseCourseCaptions,
}
public new class Config : SaxProcess_Web.Config
{
public Config(SaxSessionContext sessionContext, DbUpdateInfo dbUpdateInfo) : base(sessionContext, dbUpdateInfo)
{ }
/// <summary>Tasks to be run under this instance of the USGA Data Lookup process.</summary>
public HashSet<Task> Tasks { get; } = new HashSet<Task>();
}
public class CountryProvinceCodes
{
public CountryProvinceCodes(String countryCode, String provinceCode)
{
this.CountryCode = StringUtil.OnBlank(countryCode);
this.ProvinceCode = StringUtil.OnBlank(provinceCode);
}
public string CountryCode { get; }
public string ProvinceCode { get; }
public override bool Equals(object obj)
{
return Equals(obj as CountryProvinceCodes);
}
public bool Equals(CountryProvinceCodes other)
{
return other != null && CountryCode == other.CountryCode && ProvinceCode == other.ProvinceCode;
}
public override int GetHashCode()
{
return HashCode.Combine(CountryCode, ProvinceCode);
}
public override string ToString()
{
return $"CountryProvinceCodes({CountryCode}, {ProvinceCode})";
}
}
public class CountryData
{
public string Code { get; set; }
public string Name { get; set; }
}
public class ProvinceData
{
public string Code { get; set; }
public string Name { get; set; }
}
public class CourseData
{
public string UsgaCourseId { get; set; }
public string LegacyCrpCourseId { get; set; }
public string Caption { get; set; }
public string CourseName { get; set; }
public string FullName { get; set; }
public string UsgaFacilityId { get; set; }
public string FacilityName { get; set; }
public string Address1 { get; set; }
public string Address2 { get; set; }
public string City { get; set; }
public string ProvinceCode { get; set; }
public string EntProvinceCode { get; set; }
public string Province { get; set; }
public string CountryCode { get; set; }
public string EntCountryCode { get; set; }
public string Country { get; set; }
public string Email { get; set; }
public string Telephone { get; set; }
public List<String> Ratings { get; } = new List<string>();
}
public class MarkerData
{
public MarkerData(SysId courseId, short holes, string holesCaption)
{
this.CourseId = courseId;
this.Holes = holes;
this.HolesCaption = holesCaption;
}
public SysId CourseId { get; }
public SysId MarkerId { get; set; }
public string Caption { get { return StringUtil.TrimWhitespace(StringUtil.OnBlank(Tee) + " " + StringUtil.OnBlank(HolesCaption) + " " + StringUtil.OnBlank(Gender)); } }
public short Holes { get; set; }
public string HolesCaption { get; set; }
public string Tee { get; set; }
public string Gender { get; set; }
public short SlopeRating { get; set; }
public short Par { get; set; }
public decimal CourseRating { get; set; }
public decimal BogeyRating { get; set; }
public short Length { get; set; }
public bool IsValid(bool ignorePar = false)
{
if (!SysId.IsValid(CourseId)) return false;
// 9 hole markers do not have there own Id.
//if (!SysId.IsValid(MarkerId)) return false;
if (Holes < 1) return false;
if (StringUtil.IsBlank(Tee)) return false;
//if (StringUtil.IsBlank(Gender)) return false;
if (SlopeRating == 0) return false;
if (!ignorePar && Par == 0) return false;
if (CourseRating == 0) return false;
return true;
}
}
//-------------------------------------------------------------------
// Data members.
private Config config;
//-------------------------------------------------------------------
// Construction.
public SaxGolfUsgaDataLookup(Config config) : base(config)
{
this.config = config;
}
//-------------------------------------------------------------------
// Properties.
public override string ProcessDescription => "Golf Course Lookup";
//-------------------------------------------------------------------
// Run the process.
protected override bool Run()
{
// Execute selected tasks.
foreach (Task task in config.Tasks)
{
switch (task)
{
case Task.CleanseCourseCaptions:
CleanseCourseCaptions();
break;
}
if (!OK) break;
}
return OK;
}
//------------------------------------------------------------------
// USGA data lookup.
//------------------------------------------------------------------
/// <summary>
/// Data elements that may be required in subsequent requests etc.
/// </summary>
private class UsgaRequestData
{
public DateTime PopulatedAt { get; } = DateTime.Now;
public Dictionary<String, SacsHttpCookieData> Cookies { get; } = new Dictionary<String, SacsHttpCookieData>();
public String RequestVerificationToken { get; set; }
}
private UsgaRequestData usgaRequestData = null;
/// <summary>
/// Populate the request data from the Home Page response.
/// </summary>
private void PopulateUsgaRequestData(SacsHttpResponseData response)
{
try
{
if (!StringUtil.IsBlank(response.Data))
{
usgaRequestData = new UsgaRequestData();
// Cookies.
usgaRequestData.Cookies.AddRange(response.Cookies);
// Parse HTML.
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(response.Data);
// Locate verification token.
HtmlNode verificationTokenNode = htmlDoc.DocumentNode.SelectSingleNode("//input[@name='__RequestVerificationToken']");
if (verificationTokenNode != null)
{
usgaRequestData.RequestVerificationToken = verificationTokenNode.GetAttributeValue("value", "");
}
}
}
catch (Exception exc)
{
SaLogger.Warn(SysUtil.Fn + " " + ExceptionUtil.GetLogMessage(exc, "populating USGA request data", 0));
}
}
/// <summary>
/// Get the USGA request data to extract useful elements, eg. verification token and cookies.
/// <para>- Request data is refreshed periodically in case of expiry.</para>
/// </summary>
private UsgaRequestData GetUsgaRequestData()
{
if (usgaRequestData == null || (DateTime.Now - usgaRequestData.PopulatedAt).TotalMinutes > 15)
{
try
{
// Query countries on USGA website.
CancellationToken cancelToken = new CancellationToken();
SacsHttpRequestData request = new SacsHttpRequestData(cancelToken) { Url = "https://ncrdb.usga.org/" };
SacsHttpResponseData response = SacsHttpClient.DoRequest(request).Result;
String data = response.Data;
PopulateUsgaRequestData(response);
}
catch (Exception exc)
{
SaLogger.Warn(SysUtil.Fn + " " + ExceptionUtil.GetLogMessage(exc, "querying USGA request data", 0));
}
}
return usgaRequestData;
}
//------------------------------------------------------------------
/// <summary>
/// USGA Countries.
/// <para>- Countries are listed on the home page of the USGA site in a dropdown.</para>
/// </summary>
public List<CountryData> LookupUsgaCountries()
{
List<CountryData> usgaCountries = new List<CountryData>();
try
{
// Query Countries on USGA website.
CancellationToken cancelToken = new CancellationToken();
SacsHttpRequestData request = new SacsHttpRequestData(cancelToken) { Url = "https://ncrdb.usga.org/" };
SacsHttpResponseData response = SacsHttpClient.DoRequest(request).Result;
String data = response.Data;
PopulateUsgaRequestData(response);
if (!StringUtil.IsBlank(response.Data))
{
// Parse HTML.
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(response.Data);
// Locate countries list.
HtmlNode countriesNode = htmlDoc.GetElementbyId("ddlCountries");
if (countriesNode != null)
{
foreach (HtmlNode countryOptionNode in countriesNode.SelectNodes("./option"))
{
String countryCode = StringUtil.TrimWhitespace(countryOptionNode.GetAttributeValue("value", ""));
String countryName = StringUtil.TrimWhitespace(WebUtil.HtmlDecode(countryOptionNode.InnerText));
// Ignore blank code or name.
if (StringUtil.IsBlank(countryCode) || countryCode.EqualsCI("(Select)") || StringUtil.IsBlank(countryName)) continue;
// Ignore 'Select' item.
if (countryCode.EqualsCI("(Select)")) continue;
// Ignore US States and territories.
if (countryCode.StartsWithCI("US-") || countryCode.StartsWithCI("UM-")) continue;
// Ignore codes that aren't 3 chars.
if (countryCode.Length != 3) continue;
// Add country to list.
usgaCountries.Add(new CountryData() { Code = countryCode, Name = countryName });
}
}
else SaLogger.Warn(SysUtil.Fn + " Countries node not found.");
}
// Store countries.
List<SaxReferenceGolfMgr.UsgaCountryData> countriesData = new List<SaxReferenceGolfMgr.UsgaCountryData>();
foreach (CountryData usgaCountry in usgaCountries)
{
SaxReferenceGolfMgr.UsgaCountryData countryData = new SaxReferenceGolfMgr.UsgaCountryData();
countryData.CountryCode = usgaCountry.Code;
countryData.CountryName = usgaCountry.Name;
countriesData.Add(countryData);
}
ReferenceGolfMgr.BulkUpdateUsgaCountries(countriesData, DbUpdateInfo);
if (ReferenceGolfMgr.HasErrorState())
SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA countries: {ReferenceGolfMgr.ErrorStateMessage}.");
}
catch (Exception exc)
{
SaLogger.Warn(SysUtil.Fn + " " + ExceptionUtil.GetLogMessage(exc, "querying USGA countries", 0));
}
return usgaCountries;
}
//------------------------------------------------------------------
/// <summary>
/// USGA Provinces for country.
/// </summary>
public List<ProvinceData> LookupUsgaProvincesForCountry(String countryCode)
{
List<ProvinceData> usgaProvinces = new List<ProvinceData>();
if (StringUtil.IsBlank(countryCode)) return usgaProvinces;
UsgaRequestData usgaRequestData = GetUsgaRequestData();
using (DisposableCollection<SacsDisposable> disposableItems = new DisposableCollection<SacsDisposable>())
{
// Retrieve and validate country.
SaxGolfUsgaCountry usgaCountry = ReferenceGolfMgr.GetUsgaCountry(new SaxGolfUsgaCountry.Params() { CountryCodeFull = countryCode });
disposableItems.Add(usgaCountry);
if (usgaCountry == null)
{
SaLogger.Warn($"{SysUtil.Fn} USGA Country not found for code '{countryCode}'.");
return usgaProvinces;
}
if (usgaCountry.Priority)
{
// Clear country 'priority' flag.
usgaCountry.Priority = false;
ReferenceGolfMgr.UpdateUsgaCountry(usgaCountry, DbUpdateInfo);
}
try
{
// Query Provinces for Country on USGA website.
CancellationToken cancelToken = new CancellationToken();
SacsHttpRequestData request = new SacsHttpRequestData(cancelToken) { Url = "https://ncrdb.usga.org/NCRListing?handler=LoadStates" };
// Headers.
request.RequestHeaders.Add(HeaderNames.Accept, "*/*");
request.RequestHeaders.Add(HeaderNames.AcceptEncoding, "gzip, deflate, br, zstd");
request.RequestHeaders.Add(HeaderNames.AcceptLanguage, "en-GB,en;q=0.9");
request.RequestHeaders.Add(HeaderNames.DNT, "1");
request.RequestHeaders.Add(HeaderNames.Origin, "https://ncrdb.usga.org/");
request.RequestHeaders.Add("Priority", "u=1, i");
request.RequestHeaders.Add(HeaderNames.Referer, "https://ncrdb.usga.org/");
if (!StringUtil.IsBlank(usgaRequestData.RequestVerificationToken))
request.RequestHeaders.Add("Requestverificationtoken", usgaRequestData.RequestVerificationToken);
// Content.
request.FormData.Add(new KeyValuePair<string, string>("countryCode", countryCode));
request.ContentHeaders.Add(HeaderNames.ContentType, "application/x-www-form-urlencoded; charset=UTF-8");
// Request.
SacsHttpResponseData response = SacsHttpClient.DoRequest(request).Result;
String data = response.Data;
if (!StringUtil.IsBlank(response.Data))
{
dynamic provincesJson = SacsJsonHelper.DeserializeJsonObject(response.Data);
if (provincesJson is JArray)
{
JArray provincesArray = provincesJson as JArray;
foreach (var provinceToken in provincesArray)
{
// Province properties.
ProvinceData provinceData = new ProvinceData();
foreach (var propertyToken in provinceToken)
{
if (propertyToken is JProperty)
{
JProperty provinceProperty = propertyToken as JProperty;
String propertyName = provinceProperty.Name;
String propertyStringValue = StringUtil.TrimWhitespace(provinceProperty.Value.ToString());
if (propertyName.EqualsCI("text"))
provinceData.Name = propertyStringValue;
else if (propertyName.EqualsCI("value"))
provinceData.Code = propertyStringValue;
else if (propertyName.EqualsCI("group")) { }
else if (propertyName.EqualsCI("selected")) { }
else if (propertyName.EqualsCI("disabled")) { }
}
}
if (!StringUtil.IsBlank(provinceData.Code) && !provinceData.Code.EqualsCI("(Select)") && !StringUtil.IsBlank(provinceData.Name))
usgaProvinces.Add(provinceData);
}
}
}
// Store provinces.
List<SaxReferenceGolfMgr.UsgaProvinceData> provincesData = new List<SaxReferenceGolfMgr.UsgaProvinceData>();
foreach (ProvinceData usgaProvince in usgaProvinces)
{
SaxReferenceGolfMgr.UsgaProvinceData provinceData = new SaxReferenceGolfMgr.UsgaProvinceData();
provinceData.CountryCode = countryCode;
provinceData.ProvinceCode = usgaProvince.Code;
provinceData.ProvinceName = usgaProvince.Name;
provincesData.Add(provinceData);
}
ReferenceGolfMgr.BulkUpdateUsgaProvinces(provincesData, DbUpdateInfo);
if (!ReferenceGolfMgr.HasErrorState())
{
// Provinces stored OK = update Country.
usgaCountry.ProvincesUpdateAt = DateTime.Now;
usgaCountry.Priority = false;
ReferenceGolfMgr.UpdateUsgaCountry(usgaCountry, DbUpdateInfo);
}
else SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA provinces: {ReferenceGolfMgr.ErrorStateMessage}.");
}
catch (Exception exc)
{
SaLogger.Warn(SysUtil.Fn + " " + ExceptionUtil.GetLogMessage(exc, $"querying USGA provinces for {countryCode}", 0));
}
}
return usgaProvinces;
}
//------------------------------------------------------------------
/// <summary>
/// USGA Courses for country.
/// </summary>
public List<CourseData> LookupUsgaCoursesForCountryAndProvince(String countryCode, String provinceCode)
{
List<CourseData> usgaCourses = new List<CourseData>();
if (StringUtil.IsBlank(countryCode) || StringUtil.IsBlank(provinceCode)) return usgaCourses;
UsgaRequestData usgaRequestData = GetUsgaRequestData();
using (DisposableCollection<SacsDisposable> disposableItems = new DisposableCollection<SacsDisposable>())
{
// Retrieve and validate country.
SaxGolfUsgaCountry usgaCountry = ReferenceGolfMgr.GetUsgaCountry(new SaxGolfUsgaCountry.Params() { CountryCodeFull = countryCode });
disposableItems.Add(usgaCountry);
if (usgaCountry == null)
{
SaLogger.Warn($"{SysUtil.Fn} USGA Country not found for code '{countryCode}'.");
return usgaCourses;
}
// Retrieve and validate province.
SaxGolfUsgaProvince usgaProvince = ReferenceGolfMgr.GetUsgaProvince(new SaxGolfUsgaProvince.Params() { ProvinceCodeFull = provinceCode });
disposableItems.Add(usgaProvince);
if (usgaProvince == null)
{
SaLogger.Warn($"{SysUtil.Fn} USGA Province not found for code '{provinceCode}'.");
return usgaCourses;
}
if (usgaProvince.Priority)
{
// Clear province 'priority' flag.
usgaProvince.Priority = false;
ReferenceGolfMgr.UpdateUsgaProvince(usgaProvince, DbUpdateInfo);
}
try
{
// Query Courses for Country & Province on USGA website.
CancellationToken cancelToken = new CancellationToken();
SacsHttpRequestData request = new SacsHttpRequestData(cancelToken) { Url = "https://ncrdb.usga.org/NCRListing?handler=LoadCourses" };
// Headers.
request.RequestHeaders.Add(HeaderNames.Accept, "*/*");
request.RequestHeaders.Add(HeaderNames.AcceptEncoding, "gzip, deflate, br, zstd");
request.RequestHeaders.Add(HeaderNames.AcceptLanguage, "en-GB,en;q=0.9");
request.RequestHeaders.Add(HeaderNames.DNT, "1");
request.RequestHeaders.Add(HeaderNames.Origin, "https://ncrdb.usga.org/");
request.RequestHeaders.Add("Priority", "u=1, i");
request.RequestHeaders.Add(HeaderNames.Referer, "https://ncrdb.usga.org/");
if (!StringUtil.IsBlank(usgaRequestData.RequestVerificationToken))
request.RequestHeaders.Add("Requestverificationtoken", usgaRequestData.RequestVerificationToken);
// Content.
request.FormData.Add(new KeyValuePair<string, string>("clubName", ""));
request.FormData.Add(new KeyValuePair<string, string>("clubCity", ""));
request.FormData.Add(new KeyValuePair<string, string>("clubState", provinceCode));
request.FormData.Add(new KeyValuePair<string, string>("clubCountry", countryCode));
request.ContentHeaders.Add(HeaderNames.ContentType, "application/x-www-form-urlencoded; charset=UTF-8");
// Request.
SacsHttpResponseData response = SacsHttpClient.DoRequest(request).Result;
String data = response.Data;
if (!StringUtil.IsBlank(response.Data))
{
HashSet<String> UnrecognisedCourseProperties = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
dynamic coursesJson = SacsJsonHelper.DeserializeJsonObject(response.Data);
if (coursesJson is JArray)
{
JArray coursesArray = coursesJson as JArray;
foreach (var courseToken in coursesArray)
{
// Course properties.
CourseData courseData = new CourseData();
foreach (var propertyToken in courseToken)
{
if (propertyToken is JProperty)
{
JProperty courseProperty = propertyToken as JProperty;
String propertyName = courseProperty.Name;
String propertyStringValue = StringUtil.TrimWhitespace(courseProperty.Value.ToString());
if (propertyName.EqualsCI("courseId"))
courseData.UsgaCourseId = propertyStringValue;
else if (propertyName.EqualsCI("legacyCRPCourseId"))
courseData.LegacyCrpCourseId = propertyStringValue;
else if (propertyName.EqualsCI("courseName"))
courseData.CourseName = propertyStringValue;
else if (propertyName.EqualsCI("facilityId"))
courseData.UsgaFacilityId = propertyStringValue;
else if (propertyName.EqualsCI("facilityName"))
courseData.FacilityName = propertyStringValue;
else if (propertyName.EqualsCI("fullName"))
courseData.FullName = propertyStringValue;
else if (propertyName.EqualsCI("address1"))
courseData.Address1 = propertyStringValue;
else if (propertyName.EqualsCI("address2"))
courseData.Address2 = propertyStringValue;
else if (propertyName.EqualsCI("city"))
courseData.City = propertyStringValue;
else if (propertyName.EqualsCI("entStateCode"))
courseData.EntProvinceCode = propertyStringValue;
else if (propertyName.EqualsCI("state"))
courseData.ProvinceCode = propertyStringValue;
else if (propertyName.EqualsCI("stateDisplay"))
courseData.Province = propertyStringValue;
else if (propertyName.EqualsCI("country"))
courseData.CountryCode = propertyStringValue;
else if (propertyName.EqualsCI("entCountryCode"))
courseData.EntCountryCode = propertyStringValue;
else if (propertyName.EqualsCI("email"))
courseData.Email = propertyStringValue;
else if (propertyName.EqualsCI("telephone"))
courseData.Telephone = propertyStringValue;
else if (propertyName.EqualsCI("ratings"))
{
if (courseProperty.Value.Count() > 0)
courseData.Ratings.Add("");
}
else UnrecognisedCourseProperties.Add(propertyName);
}
}
if (!StringUtil.IsBlank(courseData.UsgaCourseId) && !StringUtil.IsBlank(courseData.CourseName))
{
// Populate caption and add to list.
SetCourseCaption(courseData);
usgaCourses.Add(courseData);
}
}
}
if (UnrecognisedCourseProperties.Any())
SaLogger.Warn(SysUtil.Fn + " Unrecognised USGA course properties: " + String.Join(", ", UnrecognisedCourseProperties));
}
// Store countries.
Dictionary<String, SaxReferenceGolfMgr.UsgaCountryData> countriesData = new Dictionary<String, SaxReferenceGolfMgr.UsgaCountryData>();
foreach (CourseData usgaCourse in usgaCourses)
{
SaxReferenceGolfMgr.UsgaCountryData countryData = new SaxReferenceGolfMgr.UsgaCountryData();
countryData.CountryCode = usgaCourse.CountryCode;
countryData.EntCountryCode = usgaCourse.EntCountryCode;
countriesData[countryData.CountryCode] = countryData;
}
ReferenceGolfMgr.BulkUpdateUsgaCountries(countriesData.Values.ToList(), DbUpdateInfo);
if (ReferenceGolfMgr.HasErrorState())
SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA countries: {ReferenceGolfMgr.ErrorStateMessage}.");
// Store provinces.
Dictionary<String, SaxReferenceGolfMgr.UsgaProvinceData> provincesData = new Dictionary<String, SaxReferenceGolfMgr.UsgaProvinceData>();
foreach (CourseData usgaCourse in usgaCourses)
{
SaxReferenceGolfMgr.UsgaProvinceData provinceData = new SaxReferenceGolfMgr.UsgaProvinceData();
provinceData.CountryCode = usgaCourse.CountryCode;
provinceData.ProvinceCode = usgaCourse.ProvinceCode;
provinceData.EntProvinceCode = usgaCourse.EntProvinceCode;
provincesData[provinceData.ProvinceCode] = provinceData;
}
ReferenceGolfMgr.BulkUpdateUsgaProvinces(provincesData.Values.ToList(), DbUpdateInfo);
if (ReferenceGolfMgr.HasErrorState())
SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA provinces: {ReferenceGolfMgr.ErrorStateMessage}.");
// Store facilities.
Dictionary<SysId, SaxReferenceGolfMgr.UsgaFacilityData> facilitiesData = new Dictionary<SysId, SaxReferenceGolfMgr.UsgaFacilityData>();
foreach (CourseData usgaCourse in usgaCourses)
{
if (StringUtil.IsBlank(usgaCourse.UsgaFacilityId)) continue;
SaxReferenceGolfMgr.UsgaFacilityData facilityData = new SaxReferenceGolfMgr.UsgaFacilityData();
facilityData.ProvinceCode = provinceCode;
facilityData.UsgaFacilityId = StringUtil.ToSysId(usgaCourse.UsgaFacilityId);
facilityData.FacilityName = usgaCourse.FacilityName;
if (!SysId.IsValid(facilityData.UsgaFacilityId))
{
SaLogger.Warn($"{SysUtil.Fn} Invalid Facility Id for {usgaCourse.CourseName}: {usgaCourse.UsgaFacilityId}.");
continue;
}
facilitiesData[facilityData.UsgaFacilityId] = facilityData;
}
ReferenceGolfMgr.BulkUpdateUsgaFacilities(facilitiesData.Values.ToList(), DbUpdateInfo);
if (ReferenceGolfMgr.HasErrorState())
SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA facilities: {ReferenceGolfMgr.ErrorStateMessage}.");
// Store courses.
Dictionary<SysId, SaxReferenceGolfMgr.UsgaCourseData> coursesData = new Dictionary<SysId, SaxReferenceGolfMgr.UsgaCourseData>();
foreach (CourseData usgaCourse in usgaCourses)
{
if (StringUtil.IsBlank(usgaCourse.UsgaCourseId)) continue;
SaxReferenceGolfMgr.UsgaCourseData courseData = new SaxReferenceGolfMgr.UsgaCourseData();
courseData.UsgaFacilityId = StringUtil.ToSysId(usgaCourse.UsgaFacilityId);
courseData.UsgaCourseId = StringUtil.ToSysId(usgaCourse.UsgaCourseId);
courseData.LegacyCrpCourseId = StringUtil.ToSysId(usgaCourse.LegacyCrpCourseId);
courseData.Caption = usgaCourse.Caption;
courseData.CourseName = usgaCourse.CourseName;
courseData.FullName = usgaCourse.FullName;
courseData.Address1 = usgaCourse.Address1;
courseData.Address2 = usgaCourse.Address2;
courseData.City = usgaCourse.City;
courseData.Email = usgaCourse.Email;
courseData.Telephone = usgaCourse.Telephone;
if (!SysId.IsValid(courseData.UsgaCourseId))
{
SaLogger.Warn($"{SysUtil.Fn} Invalid Course Id for {usgaCourse.CourseName}: {usgaCourse.UsgaCourseId}.");
continue;
}
coursesData[courseData.UsgaCourseId] = courseData;
}
ReferenceGolfMgr.BulkUpdateUsgaCourses(coursesData.Values.ToList(), DbUpdateInfo);
if (!ReferenceGolfMgr.HasErrorState())
{
// Courses stored OK - update Province.
usgaProvince.CoursesUpdateAt = DateTime.Now;
usgaProvince.Priority = false;
ReferenceGolfMgr.UpdateUsgaProvince(usgaProvince, DbUpdateInfo);
}
else SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA courses: {ReferenceGolfMgr.ErrorStateMessage}.");
}
catch (Exception exc)
{
SaLogger.Warn(SysUtil.Fn + " " + ExceptionUtil.GetLogMessage(exc, $"querying USGA courses for {countryCode}", 0));
}
}
return usgaCourses;
}
/// <summary>
/// Combine facility and course names as required to generate course caption.
/// <para>- NB Course name is sufficient if same as facility name.</para>
/// </summary>
/// <param name="course"></param>
private void SetCourseCaption(CourseData course)
{
course.Caption = GetCourseCaption(course.FacilityName, course.CourseName);
}
/// <summary>
/// Combine facility and course names as required to generate course caption.
/// <para>- NB Course name is sufficient if same as facility name.</para>
/// </summary>
/// <param name="course"></param>
private String GetCourseCaption(String facilityName, String courseName)
{
// Facility (club) name.
// - Strip id in brackets + anything following and remove generic phrases (golf club, hotel etc).
facilityName = CleanseFacilityName(Regex.Replace(facilityName, "\\s*\\(\\s*\\d+\\s*\\).*$", ""));
// Course name.
// - Remove id in brackets (occasionally found).
// - Remove generic phrases (including for facility name).
courseName = CleanseCourseName(Regex.Replace(courseName, "\\(\\s*\\d+\\s*\\)", ""));
String courseCaption = courseName;
if (!CourseNameMatchesFacility(facilityName, courseName))
courseCaption = facilityName + " - " + courseName;
return courseCaption;
}
/// <summary>
/// Cleanse Facility name by removing generic phrases, eg. golf club, hotel etc.
/// </summary>
private String CleanseFacilityName(String facilityName)
{
facilityName = Regex.Replace(facilityName, "Hotel(,\\s*Golf)?\\s*(and|&)\\s*(Golf|Country|Health|Sailing|Spa)(\\s*Club)?", "", RegexOptions.IgnoreCase);
facilityName = Regex.Replace(facilityName, "Golf(\\s*Club)?\\s*(and|&)\\s*(Hotel|Country|Fitness|Sailing|Spa)(\\s*Club)", "", RegexOptions.IgnoreCase);
facilityName = Regex.Replace(facilityName, "Golf\\s*(Club|Course|Links|Centre|Academy|Resort|Klub)(\\s*(Ltd|Limited))?", "", RegexOptions.IgnoreCase);
facilityName = Regex.Replace(facilityName, "Club\\s*de\\s*Golf", "", RegexOptions.IgnoreCase);
facilityName = Regex.Replace(facilityName, "\\s*(Golf|Country\\s*Club|Club|Ltd)\\s*$", "", RegexOptions.IgnoreCase);
return StringUtil.TrimWhitespace(facilityName);
}
/// <summary>
/// Cleanse Course name by removing generic phrases, including those defined for facility name.
/// </summary>
private String CleanseCourseName(String facilityName)
{
// Course name occasionally suffixed with 'Course' but this is meaningful in many cases (old course, new course).
//facilityName = Regex.Replace(facilityName, "\\bCourse\\b", "", RegexOptions.IgnoreCase);
return CleanseFacilityName(facilityName);
}
private bool CourseNameMatchesFacility(String facilityName, String courseName)
{
if (courseName.EqualsCI(facilityName)) return true;
// Strip selected char/symbols (especially apostrophes which are inconsistent).
if (Regex.Replace(courseName, "[\\s-',\\.]", "").EqualsCI(Regex.Replace(facilityName, "[\\s-',\\.]", ""))) return true;
// Course name can end on Golf or Course and still match facility.
if (Regex.Replace(courseName, "\\s*(Golf|Course)$", "").EqualsCI(facilityName)) return true;
// Course name can start with facility name or vice-versa.
if (courseName.StartsWithCI(facilityName) || facilityName.StartsWithCI(courseName)) return true;
return false;
}
//------------------------------------------------------------------
public bool CleanseCourseCaptions()
{
SaLogger.Info($"{SysUtil.Fn} START", true);
DateTime startedAt = DateTime.Now;
ClearProcessMessages();
int courseCount = 0;
int maxCourses = 0;
int courseCaptionsCorrected = 0;
// Verify Course captions.
using (DisposableCollection<SacsDisposable> disposables = new DisposableCollection<SacsDisposable>())
{
// Local function to allow break/return from within nested loop.
void CompareCourseCaptions()
{
// Countries.
foreach (SaxGolfUsgaCountry country in ReferenceGolfMgr.GetUsgaCountryList(new SaxGolfUsgaCountry.Params()))
{
disposables.Add(country);
// Provinces.
SaxGolfUsgaProvince.Params prmProvince = new SaxGolfUsgaProvince.Params();
prmProvince.CountryIdsIncluded.Add(country.Id);
foreach (SaxGolfUsgaProvince province in ReferenceGolfMgr.GetUsgaProvinceList(prmProvince))
{
disposables.Add(province);
// Separate disposables for each province to restrict volume handled on each disposal.
using (DisposableCollection<SacsDisposable> disposablesProvince = new DisposableCollection<SacsDisposable>())
{
// Facilities.
SaxGolfUsgaFacility.Params prmFacility = new SaxGolfUsgaFacility.Params();
prmFacility.ProvinceIdsIncluded.Add(province.Id);
foreach (SaxGolfUsgaFacility facility in ReferenceGolfMgr.GetUsgaFacilityList(prmFacility))
{
disposablesProvince.Add(facility);
// Courses.
SaxGolfUsgaCourse.Params prmCourse = new SaxGolfUsgaCourse.Params();
prmCourse.FacilityIdsIncluded.Add(facility.Id);
foreach (SaxGolfUsgaCourse course in ReferenceGolfMgr.GetUsgaCourseList(prmCourse))
{
disposablesProvince.Add(course);
courseCount++;
// Compare course caption.
String courseCaption = GetCourseCaption(facility.FacilityName, course.CourseName);
if (!course.Caption.Equals(courseCaption))
{
course.Caption = courseCaption;
if (ReferenceGolfMgr.UpdateUsgaCourse(course, DbUpdateInfo))
{
courseCaptionsCorrected++;
SaLogger.Info($"{SysUtil.Fn} Course caption corrected. Found: {course.Caption}.Corrected: {courseCaption}.");
}
else
{
SaLogger.Warn($"{SysUtil.Fn} Error updating Course caption: {ReferenceGolfMgr.ErrorStateMessage}");
return;
}
}
if (maxCourses > 0 && courseCount > maxCourses)
{
SaLogger.Info($"{SysUtil.Fn} Maximum number of courses processed ({maxCourses}).");
return;
}
}
}
}
}
}
}
// Invoke local function.
CompareCourseCaptions();
}
// Log statistics.
SaLogger.Info($"{SysUtil.Fn} Course caption corrected: {courseCaptionsCorrected}. Courses checked: {courseCount}.", true);
SaLogger.Info($"{SysUtil.Fn} END Elapsed: {DateUtil.FormatNumberOfSeconds((DateTime.Now - startedAt).TotalSeconds)}", true);
return OK;
}
//------------------------------------------------------------------
/// <summary>
/// Identify the column for each marker property.
/// </summary>
private class MarkerColumnData
{
public int ColIdx_Tee { get; set; } = -1;
public int ColIdx_Gender { get; set; } = -1;
public int ColIdx_Par { get; set; } = -1;
public int ColIdx_CourseRating { get; set; } = -1;
public int ColIdx_BogeyRating { get; set; } = -1;
public int ColIdx_SlopeRating { get; set; } = -1;
public int ColIdx_Front9 { get; set; } = -1;
public int ColIdx_Back9 { get; set; } = -1;
public int ColIdx_Front9CourseRating { get; set; } = -1;
public int ColIdx_Back9CourseRating { get; set; } = -1;
public int ColIdx_Front9BogeyRating { get; set; } = -1;
public int ColIdx_Back9BogeyRating { get; set; } = -1;
public int ColIdx_Front9SlopeRating { get; set; } = -1;
public int ColIdx_Back9SlopeRating { get; set; } = -1;
public int ColIdx_MarkerId { get; set; } = -1;
public int ColIdx_Length { get; set; } = -1;
}
/// <summary>
/// Class to allow markers list to be serialised as 'Markers'.
/// </summary>
private class MarkersData
{
public List<MarkerData> Markers { get; } = new List<MarkerData>();
}
/// <summary>
/// USGA Markers for course.
/// </summary>
public List<MarkerData> LookupUsgaMarkersForCourse(SysId usgaCourseId)
{
MarkersData markersData = new MarkersData();
if (!SysId.IsValid(usgaCourseId)) return markersData.Markers;
MarkerColumnData markerColumns = new MarkerColumnData();
using (DisposableCollection<SacsDisposable> disposableItems = new DisposableCollection<SacsDisposable>())
{
// Retrieve and validate course.
SaxGolfUsgaCourse.Params prmCourse = new SaxGolfUsgaCourse.Params();
prmCourse.UsgaCourseIdsIncluded.Add(usgaCourseId);
SaxGolfUsgaCourse usgaCourse = ReferenceGolfMgr.GetUsgaCourse(prmCourse);
disposableItems.Add(usgaCourse);
if (usgaCourse == null)
{
SaLogger.Warn($"{SysUtil.Fn} USGA Course not found for Id '{usgaCourseId}'.");
return markersData.Markers;
}
if (usgaCourse.Priority)
{
usgaCourse.Priority = false;
ReferenceGolfMgr.UpdateUsgaCourse(usgaCourse, DbUpdateInfo);
}
try
{
// Query course markers on USGA website.
CancellationToken cancelToken = new CancellationToken();
SacsHttpRequestData request = new SacsHttpRequestData(cancelToken) { Url = $"https://ncrdb.usga.org/courseTeeInfo?CourseID={usgaCourseId}" };
SacsHttpResponseData response = SacsHttpClient.DoRequest(request).Result;
String data = response.Data;
if (!StringUtil.IsBlank(response.Data))
{
// Parse HTML.
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(response.Data);
// Locate countries list.
HtmlNode teeTableNode = htmlDoc.GetElementbyId("gvTee");
if (teeTableNode != null)
{
// Identify marker columns.
int teeColumnIdx = -1;
foreach (HtmlNode teeHeaderNode in teeTableNode.SelectNodes("./tr[1]/th"))
{
teeColumnIdx++;
String headerText = StringUtil.TrimWhitespace(WebUtil.HtmlDecode(teeHeaderNode.InnerText));
if (StringUtil.IsBlank(headerText) || headerText.EqualsCI("CH"))
{
// Columns ignored.
}
else if (headerText.EqualsCI("Tee Name")) markerColumns.ColIdx_Tee = teeColumnIdx;
else if (headerText.EqualsCI("Gender")) markerColumns.ColIdx_Gender = teeColumnIdx;
else if (headerText.EqualsCI("Par")) markerColumns.ColIdx_Par = teeColumnIdx;
else if (headerText.EqualsCI("TeeId")) markerColumns.ColIdx_MarkerId = teeColumnIdx;
else if (headerText.EqualsCI("Length")) markerColumns.ColIdx_Length = teeColumnIdx;
// Specific part-course (9 holes) values.
else if (headerText.EqualsCI("RatingF9")) markerColumns.ColIdx_Front9CourseRating = teeColumnIdx;
else if (headerText.EqualsCI("RatingB9")) markerColumns.ColIdx_Back9CourseRating = teeColumnIdx;
else if (headerText.EqualsCI("Front (9)")) markerColumns.ColIdx_Front9 = teeColumnIdx;
else if (headerText.EqualsCI("Back (9)")) markerColumns.ColIdx_Back9 = teeColumnIdx;
else if (headerText.EqualsCI("Bogey Rating (F9)")) markerColumns.ColIdx_Front9BogeyRating = teeColumnIdx;
else if (headerText.EqualsCI("Bogey Rating (B9)")) markerColumns.ColIdx_Back9BogeyRating = teeColumnIdx;
else if (headerText.EqualsCI("Slope (F9)")) markerColumns.ColIdx_Front9SlopeRating = teeColumnIdx;
else if (headerText.EqualsCI("Slope (B9)")) markerColumns.ColIdx_Back9SlopeRating = teeColumnIdx;
// Less specific full-course (commonly 18 holes) values.
else if (headerText.IsMatchCI("Course Rating")) markerColumns.ColIdx_CourseRating = teeColumnIdx;
else if (headerText.IsMatchCI("Bogey Rating")) markerColumns.ColIdx_BogeyRating = teeColumnIdx;
else if (headerText.IsMatchCI("Slope Rating")) markerColumns.ColIdx_SlopeRating = teeColumnIdx;
else SaLogger.Info($"{SysUtil.Fn} Unexpected Marker column header: {headerText}");
}
foreach (HtmlNode teeRowNode in teeTableNode.SelectNodes("./tr[position()>1]"))
{
MarkerData markerData18 = new MarkerData(usgaCourseId, 18, "18");
MarkerData markerDataF9 = new MarkerData(usgaCourseId, 9, "F9");
MarkerData markerDataB9 = new MarkerData(usgaCourseId, 9, "B9");
teeColumnIdx = -1;
foreach (HtmlNode teePropertyNode in teeRowNode.SelectNodes("./td"))
{
teeColumnIdx++;
String propertyText = StringUtil.TrimWhitespace(WebUtil.HtmlDecode(teePropertyNode.InnerText));
if (markerColumns.ColIdx_Tee == teeColumnIdx)
{
markerData18.Tee = propertyText;
markerDataF9.Tee = propertyText;
markerDataB9.Tee = propertyText;
}
else if (markerColumns.ColIdx_Gender == teeColumnIdx)
{
markerData18.Gender = propertyText;
markerDataF9.Gender = propertyText;
markerDataB9.Gender = propertyText;
}
else if (markerColumns.ColIdx_Par == teeColumnIdx)
{
markerData18.Par = StringUtil.ToShort(propertyText);
if (markerData18.Par <= 50)
{
markerData18.Holes = 9;
markerData18.HolesCaption = "9";
}
}
else if (markerColumns.ColIdx_CourseRating == teeColumnIdx)
{
markerData18.CourseRating = StringUtil.ToDecimal(propertyText);
}
else if (markerColumns.ColIdx_BogeyRating == teeColumnIdx)
{
markerData18.BogeyRating = StringUtil.ToDecimal(propertyText);
}
else if (markerColumns.ColIdx_SlopeRating == teeColumnIdx)
{
markerData18.SlopeRating = StringUtil.ToShort(propertyText);
}
else if (markerColumns.ColIdx_Front9CourseRating == teeColumnIdx)
{
if (markerDataF9.CourseRating == 0)
markerDataF9.CourseRating = StringUtil.ToDecimal(propertyText);
}
else if (markerColumns.ColIdx_Back9CourseRating == teeColumnIdx)
{
if (markerDataB9.CourseRating == 0)
markerDataB9.CourseRating = StringUtil.ToDecimal(propertyText);
}
else if (markerColumns.ColIdx_Front9 == teeColumnIdx)
{
// Format: CourseRating / SlopeRating.
markerDataF9.CourseRating = StringUtil.ToDecimal(Regex.Replace(propertyText, "\\s*/.*$", ""));
markerDataF9.SlopeRating = StringUtil.ToShort(Regex.Replace(propertyText, "^.*/\\s*", ""));
}
else if (markerColumns.ColIdx_Back9 == teeColumnIdx)
{
// Format: CourseRating / SlopeRating.
markerDataB9.CourseRating = StringUtil.ToDecimal(Regex.Replace(propertyText, "\\s*/.*$", ""));
markerDataB9.SlopeRating = StringUtil.ToShort(Regex.Replace(propertyText, "^.*/\\s*", ""));
}
else if (markerColumns.ColIdx_Front9BogeyRating == teeColumnIdx)
{
markerDataF9.BogeyRating = StringUtil.ToDecimal(propertyText);
}
else if (markerColumns.ColIdx_Back9BogeyRating == teeColumnIdx)
{
markerDataB9.BogeyRating = StringUtil.ToDecimal(propertyText);
}
else if (markerColumns.ColIdx_Front9SlopeRating == teeColumnIdx)
{
if (markerDataF9.SlopeRating == 0)
markerDataF9.SlopeRating = StringUtil.ToShort(propertyText);
}
else if (markerColumns.ColIdx_Back9SlopeRating == teeColumnIdx)
{
// Back 9 Slope Rating populated with value for Front 9.
//markerDataB9.SlopeRating = StringUtil.ToInt(propertyText);
}
else if (markerColumns.ColIdx_MarkerId == teeColumnIdx)
{
markerData18.MarkerId = StringUtil.ToSysId(propertyText);
// Unclear if Tee Id is relevant to 9 holes.
//markerDataF9.MarkerId = StringUtil.ToSysId(propertyText);
//markerDataB9.MarkerId = StringUtil.ToSysId(propertyText);
}
else if (markerColumns.ColIdx_Length == teeColumnIdx)
{
markerData18.Length = StringUtil.ToShort(propertyText);
}
}
// Front and Back 9 likely missing par values - derive from course ratings.
if (markerData18.IsValid() && markerDataF9.IsValid(true) && markerDataF9.Par == 0 && markerDataB9.IsValid(true) && markerDataB9.Par == 0)
{
decimal combined9sCourseRating = markerDataF9.CourseRating + markerDataB9.CourseRating;
markerDataF9.Par = (short)Math.Round(markerData18.Par * markerDataF9.CourseRating / combined9sCourseRating);
markerDataB9.Par = (short)(markerData18.Par - markerDataF9.Par);
}
// Add valid markers to list.
if (markerData18.IsValid())
markersData.Markers.Add(markerData18);
if (markerDataF9.IsValid())
markersData.Markers.Add(markerDataF9);
if (markerDataB9.IsValid())
markersData.Markers.Add(markerDataB9);
}
}
else SaLogger.Warn(SysUtil.Fn + " Tee table not found.");
}
// Store markers.
List<SaxReferenceGolfMgr.UsgaMarkerData> markersUpdateData = new List<SaxReferenceGolfMgr.UsgaMarkerData>();
foreach (MarkerData usgaMarker in markersData.Markers)
{
if (!SysId.IsValid(usgaMarker.CourseId)) continue;
//if (!SysId.IsValid(usgaMarker.MarkerId)) continue;
if (StringUtil.IsBlank(usgaMarker.Caption)) continue;
SaxReferenceGolfMgr.UsgaMarkerData markerData = new SaxReferenceGolfMgr.UsgaMarkerData();
markerData.UsgaCourseId = usgaMarker.CourseId;
markerData.UsgaMarkerId = usgaMarker.MarkerId;
markerData.Caption = usgaMarker.Caption;
markerData.Holes = usgaMarker.Holes;
markerData.HolesCaption = usgaMarker.HolesCaption;
markerData.Tee = usgaMarker.Tee;
markerData.Gender = usgaMarker.Gender;
markerData.SlopeRating = usgaMarker.SlopeRating;
markerData.Par = usgaMarker.Par;
markerData.CourseRating = usgaMarker.CourseRating;
markerData.BogeyRating = usgaMarker.BogeyRating;
markerData.Length = usgaMarker.Length;
markersUpdateData.Add(markerData);
}
ReferenceGolfMgr.BulkUpdateUsgaMarkers(markersUpdateData, DbUpdateInfo);
if (!ReferenceGolfMgr.HasErrorState())
{
// Markers stored OK - update Course.
if (markersData.Markers.Any())
usgaCourse.MarkersData = JsonConvert.SerializeObject(markersData);
usgaCourse.MarkersUpdateAt = DateTime.Now;
usgaCourse.Priority = false;
ReferenceGolfMgr.UpdateUsgaCourse(usgaCourse, DbUpdateInfo);
}
else SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA markers: {ReferenceGolfMgr.ErrorStateMessage}.");
}
catch (Exception exc)
{
SaLogger.Warn(SysUtil.Fn + " " + ExceptionUtil.GetLogMessage(exc, $"querying USGA markers for course Id: {usgaCourseId}", 0));
}
}
return markersData.Markers;
}
//------------------------------------------------------------------
}
//======================================================================
}