using DocumentFormat.OpenXml.InkML;
using HtmlAgilityPack;
using MailKit;
using Microsoft.Net.Http.Headers;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Sarand.Net.Standard.Base.App;
using Sarand.Net.Standard.Base.Bus;
using Sarand.Net.Standard.Base.Core;
using Sarand.Net.Standard.Base.Dbas;
using Sarand.Net.Standard.Base.Helpers;
using Sarand.Net.Standard.Base.Net;
using Sarand.Net.Standard.Base.Utils;
using Saxonite.Net.Standard.Base.App;
using Saxonite.Net.Standard.Base.Bus.Reference;
using Saxonite.Net.Standard.Base.Mgr.Reference;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using static Saxonite.Net.Core.Web.Base.Procs.SaxGolfUsgaDataLookup;

namespace Saxonite.Net.Core.Web.Base.Procs
{
	//======================================================================
	// USGA course rating & slope database: https://ncrdb.usga.org/
	// https://ncrdb.usga.org/courseTeeInfo.aspx?CourseID=14758
	// Ref. https://zylalabs.com/blog/enhance-your-game-with-a-free-golf-api
	public partial class SaxGolfUsgaDataLookup : SaxProcess_Web
	{
		//-------------------------------------------------------------------
		// Definitions.
		public enum Task
		{
			CleanseCourseCaptions,
		}

		public new class Config : SaxProcess_Web.Config
		{
			public Config(SaxSessionContext sessionContext, DbUpdateInfo dbUpdateInfo) : base(sessionContext, dbUpdateInfo)
			{ }
			/// <summary>Tasks to be run under this instance of the USGA Data Lookup process.</summary>
			public HashSet<Task> Tasks { get; } = new HashSet<Task>();
		}

		public class CountryProvinceCodes
		{
			public CountryProvinceCodes(String countryCode, String provinceCode)
			{
				this.CountryCode = StringUtil.OnBlank(countryCode);
				this.ProvinceCode = StringUtil.OnBlank(provinceCode);
			}
			public string CountryCode { get; }
			public string ProvinceCode { get; }
			public override bool Equals(object obj)
			{
				return Equals(obj as CountryProvinceCodes);
			}
			public bool Equals(CountryProvinceCodes other)
			{
				return other != null && CountryCode == other.CountryCode && ProvinceCode == other.ProvinceCode;
			}
			public override int GetHashCode()
			{
				return HashCode.Combine(CountryCode, ProvinceCode);
			}
			public override string ToString()
			{
				return $"CountryProvinceCodes({CountryCode}, {ProvinceCode})";
			}
		}

		public class CountryData
		{
			public string Code { get; set; }
			public string Name { get; set; }
		}

		public class ProvinceData
		{
			public string Code { get; set; }
			public string Name { get; set; }
		}

		public class CourseData
		{
			public string UsgaCourseId { get; set; }
			public string LegacyCrpCourseId { get; set; }
			public string Caption { get; set; }
			public string CourseName { get; set; }
			public string FullName { get; set; }
			public string UsgaFacilityId { get; set; }
			public string FacilityName { get; set; }
			public string Address1 { get; set; }
			public string Address2 { get; set; }
			public string City { get; set; }
			public string ProvinceCode { get; set; }
			public string EntProvinceCode { get; set; }
			public string Province { get; set; }
			public string CountryCode { get; set; }
			public string EntCountryCode { get; set; }
			public string Country { get; set; }
			public string Email { get; set; }
			public string Telephone { get; set; }
			public List<String> Ratings { get; } = new List<string>();
		}

		public class MarkerData
		{
			public MarkerData(SysId courseId, short holes, string holesCaption)
			{
				this.CourseId = courseId;
				this.Holes = holes;
				this.HolesCaption = holesCaption;
			}
			public SysId CourseId { get; }
			public SysId MarkerId { get; set; }
			public string Caption { get { return StringUtil.TrimWhitespace(StringUtil.OnBlank(Tee) + " " + StringUtil.OnBlank(HolesCaption) + " " + StringUtil.OnBlank(Gender)); } }
			public short Holes { get; set; }
			public string HolesCaption { get; set; }
			public string Tee { get; set; }
			public string Gender { get; set; }
			public short SlopeRating { get; set; }
			public short Par { get; set; }
			public decimal CourseRating { get; set; }
			public decimal BogeyRating { get; set; }
			public short Length { get; set; }
			public bool IsValid(bool ignorePar = false)
			{
				if (!SysId.IsValid(CourseId)) return false;
				// 9 hole markers do not have there own Id.
				//if (!SysId.IsValid(MarkerId)) return false;
				if (Holes < 1) return false;
				if (StringUtil.IsBlank(Tee)) return false;
				//if (StringUtil.IsBlank(Gender)) return false;
				if (SlopeRating == 0) return false;
				if (!ignorePar && Par == 0) return false;
				if (CourseRating == 0) return false;
				return true;
			}
		}

		//-------------------------------------------------------------------
		// Data members.
		private Config config;

		//-------------------------------------------------------------------
		// Construction.
		public SaxGolfUsgaDataLookup(Config config) : base(config)
		{
			this.config = config;
		}

		//-------------------------------------------------------------------
		// Properties.

		public override string ProcessDescription => "Golf Course Lookup";

		//-------------------------------------------------------------------
		// Run the process.
		protected override bool Run()
		{
			// Execute selected tasks.
			foreach (Task task in config.Tasks)
			{
				switch (task)
				{
					case Task.CleanseCourseCaptions:
						CleanseCourseCaptions();
						break;
				}
				if (!OK) break;
			}

			return OK;
		}

		//------------------------------------------------------------------
		// USGA data lookup.
		//------------------------------------------------------------------

		/// <summary>
		/// Data elements that may be required in subsequent requests etc.
		/// </summary>
		private class UsgaRequestData
		{
			public DateTime PopulatedAt { get; } = DateTime.Now;
			public Dictionary<String, SacsHttpCookieData> Cookies { get; } = new Dictionary<String, SacsHttpCookieData>();
			public String RequestVerificationToken { get; set; }
		}
		private UsgaRequestData usgaRequestData = null;
		/// <summary>
		/// Populate the request data from the Home Page response.
		/// </summary>
		private void PopulateUsgaRequestData(SacsHttpResponseData response)
		{
			try
			{
				if (!StringUtil.IsBlank(response.Data))
				{
					usgaRequestData = new UsgaRequestData();

					// Cookies.
					usgaRequestData.Cookies.AddRange(response.Cookies);

					// Parse HTML.
					HtmlDocument htmlDoc = new HtmlDocument();
					htmlDoc.LoadHtml(response.Data);

					// Locate verification token.
					HtmlNode verificationTokenNode = htmlDoc.DocumentNode.SelectSingleNode("//input[@name='__RequestVerificationToken']");
					if (verificationTokenNode != null)
					{
						usgaRequestData.RequestVerificationToken = verificationTokenNode.GetAttributeValue("value", "");
					}
				}

			}
			catch (Exception exc)
			{
				SaLogger.Warn(SysUtil.Fn + " " + ExceptionUtil.GetLogMessage(exc, "populating USGA request data", 0));
			}
		}
		/// <summary>
		/// Get the USGA request data to extract useful elements, eg. verification token and cookies.
		/// <para>- Request data is refreshed periodically in case of expiry.</para>
		/// </summary>
		private UsgaRequestData GetUsgaRequestData()
		{
			if (usgaRequestData == null || (DateTime.Now - usgaRequestData.PopulatedAt).TotalMinutes > 15)
			{
				try
				{

					// Query countries on USGA website.
					CancellationToken cancelToken = new CancellationToken();
					SacsHttpRequestData request = new SacsHttpRequestData(cancelToken) { Url = "https://ncrdb.usga.org/" };
					SacsHttpResponseData response = SacsHttpClient.DoRequest(request).Result;
					String data = response.Data;

					PopulateUsgaRequestData(response);

				}
				catch (Exception exc)
				{
					SaLogger.Warn(SysUtil.Fn + " " + ExceptionUtil.GetLogMessage(exc, "querying USGA request data", 0));
				}
			}
			return usgaRequestData;
		}

		//------------------------------------------------------------------
		/// <summary>
		/// USGA Countries.
		/// <para>- Countries are listed on the home page of the USGA site in a dropdown.</para>
		/// </summary>
		public List<CountryData> LookupUsgaCountries()
		{
			List<CountryData> usgaCountries = new List<CountryData>();

			try
			{

				// Query Countries on USGA website.
				CancellationToken cancelToken = new CancellationToken();
				SacsHttpRequestData request = new SacsHttpRequestData(cancelToken) { Url = "https://ncrdb.usga.org/" };
				SacsHttpResponseData response = SacsHttpClient.DoRequest(request).Result;
				String data = response.Data;
				PopulateUsgaRequestData(response);

				if (!StringUtil.IsBlank(response.Data))
				{
					// Parse HTML.
					HtmlDocument htmlDoc = new HtmlDocument();
					htmlDoc.LoadHtml(response.Data);

					// Locate countries list.
					HtmlNode countriesNode = htmlDoc.GetElementbyId("ddlCountries");
					if (countriesNode != null)
					{
						foreach (HtmlNode countryOptionNode in countriesNode.SelectNodes("./option"))
						{
							String countryCode = StringUtil.TrimWhitespace(countryOptionNode.GetAttributeValue("value", ""));
							String countryName = StringUtil.TrimWhitespace(WebUtil.HtmlDecode(countryOptionNode.InnerText));
							// Ignore blank code or name.
							if (StringUtil.IsBlank(countryCode) || countryCode.EqualsCI("(Select)") || StringUtil.IsBlank(countryName)) continue;
							// Ignore 'Select' item.
							if (countryCode.EqualsCI("(Select)")) continue;
							// Ignore US States and territories.
							if (countryCode.StartsWithCI("US-") || countryCode.StartsWithCI("UM-")) continue;
							// Ignore codes that aren't 3 chars.
							if (countryCode.Length != 3) continue;
							// Add country to list.
							usgaCountries.Add(new CountryData() { Code = countryCode, Name = countryName });
						}
					}
					else SaLogger.Warn(SysUtil.Fn + " Countries node not found.");
				}

				// Store countries.
				List<SaxReferenceGolfMgr.UsgaCountryData> countriesData = new List<SaxReferenceGolfMgr.UsgaCountryData>();
				foreach (CountryData usgaCountry in usgaCountries)
				{
					SaxReferenceGolfMgr.UsgaCountryData countryData = new SaxReferenceGolfMgr.UsgaCountryData();
					countryData.CountryCode = usgaCountry.Code;
					countryData.CountryName = usgaCountry.Name;
					countriesData.Add(countryData);
				}
				ReferenceGolfMgr.BulkUpdateUsgaCountries(countriesData, DbUpdateInfo);
				if (ReferenceGolfMgr.HasErrorState())
					SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA countries: {ReferenceGolfMgr.ErrorStateMessage}.");

			}
			catch (Exception exc)
			{
				SaLogger.Warn(SysUtil.Fn + " " + ExceptionUtil.GetLogMessage(exc, "querying USGA countries", 0));
			}
			return usgaCountries;
		}

		//------------------------------------------------------------------
		/// <summary>
		/// USGA Provinces for country.
		/// </summary>
		public List<ProvinceData> LookupUsgaProvincesForCountry(String countryCode)
		{
			List<ProvinceData> usgaProvinces = new List<ProvinceData>();
			if (StringUtil.IsBlank(countryCode)) return usgaProvinces;

			UsgaRequestData usgaRequestData = GetUsgaRequestData();

			using (DisposableCollection<SacsDisposable> disposableItems = new DisposableCollection<SacsDisposable>())
			{
				// Retrieve and validate country.
				SaxGolfUsgaCountry usgaCountry = ReferenceGolfMgr.GetUsgaCountry(new SaxGolfUsgaCountry.Params() { CountryCodeFull = countryCode });
				disposableItems.Add(usgaCountry);
				if (usgaCountry == null)
				{
					SaLogger.Warn($"{SysUtil.Fn} USGA Country not found for code '{countryCode}'.");
					return usgaProvinces;
				}
				if (usgaCountry.Priority)
				{
					// Clear country 'priority' flag.
					usgaCountry.Priority = false;
					ReferenceGolfMgr.UpdateUsgaCountry(usgaCountry, DbUpdateInfo);
				}

				try
				{
					// Query Provinces for Country on USGA website.
					CancellationToken cancelToken = new CancellationToken();
					SacsHttpRequestData request = new SacsHttpRequestData(cancelToken) { Url = "https://ncrdb.usga.org/NCRListing?handler=LoadStates" };
					// Headers.
					request.RequestHeaders.Add(HeaderNames.Accept, "*/*");
					request.RequestHeaders.Add(HeaderNames.AcceptEncoding, "gzip, deflate, br, zstd");
					request.RequestHeaders.Add(HeaderNames.AcceptLanguage, "en-GB,en;q=0.9");
					request.RequestHeaders.Add(HeaderNames.DNT, "1");
					request.RequestHeaders.Add(HeaderNames.Origin, "https://ncrdb.usga.org/");
					request.RequestHeaders.Add("Priority", "u=1, i");
					request.RequestHeaders.Add(HeaderNames.Referer, "https://ncrdb.usga.org/");
					if (!StringUtil.IsBlank(usgaRequestData.RequestVerificationToken))
						request.RequestHeaders.Add("Requestverificationtoken", usgaRequestData.RequestVerificationToken);
					// Content.
					request.FormData.Add(new KeyValuePair<string, string>("countryCode", countryCode));
					request.ContentHeaders.Add(HeaderNames.ContentType, "application/x-www-form-urlencoded; charset=UTF-8");
					// Request.
					SacsHttpResponseData response = SacsHttpClient.DoRequest(request).Result;
					String data = response.Data;

					if (!StringUtil.IsBlank(response.Data))
					{
						dynamic provincesJson = SacsJsonHelper.DeserializeJsonObject(response.Data);
						if (provincesJson is JArray)
						{
							JArray provincesArray = provincesJson as JArray;
							foreach (var provinceToken in provincesArray)
							{
								// Province properties.
								ProvinceData provinceData = new ProvinceData();
								foreach (var propertyToken in provinceToken)
								{
									if (propertyToken is JProperty)
									{
										JProperty provinceProperty = propertyToken as JProperty;
										String propertyName = provinceProperty.Name;
										String propertyStringValue = StringUtil.TrimWhitespace(provinceProperty.Value.ToString());
										if (propertyName.EqualsCI("text"))
											provinceData.Name = propertyStringValue;
										else if (propertyName.EqualsCI("value"))
											provinceData.Code = propertyStringValue;
										else if (propertyName.EqualsCI("group")) { }
										else if (propertyName.EqualsCI("selected")) { }
										else if (propertyName.EqualsCI("disabled")) { }
									}
								}
								if (!StringUtil.IsBlank(provinceData.Code) && !provinceData.Code.EqualsCI("(Select)") && !StringUtil.IsBlank(provinceData.Name))
									usgaProvinces.Add(provinceData);
							}
						}
					}

					// Store provinces.
					List<SaxReferenceGolfMgr.UsgaProvinceData> provincesData = new List<SaxReferenceGolfMgr.UsgaProvinceData>();
					foreach (ProvinceData usgaProvince in usgaProvinces)
					{
						SaxReferenceGolfMgr.UsgaProvinceData provinceData = new SaxReferenceGolfMgr.UsgaProvinceData();
						provinceData.CountryCode = countryCode;
						provinceData.ProvinceCode = usgaProvince.Code;
						provinceData.ProvinceName = usgaProvince.Name;
						provincesData.Add(provinceData);
					}
					ReferenceGolfMgr.BulkUpdateUsgaProvinces(provincesData, DbUpdateInfo);
					if (!ReferenceGolfMgr.HasErrorState())
					{
						// Provinces stored OK = update Country.
						usgaCountry.ProvincesUpdateAt = DateTime.Now;
						usgaCountry.Priority = false;
						ReferenceGolfMgr.UpdateUsgaCountry(usgaCountry, DbUpdateInfo);
					}
					else SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA provinces: {ReferenceGolfMgr.ErrorStateMessage}.");
				}
				catch (Exception exc)
				{
					SaLogger.Warn(SysUtil.Fn + " " + ExceptionUtil.GetLogMessage(exc, $"querying USGA provinces for {countryCode}", 0));
				}
			}
			return usgaProvinces;
		}

		//------------------------------------------------------------------
		/// <summary>
		/// USGA Courses for country.
		/// </summary>
		public List<CourseData> LookupUsgaCoursesForCountryAndProvince(String countryCode, String provinceCode)
		{
			List<CourseData> usgaCourses = new List<CourseData>();
			if (StringUtil.IsBlank(countryCode) || StringUtil.IsBlank(provinceCode)) return usgaCourses;

			UsgaRequestData usgaRequestData = GetUsgaRequestData();

			using (DisposableCollection<SacsDisposable> disposableItems = new DisposableCollection<SacsDisposable>())
			{
				// Retrieve and validate country.
				SaxGolfUsgaCountry usgaCountry = ReferenceGolfMgr.GetUsgaCountry(new SaxGolfUsgaCountry.Params() { CountryCodeFull = countryCode });
				disposableItems.Add(usgaCountry);
				if (usgaCountry == null)
				{
					SaLogger.Warn($"{SysUtil.Fn} USGA Country not found for code '{countryCode}'.");
					return usgaCourses;
				}
				// Retrieve and validate province.
				SaxGolfUsgaProvince usgaProvince = ReferenceGolfMgr.GetUsgaProvince(new SaxGolfUsgaProvince.Params() { ProvinceCodeFull = provinceCode });
				disposableItems.Add(usgaProvince);
				if (usgaProvince == null)
				{
					SaLogger.Warn($"{SysUtil.Fn} USGA Province not found for code '{provinceCode}'.");
					return usgaCourses;
				}
				if (usgaProvince.Priority)
				{
					// Clear province 'priority' flag.
					usgaProvince.Priority = false;
					ReferenceGolfMgr.UpdateUsgaProvince(usgaProvince, DbUpdateInfo);
				}

				try
				{
					// Query Courses for Country & Province on USGA website.
					CancellationToken cancelToken = new CancellationToken();
					SacsHttpRequestData request = new SacsHttpRequestData(cancelToken) { Url = "https://ncrdb.usga.org/NCRListing?handler=LoadCourses" };
					// Headers.
					request.RequestHeaders.Add(HeaderNames.Accept, "*/*");
					request.RequestHeaders.Add(HeaderNames.AcceptEncoding, "gzip, deflate, br, zstd");
					request.RequestHeaders.Add(HeaderNames.AcceptLanguage, "en-GB,en;q=0.9");
					request.RequestHeaders.Add(HeaderNames.DNT, "1");
					request.RequestHeaders.Add(HeaderNames.Origin, "https://ncrdb.usga.org/");
					request.RequestHeaders.Add("Priority", "u=1, i");
					request.RequestHeaders.Add(HeaderNames.Referer, "https://ncrdb.usga.org/");
					if (!StringUtil.IsBlank(usgaRequestData.RequestVerificationToken))
						request.RequestHeaders.Add("Requestverificationtoken", usgaRequestData.RequestVerificationToken);
					// Content.
					request.FormData.Add(new KeyValuePair<string, string>("clubName", ""));
					request.FormData.Add(new KeyValuePair<string, string>("clubCity", ""));
					request.FormData.Add(new KeyValuePair<string, string>("clubState", provinceCode));
					request.FormData.Add(new KeyValuePair<string, string>("clubCountry", countryCode));
					request.ContentHeaders.Add(HeaderNames.ContentType, "application/x-www-form-urlencoded; charset=UTF-8");
					// Request.
					SacsHttpResponseData response = SacsHttpClient.DoRequest(request).Result;
					String data = response.Data;

					if (!StringUtil.IsBlank(response.Data))
					{
						HashSet<String> UnrecognisedCourseProperties = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
						dynamic coursesJson = SacsJsonHelper.DeserializeJsonObject(response.Data);
						if (coursesJson is JArray)
						{
							JArray coursesArray = coursesJson as JArray;
							foreach (var courseToken in coursesArray)
							{
								// Course properties.
								CourseData courseData = new CourseData();
								foreach (var propertyToken in courseToken)
								{
									if (propertyToken is JProperty)
									{
										JProperty courseProperty = propertyToken as JProperty;
										String propertyName = courseProperty.Name;
										String propertyStringValue = StringUtil.TrimWhitespace(courseProperty.Value.ToString());
										if (propertyName.EqualsCI("courseId"))
											courseData.UsgaCourseId = propertyStringValue;
										else if (propertyName.EqualsCI("legacyCRPCourseId"))
											courseData.LegacyCrpCourseId = propertyStringValue;
										else if (propertyName.EqualsCI("courseName"))
											courseData.CourseName = propertyStringValue;
										else if (propertyName.EqualsCI("facilityId"))
											courseData.UsgaFacilityId = propertyStringValue;
										else if (propertyName.EqualsCI("facilityName"))
											courseData.FacilityName = propertyStringValue;
										else if (propertyName.EqualsCI("fullName"))
											courseData.FullName = propertyStringValue;
										else if (propertyName.EqualsCI("address1"))
											courseData.Address1 = propertyStringValue;
										else if (propertyName.EqualsCI("address2"))
											courseData.Address2 = propertyStringValue;
										else if (propertyName.EqualsCI("city"))
											courseData.City = propertyStringValue;
										else if (propertyName.EqualsCI("entStateCode"))
											courseData.EntProvinceCode = propertyStringValue;
										else if (propertyName.EqualsCI("state"))
											courseData.ProvinceCode = propertyStringValue;
										else if (propertyName.EqualsCI("stateDisplay"))
											courseData.Province = propertyStringValue;
										else if (propertyName.EqualsCI("country"))
											courseData.CountryCode = propertyStringValue;
										else if (propertyName.EqualsCI("entCountryCode"))
											courseData.EntCountryCode = propertyStringValue;
										else if (propertyName.EqualsCI("email"))
											courseData.Email = propertyStringValue;
										else if (propertyName.EqualsCI("telephone"))
											courseData.Telephone = propertyStringValue;
										else if (propertyName.EqualsCI("ratings"))
										{
											if (courseProperty.Value.Count() > 0)
												courseData.Ratings.Add("");
										}
										else UnrecognisedCourseProperties.Add(propertyName);
									}
								}
								if (!StringUtil.IsBlank(courseData.UsgaCourseId) && !StringUtil.IsBlank(courseData.CourseName))
								{
									// Populate caption and add to list.
									SetCourseCaption(courseData);
									usgaCourses.Add(courseData);
								}
							}
						}
						if (UnrecognisedCourseProperties.Any())
							SaLogger.Warn(SysUtil.Fn + " Unrecognised USGA course properties: " + String.Join(", ", UnrecognisedCourseProperties));
					}

					// Store countries.
					Dictionary<String, SaxReferenceGolfMgr.UsgaCountryData> countriesData = new Dictionary<String, SaxReferenceGolfMgr.UsgaCountryData>();
					foreach (CourseData usgaCourse in usgaCourses)
					{
						SaxReferenceGolfMgr.UsgaCountryData countryData = new SaxReferenceGolfMgr.UsgaCountryData();
						countryData.CountryCode = usgaCourse.CountryCode;
						countryData.EntCountryCode = usgaCourse.EntCountryCode;
						countriesData[countryData.CountryCode] = countryData;
					}
					ReferenceGolfMgr.BulkUpdateUsgaCountries(countriesData.Values.ToList(), DbUpdateInfo);
					if (ReferenceGolfMgr.HasErrorState())
						SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA countries: {ReferenceGolfMgr.ErrorStateMessage}.");

					// Store provinces.
					Dictionary<String, SaxReferenceGolfMgr.UsgaProvinceData> provincesData = new Dictionary<String, SaxReferenceGolfMgr.UsgaProvinceData>();
					foreach (CourseData usgaCourse in usgaCourses)
					{
						SaxReferenceGolfMgr.UsgaProvinceData provinceData = new SaxReferenceGolfMgr.UsgaProvinceData();
						provinceData.CountryCode = usgaCourse.CountryCode;
						provinceData.ProvinceCode = usgaCourse.ProvinceCode;
						provinceData.EntProvinceCode = usgaCourse.EntProvinceCode;
						provincesData[provinceData.ProvinceCode] = provinceData;
					}
					ReferenceGolfMgr.BulkUpdateUsgaProvinces(provincesData.Values.ToList(), DbUpdateInfo);
					if (ReferenceGolfMgr.HasErrorState())
						SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA provinces: {ReferenceGolfMgr.ErrorStateMessage}.");

					// Store facilities.
					Dictionary<SysId, SaxReferenceGolfMgr.UsgaFacilityData> facilitiesData = new Dictionary<SysId, SaxReferenceGolfMgr.UsgaFacilityData>();
					foreach (CourseData usgaCourse in usgaCourses)
					{
						if (StringUtil.IsBlank(usgaCourse.UsgaFacilityId)) continue;
						SaxReferenceGolfMgr.UsgaFacilityData facilityData = new SaxReferenceGolfMgr.UsgaFacilityData();
						facilityData.ProvinceCode = provinceCode;
						facilityData.UsgaFacilityId = StringUtil.ToSysId(usgaCourse.UsgaFacilityId);
						facilityData.FacilityName = usgaCourse.FacilityName;
						if (!SysId.IsValid(facilityData.UsgaFacilityId))
						{
							SaLogger.Warn($"{SysUtil.Fn} Invalid Facility Id for {usgaCourse.CourseName}: {usgaCourse.UsgaFacilityId}.");
							continue;
						}
						facilitiesData[facilityData.UsgaFacilityId] = facilityData;
					}
					ReferenceGolfMgr.BulkUpdateUsgaFacilities(facilitiesData.Values.ToList(), DbUpdateInfo);
					if (ReferenceGolfMgr.HasErrorState())
						SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA facilities: {ReferenceGolfMgr.ErrorStateMessage}.");

					// Store courses.
					Dictionary<SysId, SaxReferenceGolfMgr.UsgaCourseData> coursesData = new Dictionary<SysId, SaxReferenceGolfMgr.UsgaCourseData>();
					foreach (CourseData usgaCourse in usgaCourses)
					{
						if (StringUtil.IsBlank(usgaCourse.UsgaCourseId)) continue;
						SaxReferenceGolfMgr.UsgaCourseData courseData = new SaxReferenceGolfMgr.UsgaCourseData();
						courseData.UsgaFacilityId = StringUtil.ToSysId(usgaCourse.UsgaFacilityId);
						courseData.UsgaCourseId = StringUtil.ToSysId(usgaCourse.UsgaCourseId);
						courseData.LegacyCrpCourseId = StringUtil.ToSysId(usgaCourse.LegacyCrpCourseId);
						courseData.Caption = usgaCourse.Caption;
						courseData.CourseName = usgaCourse.CourseName;
						courseData.FullName = usgaCourse.FullName;
						courseData.Address1 = usgaCourse.Address1;
						courseData.Address2 = usgaCourse.Address2;
						courseData.City = usgaCourse.City;
						courseData.Email = usgaCourse.Email;
						courseData.Telephone = usgaCourse.Telephone;
						if (!SysId.IsValid(courseData.UsgaCourseId))
						{
							SaLogger.Warn($"{SysUtil.Fn} Invalid Course Id for {usgaCourse.CourseName}: {usgaCourse.UsgaCourseId}.");
							continue;
						}
						coursesData[courseData.UsgaCourseId] = courseData;
					}
					ReferenceGolfMgr.BulkUpdateUsgaCourses(coursesData.Values.ToList(), DbUpdateInfo);
					if (!ReferenceGolfMgr.HasErrorState())
					{
						// Courses stored OK - update Province.
						usgaProvince.CoursesUpdateAt = DateTime.Now;
						usgaProvince.Priority = false;
						ReferenceGolfMgr.UpdateUsgaProvince(usgaProvince, DbUpdateInfo);
					}
					else SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA courses: {ReferenceGolfMgr.ErrorStateMessage}.");
				}
				catch (Exception exc)
				{
					SaLogger.Warn(SysUtil.Fn + " " + ExceptionUtil.GetLogMessage(exc, $"querying USGA courses for {countryCode}", 0));
				}
			}
			return usgaCourses;
		}

		/// <summary>
		/// Combine facility and course names as required to generate course caption.
		/// <para>- NB Course name is sufficient if same as facility name.</para>
		/// </summary>
		/// <param name="course"></param>
		private void SetCourseCaption(CourseData course)
		{
			course.Caption = GetCourseCaption(course.FacilityName, course.CourseName);
		}

		/// <summary>
		/// Combine facility and course names as required to generate course caption.
		/// <para>- NB Course name is sufficient if same as facility name.</para>
		/// </summary>
		/// <param name="course"></param>
		private String GetCourseCaption(String facilityName, String courseName)
		{
			// Facility (club) name.
			// - Strip id in brackets + anything following and remove generic phrases (golf club, hotel etc).
			facilityName = CleanseFacilityName(Regex.Replace(facilityName, "\\s*\\(\\s*\\d+\\s*\\).*$", ""));
			// Course name.
			// - Remove id in brackets (occasionally found).
			// - Remove generic phrases (including for facility name).
			courseName = CleanseCourseName(Regex.Replace(courseName, "\\(\\s*\\d+\\s*\\)", ""));
			String courseCaption = courseName;
			if (!CourseNameMatchesFacility(facilityName, courseName))
				courseCaption = facilityName + " - " + courseName;
			return courseCaption;
		}

		/// <summary>
		/// Cleanse Facility name by removing generic phrases, eg. golf club, hotel etc.
		/// </summary>
		private String CleanseFacilityName(String facilityName)
		{
			facilityName = Regex.Replace(facilityName, "Hotel(,\\s*Golf)?\\s*(and|&)\\s*(Golf|Country|Health|Sailing|Spa)(\\s*Club)?", "", RegexOptions.IgnoreCase);
			facilityName = Regex.Replace(facilityName, "Golf(\\s*Club)?\\s*(and|&)\\s*(Hotel|Country|Fitness|Sailing|Spa)(\\s*Club)", "", RegexOptions.IgnoreCase);
			facilityName = Regex.Replace(facilityName, "Golf\\s*(Club|Course|Links|Centre|Academy|Resort|Klub)(\\s*(Ltd|Limited))?", "", RegexOptions.IgnoreCase);
			facilityName = Regex.Replace(facilityName, "Club\\s*de\\s*Golf", "", RegexOptions.IgnoreCase);
			facilityName = Regex.Replace(facilityName, "\\s*(Golf|Country\\s*Club|Club|Ltd)\\s*$", "", RegexOptions.IgnoreCase);
			return StringUtil.TrimWhitespace(facilityName);
		}

		/// <summary>
		/// Cleanse Course name by removing generic phrases, including those defined for facility name.
		/// </summary>
		private String CleanseCourseName(String facilityName)
		{
			// Course name occasionally suffixed with 'Course' but this is meaningful in many cases (old course, new course).
			//facilityName = Regex.Replace(facilityName, "\\bCourse\\b", "", RegexOptions.IgnoreCase);
			return CleanseFacilityName(facilityName);
		}

		private bool CourseNameMatchesFacility(String facilityName, String courseName)
		{
			if (courseName.EqualsCI(facilityName)) return true;
			// Strip selected char/symbols (especially apostrophes which are inconsistent).
			if (Regex.Replace(courseName, "[\\s-',\\.]", "").EqualsCI(Regex.Replace(facilityName, "[\\s-',\\.]", ""))) return true;
			// Course name can end on Golf or Course and still match facility.
			if (Regex.Replace(courseName, "\\s*(Golf|Course)$", "").EqualsCI(facilityName)) return true;
			// Course name can start with facility name or vice-versa.
			if (courseName.StartsWithCI(facilityName) || facilityName.StartsWithCI(courseName)) return true;
			return false;
		}

		//------------------------------------------------------------------
		public bool CleanseCourseCaptions()
		{
			SaLogger.Info($"{SysUtil.Fn} START", true);
			DateTime startedAt = DateTime.Now;
			ClearProcessMessages();

			int courseCount = 0;
			int maxCourses = 0;
			int courseCaptionsCorrected = 0;

			// Verify Course captions.
			using (DisposableCollection<SacsDisposable> disposables = new DisposableCollection<SacsDisposable>())
			{
				// Local function to allow break/return from within nested loop.
				void CompareCourseCaptions()
				{
					// Countries.
					foreach (SaxGolfUsgaCountry country in ReferenceGolfMgr.GetUsgaCountryList(new SaxGolfUsgaCountry.Params()))
					{
						disposables.Add(country);

						// Provinces.
						SaxGolfUsgaProvince.Params prmProvince = new SaxGolfUsgaProvince.Params();
						prmProvince.CountryIdsIncluded.Add(country.Id);
						foreach (SaxGolfUsgaProvince province in ReferenceGolfMgr.GetUsgaProvinceList(prmProvince))
						{
							disposables.Add(province);

							// Separate disposables for each province to restrict volume handled on each disposal.
							using (DisposableCollection<SacsDisposable> disposablesProvince = new DisposableCollection<SacsDisposable>())
							{

								// Facilities.
								SaxGolfUsgaFacility.Params prmFacility = new SaxGolfUsgaFacility.Params();
								prmFacility.ProvinceIdsIncluded.Add(province.Id);
								foreach (SaxGolfUsgaFacility facility in ReferenceGolfMgr.GetUsgaFacilityList(prmFacility))
								{
									disposablesProvince.Add(facility);

									// Courses.
									SaxGolfUsgaCourse.Params prmCourse = new SaxGolfUsgaCourse.Params();
									prmCourse.FacilityIdsIncluded.Add(facility.Id);
									foreach (SaxGolfUsgaCourse course in ReferenceGolfMgr.GetUsgaCourseList(prmCourse))
									{
										disposablesProvince.Add(course);
										courseCount++;

										// Compare course caption.
										String courseCaption = GetCourseCaption(facility.FacilityName, course.CourseName);
										if (!course.Caption.Equals(courseCaption))
										{
											course.Caption = courseCaption;
											if (ReferenceGolfMgr.UpdateUsgaCourse(course, DbUpdateInfo))
											{
												courseCaptionsCorrected++;
												SaLogger.Info($"{SysUtil.Fn} Course caption corrected. Found: {course.Caption}.Corrected: {courseCaption}.");
											}
											else
											{
												SaLogger.Warn($"{SysUtil.Fn} Error updating Course caption: {ReferenceGolfMgr.ErrorStateMessage}");
												return;
											}
										}

										if (maxCourses > 0 && courseCount > maxCourses)
										{
											SaLogger.Info($"{SysUtil.Fn} Maximum number of courses processed ({maxCourses}).");
											return;
										}
									}
								}
							}
						}
					}
				}
				// Invoke local function.
				CompareCourseCaptions();
			}

			// Log statistics.
			SaLogger.Info($"{SysUtil.Fn} Course caption corrected: {courseCaptionsCorrected}. Courses checked: {courseCount}.", true);

			SaLogger.Info($"{SysUtil.Fn} END Elapsed: {DateUtil.FormatNumberOfSeconds((DateTime.Now - startedAt).TotalSeconds)}", true);
			return OK;
		}

		//------------------------------------------------------------------
		/// <summary>
		/// Identify the column for each marker property.
		/// </summary>
		private class MarkerColumnData
		{
			public int ColIdx_Tee { get; set; } = -1;
			public int ColIdx_Gender { get; set; } = -1;
			public int ColIdx_Par { get; set; } = -1;
			public int ColIdx_CourseRating { get; set; } = -1;
			public int ColIdx_BogeyRating { get; set; } = -1;
			public int ColIdx_SlopeRating { get; set; } = -1;
			public int ColIdx_Front9 { get; set; } = -1;
			public int ColIdx_Back9 { get; set; } = -1;
			public int ColIdx_Front9CourseRating { get; set; } = -1;
			public int ColIdx_Back9CourseRating { get; set; } = -1;
			public int ColIdx_Front9BogeyRating { get; set; } = -1;
			public int ColIdx_Back9BogeyRating { get; set; } = -1;
			public int ColIdx_Front9SlopeRating { get; set; } = -1;
			public int ColIdx_Back9SlopeRating { get; set; } = -1;
			public int ColIdx_MarkerId { get; set; } = -1;
			public int ColIdx_Length { get; set; } = -1;
		}
		/// <summary>
		/// Class to allow markers list to be serialised as 'Markers'.
		/// </summary>
		private class MarkersData
		{
			public List<MarkerData> Markers { get; } = new List<MarkerData>();
		}
		/// <summary>
		/// USGA Markers for course.
		/// </summary>
		public List<MarkerData> LookupUsgaMarkersForCourse(SysId usgaCourseId)
		{
			MarkersData markersData = new MarkersData();
			if (!SysId.IsValid(usgaCourseId)) return markersData.Markers;
			MarkerColumnData markerColumns = new MarkerColumnData();

			using (DisposableCollection<SacsDisposable> disposableItems = new DisposableCollection<SacsDisposable>())
			{
				// Retrieve and validate course.
				SaxGolfUsgaCourse.Params prmCourse = new SaxGolfUsgaCourse.Params();
				prmCourse.UsgaCourseIdsIncluded.Add(usgaCourseId);
				SaxGolfUsgaCourse usgaCourse = ReferenceGolfMgr.GetUsgaCourse(prmCourse);
				disposableItems.Add(usgaCourse);
				if (usgaCourse == null)
				{
					SaLogger.Warn($"{SysUtil.Fn} USGA Course not found for Id '{usgaCourseId}'.");
					return markersData.Markers;
				}
				if (usgaCourse.Priority)
				{
					usgaCourse.Priority = false;
					ReferenceGolfMgr.UpdateUsgaCourse(usgaCourse, DbUpdateInfo);
				}

				try
				{
					// Query course markers on USGA website.
					CancellationToken cancelToken = new CancellationToken();
					SacsHttpRequestData request = new SacsHttpRequestData(cancelToken) { Url = $"https://ncrdb.usga.org/courseTeeInfo?CourseID={usgaCourseId}" };
					SacsHttpResponseData response = SacsHttpClient.DoRequest(request).Result;
					String data = response.Data;

					if (!StringUtil.IsBlank(response.Data))
					{
						// Parse HTML.
						HtmlDocument htmlDoc = new HtmlDocument();
						htmlDoc.LoadHtml(response.Data);

						// Locate countries list.
						HtmlNode teeTableNode = htmlDoc.GetElementbyId("gvTee");
						if (teeTableNode != null)
						{
							// Identify marker columns.
							int teeColumnIdx = -1;
							foreach (HtmlNode teeHeaderNode in teeTableNode.SelectNodes("./tr[1]/th"))
							{
								teeColumnIdx++;
								String headerText = StringUtil.TrimWhitespace(WebUtil.HtmlDecode(teeHeaderNode.InnerText));
								if (StringUtil.IsBlank(headerText) || headerText.EqualsCI("CH"))
								{
									// Columns ignored.
								}
								else if (headerText.EqualsCI("Tee Name")) markerColumns.ColIdx_Tee = teeColumnIdx;
								else if (headerText.EqualsCI("Gender")) markerColumns.ColIdx_Gender = teeColumnIdx;
								else if (headerText.EqualsCI("Par")) markerColumns.ColIdx_Par = teeColumnIdx;
								else if (headerText.EqualsCI("TeeId")) markerColumns.ColIdx_MarkerId = teeColumnIdx;
								else if (headerText.EqualsCI("Length")) markerColumns.ColIdx_Length = teeColumnIdx;
								// Specific part-course (9 holes) values.
								else if (headerText.EqualsCI("RatingF9")) markerColumns.ColIdx_Front9CourseRating = teeColumnIdx;
								else if (headerText.EqualsCI("RatingB9")) markerColumns.ColIdx_Back9CourseRating = teeColumnIdx;
								else if (headerText.EqualsCI("Front (9)")) markerColumns.ColIdx_Front9 = teeColumnIdx;
								else if (headerText.EqualsCI("Back (9)")) markerColumns.ColIdx_Back9 = teeColumnIdx;
								else if (headerText.EqualsCI("Bogey Rating (F9)")) markerColumns.ColIdx_Front9BogeyRating = teeColumnIdx;
								else if (headerText.EqualsCI("Bogey Rating (B9)")) markerColumns.ColIdx_Back9BogeyRating = teeColumnIdx;
								else if (headerText.EqualsCI("Slope (F9)")) markerColumns.ColIdx_Front9SlopeRating = teeColumnIdx;
								else if (headerText.EqualsCI("Slope (B9)")) markerColumns.ColIdx_Back9SlopeRating = teeColumnIdx;
								// Less specific full-course (commonly 18 holes) values.
								else if (headerText.IsMatchCI("Course Rating")) markerColumns.ColIdx_CourseRating = teeColumnIdx;
								else if (headerText.IsMatchCI("Bogey Rating")) markerColumns.ColIdx_BogeyRating = teeColumnIdx;
								else if (headerText.IsMatchCI("Slope Rating")) markerColumns.ColIdx_SlopeRating = teeColumnIdx;
								else SaLogger.Info($"{SysUtil.Fn} Unexpected Marker column header: {headerText}");
							}
							foreach (HtmlNode teeRowNode in teeTableNode.SelectNodes("./tr[position()>1]"))
							{
								MarkerData markerData18 = new MarkerData(usgaCourseId, 18, "18");
								MarkerData markerDataF9 = new MarkerData(usgaCourseId, 9, "F9");
								MarkerData markerDataB9 = new MarkerData(usgaCourseId, 9, "B9");
								teeColumnIdx = -1;
								foreach (HtmlNode teePropertyNode in teeRowNode.SelectNodes("./td"))
								{
									teeColumnIdx++;
									String propertyText = StringUtil.TrimWhitespace(WebUtil.HtmlDecode(teePropertyNode.InnerText));
									if (markerColumns.ColIdx_Tee == teeColumnIdx)
									{
										markerData18.Tee = propertyText;
										markerDataF9.Tee = propertyText;
										markerDataB9.Tee = propertyText;
									}
									else if (markerColumns.ColIdx_Gender == teeColumnIdx)
									{
										markerData18.Gender = propertyText;
										markerDataF9.Gender = propertyText;
										markerDataB9.Gender = propertyText;
									}
									else if (markerColumns.ColIdx_Par == teeColumnIdx)
									{
										markerData18.Par = StringUtil.ToShort(propertyText);
										if (markerData18.Par <= 50)
										{
											markerData18.Holes = 9;
											markerData18.HolesCaption = "9";
										}
									}
									else if (markerColumns.ColIdx_CourseRating == teeColumnIdx)
									{
										markerData18.CourseRating = StringUtil.ToDecimal(propertyText);
									}
									else if (markerColumns.ColIdx_BogeyRating == teeColumnIdx)
									{
										markerData18.BogeyRating = StringUtil.ToDecimal(propertyText);
									}
									else if (markerColumns.ColIdx_SlopeRating == teeColumnIdx)
									{
										markerData18.SlopeRating = StringUtil.ToShort(propertyText);
									}
									else if (markerColumns.ColIdx_Front9CourseRating == teeColumnIdx)
									{
										if (markerDataF9.CourseRating == 0)
											markerDataF9.CourseRating = StringUtil.ToDecimal(propertyText);
									}
									else if (markerColumns.ColIdx_Back9CourseRating == teeColumnIdx)
									{
										if (markerDataB9.CourseRating == 0)
											markerDataB9.CourseRating = StringUtil.ToDecimal(propertyText);
									}
									else if (markerColumns.ColIdx_Front9 == teeColumnIdx)
									{
										// Format: CourseRating / SlopeRating.
										markerDataF9.CourseRating = StringUtil.ToDecimal(Regex.Replace(propertyText, "\\s*/.*$", ""));
										markerDataF9.SlopeRating = StringUtil.ToShort(Regex.Replace(propertyText, "^.*/\\s*", ""));
									}
									else if (markerColumns.ColIdx_Back9 == teeColumnIdx)
									{
										// Format: CourseRating / SlopeRating.
										markerDataB9.CourseRating = StringUtil.ToDecimal(Regex.Replace(propertyText, "\\s*/.*$", ""));
										markerDataB9.SlopeRating = StringUtil.ToShort(Regex.Replace(propertyText, "^.*/\\s*", ""));
									}
									else if (markerColumns.ColIdx_Front9BogeyRating == teeColumnIdx)
									{
										markerDataF9.BogeyRating = StringUtil.ToDecimal(propertyText);
									}
									else if (markerColumns.ColIdx_Back9BogeyRating == teeColumnIdx)
									{
										markerDataB9.BogeyRating = StringUtil.ToDecimal(propertyText);
									}
									else if (markerColumns.ColIdx_Front9SlopeRating == teeColumnIdx)
									{
										if (markerDataF9.SlopeRating == 0)
											markerDataF9.SlopeRating = StringUtil.ToShort(propertyText);
									}
									else if (markerColumns.ColIdx_Back9SlopeRating == teeColumnIdx)
									{
										// Back 9 Slope Rating populated with value for Front 9.
										//markerDataB9.SlopeRating = StringUtil.ToInt(propertyText);
									}
									else if (markerColumns.ColIdx_MarkerId == teeColumnIdx)
									{
										markerData18.MarkerId = StringUtil.ToSysId(propertyText);
										// Unclear if Tee Id is relevant to 9 holes.
										//markerDataF9.MarkerId = StringUtil.ToSysId(propertyText);
										//markerDataB9.MarkerId = StringUtil.ToSysId(propertyText);
									}
									else if (markerColumns.ColIdx_Length == teeColumnIdx)
									{
										markerData18.Length = StringUtil.ToShort(propertyText);
									}
								}

								// Front and Back 9 likely missing par values - derive from course ratings.
								if (markerData18.IsValid() && markerDataF9.IsValid(true) && markerDataF9.Par == 0 && markerDataB9.IsValid(true) && markerDataB9.Par == 0)
								{
									decimal combined9sCourseRating = markerDataF9.CourseRating + markerDataB9.CourseRating;
									markerDataF9.Par = (short)Math.Round(markerData18.Par * markerDataF9.CourseRating / combined9sCourseRating);
									markerDataB9.Par = (short)(markerData18.Par - markerDataF9.Par);
								}

								// Add valid markers to list.
								if (markerData18.IsValid())
									markersData.Markers.Add(markerData18);
								if (markerDataF9.IsValid())
									markersData.Markers.Add(markerDataF9);
								if (markerDataB9.IsValid())
									markersData.Markers.Add(markerDataB9);
							}
						}
						else SaLogger.Warn(SysUtil.Fn + " Tee table not found.");
					}

					// Store markers.
					List<SaxReferenceGolfMgr.UsgaMarkerData> markersUpdateData = new List<SaxReferenceGolfMgr.UsgaMarkerData>();
					foreach (MarkerData usgaMarker in markersData.Markers)
					{
						if (!SysId.IsValid(usgaMarker.CourseId)) continue;
						//if (!SysId.IsValid(usgaMarker.MarkerId)) continue;
						if (StringUtil.IsBlank(usgaMarker.Caption)) continue;
						SaxReferenceGolfMgr.UsgaMarkerData markerData = new SaxReferenceGolfMgr.UsgaMarkerData();
						markerData.UsgaCourseId = usgaMarker.CourseId;
						markerData.UsgaMarkerId = usgaMarker.MarkerId;
						markerData.Caption = usgaMarker.Caption;
						markerData.Holes = usgaMarker.Holes;
						markerData.HolesCaption = usgaMarker.HolesCaption;
						markerData.Tee = usgaMarker.Tee;
						markerData.Gender = usgaMarker.Gender;
						markerData.SlopeRating = usgaMarker.SlopeRating;
						markerData.Par = usgaMarker.Par;
						markerData.CourseRating = usgaMarker.CourseRating;
						markerData.BogeyRating = usgaMarker.BogeyRating;
						markerData.Length = usgaMarker.Length;
						markersUpdateData.Add(markerData);
					}
					ReferenceGolfMgr.BulkUpdateUsgaMarkers(markersUpdateData, DbUpdateInfo);
					if (!ReferenceGolfMgr.HasErrorState())
					{
						// Markers stored OK - update Course.
						if (markersData.Markers.Any())
							usgaCourse.MarkersData = JsonConvert.SerializeObject(markersData);
						usgaCourse.MarkersUpdateAt = DateTime.Now;
						usgaCourse.Priority = false;
						ReferenceGolfMgr.UpdateUsgaCourse(usgaCourse, DbUpdateInfo);
					}
					else SaLogger.Warn($"{SysUtil.Fn} Error bulk updating USGA markers: {ReferenceGolfMgr.ErrorStateMessage}.");
				}
				catch (Exception exc)
				{
					SaLogger.Warn(SysUtil.Fn + " " + ExceptionUtil.GetLogMessage(exc, $"querying USGA markers for course Id: {usgaCourseId}", 0));
				}
			}
			return markersData.Markers;
		}

		//------------------------------------------------------------------
	}
	//======================================================================
}