using Microsoft.Net.Http.Headers; using Sarand.Net.Standard.Base.App; using Sarand.Net.Standard.Base.Core; using Sarand.Net.Standard.Base.Tasks; using Sarand.Net.Standard.Base.Utils; using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.IO.Compression; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; using System.Text; using System.Threading; using System.Threading.Tasks; using ZstdNet; namespace Sarand.Net.Standard.Base.Net { //======================================================================= public class SacsHttpRequestData : SacsTaskData { public SacsHttpRequestData(CancellationToken? cancellationToken) : base(cancellationToken) { } /// URL to send request to. public String Url { get; set; } /// /// Request timeout. Default: 5 secs. /// NB Not currently supported as can only be changed when static HTTP Client not in use. /// public int TimeoutSecs { get; set; } = SacsHttpClient.RequestTimeoutSecs_Default; /// /// Headers to send with request. /// Accept, Accept-Charset, Accept-Encoding, Accept-Language, User-Agent, Transfer-Encoding, Pragma, Expect etc. /// public Dictionary> RequestHeaders { get; } = new Dictionary>(); /// /// Content Headers to send with request. /// Content-Length (bytes), Content-Type (media type), Content-Encoding (compression algorithm), Content-Language etc. /// Content-Type header is defaulted for POST request if not specified. /// public Dictionary> ContentHeaders { get; } = new Dictionary>(); /// /// Form values to post as 'URL Encoded'. /// Content-Type defaults to 'application/x-www-form-urlencoded; charset=utf-8'. /// public List> FormData { get; } = new List>(); /// /// Data to POST. Ignored if FormData populated. /// Content-Type defaults to 'text/plain; charset=utf-8'. /// public String PostData { get; set; } /// Use POST request rather than GET even if no data to post. public bool ForcePost { get; set; } } //======================================================================= public class SacsHttpCookieData { public String HeaderValue { get; set; } public String Name { get; set; } public String Value { get; set; } public DateTime Expires { get; set; } public long MaxAge { get; set; } public String Path { get; set; } public String Domain { get; set; } public bool Secure { get; set; } public bool HttpOnly { get; set; } public Dictionary Properties { get; } = new Dictionary(); } //======================================================================= public class SacsHttpResponseData { public DateTime RequestStarted { get; set; } public DateTime RequestCompleted { get; set; } /// Response headers. public Dictionary> ResponseHeaders { get; } = new Dictionary>(); /// Response content headers. public Dictionary> ContentHeaders { get; } = new Dictionary>(); /// Response cookies from 'Set-Cookie' header. public Dictionary Cookies { get; } = new Dictionary(); /// Indicates whether Status Code represents success. public bool Success { get; set; } /// HTTP Status Code. public HttpStatusCode StatusCode { get; set; } /// HTTP message version. Default: 1.1 public Version Version { get; set; } /// HTTP Reason Code. public String ReasonPhrase { get; set; } = ""; /// Response data. public String Data { get; set; } = ""; /// Elapsed time for request. public TimeSpan Elapsed { get; set; } /// Exception thrown during handling of request. public Exception Exception { get; set; } /// Error state message - generally exception type and message. public String ErrorStateMessage { get; set; } } //======================================================================= public static class SacsHttpClient { //------------------------------------------------------------------- // Data members. private static readonly HttpClient httpClient; //------------------------------------------------------------------- // Static initialisation. static SacsHttpClient() { // Configure client. httpClient = new HttpClient(); httpClient.Timeout = new TimeSpan(0, 0, RequestTimeoutSecs_Default); ProductInfoHeaderValue productInfoHeaderValue = new ProductInfoHeaderValue("Saratoga", "1.2.3"); httpClient.DefaultRequestHeaders.UserAgent.Add(productInfoHeaderValue); } //------------------------------------------------------------------- // Properties. public static HttpClient HttpClient { get { return httpClient; } } public static int RequestTimeoutSecs_Default { get; } = 5; public static String HeaderName_ContentType { get; } = HeaderNames.ContentType; //------------------------------------------------------------------- public static async Task DoRequest(SacsHttpRequestData requestData) { SacsHttpResponseData responseData = new SacsHttpResponseData(); requestData.ThrowIfCancellationRequested(); // Construct content to post. HttpContent postContent = null; if (requestData.FormData.Count > 0) { postContent = new FormUrlEncodedContent(requestData.FormData); if (!requestData.ContentHeaders.ContainsKey(HeaderName_ContentType)) requestData.ContentHeaders.Add(HeaderName_ContentType, "application/x-www-form-urlencoded; charset=utf-8"); } else if (!StringUtil.IsEmpty(requestData.PostData) || requestData.ForcePost) { postContent = new StringContent(StringUtil.EmptyOnNull(requestData.PostData), Encoding.UTF8); if (!requestData.ContentHeaders.ContainsKey(HeaderName_ContentType)) requestData.ContentHeaders.Add(HeaderName_ContentType, "text/plain; charset=utf-8"); } requestData.ThrowIfCancellationRequested(); // Make request. DateTime startedAt = DateTime.Now; try { // Request. using (HttpRequestMessage request = new HttpRequestMessage()) { request.RequestUri = new Uri(requestData.Url); // Request headers. foreach (String headerName in requestData.RequestHeaders.Keys) { if (request.Headers.Contains(headerName)) request.Headers.Remove(headerName); request.Headers.Add(headerName, requestData.RequestHeaders[headerName]); } // Default request headers. if (httpClient.DefaultRequestHeaders.Accept.Count == 0 && request.Headers.Accept.Count == 0) request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("text/*")); // Configure request. HttpResponseMessage response = null; // Send request. if (postContent != null) { // POST. request.Method = HttpMethod.Post; request.Content = postContent; // Content headers. foreach (String headerName in requestData.ContentHeaders.Keys) { if (request.Content.Headers.Contains(headerName)) request.Content.Headers.Remove(headerName); request.Content.Headers.Add(headerName, requestData.ContentHeaders[headerName]); } // Send request. responseData.RequestStarted = DateTime.Now; response = await httpClient.SendAsync(request, requestData.CancellationToken); responseData.RequestCompleted = DateTime.Now; } else { // GET. request.Method = HttpMethod.Get; responseData.RequestStarted = DateTime.Now; response = await httpClient.SendAsync(request, requestData.CancellationToken); responseData.RequestCompleted = DateTime.Now; } // Extract response data. responseData.Version = response.Version; responseData.Success = response.IsSuccessStatusCode; responseData.StatusCode = response.StatusCode; responseData.ReasonPhrase = response.ReasonPhrase; // Extract response headers. if (response.Headers != null) { foreach (var header in response.Headers) responseData.ResponseHeaders[header.Key] = new List(header.Value); } // Extract response content headers. if (response.Content != null && response.Content.Headers != null) { foreach (var header in response.Content.Headers) responseData.ContentHeaders[header.Key] = new List(header.Value); } // Extract response cookies. if (response.Headers.Contains(HeaderNames.SetCookie)) { foreach (var cookieHeader in response.Headers.GetValues(HeaderNames.SetCookie)) { SacsHttpCookieData cookie = ExtractCookieFromHeader(cookieHeader); if (cookie != null && !StringUtil.IsBlank(cookie.Name)) responseData.Cookies.Add(cookie.Name, cookie); } } // Extract response payload. var contentEncodings = responseData.ContentHeaders.GetValueOrDefault(HeaderNames.ContentEncoding, new List()); using (var responseStream = await response.Content.ReadAsStreamAsync()) { // Check if response needs to be decompressed, as directed by 'content-encoding' header. Stream decompressedStream = null; if (contentEncodings.ContainsCI("gzip")) decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress); else if (contentEncodings.ContainsCI("deflate")) decompressedStream = new DeflateStream(responseStream, CompressionMode.Decompress); else if (contentEncodings.ContainsCI("br")) decompressedStream = new BrotliStream(responseStream, CompressionMode.Decompress); else if (contentEncodings.ContainsCI("zstd")) decompressedStream = new DecompressionStream(responseStream); // Read the response payload, decompressed if necessary. using (var streamReader = new StreamReader(decompressedStream != null ? decompressedStream : responseStream)) responseData.Data = await streamReader.ReadToEndAsync(); // Clean-up. if (decompressedStream != null) decompressedStream.Dispose(); } // Clean-up. if (response != null) response.Dispose(); } } catch (Exception exc) { // Ignore if task has been cancelled. if (requestData.IsCancellationRequested) throw exc; // Record exception. responseData.Exception = exc; if (exc.InnerException != null) { exc = exc.InnerException; } String msg = ExceptionUtil.GetErrorStateMessage(exc); SaLogger.Warn(SysUtil.Fn + " " + msg + ExceptionUtil.GetStackEntries(exc, 6).Concat("", "\n\t")); responseData.ErrorStateMessage = msg; } requestData.ThrowIfCancellationRequested(); // Complete response data. responseData.Elapsed = DateTime.Now - startedAt; return responseData; } //------------------------------------------------------------------- // Cookies. //------------------------------------------------------------------- public static SacsHttpCookieData ExtractCookieFromHeader(String cookieHeader) { SacsHttpCookieData cookie = null; String[] cookieParts = cookieHeader.Split(';'); if (cookieParts.Count() > 1) { cookie = new SacsHttpCookieData(); cookie.HeaderValue = cookieHeader; // Name and value. String[] cookieNameValue = cookieParts[0].Trim().Split(new char[] { '=' }, 2); cookie.Name = cookieNameValue[0].Trim(); cookie.Value = cookieNameValue.Count() > 1 ? cookieNameValue[1].Trim() : ""; // Other properties. for (int i = 1; i < cookieParts.Length; i++) { string[] properties = cookieParts[i].Trim().Split(new char[] { '=' }, 2); String propertyName = properties[0].Trim(); String propertyValue = properties.Count() > 1 ? properties[1].Trim() : "True"; cookie.Properties.Add(propertyName, propertyValue); if (propertyName.EqualsCI("Domain")) cookie.Domain = propertyValue; else if (propertyName.EqualsCI("Path")) cookie.Path = propertyValue; else if (propertyName.EqualsCI("Expires")) { cookie.Expires = DateUtil.ParseExact(propertyValue, "ddd, d MMM yyyy HH:mm:ss GMT"); if (!DateUtil.IsValid(cookie.Expires)) { DateTime expires; DateTime.TryParse(propertyValue, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out expires); cookie.Expires = expires; } } else if (propertyName.EqualsCI("Max-Age")) cookie.MaxAge = StringUtil.ToLong(propertyValue); else if (propertyName.EqualsCI("Secure")) cookie.Secure = StringUtil.ToBool(propertyValue); else if (propertyName.EqualsCI("HttpOnly")) cookie.HttpOnly = StringUtil.ToBool(propertyValue); } } return cookie; } //------------------------------------------------------------------- // R e s p o n s e S t a t u s D e s c r i p t i o n //------------------------------------------------------------------- public static String GetResponseStatusDescription(HttpStatusCode httpStatus) { // Ref. https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml switch (httpStatus) { // 1xx: Informational - Request received, continuing process. case HttpStatusCode.Continue: return "Continue"; // 2xx: Success - The action was successfully received, understood, and accepted. case HttpStatusCode.OK: return "OK"; case HttpStatusCode.Created: return "Created"; case HttpStatusCode.Accepted: return "Accepted"; // 3xx: Redirection - Further action must be taken in order to complete the request. case HttpStatusCode.MultipleChoices: return "Multiple choices"; case HttpStatusCode.MovedPermanently: return "Moved permanently"; case HttpStatusCode.Found: return "Found"; // 4xx: Client Error -The request contains bad syntax or cannot be fulfilled. case HttpStatusCode.BadRequest: return "Bad request"; case HttpStatusCode.Unauthorized: return "Unauthorized"; case HttpStatusCode.PaymentRequired: return "Payment required"; case HttpStatusCode.Forbidden: return "Forbidden"; case HttpStatusCode.NotFound: return "Not found"; case HttpStatusCode.MethodNotAllowed: return "Method not allowed"; case HttpStatusCode.NotAcceptable: return "Not acceptable"; // 5xx: Server Error -The server failed to fulfill an apparently valid request. case HttpStatusCode.InternalServerError: return "Internal server error"; } return httpStatus.ToString(); } //------------------------------------------------------------------- } //======================================================================= }